From 6e68b42128080a16423c7b155032cad81147835e Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 27 Jan 2026 10:57:27 +0800 Subject: [PATCH] Enhance Agent Documentation and Add Extract Command Functionality - Updated README files to include new commands for running tests and extracting results for review. - Introduced the `yao agent extract` command to facilitate extraction of test results from JSONL files into Markdown or JSON formats. - Enhanced the `FormatAvailableResources` function to support localization and detailed information for agents and MCP tools. - Improved output formatting for better readability and usability in test result documentation. --- agent/README.md | 7 + agent/robot/executor/standard/input.go | 153 ++++++- .../standard/input_integration_test.go | 372 ++++++++++++++++++ agent/test/README.md | 45 +++ agent/test/extract.go | 189 +++++++++ cmd/agent/agent.go | 5 + cmd/agent/extract.go | 87 ++++ cmd/root.go | 1 + 8 files changed, 847 insertions(+), 12 deletions(-) create mode 100644 agent/robot/executor/standard/input_integration_test.go create mode 100644 agent/test/extract.go create mode 100644 cmd/agent/extract.go diff --git a/agent/README.md b/agent/README.md index 74018c48..edef1170 100644 --- a/agent/README.md +++ b/agent/README.md @@ -69,7 +69,14 @@ function Next(ctx: agent.Context, payload: agent.Payload): agent.Next { ### 3. Test (Optional) ```bash +# Run tests yao agent test -i "Hello, how are you?" + +# Run tests from JSONL file +yao agent test -i tests/inputs.jsonl -v + +# Extract results for review +yao agent extract output-*.jsonl ``` ### 4. Run diff --git a/agent/robot/executor/standard/input.go b/agent/robot/executor/standard/input.go index d02d08dd..e1ac8242 100644 --- a/agent/robot/executor/standard/input.go +++ b/agent/robot/executor/standard/input.go @@ -1,11 +1,15 @@ package standard import ( + "context" "encoding/json" "fmt" "strings" + "github.com/yaoapp/gou/mcp" + "github.com/yaoapp/yao/agent/assistant" agentcontext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/i18n" robottypes "github.com/yaoapp/yao/agent/robot/types" ) @@ -115,6 +119,15 @@ func (f *InputFormatter) FormatRobotIdentity(robot *robottypes.Robot) string { // This is critical for generating achievable goals - without knowing available tools, // the agent might generate goals that cannot be accomplished func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) string { + locale := "en" // default locale + if robot != nil && robot.Config != nil { + locale = robot.Config.GetDefaultLocale() + } + return f.FormatAvailableResourcesWithLocale(robot, locale) +} + +// FormatAvailableResourcesWithLocale formats available resources with specific locale for i18n support +func (f *InputFormatter) FormatAvailableResourcesWithLocale(robot *robottypes.Robot, locale string) string { if robot == nil || robot.Config == nil { return "" } @@ -122,36 +135,152 @@ func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) strin var sb strings.Builder hasContent := false - // Available Agents + // Available Agents - with detailed information (name, description) if robot.Config.Resources != nil && len(robot.Config.Resources.Agents) > 0 { if !hasContent { sb.WriteString("## Available Resources\n\n") hasContent = true } sb.WriteString("### Agents\n") - sb.WriteString("These are the AI assistants you can delegate tasks to:\n") - for _, agent := range robot.Config.Resources.Agents { - sb.WriteString(fmt.Sprintf("- **%s**\n", agent)) + sb.WriteString("These are the AI assistants you can delegate tasks to:\n\n") + for _, agentID := range robot.Config.Resources.Agents { + // Try to get agent details + ast, err := assistant.Get(agentID) + if err != nil { + // Fallback to just ID if agent not found + sb.WriteString(fmt.Sprintf("- **%s**\n", agentID)) + continue + } + + // Get localized name and description + name := i18n.Translate(agentID, locale, ast.Name).(string) + description := "" + if ast.Description != "" { + description = i18n.Translate(agentID, locale, ast.Description).(string) + } + + // Format agent info + sb.WriteString(fmt.Sprintf("- **%s** (`%s`)\n", name, agentID)) + if description != "" { + sb.WriteString(fmt.Sprintf(" - %s\n", description)) + } } sb.WriteString("\n") } - // Available MCP Tools + // Available MCP Tools - with detailed tool information if robot.Config.Resources != nil && len(robot.Config.Resources.MCP) > 0 { if !hasContent { sb.WriteString("## Available Resources\n\n") hasContent = true } sb.WriteString("### MCP Tools\n") - sb.WriteString("These are the external tools and services you can use:\n") - for _, mcp := range robot.Config.Resources.MCP { - if len(mcp.Tools) > 0 { - sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcp.ID, strings.Join(mcp.Tools, ", "))) - } else { - sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcp.ID)) + sb.WriteString("These are the external tools and services you can use:\n\n") + for _, mcpConfig := range robot.Config.Resources.MCP { + // Try to get MCP client and list tools + client, err := mcp.Select(mcpConfig.ID) + if err != nil { + // Fallback to basic info if client not found + if len(mcpConfig.Tools) > 0 { + sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcpConfig.ID, strings.Join(mcpConfig.Tools, ", "))) + } else { + sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcpConfig.ID)) + } + continue } + + // Get client info for name and description + clientInfo := client.Info() + clientName := mcpConfig.ID + clientDesc := "" + if clientInfo != nil { + if clientInfo.Name != "" { + clientName = clientInfo.Name + } + if clientInfo.Description != "" { + clientDesc = clientInfo.Description + } + } + + // Write MCP header + if clientDesc != "" { + sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n", clientName, mcpConfig.ID)) + sb.WriteString(fmt.Sprintf("%s\n\n", clientDesc)) + } else { + sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n\n", clientName, mcpConfig.ID)) + } + + // Try to list tools with context + ctx := context.Background() + toolsResp, err := client.ListTools(ctx, "") + if err != nil || toolsResp == nil { + // Fallback to configured tools + if len(mcpConfig.Tools) > 0 { + sb.WriteString("Available tools: ") + sb.WriteString(strings.Join(mcpConfig.Tools, ", ")) + sb.WriteString("\n\n") + } else { + sb.WriteString("All tools available\n\n") + } + continue + } + + // Filter tools if specific tools are configured + toolsToShow := toolsResp.Tools + if len(mcpConfig.Tools) > 0 { + // Create a map for quick lookup + allowedTools := make(map[string]bool) + for _, t := range mcpConfig.Tools { + allowedTools[t] = true + } + // Filter tools + var filteredTools []struct { + Name string + Description string + } + for _, tool := range toolsResp.Tools { + if allowedTools[tool.Name] { + filteredTools = append(filteredTools, struct { + Name string + Description string + }{tool.Name, tool.Description}) + } + } + // Write filtered tools + if len(filteredTools) > 0 { + sb.WriteString("| Tool | Description |\n") + sb.WriteString("|------|-------------|\n") + for _, tool := range filteredTools { + desc := tool.Description + if len(desc) > 100 { + desc = desc[:97] + "..." + } + // Escape pipe characters in description + desc = strings.ReplaceAll(desc, "|", "\\|") + sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc)) + } + } else { + sb.WriteString("Configured tools: ") + sb.WriteString(strings.Join(mcpConfig.Tools, ", ")) + } + } else if len(toolsToShow) > 0 { + // Show all available tools + sb.WriteString("| Tool | Description |\n") + sb.WriteString("|------|-------------|\n") + for _, tool := range toolsToShow { + desc := tool.Description + if len(desc) > 100 { + desc = desc[:97] + "..." + } + // Escape pipe characters in description + desc = strings.ReplaceAll(desc, "|", "\\|") + sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc)) + } + } else { + sb.WriteString("No tools available\n") + } + sb.WriteString("\n") } - sb.WriteString("\n") } // Available Knowledge Base diff --git a/agent/robot/executor/standard/input_integration_test.go b/agent/robot/executor/standard/input_integration_test.go new file mode 100644 index 00000000..47297af5 --- /dev/null +++ b/agent/robot/executor/standard/input_integration_test.go @@ -0,0 +1,372 @@ +package standard_test + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/yao/agent/robot/executor/standard" + "github.com/yaoapp/yao/agent/robot/types" + "github.com/yaoapp/yao/agent/testutils" +) + +// ============================================================================ +// InputFormatter Integration Tests with Real Data +// These tests use the yao-dev-app environment with real assistants and MCPs +// ============================================================================ + +func TestFormatAvailableResourcesIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + formatter := standard.NewInputFormatter() + + t.Run("formats_real_agents_with_details", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-agents", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Robot", + Duties: []string{"Testing agent formatting"}, + }, + DefaultLocale: "en", + Resources: &types.Resources{ + // Use real agents from yao-dev-app/assistants + Agents: []string{ + "experts.data-analyst", + "experts.text-writer", + "experts.summarizer", + }, + }, + }, + } + + result := formatter.FormatAvailableResources(robot) + + // Verify structure + assert.Contains(t, result, "## Available Resources") + assert.Contains(t, result, "### Agents") + assert.Contains(t, result, "These are the AI assistants you can delegate tasks to:") + + // Verify real agent details are included + // experts.data-analyst should show name and description + assert.Contains(t, result, "experts.data-analyst") + assert.Contains(t, result, "Data Analyst Expert") // Name from package.yao + + // experts.text-writer + assert.Contains(t, result, "experts.text-writer") + + // experts.summarizer + assert.Contains(t, result, "experts.summarizer") + + // Verify important note is present + assert.Contains(t, result, "Only plan goals and tasks that can be accomplished") + + t.Logf("Formatted agents result:\n%s", result) + }) + + t.Run("formats_real_mcp_with_tool_details", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-mcp", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Robot", + Duties: []string{"Testing MCP formatting"}, + }, + DefaultLocale: "en", + Resources: &types.Resources{ + // Use real MCPs from yao-dev-app/mcps + MCP: []types.MCPConfig{ + {ID: "echo", Tools: []string{"ping", "status"}}, // Specific tools + {ID: "echo"}, // All tools + }, + }, + }, + } + + result := formatter.FormatAvailableResources(robot) + + // Verify structure + assert.Contains(t, result, "## Available Resources") + assert.Contains(t, result, "### MCP Tools") + assert.Contains(t, result, "These are the external tools and services you can use:") + + // Verify MCP details + assert.Contains(t, result, "echo") + + // Verify important note is present + assert.Contains(t, result, "Only plan goals and tasks that can be accomplished") + + t.Logf("Formatted MCP result:\n%s", result) + }) + + t.Run("formats_combined_resources", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-combined", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Sales Analyst Robot", + Duties: []string{"Analyze sales data", "Generate reports"}, + Rules: []string{"Be accurate", "Be concise"}, + }, + DefaultLocale: "en", + Resources: &types.Resources{ + Agents: []string{ + "experts.data-analyst", + "experts.summarizer", + }, + MCP: []types.MCPConfig{ + {ID: "echo", Tools: []string{"ping", "echo"}}, + }, + }, + KB: &types.KB{ + Collections: []string{"sales-policies", "product-catalog"}, + }, + DB: &types.DB{ + Models: []string{"sales", "customers", "orders"}, + }, + }, + } + + result := formatter.FormatAvailableResources(robot) + + // Verify all sections are present + assert.Contains(t, result, "## Available Resources") + assert.Contains(t, result, "### Agents") + assert.Contains(t, result, "### MCP Tools") + assert.Contains(t, result, "### Knowledge Base") + assert.Contains(t, result, "### Database") + + // Verify agents + assert.Contains(t, result, "experts.data-analyst") + assert.Contains(t, result, "experts.summarizer") + + // Verify MCP + assert.Contains(t, result, "echo") + + // Verify KB + assert.Contains(t, result, "sales-policies") + assert.Contains(t, result, "product-catalog") + + // Verify DB + assert.Contains(t, result, "sales") + assert.Contains(t, result, "customers") + assert.Contains(t, result, "orders") + + t.Logf("Formatted combined resources result:\n%s", result) + }) + + t.Run("handles_locale_zh", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-zh", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "测试机器人", + Duties: []string{"测试国际化"}, + }, + DefaultLocale: "zh-cn", + Resources: &types.Resources{ + // Use agents that have zh-cn locales + Agents: []string{ + "hello", // This agent has locales/zh-cn.yml + "mohe", // This agent also has locales/zh-cn.yml + }, + }, + }, + } + + result := formatter.FormatAvailableResourcesWithLocale(robot, "zh-cn") + + // Verify structure + assert.Contains(t, result, "## Available Resources") + assert.Contains(t, result, "### Agents") + + // Verify agents are listed + assert.Contains(t, result, "hello") + assert.Contains(t, result, "mohe") + + t.Logf("Formatted zh-cn result:\n%s", result) + }) + + t.Run("gracefully_handles_missing_agents", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-missing", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Robot", + }, + Resources: &types.Resources{ + Agents: []string{ + "non-existent-agent", + "experts.data-analyst", // This one exists + "another-missing-agent", + }, + }, + }, + } + + result := formatter.FormatAvailableResources(robot) + + // Should not panic, should include fallback for missing agents + assert.Contains(t, result, "## Available Resources") + assert.Contains(t, result, "### Agents") + + // Missing agents should still be listed with just ID + assert.Contains(t, result, "non-existent-agent") + assert.Contains(t, result, "another-missing-agent") + + // Existing agent should have full details + assert.Contains(t, result, "experts.data-analyst") + assert.Contains(t, result, "Data Analyst Expert") + + t.Logf("Formatted with missing agents:\n%s", result) + }) + + t.Run("gracefully_handles_missing_mcp", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-missing-mcp", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Robot", + }, + Resources: &types.Resources{ + MCP: []types.MCPConfig{ + {ID: "non-existent-mcp", Tools: []string{"tool1", "tool2"}}, + {ID: "echo"}, // This one exists + }, + }, + }, + } + + result := formatter.FormatAvailableResources(robot) + + // Should not panic, should include fallback for missing MCP + assert.Contains(t, result, "## Available Resources") + assert.Contains(t, result, "### MCP Tools") + + // Missing MCP should still be listed with fallback + assert.Contains(t, result, "non-existent-mcp") + assert.Contains(t, result, "tool1, tool2") + + // Existing MCP should have details + assert.Contains(t, result, "echo") + + t.Logf("Formatted with missing MCP:\n%s", result) + }) +} + +func TestFormatAvailableResourcesTableFormat(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + formatter := standard.NewInputFormatter() + + t.Run("mcp_tools_in_table_format", func(t *testing.T) { + robot := &types.Robot{ + MemberID: "test-robot-table", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Robot", + }, + Resources: &types.Resources{ + MCP: []types.MCPConfig{ + {ID: "echo"}, // All tools - should show table + }, + }, + }, + } + + result := formatter.FormatAvailableResources(robot) + + // Check if table format is used when tools are available + // Table headers: | Tool | Description | + if strings.Contains(result, "| Tool | Description |") { + assert.Contains(t, result, "|------|-------------|") + t.Logf("MCP tools displayed in table format:\n%s", result) + } else { + // Fallback format + t.Logf("MCP tools displayed in fallback format:\n%s", result) + } + }) +} + +func TestFormatClockContextWithRobotIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + formatter := standard.NewInputFormatter() + + t.Run("full_context_for_inspiration", func(t *testing.T) { + // Create a realistic robot configuration + robot := &types.Robot{ + MemberID: "sales-robot-001", + TeamID: "team-001", + DisplayName: "Sales Analyst Robot", + AutonomousMode: true, + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Sales Analyst", + Duties: []string{"Monitor sales performance", "Generate daily reports", "Alert on anomalies"}, + Rules: []string{"Only use approved data sources", "Maintain confidentiality"}, + }, + DefaultLocale: "en", + Resources: &types.Resources{ + Agents: []string{ + "experts.data-analyst", + "experts.summarizer", + }, + MCP: []types.MCPConfig{ + {ID: "echo", Tools: []string{"ping"}}, + }, + }, + }, + } + + // Create clock context + clock := types.NewClockContext(time.Now(), "UTC") + + // Format clock context (includes robot identity) + clockContent := formatter.FormatClockContext(clock, robot) + + // Format available resources + resourcesContent := formatter.FormatAvailableResources(robot) + + // Combine for full context (as done in inspiration.go) + fullContext := clockContent + "\n\n" + resourcesContent + + // Verify full context contains all necessary information + require.NotEmpty(t, fullContext) + + // Time context + assert.Contains(t, fullContext, "## Current Time Context") + assert.Contains(t, fullContext, "### Time Markers") + + // Robot identity + assert.Contains(t, fullContext, "## Robot Identity") + assert.Contains(t, fullContext, "Sales Analyst") + assert.Contains(t, fullContext, "Monitor sales performance") + assert.Contains(t, fullContext, "Only use approved data sources") + + // Available resources + assert.Contains(t, fullContext, "## Available Resources") + assert.Contains(t, fullContext, "### Agents") + assert.Contains(t, fullContext, "experts.data-analyst") + assert.Contains(t, fullContext, "### MCP Tools") + + t.Logf("Full context for inspiration:\n%s", fullContext) + }) +} diff --git a/agent/test/README.md b/agent/test/README.md index f3f7039c..a4d0db8b 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -1428,6 +1428,51 @@ Simulates user behavior for dynamic mode testing. yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent ``` +## Extract Command + +Extract test results from output JSONL file to individual Markdown or JSON files for human review: + +```bash +# Extract to Markdown files (default) +yao agent extract output-20260127104118.jsonl + +# Specify output directory +yao agent extract output.jsonl -o ./reports/ + +# Extract to JSON format +yao agent extract output.jsonl --format json +``` + +### Extract Command Options + +| Flag | Description | Default | +| ---------- | ---------------------------------------- | ---------- | +| `-o` | Output directory | same as input | +| `--format` | Output format: `markdown`, `json` | `markdown` | + +### Output Format (Markdown) + +Each test result is extracted to a separate file: + +```markdown +# T001-销售分析师-月末周五 + +**Status**: ✅ PASSED + +**Duration**: 16743ms + +## Input +(Full input content in markdown code block) + +## Output +(Agent's response content) +``` + +This is useful for: +- Human review of agent outputs +- Comparing results across test runs +- Documentation and reporting + ## Exit Codes | Code | Description | diff --git a/agent/test/extract.go b/agent/test/extract.go new file mode 100644 index 00000000..babc52ee --- /dev/null +++ b/agent/test/extract.go @@ -0,0 +1,189 @@ +package test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + jsoniter "github.com/json-iterator/go" +) + +// ExtractOptions represents options for extracting test results +type ExtractOptions struct { + // InputFile is the path to the output JSONL file from test run + InputFile string + + // OutputDir is the directory to write extracted files (default: same as input file) + OutputDir string + + // Format is the output format: "markdown" (default), "json" + Format string +} + +// Extractor extracts test results to individual files for review +type Extractor struct { + opts *ExtractOptions +} + +// NewExtractor creates a new extractor +func NewExtractor(opts *ExtractOptions) *Extractor { + if opts.Format == "" { + opts.Format = "markdown" + } + if opts.OutputDir == "" { + opts.OutputDir = filepath.Dir(opts.InputFile) + } + return &Extractor{opts: opts} +} + +// Extract reads the test output file and extracts results to individual files +func (e *Extractor) Extract() ([]string, error) { + // Read the JSONL file + data, err := os.ReadFile(e.opts.InputFile) + if err != nil { + return nil, fmt.Errorf("failed to read input file: %w", err) + } + + // Parse the JSON (the output file is a single JSON object, not JSONL) + var report Report + if err := jsoniter.Unmarshal(data, &report); err != nil { + return nil, fmt.Errorf("failed to parse test report: %w", err) + } + + // Create output directory if it doesn't exist + if err := os.MkdirAll(e.opts.OutputDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create output directory: %w", err) + } + + var extractedFiles []string + + // Extract each result + for _, result := range report.Results { + var filename string + var content string + + switch e.opts.Format { + case "markdown": + filename = filepath.Join(e.opts.OutputDir, result.ID+".md") + content = e.formatMarkdown(result) + case "json": + filename = filepath.Join(e.opts.OutputDir, result.ID+".json") + jsonBytes, err := jsoniter.MarshalIndent(result, "", " ") + if err != nil { + return extractedFiles, fmt.Errorf("failed to marshal result %s: %w", result.ID, err) + } + content = string(jsonBytes) + default: + return nil, fmt.Errorf("unsupported format: %s", e.opts.Format) + } + + if err := os.WriteFile(filename, []byte(content), 0644); err != nil { + return extractedFiles, fmt.Errorf("failed to write file %s: %w", filename, err) + } + + extractedFiles = append(extractedFiles, filename) + } + + return extractedFiles, nil +} + +// formatMarkdown formats a single test result as Markdown +func (e *Extractor) formatMarkdown(result *Result) string { + var sb strings.Builder + + // Title + sb.WriteString(fmt.Sprintf("# %s\n\n", result.ID)) + + // Status badge + switch result.Status { + case StatusPassed: + sb.WriteString("**Status**: ✅ PASSED\n\n") + case StatusFailed: + sb.WriteString("**Status**: ❌ FAILED\n\n") + case StatusError: + sb.WriteString("**Status**: ⚠️ ERROR\n\n") + case StatusTimeout: + sb.WriteString("**Status**: ⏱️ TIMEOUT\n\n") + case StatusSkipped: + sb.WriteString("**Status**: ⏭️ SKIPPED\n\n") + } + + // Duration + sb.WriteString(fmt.Sprintf("**Duration**: %dms\n\n", result.DurationMs)) + + // Error (if any) + if result.Error != "" { + sb.WriteString("## Error\n\n") + sb.WriteString("```\n") + sb.WriteString(result.Error) + sb.WriteString("\n```\n\n") + } + + // Input + sb.WriteString("## Input\n\n") + sb.WriteString("```markdown\n") + sb.WriteString(formatInputAsString(result.Input)) + sb.WriteString("\n```\n\n") + + // Output + sb.WriteString("## Output\n\n") + output := formatOutputAsString(result.Output) + // Remove markdown code block wrapper if present + output = strings.TrimPrefix(output, "```markdown\n") + output = strings.TrimSuffix(output, "\n```") + output = strings.TrimSuffix(output, "```") + sb.WriteString(output) + sb.WriteString("\n") + + return sb.String() +} + +// formatInputAsString converts input to string format +func formatInputAsString(input interface{}) string { + switch v := input.(type) { + case string: + return v + case map[string]interface{}: + // Single message format + if content, ok := v["content"].(string); ok { + return content + } + // Fallback to JSON + jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ") + return string(jsonBytes) + case []interface{}: + // Array of messages - extract content from last user message + for i := len(v) - 1; i >= 0; i-- { + if msg, ok := v[i].(map[string]interface{}); ok { + if role, ok := msg["role"].(string); ok && role == "user" { + if content, ok := msg["content"].(string); ok { + return content + } + } + } + } + // Fallback to JSON + jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ") + return string(jsonBytes) + default: + jsonBytes, _ := jsoniter.MarshalIndent(input, "", " ") + return string(jsonBytes) + } +} + +// formatOutputAsString converts output to string format +func formatOutputAsString(output interface{}) string { + switch v := output.(type) { + case string: + return v + case map[string]interface{}, []interface{}: + jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ") + return string(jsonBytes) + default: + if output == nil { + return "(no output)" + } + return fmt.Sprintf("%v", output) + } +} diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index b7eb2549..8fbde5e6 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -35,6 +35,11 @@ var langs = map[string]string{ "Error: agent (-n) is required when using direct message input and not in an agent directory": "错误: 使用直接消息输入且不在智能体目录时需要指定 -n 参数", "Hint: Make sure you're in a Yao application directory or specify --app flag": "提示: 确保在 Yao 应用目录中或使用 --app 参数指定", "Error: invalid timeout format": "错误: 无效的超时格式", + // Extract command + "Extract test results to individual files for review": "提取测试结果到单独的文件供审查", + "Extract test results from output JSONL file to individual Markdown or JSON files": "从输出 JSONL 文件中提取测试结果到单独的 Markdown 或 JSON 文件", + "Output directory (default: same as input file)": "输出目录 (默认: 与输入文件相同)", + "Output format: markdown, json": "输出格式: markdown, json", } // L Language switch diff --git a/cmd/agent/extract.go b/cmd/agent/extract.go new file mode 100644 index 00000000..2f7b49c8 --- /dev/null +++ b/cmd/agent/extract.go @@ -0,0 +1,87 @@ +package agent + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/yaoapp/yao/agent/test" +) + +// Extract command flags +var ( + extractOutput string + extractFormat string +) + +// ExtractCmd is the agent extract command +var ExtractCmd = &cobra.Command{ + Use: "extract ", + Short: L("Extract test results to individual files for review"), + Long: L("Extract test results from output JSONL file to individual Markdown or JSON files"), + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + inputFile := args[0] + + // Resolve absolute path + absPath, err := filepath.Abs(inputFile) + if err != nil { + color.Red("Error: %s\n", err.Error()) + os.Exit(1) + } + + // Check if file exists + if _, err := os.Stat(absPath); os.IsNotExist(err) { + color.Red("Error: file not found: %s\n", absPath) + os.Exit(1) + } + + // Build extract options + opts := &test.ExtractOptions{ + InputFile: absPath, + OutputDir: extractOutput, + Format: extractFormat, + } + + // Create extractor and run + extractor := test.NewExtractor(opts) + files, err := extractor.Extract() + if err != nil { + color.Red("Error: %s\n", err.Error()) + os.Exit(1) + } + + // Print results + fmt.Println() + color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════") + color.New(color.FgGreen, color.Bold).Println(" Extract Complete") + color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════") + fmt.Println() + + for _, file := range files { + color.New(color.FgGreen).Printf("✓ ") + fmt.Printf("Written: %s\n", filepath.Base(file)) + } + + fmt.Println() + color.New(color.FgWhite).Printf(" Total: ") + color.New(color.FgCyan).Printf("%d files\n", len(files)) + + if extractOutput != "" { + color.New(color.FgWhite).Printf(" Output: ") + color.New(color.FgCyan).Printf("%s\n", extractOutput) + } else { + color.New(color.FgWhite).Printf(" Output: ") + color.New(color.FgCyan).Printf("%s\n", filepath.Dir(absPath)) + } + fmt.Println() + }, +} + +func init() { + // Extract command flags + ExtractCmd.Flags().StringVarP(&extractOutput, "output", "o", "", L("Output directory (default: same as input file)")) + ExtractCmd.Flags().StringVar(&extractFormat, "format", "markdown", L("Output format: markdown, json")) +} diff --git a/cmd/root.go b/cmd/root.go index b89f4c96..55ba5095 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -152,6 +152,7 @@ func init() { // Agent agentCmd.AddCommand(agent.TestCmd) + agentCmd.AddCommand(agent.ExtractCmd) rootCmd.AddCommand( versionCmd,