Enhance Agent Documentation and Add Extract Command Functionality
- Updated README files to include new commands for running tests and extracting results for review. - Introduced the `yao agent extract` command to facilitate extraction of test results from JSONL files into Markdown or JSON formats. - Enhanced the `FormatAvailableResources` function to support localization and detailed information for agents and MCP tools. - Improved output formatting for better readability and usability in test result documentation.
This commit is contained in:
Max
parent
00b6b5ee0e
commit
6e68b42128
8 changed files with 847 additions and 12 deletions
|
|
@ -69,7 +69,14 @@ function Next(ctx: agent.Context, payload: agent.Payload): agent.Next {
|
|||
### 3. Test (Optional)
|
||||
|
||||
```bash
|
||||
# Run tests
|
||||
yao agent test -i "Hello, how are you?"
|
||||
|
||||
# Run tests from JSONL file
|
||||
yao agent test -i tests/inputs.jsonl -v
|
||||
|
||||
# Extract results for review
|
||||
yao agent extract output-*.jsonl
|
||||
```
|
||||
|
||||
### 4. Run
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
package standard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/gou/mcp"
|
||||
"github.com/yaoapp/yao/agent/assistant"
|
||||
agentcontext "github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/i18n"
|
||||
robottypes "github.com/yaoapp/yao/agent/robot/types"
|
||||
)
|
||||
|
||||
|
|
@ -115,6 +119,15 @@ func (f *InputFormatter) FormatRobotIdentity(robot *robottypes.Robot) string {
|
|||
// This is critical for generating achievable goals - without knowing available tools,
|
||||
// the agent might generate goals that cannot be accomplished
|
||||
func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) string {
|
||||
locale := "en" // default locale
|
||||
if robot != nil && robot.Config != nil {
|
||||
locale = robot.Config.GetDefaultLocale()
|
||||
}
|
||||
return f.FormatAvailableResourcesWithLocale(robot, locale)
|
||||
}
|
||||
|
||||
// FormatAvailableResourcesWithLocale formats available resources with specific locale for i18n support
|
||||
func (f *InputFormatter) FormatAvailableResourcesWithLocale(robot *robottypes.Robot, locale string) string {
|
||||
if robot == nil || robot.Config == nil {
|
||||
return ""
|
||||
}
|
||||
|
|
@ -122,36 +135,152 @@ func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) strin
|
|||
var sb strings.Builder
|
||||
hasContent := false
|
||||
|
||||
// Available Agents
|
||||
// Available Agents - with detailed information (name, description)
|
||||
if robot.Config.Resources != nil && len(robot.Config.Resources.Agents) > 0 {
|
||||
if !hasContent {
|
||||
sb.WriteString("## Available Resources\n\n")
|
||||
hasContent = true
|
||||
}
|
||||
sb.WriteString("### Agents\n")
|
||||
sb.WriteString("These are the AI assistants you can delegate tasks to:\n")
|
||||
for _, agent := range robot.Config.Resources.Agents {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**\n", agent))
|
||||
sb.WriteString("These are the AI assistants you can delegate tasks to:\n\n")
|
||||
for _, agentID := range robot.Config.Resources.Agents {
|
||||
// Try to get agent details
|
||||
ast, err := assistant.Get(agentID)
|
||||
if err != nil {
|
||||
// Fallback to just ID if agent not found
|
||||
sb.WriteString(fmt.Sprintf("- **%s**\n", agentID))
|
||||
continue
|
||||
}
|
||||
|
||||
// Get localized name and description
|
||||
name := i18n.Translate(agentID, locale, ast.Name).(string)
|
||||
description := ""
|
||||
if ast.Description != "" {
|
||||
description = i18n.Translate(agentID, locale, ast.Description).(string)
|
||||
}
|
||||
|
||||
// Format agent info
|
||||
sb.WriteString(fmt.Sprintf("- **%s** (`%s`)\n", name, agentID))
|
||||
if description != "" {
|
||||
sb.WriteString(fmt.Sprintf(" - %s\n", description))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
// Available MCP Tools
|
||||
// Available MCP Tools - with detailed tool information
|
||||
if robot.Config.Resources != nil && len(robot.Config.Resources.MCP) > 0 {
|
||||
if !hasContent {
|
||||
sb.WriteString("## Available Resources\n\n")
|
||||
hasContent = true
|
||||
}
|
||||
sb.WriteString("### MCP Tools\n")
|
||||
sb.WriteString("These are the external tools and services you can use:\n")
|
||||
for _, mcp := range robot.Config.Resources.MCP {
|
||||
if len(mcp.Tools) > 0 {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcp.ID, strings.Join(mcp.Tools, ", ")))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcp.ID))
|
||||
sb.WriteString("These are the external tools and services you can use:\n\n")
|
||||
for _, mcpConfig := range robot.Config.Resources.MCP {
|
||||
// Try to get MCP client and list tools
|
||||
client, err := mcp.Select(mcpConfig.ID)
|
||||
if err != nil {
|
||||
// Fallback to basic info if client not found
|
||||
if len(mcpConfig.Tools) > 0 {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcpConfig.ID, strings.Join(mcpConfig.Tools, ", ")))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcpConfig.ID))
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Get client info for name and description
|
||||
clientInfo := client.Info()
|
||||
clientName := mcpConfig.ID
|
||||
clientDesc := ""
|
||||
if clientInfo != nil {
|
||||
if clientInfo.Name != "" {
|
||||
clientName = clientInfo.Name
|
||||
}
|
||||
if clientInfo.Description != "" {
|
||||
clientDesc = clientInfo.Description
|
||||
}
|
||||
}
|
||||
|
||||
// Write MCP header
|
||||
if clientDesc != "" {
|
||||
sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n", clientName, mcpConfig.ID))
|
||||
sb.WriteString(fmt.Sprintf("%s\n\n", clientDesc))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n\n", clientName, mcpConfig.ID))
|
||||
}
|
||||
|
||||
// Try to list tools with context
|
||||
ctx := context.Background()
|
||||
toolsResp, err := client.ListTools(ctx, "")
|
||||
if err != nil || toolsResp == nil {
|
||||
// Fallback to configured tools
|
||||
if len(mcpConfig.Tools) > 0 {
|
||||
sb.WriteString("Available tools: ")
|
||||
sb.WriteString(strings.Join(mcpConfig.Tools, ", "))
|
||||
sb.WriteString("\n\n")
|
||||
} else {
|
||||
sb.WriteString("All tools available\n\n")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Filter tools if specific tools are configured
|
||||
toolsToShow := toolsResp.Tools
|
||||
if len(mcpConfig.Tools) > 0 {
|
||||
// Create a map for quick lookup
|
||||
allowedTools := make(map[string]bool)
|
||||
for _, t := range mcpConfig.Tools {
|
||||
allowedTools[t] = true
|
||||
}
|
||||
// Filter tools
|
||||
var filteredTools []struct {
|
||||
Name string
|
||||
Description string
|
||||
}
|
||||
for _, tool := range toolsResp.Tools {
|
||||
if allowedTools[tool.Name] {
|
||||
filteredTools = append(filteredTools, struct {
|
||||
Name string
|
||||
Description string
|
||||
}{tool.Name, tool.Description})
|
||||
}
|
||||
}
|
||||
// Write filtered tools
|
||||
if len(filteredTools) > 0 {
|
||||
sb.WriteString("| Tool | Description |\n")
|
||||
sb.WriteString("|------|-------------|\n")
|
||||
for _, tool := range filteredTools {
|
||||
desc := tool.Description
|
||||
if len(desc) > 100 {
|
||||
desc = desc[:97] + "..."
|
||||
}
|
||||
// Escape pipe characters in description
|
||||
desc = strings.ReplaceAll(desc, "|", "\\|")
|
||||
sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc))
|
||||
}
|
||||
} else {
|
||||
sb.WriteString("Configured tools: ")
|
||||
sb.WriteString(strings.Join(mcpConfig.Tools, ", "))
|
||||
}
|
||||
} else if len(toolsToShow) > 0 {
|
||||
// Show all available tools
|
||||
sb.WriteString("| Tool | Description |\n")
|
||||
sb.WriteString("|------|-------------|\n")
|
||||
for _, tool := range toolsToShow {
|
||||
desc := tool.Description
|
||||
if len(desc) > 100 {
|
||||
desc = desc[:97] + "..."
|
||||
}
|
||||
// Escape pipe characters in description
|
||||
desc = strings.ReplaceAll(desc, "|", "\\|")
|
||||
sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc))
|
||||
}
|
||||
} else {
|
||||
sb.WriteString("No tools available\n")
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
// Available Knowledge Base
|
||||
|
|
|
|||
372
agent/robot/executor/standard/input_integration_test.go
Normal file
372
agent/robot/executor/standard/input_integration_test.go
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
package standard_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/yaoapp/yao/agent/robot/executor/standard"
|
||||
"github.com/yaoapp/yao/agent/robot/types"
|
||||
"github.com/yaoapp/yao/agent/testutils"
|
||||
)
|
||||
|
||||
// ============================================================================
|
||||
// InputFormatter Integration Tests with Real Data
|
||||
// These tests use the yao-dev-app environment with real assistants and MCPs
|
||||
// ============================================================================
|
||||
|
||||
func TestFormatAvailableResourcesIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
formatter := standard.NewInputFormatter()
|
||||
|
||||
t.Run("formats_real_agents_with_details", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-agents",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
Duties: []string{"Testing agent formatting"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
// Use real agents from yao-dev-app/assistants
|
||||
Agents: []string{
|
||||
"experts.data-analyst",
|
||||
"experts.text-writer",
|
||||
"experts.summarizer",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Verify structure
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
assert.Contains(t, result, "These are the AI assistants you can delegate tasks to:")
|
||||
|
||||
// Verify real agent details are included
|
||||
// experts.data-analyst should show name and description
|
||||
assert.Contains(t, result, "experts.data-analyst")
|
||||
assert.Contains(t, result, "Data Analyst Expert") // Name from package.yao
|
||||
|
||||
// experts.text-writer
|
||||
assert.Contains(t, result, "experts.text-writer")
|
||||
|
||||
// experts.summarizer
|
||||
assert.Contains(t, result, "experts.summarizer")
|
||||
|
||||
// Verify important note is present
|
||||
assert.Contains(t, result, "Only plan goals and tasks that can be accomplished")
|
||||
|
||||
t.Logf("Formatted agents result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("formats_real_mcp_with_tool_details", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-mcp",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
Duties: []string{"Testing MCP formatting"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
// Use real MCPs from yao-dev-app/mcps
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo", Tools: []string{"ping", "status"}}, // Specific tools
|
||||
{ID: "echo"}, // All tools
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Verify structure
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### MCP Tools")
|
||||
assert.Contains(t, result, "These are the external tools and services you can use:")
|
||||
|
||||
// Verify MCP details
|
||||
assert.Contains(t, result, "echo")
|
||||
|
||||
// Verify important note is present
|
||||
assert.Contains(t, result, "Only plan goals and tasks that can be accomplished")
|
||||
|
||||
t.Logf("Formatted MCP result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("formats_combined_resources", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-combined",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Sales Analyst Robot",
|
||||
Duties: []string{"Analyze sales data", "Generate reports"},
|
||||
Rules: []string{"Be accurate", "Be concise"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
Agents: []string{
|
||||
"experts.data-analyst",
|
||||
"experts.summarizer",
|
||||
},
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo", Tools: []string{"ping", "echo"}},
|
||||
},
|
||||
},
|
||||
KB: &types.KB{
|
||||
Collections: []string{"sales-policies", "product-catalog"},
|
||||
},
|
||||
DB: &types.DB{
|
||||
Models: []string{"sales", "customers", "orders"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Verify all sections are present
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
assert.Contains(t, result, "### MCP Tools")
|
||||
assert.Contains(t, result, "### Knowledge Base")
|
||||
assert.Contains(t, result, "### Database")
|
||||
|
||||
// Verify agents
|
||||
assert.Contains(t, result, "experts.data-analyst")
|
||||
assert.Contains(t, result, "experts.summarizer")
|
||||
|
||||
// Verify MCP
|
||||
assert.Contains(t, result, "echo")
|
||||
|
||||
// Verify KB
|
||||
assert.Contains(t, result, "sales-policies")
|
||||
assert.Contains(t, result, "product-catalog")
|
||||
|
||||
// Verify DB
|
||||
assert.Contains(t, result, "sales")
|
||||
assert.Contains(t, result, "customers")
|
||||
assert.Contains(t, result, "orders")
|
||||
|
||||
t.Logf("Formatted combined resources result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("handles_locale_zh", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-zh",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "测试机器人",
|
||||
Duties: []string{"测试国际化"},
|
||||
},
|
||||
DefaultLocale: "zh-cn",
|
||||
Resources: &types.Resources{
|
||||
// Use agents that have zh-cn locales
|
||||
Agents: []string{
|
||||
"hello", // This agent has locales/zh-cn.yml
|
||||
"mohe", // This agent also has locales/zh-cn.yml
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResourcesWithLocale(robot, "zh-cn")
|
||||
|
||||
// Verify structure
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
|
||||
// Verify agents are listed
|
||||
assert.Contains(t, result, "hello")
|
||||
assert.Contains(t, result, "mohe")
|
||||
|
||||
t.Logf("Formatted zh-cn result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("gracefully_handles_missing_agents", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-missing",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
},
|
||||
Resources: &types.Resources{
|
||||
Agents: []string{
|
||||
"non-existent-agent",
|
||||
"experts.data-analyst", // This one exists
|
||||
"another-missing-agent",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Should not panic, should include fallback for missing agents
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
|
||||
// Missing agents should still be listed with just ID
|
||||
assert.Contains(t, result, "non-existent-agent")
|
||||
assert.Contains(t, result, "another-missing-agent")
|
||||
|
||||
// Existing agent should have full details
|
||||
assert.Contains(t, result, "experts.data-analyst")
|
||||
assert.Contains(t, result, "Data Analyst Expert")
|
||||
|
||||
t.Logf("Formatted with missing agents:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("gracefully_handles_missing_mcp", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-missing-mcp",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
},
|
||||
Resources: &types.Resources{
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "non-existent-mcp", Tools: []string{"tool1", "tool2"}},
|
||||
{ID: "echo"}, // This one exists
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Should not panic, should include fallback for missing MCP
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### MCP Tools")
|
||||
|
||||
// Missing MCP should still be listed with fallback
|
||||
assert.Contains(t, result, "non-existent-mcp")
|
||||
assert.Contains(t, result, "tool1, tool2")
|
||||
|
||||
// Existing MCP should have details
|
||||
assert.Contains(t, result, "echo")
|
||||
|
||||
t.Logf("Formatted with missing MCP:\n%s", result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFormatAvailableResourcesTableFormat(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
formatter := standard.NewInputFormatter()
|
||||
|
||||
t.Run("mcp_tools_in_table_format", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-table",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
},
|
||||
Resources: &types.Resources{
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo"}, // All tools - should show table
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Check if table format is used when tools are available
|
||||
// Table headers: | Tool | Description |
|
||||
if strings.Contains(result, "| Tool | Description |") {
|
||||
assert.Contains(t, result, "|------|-------------|")
|
||||
t.Logf("MCP tools displayed in table format:\n%s", result)
|
||||
} else {
|
||||
// Fallback format
|
||||
t.Logf("MCP tools displayed in fallback format:\n%s", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFormatClockContextWithRobotIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
formatter := standard.NewInputFormatter()
|
||||
|
||||
t.Run("full_context_for_inspiration", func(t *testing.T) {
|
||||
// Create a realistic robot configuration
|
||||
robot := &types.Robot{
|
||||
MemberID: "sales-robot-001",
|
||||
TeamID: "team-001",
|
||||
DisplayName: "Sales Analyst Robot",
|
||||
AutonomousMode: true,
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Sales Analyst",
|
||||
Duties: []string{"Monitor sales performance", "Generate daily reports", "Alert on anomalies"},
|
||||
Rules: []string{"Only use approved data sources", "Maintain confidentiality"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
Agents: []string{
|
||||
"experts.data-analyst",
|
||||
"experts.summarizer",
|
||||
},
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo", Tools: []string{"ping"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Create clock context
|
||||
clock := types.NewClockContext(time.Now(), "UTC")
|
||||
|
||||
// Format clock context (includes robot identity)
|
||||
clockContent := formatter.FormatClockContext(clock, robot)
|
||||
|
||||
// Format available resources
|
||||
resourcesContent := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Combine for full context (as done in inspiration.go)
|
||||
fullContext := clockContent + "\n\n" + resourcesContent
|
||||
|
||||
// Verify full context contains all necessary information
|
||||
require.NotEmpty(t, fullContext)
|
||||
|
||||
// Time context
|
||||
assert.Contains(t, fullContext, "## Current Time Context")
|
||||
assert.Contains(t, fullContext, "### Time Markers")
|
||||
|
||||
// Robot identity
|
||||
assert.Contains(t, fullContext, "## Robot Identity")
|
||||
assert.Contains(t, fullContext, "Sales Analyst")
|
||||
assert.Contains(t, fullContext, "Monitor sales performance")
|
||||
assert.Contains(t, fullContext, "Only use approved data sources")
|
||||
|
||||
// Available resources
|
||||
assert.Contains(t, fullContext, "## Available Resources")
|
||||
assert.Contains(t, fullContext, "### Agents")
|
||||
assert.Contains(t, fullContext, "experts.data-analyst")
|
||||
assert.Contains(t, fullContext, "### MCP Tools")
|
||||
|
||||
t.Logf("Full context for inspiration:\n%s", fullContext)
|
||||
})
|
||||
}
|
||||
|
|
@ -1428,6 +1428,51 @@ Simulates user behavior for dynamic mode testing.
|
|||
yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent
|
||||
```
|
||||
|
||||
## Extract Command
|
||||
|
||||
Extract test results from output JSONL file to individual Markdown or JSON files for human review:
|
||||
|
||||
```bash
|
||||
# Extract to Markdown files (default)
|
||||
yao agent extract output-20260127104118.jsonl
|
||||
|
||||
# Specify output directory
|
||||
yao agent extract output.jsonl -o ./reports/
|
||||
|
||||
# Extract to JSON format
|
||||
yao agent extract output.jsonl --format json
|
||||
```
|
||||
|
||||
### Extract Command Options
|
||||
|
||||
| Flag | Description | Default |
|
||||
| ---------- | ---------------------------------------- | ---------- |
|
||||
| `-o` | Output directory | same as input |
|
||||
| `--format` | Output format: `markdown`, `json` | `markdown` |
|
||||
|
||||
### Output Format (Markdown)
|
||||
|
||||
Each test result is extracted to a separate file:
|
||||
|
||||
```markdown
|
||||
# T001-销售分析师-月末周五
|
||||
|
||||
**Status**: ✅ PASSED
|
||||
|
||||
**Duration**: 16743ms
|
||||
|
||||
## Input
|
||||
(Full input content in markdown code block)
|
||||
|
||||
## Output
|
||||
(Agent's response content)
|
||||
```
|
||||
|
||||
This is useful for:
|
||||
- Human review of agent outputs
|
||||
- Comparing results across test runs
|
||||
- Documentation and reporting
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Description |
|
||||
|
|
|
|||
189
agent/test/extract.go
Normal file
189
agent/test/extract.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
// ExtractOptions represents options for extracting test results
|
||||
type ExtractOptions struct {
|
||||
// InputFile is the path to the output JSONL file from test run
|
||||
InputFile string
|
||||
|
||||
// OutputDir is the directory to write extracted files (default: same as input file)
|
||||
OutputDir string
|
||||
|
||||
// Format is the output format: "markdown" (default), "json"
|
||||
Format string
|
||||
}
|
||||
|
||||
// Extractor extracts test results to individual files for review
|
||||
type Extractor struct {
|
||||
opts *ExtractOptions
|
||||
}
|
||||
|
||||
// NewExtractor creates a new extractor
|
||||
func NewExtractor(opts *ExtractOptions) *Extractor {
|
||||
if opts.Format == "" {
|
||||
opts.Format = "markdown"
|
||||
}
|
||||
if opts.OutputDir == "" {
|
||||
opts.OutputDir = filepath.Dir(opts.InputFile)
|
||||
}
|
||||
return &Extractor{opts: opts}
|
||||
}
|
||||
|
||||
// Extract reads the test output file and extracts results to individual files
|
||||
func (e *Extractor) Extract() ([]string, error) {
|
||||
// Read the JSONL file
|
||||
data, err := os.ReadFile(e.opts.InputFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read input file: %w", err)
|
||||
}
|
||||
|
||||
// Parse the JSON (the output file is a single JSON object, not JSONL)
|
||||
var report Report
|
||||
if err := jsoniter.Unmarshal(data, &report); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse test report: %w", err)
|
||||
}
|
||||
|
||||
// Create output directory if it doesn't exist
|
||||
if err := os.MkdirAll(e.opts.OutputDir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
|
||||
var extractedFiles []string
|
||||
|
||||
// Extract each result
|
||||
for _, result := range report.Results {
|
||||
var filename string
|
||||
var content string
|
||||
|
||||
switch e.opts.Format {
|
||||
case "markdown":
|
||||
filename = filepath.Join(e.opts.OutputDir, result.ID+".md")
|
||||
content = e.formatMarkdown(result)
|
||||
case "json":
|
||||
filename = filepath.Join(e.opts.OutputDir, result.ID+".json")
|
||||
jsonBytes, err := jsoniter.MarshalIndent(result, "", " ")
|
||||
if err != nil {
|
||||
return extractedFiles, fmt.Errorf("failed to marshal result %s: %w", result.ID, err)
|
||||
}
|
||||
content = string(jsonBytes)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported format: %s", e.opts.Format)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filename, []byte(content), 0644); err != nil {
|
||||
return extractedFiles, fmt.Errorf("failed to write file %s: %w", filename, err)
|
||||
}
|
||||
|
||||
extractedFiles = append(extractedFiles, filename)
|
||||
}
|
||||
|
||||
return extractedFiles, nil
|
||||
}
|
||||
|
||||
// formatMarkdown formats a single test result as Markdown
|
||||
func (e *Extractor) formatMarkdown(result *Result) string {
|
||||
var sb strings.Builder
|
||||
|
||||
// Title
|
||||
sb.WriteString(fmt.Sprintf("# %s\n\n", result.ID))
|
||||
|
||||
// Status badge
|
||||
switch result.Status {
|
||||
case StatusPassed:
|
||||
sb.WriteString("**Status**: ✅ PASSED\n\n")
|
||||
case StatusFailed:
|
||||
sb.WriteString("**Status**: ❌ FAILED\n\n")
|
||||
case StatusError:
|
||||
sb.WriteString("**Status**: ⚠️ ERROR\n\n")
|
||||
case StatusTimeout:
|
||||
sb.WriteString("**Status**: ⏱️ TIMEOUT\n\n")
|
||||
case StatusSkipped:
|
||||
sb.WriteString("**Status**: ⏭️ SKIPPED\n\n")
|
||||
}
|
||||
|
||||
// Duration
|
||||
sb.WriteString(fmt.Sprintf("**Duration**: %dms\n\n", result.DurationMs))
|
||||
|
||||
// Error (if any)
|
||||
if result.Error != "" {
|
||||
sb.WriteString("## Error\n\n")
|
||||
sb.WriteString("```\n")
|
||||
sb.WriteString(result.Error)
|
||||
sb.WriteString("\n```\n\n")
|
||||
}
|
||||
|
||||
// Input
|
||||
sb.WriteString("## Input\n\n")
|
||||
sb.WriteString("```markdown\n")
|
||||
sb.WriteString(formatInputAsString(result.Input))
|
||||
sb.WriteString("\n```\n\n")
|
||||
|
||||
// Output
|
||||
sb.WriteString("## Output\n\n")
|
||||
output := formatOutputAsString(result.Output)
|
||||
// Remove markdown code block wrapper if present
|
||||
output = strings.TrimPrefix(output, "```markdown\n")
|
||||
output = strings.TrimSuffix(output, "\n```")
|
||||
output = strings.TrimSuffix(output, "```")
|
||||
sb.WriteString(output)
|
||||
sb.WriteString("\n")
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// formatInputAsString converts input to string format
|
||||
func formatInputAsString(input interface{}) string {
|
||||
switch v := input.(type) {
|
||||
case string:
|
||||
return v
|
||||
case map[string]interface{}:
|
||||
// Single message format
|
||||
if content, ok := v["content"].(string); ok {
|
||||
return content
|
||||
}
|
||||
// Fallback to JSON
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||
return string(jsonBytes)
|
||||
case []interface{}:
|
||||
// Array of messages - extract content from last user message
|
||||
for i := len(v) - 1; i >= 0; i-- {
|
||||
if msg, ok := v[i].(map[string]interface{}); ok {
|
||||
if role, ok := msg["role"].(string); ok && role == "user" {
|
||||
if content, ok := msg["content"].(string); ok {
|
||||
return content
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback to JSON
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||
return string(jsonBytes)
|
||||
default:
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(input, "", " ")
|
||||
return string(jsonBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// formatOutputAsString converts output to string format
|
||||
func formatOutputAsString(output interface{}) string {
|
||||
switch v := output.(type) {
|
||||
case string:
|
||||
return v
|
||||
case map[string]interface{}, []interface{}:
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||
return string(jsonBytes)
|
||||
default:
|
||||
if output == nil {
|
||||
return "(no output)"
|
||||
}
|
||||
return fmt.Sprintf("%v", output)
|
||||
}
|
||||
}
|
||||
|
|
@ -35,6 +35,11 @@ var langs = map[string]string{
|
|||
"Error: agent (-n) is required when using direct message input and not in an agent directory": "错误: 使用直接消息输入且不在智能体目录时需要指定 -n 参数",
|
||||
"Hint: Make sure you're in a Yao application directory or specify --app flag": "提示: 确保在 Yao 应用目录中或使用 --app 参数指定",
|
||||
"Error: invalid timeout format": "错误: 无效的超时格式",
|
||||
// Extract command
|
||||
"Extract test results to individual files for review": "提取测试结果到单独的文件供审查",
|
||||
"Extract test results from output JSONL file to individual Markdown or JSON files": "从输出 JSONL 文件中提取测试结果到单独的 Markdown 或 JSON 文件",
|
||||
"Output directory (default: same as input file)": "输出目录 (默认: 与输入文件相同)",
|
||||
"Output format: markdown, json": "输出格式: markdown, json",
|
||||
}
|
||||
|
||||
// L Language switch
|
||||
|
|
|
|||
87
cmd/agent/extract.go
Normal file
87
cmd/agent/extract.go
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/yaoapp/yao/agent/test"
|
||||
)
|
||||
|
||||
// Extract command flags
|
||||
var (
|
||||
extractOutput string
|
||||
extractFormat string
|
||||
)
|
||||
|
||||
// ExtractCmd is the agent extract command
|
||||
var ExtractCmd = &cobra.Command{
|
||||
Use: "extract <output-file.jsonl>",
|
||||
Short: L("Extract test results to individual files for review"),
|
||||
Long: L("Extract test results from output JSONL file to individual Markdown or JSON files"),
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
inputFile := args[0]
|
||||
|
||||
// Resolve absolute path
|
||||
absPath, err := filepath.Abs(inputFile)
|
||||
if err != nil {
|
||||
color.Red("Error: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(absPath); os.IsNotExist(err) {
|
||||
color.Red("Error: file not found: %s\n", absPath)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Build extract options
|
||||
opts := &test.ExtractOptions{
|
||||
InputFile: absPath,
|
||||
OutputDir: extractOutput,
|
||||
Format: extractFormat,
|
||||
}
|
||||
|
||||
// Create extractor and run
|
||||
extractor := test.NewExtractor(opts)
|
||||
files, err := extractor.Extract()
|
||||
if err != nil {
|
||||
color.Red("Error: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Print results
|
||||
fmt.Println()
|
||||
color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════")
|
||||
color.New(color.FgGreen, color.Bold).Println(" Extract Complete")
|
||||
color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════")
|
||||
fmt.Println()
|
||||
|
||||
for _, file := range files {
|
||||
color.New(color.FgGreen).Printf("✓ ")
|
||||
fmt.Printf("Written: %s\n", filepath.Base(file))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
color.New(color.FgWhite).Printf(" Total: ")
|
||||
color.New(color.FgCyan).Printf("%d files\n", len(files))
|
||||
|
||||
if extractOutput != "" {
|
||||
color.New(color.FgWhite).Printf(" Output: ")
|
||||
color.New(color.FgCyan).Printf("%s\n", extractOutput)
|
||||
} else {
|
||||
color.New(color.FgWhite).Printf(" Output: ")
|
||||
color.New(color.FgCyan).Printf("%s\n", filepath.Dir(absPath))
|
||||
}
|
||||
fmt.Println()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Extract command flags
|
||||
ExtractCmd.Flags().StringVarP(&extractOutput, "output", "o", "", L("Output directory (default: same as input file)"))
|
||||
ExtractCmd.Flags().StringVar(&extractFormat, "format", "markdown", L("Output format: markdown, json"))
|
||||
}
|
||||
|
|
@ -152,6 +152,7 @@ func init() {
|
|||
|
||||
// Agent
|
||||
agentCmd.AddCommand(agent.TestCmd)
|
||||
agentCmd.AddCommand(agent.ExtractCmd)
|
||||
|
||||
rootCmd.AddCommand(
|
||||
versionCmd,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue