Merge pull request #1433 from trheyi/main
Enhance Agent Documentation and Add Extract Command Functionality
This commit is contained in:
Max
GitHub
commit
ab97ad730d
18 changed files with 1252 additions and 58 deletions
|
|
@ -69,7 +69,14 @@ function Next(ctx: agent.Context, payload: agent.Payload): agent.Next {
|
|||
### 3. Test (Optional)
|
||||
|
||||
```bash
|
||||
# Run tests
|
||||
yao agent test -i "Hello, how are you?"
|
||||
|
||||
# Run tests from JSONL file
|
||||
yao agent test -i tests/inputs.jsonl -v
|
||||
|
||||
# Extract results for review
|
||||
yao agent extract output-*.jsonl
|
||||
```
|
||||
|
||||
### 4. Run
|
||||
|
|
|
|||
|
|
@ -431,9 +431,16 @@ For each task:
|
|||
| Type | ExecutorID Format | Example |
|
||||
|------|-------------------|---------|
|
||||
| `assistant` | Agent ID | `experts.text-writer` |
|
||||
| `mcp` | `clientID.toolName` | `filesystem.read_file` |
|
||||
| `mcp` | `mcp_server.mcp_tool` | `ark.image.text2img.generate` |
|
||||
| `process` | Process name | `models.user.Find` |
|
||||
|
||||
**MCP Task Fields:**
|
||||
|
||||
For MCP tasks, three fields are required:
|
||||
- `executor_id`: Combined format `mcp_server.mcp_tool`
|
||||
- `mcp_server`: MCP server/client ID (e.g., `ark.image.text2img`)
|
||||
- `mcp_tool`: Tool name within the server (e.g., `generate`)
|
||||
|
||||
**Multi-Turn Conversation Flow:**
|
||||
|
||||
For assistant tasks, P3 uses a multi-turn conversation approach:
|
||||
|
|
|
|||
|
|
@ -1322,9 +1322,13 @@ type Task struct {
|
|||
|
||||
// Executor
|
||||
ExecutorType ExecutorType `json:"executor_type"`
|
||||
ExecutorID string `json:"executor_id"`
|
||||
ExecutorID string `json:"executor_id"` // unified ID: agent/assistant/process ID, or "mcp_server.mcp_tool" for MCP
|
||||
Args []any `json:"args,omitempty"`
|
||||
|
||||
// MCP-specific fields (required when executor_type is "mcp")
|
||||
MCPServer string `json:"mcp_server,omitempty"` // MCP server/client ID (e.g., "ark.image.text2img")
|
||||
MCPTool string `json:"mcp_tool,omitempty"` // MCP tool name (e.g., "generate")
|
||||
|
||||
// Validation (defined in P2, used in P3)
|
||||
ExpectedOutput string `json:"expected_output,omitempty"` // what the task should produce
|
||||
// ValidationRules supports two formats:
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ func loadRobotFromDB(memberID string) (*types.Robot, error) {
|
|||
Select: []interface{}{
|
||||
"id", "member_id", "team_id", "display_name", "bio",
|
||||
"system_prompt", "robot_status", "autonomous_mode",
|
||||
"robot_config", "robot_email",
|
||||
"robot_config", "robot_email", "agents", "mcp_servers",
|
||||
},
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "member_id", Value: memberID},
|
||||
|
|
@ -228,7 +228,7 @@ func listRobotsFromDB(query *ListQuery) (*ListResult, error) {
|
|||
Select: []interface{}{
|
||||
"id", "member_id", "team_id", "display_name", "bio",
|
||||
"system_prompt", "robot_status", "autonomous_mode",
|
||||
"robot_config", "robot_email",
|
||||
"robot_config", "robot_email", "agents", "mcp_servers",
|
||||
},
|
||||
Wheres: wheres,
|
||||
Orders: orders,
|
||||
|
|
|
|||
2
agent/robot/cache/load.go
vendored
2
agent/robot/cache/load.go
vendored
|
|
@ -24,6 +24,8 @@ var memberFields = []interface{}{
|
|||
"autonomous_mode",
|
||||
"robot_config",
|
||||
"robot_email",
|
||||
"agents",
|
||||
"mcp_servers",
|
||||
}
|
||||
|
||||
// SetMemberModel sets the member model name
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
package standard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/gou/mcp"
|
||||
"github.com/yaoapp/yao/agent/assistant"
|
||||
agentcontext "github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/i18n"
|
||||
robottypes "github.com/yaoapp/yao/agent/robot/types"
|
||||
)
|
||||
|
||||
|
|
@ -39,40 +43,47 @@ func (f *InputFormatter) FormatClockContext(clock *robottypes.ClockContext, robo
|
|||
sb.WriteString(fmt.Sprintf("- **Day**: %s\n", clock.DayOfWeek))
|
||||
sb.WriteString(fmt.Sprintf("- **Date**: %d/%d/%d\n", clock.Year, clock.Month, clock.DayOfMonth))
|
||||
sb.WriteString(fmt.Sprintf("- **Week**: %d of year\n", clock.WeekOfYear))
|
||||
sb.WriteString(fmt.Sprintf("- **Hour**: %d\n", clock.Hour))
|
||||
sb.WriteString(fmt.Sprintf("- **Timezone**: %s\n", clock.TZ))
|
||||
|
||||
// Time markers
|
||||
// Time markers - show all markers with check/cross for context awareness
|
||||
sb.WriteString("\n### Time Markers\n")
|
||||
if clock.IsWeekend {
|
||||
sb.WriteString("- ✓ Weekend\n")
|
||||
}
|
||||
if clock.IsMonthStart {
|
||||
sb.WriteString("- ✓ Month Start (1st-3rd)\n")
|
||||
}
|
||||
if clock.IsMonthEnd {
|
||||
sb.WriteString("- ✓ Month End (last 3 days)\n")
|
||||
}
|
||||
if clock.IsQuarterEnd {
|
||||
sb.WriteString("- ✓ Quarter End\n")
|
||||
}
|
||||
if clock.IsYearEnd {
|
||||
sb.WriteString("- ✓ Year End\n")
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("- %s Weekend\n", boolMark(clock.IsWeekend)))
|
||||
sb.WriteString(fmt.Sprintf("- %s Month Start (1st-3rd)\n", boolMark(clock.IsMonthStart)))
|
||||
sb.WriteString(fmt.Sprintf("- %s Month End (last 3 days)\n", boolMark(clock.IsMonthEnd)))
|
||||
sb.WriteString(fmt.Sprintf("- %s Quarter End\n", boolMark(clock.IsQuarterEnd)))
|
||||
sb.WriteString(fmt.Sprintf("- %s Year End\n", boolMark(clock.IsYearEnd)))
|
||||
|
||||
// Robot identity section (if available)
|
||||
if robot != nil && robot.Config != nil && robot.Config.Identity != nil {
|
||||
sb.WriteString("\n## Robot Identity\n\n")
|
||||
sb.WriteString(fmt.Sprintf("- **Role**: %s\n", robot.Config.Identity.Role))
|
||||
if len(robot.Config.Identity.Duties) > 0 {
|
||||
sb.WriteString("- **Duties**:\n")
|
||||
for _, duty := range robot.Config.Identity.Duties {
|
||||
sb.WriteString(fmt.Sprintf(" - %s\n", duty))
|
||||
// Robot identity section
|
||||
// Priority: Config.Identity > Robot fields (DisplayName, Bio, SystemPrompt)
|
||||
if robot != nil {
|
||||
if robot.Config != nil && robot.Config.Identity != nil {
|
||||
// Use structured identity from config
|
||||
sb.WriteString("\n## Robot Identity\n\n")
|
||||
sb.WriteString(fmt.Sprintf("- **Role**: %s\n", robot.Config.Identity.Role))
|
||||
if len(robot.Config.Identity.Duties) > 0 {
|
||||
sb.WriteString("- **Duties**:\n")
|
||||
for _, duty := range robot.Config.Identity.Duties {
|
||||
sb.WriteString(fmt.Sprintf(" - %s\n", duty))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(robot.Config.Identity.Rules) > 0 {
|
||||
sb.WriteString("- **Rules**:\n")
|
||||
for _, rule := range robot.Config.Identity.Rules {
|
||||
sb.WriteString(fmt.Sprintf(" - %s\n", rule))
|
||||
if len(robot.Config.Identity.Rules) > 0 {
|
||||
sb.WriteString("- **Rules**:\n")
|
||||
for _, rule := range robot.Config.Identity.Rules {
|
||||
sb.WriteString(fmt.Sprintf(" - %s\n", rule))
|
||||
}
|
||||
}
|
||||
} else if robot.DisplayName != "" || robot.Bio != "" || robot.SystemPrompt != "" {
|
||||
// Fallback: build identity from Robot fields (from __yao.member table)
|
||||
sb.WriteString("\n## Robot Identity\n\n")
|
||||
if robot.DisplayName != "" {
|
||||
sb.WriteString(fmt.Sprintf("- **Role**: %s\n", robot.DisplayName))
|
||||
}
|
||||
if robot.Bio != "" {
|
||||
sb.WriteString(fmt.Sprintf("- **Description**: %s\n", robot.Bio))
|
||||
}
|
||||
if robot.SystemPrompt != "" {
|
||||
sb.WriteString(fmt.Sprintf("- **Instructions**:\n%s\n", robot.SystemPrompt))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -80,6 +91,14 @@ func (f *InputFormatter) FormatClockContext(clock *robottypes.ClockContext, robo
|
|||
return sb.String()
|
||||
}
|
||||
|
||||
// boolMark returns ✓ for true and ✗ for false
|
||||
func boolMark(v bool) string {
|
||||
if v {
|
||||
return "✓"
|
||||
}
|
||||
return "✗"
|
||||
}
|
||||
|
||||
// FormatRobotIdentity formats robot identity as user message content
|
||||
// Used to provide context about the robot's role and duties
|
||||
func (f *InputFormatter) FormatRobotIdentity(robot *robottypes.Robot) string {
|
||||
|
|
@ -115,6 +134,15 @@ func (f *InputFormatter) FormatRobotIdentity(robot *robottypes.Robot) string {
|
|||
// This is critical for generating achievable goals - without knowing available tools,
|
||||
// the agent might generate goals that cannot be accomplished
|
||||
func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) string {
|
||||
locale := "en" // default locale
|
||||
if robot != nil && robot.Config != nil {
|
||||
locale = robot.Config.GetDefaultLocale()
|
||||
}
|
||||
return f.FormatAvailableResourcesWithLocale(robot, locale)
|
||||
}
|
||||
|
||||
// FormatAvailableResourcesWithLocale formats available resources with specific locale for i18n support
|
||||
func (f *InputFormatter) FormatAvailableResourcesWithLocale(robot *robottypes.Robot, locale string) string {
|
||||
if robot == nil || robot.Config == nil {
|
||||
return ""
|
||||
}
|
||||
|
|
@ -122,36 +150,152 @@ func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) strin
|
|||
var sb strings.Builder
|
||||
hasContent := false
|
||||
|
||||
// Available Agents
|
||||
// Available Agents - with detailed information (name, description)
|
||||
if robot.Config.Resources != nil && len(robot.Config.Resources.Agents) > 0 {
|
||||
if !hasContent {
|
||||
sb.WriteString("## Available Resources\n\n")
|
||||
hasContent = true
|
||||
}
|
||||
sb.WriteString("### Agents\n")
|
||||
sb.WriteString("These are the AI assistants you can delegate tasks to:\n")
|
||||
for _, agent := range robot.Config.Resources.Agents {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**\n", agent))
|
||||
sb.WriteString("These are the AI assistants you can delegate tasks to:\n\n")
|
||||
for _, agentID := range robot.Config.Resources.Agents {
|
||||
// Try to get agent details
|
||||
ast, err := assistant.Get(agentID)
|
||||
if err != nil {
|
||||
// Fallback to just ID if agent not found
|
||||
sb.WriteString(fmt.Sprintf("- **%s**\n", agentID))
|
||||
continue
|
||||
}
|
||||
|
||||
// Get localized name and description
|
||||
name := i18n.Translate(agentID, locale, ast.Name).(string)
|
||||
description := ""
|
||||
if ast.Description != "" {
|
||||
description = i18n.Translate(agentID, locale, ast.Description).(string)
|
||||
}
|
||||
|
||||
// Format agent info
|
||||
sb.WriteString(fmt.Sprintf("- **%s** (`%s`)\n", name, agentID))
|
||||
if description != "" {
|
||||
sb.WriteString(fmt.Sprintf(" - %s\n", description))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
// Available MCP Tools
|
||||
// Available MCP Tools - with detailed tool information
|
||||
if robot.Config.Resources != nil && len(robot.Config.Resources.MCP) > 0 {
|
||||
if !hasContent {
|
||||
sb.WriteString("## Available Resources\n\n")
|
||||
hasContent = true
|
||||
}
|
||||
sb.WriteString("### MCP Tools\n")
|
||||
sb.WriteString("These are the external tools and services you can use:\n")
|
||||
for _, mcp := range robot.Config.Resources.MCP {
|
||||
if len(mcp.Tools) > 0 {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcp.ID, strings.Join(mcp.Tools, ", ")))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcp.ID))
|
||||
sb.WriteString("These are the external tools and services you can use:\n\n")
|
||||
for _, mcpConfig := range robot.Config.Resources.MCP {
|
||||
// Try to get MCP client and list tools
|
||||
client, err := mcp.Select(mcpConfig.ID)
|
||||
if err != nil {
|
||||
// Fallback to basic info if client not found
|
||||
if len(mcpConfig.Tools) > 0 {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcpConfig.ID, strings.Join(mcpConfig.Tools, ", ")))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcpConfig.ID))
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Get client info for name and description
|
||||
clientInfo := client.Info()
|
||||
clientName := mcpConfig.ID
|
||||
clientDesc := ""
|
||||
if clientInfo != nil {
|
||||
if clientInfo.Name != "" {
|
||||
clientName = clientInfo.Name
|
||||
}
|
||||
if clientInfo.Description != "" {
|
||||
clientDesc = clientInfo.Description
|
||||
}
|
||||
}
|
||||
|
||||
// Write MCP header
|
||||
if clientDesc != "" {
|
||||
sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n", clientName, mcpConfig.ID))
|
||||
sb.WriteString(fmt.Sprintf("%s\n\n", clientDesc))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n\n", clientName, mcpConfig.ID))
|
||||
}
|
||||
|
||||
// Try to list tools with context
|
||||
ctx := context.Background()
|
||||
toolsResp, err := client.ListTools(ctx, "")
|
||||
if err != nil || toolsResp == nil {
|
||||
// Fallback to configured tools
|
||||
if len(mcpConfig.Tools) > 0 {
|
||||
sb.WriteString("Available tools: ")
|
||||
sb.WriteString(strings.Join(mcpConfig.Tools, ", "))
|
||||
sb.WriteString("\n\n")
|
||||
} else {
|
||||
sb.WriteString("All tools available\n\n")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Filter tools if specific tools are configured
|
||||
toolsToShow := toolsResp.Tools
|
||||
if len(mcpConfig.Tools) > 0 {
|
||||
// Create a map for quick lookup
|
||||
allowedTools := make(map[string]bool)
|
||||
for _, t := range mcpConfig.Tools {
|
||||
allowedTools[t] = true
|
||||
}
|
||||
// Filter tools
|
||||
var filteredTools []struct {
|
||||
Name string
|
||||
Description string
|
||||
}
|
||||
for _, tool := range toolsResp.Tools {
|
||||
if allowedTools[tool.Name] {
|
||||
filteredTools = append(filteredTools, struct {
|
||||
Name string
|
||||
Description string
|
||||
}{tool.Name, tool.Description})
|
||||
}
|
||||
}
|
||||
// Write filtered tools
|
||||
if len(filteredTools) > 0 {
|
||||
sb.WriteString("| Tool | Description |\n")
|
||||
sb.WriteString("|------|-------------|\n")
|
||||
for _, tool := range filteredTools {
|
||||
desc := tool.Description
|
||||
if len(desc) > 100 {
|
||||
desc = desc[:97] + "..."
|
||||
}
|
||||
// Escape pipe characters in description
|
||||
desc = strings.ReplaceAll(desc, "|", "\\|")
|
||||
sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc))
|
||||
}
|
||||
} else {
|
||||
sb.WriteString("Configured tools: ")
|
||||
sb.WriteString(strings.Join(mcpConfig.Tools, ", "))
|
||||
}
|
||||
} else if len(toolsToShow) > 0 {
|
||||
// Show all available tools
|
||||
sb.WriteString("| Tool | Description |\n")
|
||||
sb.WriteString("|------|-------------|\n")
|
||||
for _, tool := range toolsToShow {
|
||||
desc := tool.Description
|
||||
if len(desc) > 100 {
|
||||
desc = desc[:97] + "..."
|
||||
}
|
||||
// Escape pipe characters in description
|
||||
desc = strings.ReplaceAll(desc, "|", "\\|")
|
||||
sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc))
|
||||
}
|
||||
} else {
|
||||
sb.WriteString("No tools available\n")
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
// Available Knowledge Base
|
||||
|
|
|
|||
372
agent/robot/executor/standard/input_integration_test.go
Normal file
372
agent/robot/executor/standard/input_integration_test.go
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
package standard_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/yaoapp/yao/agent/robot/executor/standard"
|
||||
"github.com/yaoapp/yao/agent/robot/types"
|
||||
"github.com/yaoapp/yao/agent/testutils"
|
||||
)
|
||||
|
||||
// ============================================================================
|
||||
// InputFormatter Integration Tests with Real Data
|
||||
// These tests use the yao-dev-app environment with real assistants and MCPs
|
||||
// ============================================================================
|
||||
|
||||
func TestFormatAvailableResourcesIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
formatter := standard.NewInputFormatter()
|
||||
|
||||
t.Run("formats_real_agents_with_details", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-agents",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
Duties: []string{"Testing agent formatting"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
// Use real agents from yao-dev-app/assistants
|
||||
Agents: []string{
|
||||
"experts.data-analyst",
|
||||
"experts.text-writer",
|
||||
"experts.summarizer",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Verify structure
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
assert.Contains(t, result, "These are the AI assistants you can delegate tasks to:")
|
||||
|
||||
// Verify real agent details are included
|
||||
// experts.data-analyst should show name and description
|
||||
assert.Contains(t, result, "experts.data-analyst")
|
||||
assert.Contains(t, result, "Data Analyst Expert") // Name from package.yao
|
||||
|
||||
// experts.text-writer
|
||||
assert.Contains(t, result, "experts.text-writer")
|
||||
|
||||
// experts.summarizer
|
||||
assert.Contains(t, result, "experts.summarizer")
|
||||
|
||||
// Verify important note is present
|
||||
assert.Contains(t, result, "Only plan goals and tasks that can be accomplished")
|
||||
|
||||
t.Logf("Formatted agents result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("formats_real_mcp_with_tool_details", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-mcp",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
Duties: []string{"Testing MCP formatting"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
// Use real MCPs from yao-dev-app/mcps
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo", Tools: []string{"ping", "status"}}, // Specific tools
|
||||
{ID: "echo"}, // All tools
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Verify structure
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### MCP Tools")
|
||||
assert.Contains(t, result, "These are the external tools and services you can use:")
|
||||
|
||||
// Verify MCP details
|
||||
assert.Contains(t, result, "echo")
|
||||
|
||||
// Verify important note is present
|
||||
assert.Contains(t, result, "Only plan goals and tasks that can be accomplished")
|
||||
|
||||
t.Logf("Formatted MCP result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("formats_combined_resources", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-combined",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Sales Analyst Robot",
|
||||
Duties: []string{"Analyze sales data", "Generate reports"},
|
||||
Rules: []string{"Be accurate", "Be concise"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
Agents: []string{
|
||||
"experts.data-analyst",
|
||||
"experts.summarizer",
|
||||
},
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo", Tools: []string{"ping", "echo"}},
|
||||
},
|
||||
},
|
||||
KB: &types.KB{
|
||||
Collections: []string{"sales-policies", "product-catalog"},
|
||||
},
|
||||
DB: &types.DB{
|
||||
Models: []string{"sales", "customers", "orders"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Verify all sections are present
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
assert.Contains(t, result, "### MCP Tools")
|
||||
assert.Contains(t, result, "### Knowledge Base")
|
||||
assert.Contains(t, result, "### Database")
|
||||
|
||||
// Verify agents
|
||||
assert.Contains(t, result, "experts.data-analyst")
|
||||
assert.Contains(t, result, "experts.summarizer")
|
||||
|
||||
// Verify MCP
|
||||
assert.Contains(t, result, "echo")
|
||||
|
||||
// Verify KB
|
||||
assert.Contains(t, result, "sales-policies")
|
||||
assert.Contains(t, result, "product-catalog")
|
||||
|
||||
// Verify DB
|
||||
assert.Contains(t, result, "sales")
|
||||
assert.Contains(t, result, "customers")
|
||||
assert.Contains(t, result, "orders")
|
||||
|
||||
t.Logf("Formatted combined resources result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("handles_locale_zh", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-zh",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "测试机器人",
|
||||
Duties: []string{"测试国际化"},
|
||||
},
|
||||
DefaultLocale: "zh-cn",
|
||||
Resources: &types.Resources{
|
||||
// Use agents that have zh-cn locales
|
||||
Agents: []string{
|
||||
"hello", // This agent has locales/zh-cn.yml
|
||||
"mohe", // This agent also has locales/zh-cn.yml
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResourcesWithLocale(robot, "zh-cn")
|
||||
|
||||
// Verify structure
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
|
||||
// Verify agents are listed
|
||||
assert.Contains(t, result, "hello")
|
||||
assert.Contains(t, result, "mohe")
|
||||
|
||||
t.Logf("Formatted zh-cn result:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("gracefully_handles_missing_agents", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-missing",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
},
|
||||
Resources: &types.Resources{
|
||||
Agents: []string{
|
||||
"non-existent-agent",
|
||||
"experts.data-analyst", // This one exists
|
||||
"another-missing-agent",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Should not panic, should include fallback for missing agents
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### Agents")
|
||||
|
||||
// Missing agents should still be listed with just ID
|
||||
assert.Contains(t, result, "non-existent-agent")
|
||||
assert.Contains(t, result, "another-missing-agent")
|
||||
|
||||
// Existing agent should have full details
|
||||
assert.Contains(t, result, "experts.data-analyst")
|
||||
assert.Contains(t, result, "Data Analyst Expert")
|
||||
|
||||
t.Logf("Formatted with missing agents:\n%s", result)
|
||||
})
|
||||
|
||||
t.Run("gracefully_handles_missing_mcp", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-missing-mcp",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
},
|
||||
Resources: &types.Resources{
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "non-existent-mcp", Tools: []string{"tool1", "tool2"}},
|
||||
{ID: "echo"}, // This one exists
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Should not panic, should include fallback for missing MCP
|
||||
assert.Contains(t, result, "## Available Resources")
|
||||
assert.Contains(t, result, "### MCP Tools")
|
||||
|
||||
// Missing MCP should still be listed with fallback
|
||||
assert.Contains(t, result, "non-existent-mcp")
|
||||
assert.Contains(t, result, "tool1, tool2")
|
||||
|
||||
// Existing MCP should have details
|
||||
assert.Contains(t, result, "echo")
|
||||
|
||||
t.Logf("Formatted with missing MCP:\n%s", result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFormatAvailableResourcesTableFormat(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
formatter := standard.NewInputFormatter()
|
||||
|
||||
t.Run("mcp_tools_in_table_format", func(t *testing.T) {
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot-table",
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Test Robot",
|
||||
},
|
||||
Resources: &types.Resources{
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo"}, // All tools - should show table
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Check if table format is used when tools are available
|
||||
// Table headers: | Tool | Description |
|
||||
if strings.Contains(result, "| Tool | Description |") {
|
||||
assert.Contains(t, result, "|------|-------------|")
|
||||
t.Logf("MCP tools displayed in table format:\n%s", result)
|
||||
} else {
|
||||
// Fallback format
|
||||
t.Logf("MCP tools displayed in fallback format:\n%s", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFormatClockContextWithRobotIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
formatter := standard.NewInputFormatter()
|
||||
|
||||
t.Run("full_context_for_inspiration", func(t *testing.T) {
|
||||
// Create a realistic robot configuration
|
||||
robot := &types.Robot{
|
||||
MemberID: "sales-robot-001",
|
||||
TeamID: "team-001",
|
||||
DisplayName: "Sales Analyst Robot",
|
||||
AutonomousMode: true,
|
||||
Config: &types.Config{
|
||||
Identity: &types.Identity{
|
||||
Role: "Sales Analyst",
|
||||
Duties: []string{"Monitor sales performance", "Generate daily reports", "Alert on anomalies"},
|
||||
Rules: []string{"Only use approved data sources", "Maintain confidentiality"},
|
||||
},
|
||||
DefaultLocale: "en",
|
||||
Resources: &types.Resources{
|
||||
Agents: []string{
|
||||
"experts.data-analyst",
|
||||
"experts.summarizer",
|
||||
},
|
||||
MCP: []types.MCPConfig{
|
||||
{ID: "echo", Tools: []string{"ping"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Create clock context
|
||||
clock := types.NewClockContext(time.Now(), "UTC")
|
||||
|
||||
// Format clock context (includes robot identity)
|
||||
clockContent := formatter.FormatClockContext(clock, robot)
|
||||
|
||||
// Format available resources
|
||||
resourcesContent := formatter.FormatAvailableResources(robot)
|
||||
|
||||
// Combine for full context (as done in inspiration.go)
|
||||
fullContext := clockContent + "\n\n" + resourcesContent
|
||||
|
||||
// Verify full context contains all necessary information
|
||||
require.NotEmpty(t, fullContext)
|
||||
|
||||
// Time context
|
||||
assert.Contains(t, fullContext, "## Current Time Context")
|
||||
assert.Contains(t, fullContext, "### Time Markers")
|
||||
|
||||
// Robot identity
|
||||
assert.Contains(t, fullContext, "## Robot Identity")
|
||||
assert.Contains(t, fullContext, "Sales Analyst")
|
||||
assert.Contains(t, fullContext, "Monitor sales performance")
|
||||
assert.Contains(t, fullContext, "Only use approved data sources")
|
||||
|
||||
// Available resources
|
||||
assert.Contains(t, fullContext, "## Available Resources")
|
||||
assert.Contains(t, fullContext, "### Agents")
|
||||
assert.Contains(t, fullContext, "experts.data-analyst")
|
||||
assert.Contains(t, fullContext, "### MCP Tools")
|
||||
|
||||
t.Logf("Full context for inspiration:\n%s", fullContext)
|
||||
})
|
||||
}
|
||||
|
|
@ -28,9 +28,25 @@ func TestInputFormatterFormatClockContext(t *testing.T) {
|
|||
assert.Contains(t, result, "2024-01-15 09:30:00")
|
||||
assert.Contains(t, result, "Monday")
|
||||
assert.Contains(t, result, "UTC")
|
||||
assert.Contains(t, result, "**Hour**: 9")
|
||||
assert.Contains(t, result, "### Time Markers")
|
||||
})
|
||||
|
||||
t.Run("shows all time markers with check/cross", func(t *testing.T) {
|
||||
// Regular weekday, not month start/end
|
||||
now := time.Date(2024, 1, 15, 14, 0, 0, 0, time.UTC)
|
||||
clock := types.NewClockContext(now, "UTC")
|
||||
|
||||
result := formatter.FormatClockContext(clock, nil)
|
||||
|
||||
// Should show all markers, even when false
|
||||
assert.Contains(t, result, "✗ Weekend")
|
||||
assert.Contains(t, result, "✗ Month Start")
|
||||
assert.Contains(t, result, "✗ Month End")
|
||||
assert.Contains(t, result, "✗ Quarter End")
|
||||
assert.Contains(t, result, "✗ Year End")
|
||||
})
|
||||
|
||||
t.Run("includes robot identity when provided", func(t *testing.T) {
|
||||
now := time.Now()
|
||||
clock := types.NewClockContext(now, "UTC")
|
||||
|
|
@ -58,6 +74,26 @@ func TestInputFormatterFormatClockContext(t *testing.T) {
|
|||
assert.Empty(t, result)
|
||||
})
|
||||
|
||||
t.Run("uses DisplayName/Bio/SystemPrompt when Identity is nil", func(t *testing.T) {
|
||||
now := time.Now()
|
||||
clock := types.NewClockContext(now, "UTC")
|
||||
robot := &types.Robot{
|
||||
MemberID: "test-robot",
|
||||
DisplayName: "SEO Specialist",
|
||||
Bio: "Focuses on content optimization",
|
||||
SystemPrompt: "You are an SEO assistant.\n\n## Core Duties\n- Analyze keywords",
|
||||
Config: &types.Config{}, // Identity is nil
|
||||
}
|
||||
|
||||
result := formatter.FormatClockContext(clock, robot)
|
||||
|
||||
assert.Contains(t, result, "## Robot Identity")
|
||||
assert.Contains(t, result, "SEO Specialist")
|
||||
assert.Contains(t, result, "Focuses on content optimization")
|
||||
assert.Contains(t, result, "SEO assistant")
|
||||
assert.Contains(t, result, "Analyze keywords")
|
||||
})
|
||||
|
||||
t.Run("marks weekend correctly", func(t *testing.T) {
|
||||
// Saturday
|
||||
saturday := time.Date(2024, 1, 13, 10, 0, 0, 0, time.UTC)
|
||||
|
|
|
|||
|
|
@ -84,9 +84,24 @@ func (r *Runner) ExecuteWithRetry(task *robottypes.Task, taskCtx *RunnerContext)
|
|||
}
|
||||
|
||||
result.Output = output
|
||||
|
||||
// For MCP tasks: only validate structure (no semantic validation needed)
|
||||
// MCP tools return structured data - if execution succeeded, the result is valid
|
||||
if task.ExecutorType == robottypes.ExecutorMCP {
|
||||
validation := r.validateMCPOutput(task, output)
|
||||
result.Validation = validation
|
||||
result.Success = validation.Passed
|
||||
result.Duration = time.Since(startTime).Milliseconds()
|
||||
if !result.Success && validation != nil {
|
||||
result.Error = fmt.Sprintf("validation failed: %v", validation.Issues)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// For Process tasks: use full validation (semantic validation may still be useful)
|
||||
validation := r.validator.ValidateWithContext(task, output, nil)
|
||||
result.Validation = validation
|
||||
// For non-assistant tasks (MCP, Process):
|
||||
// For Process tasks:
|
||||
// - No multi-turn conversation, so Complete is determined by validation alone
|
||||
// - Success if passed OR score meets threshold (for partial success scenarios)
|
||||
result.Success = validation.Complete || (validation.Passed && validation.Score >= r.config.ValidationThreshold)
|
||||
|
|
@ -252,20 +267,18 @@ func (r *Runner) generateDefaultReply(validation *robottypes.ValidationResult, t
|
|||
}
|
||||
|
||||
// ExecuteMCPTask executes a task using an MCP tool
|
||||
// ExecutorID format: "mcpClientID.toolName" (e.g., "filesystem.read_file")
|
||||
// Requires task.MCPServer and task.MCPTool fields to be set
|
||||
// executor_id is the combined form: "mcp_server.mcp_tool" (e.g., "ark.image.text2img.generate")
|
||||
func (r *Runner) ExecuteMCPTask(task *robottypes.Task, taskCtx *RunnerContext) (interface{}, error) {
|
||||
// Parse MCP executor ID (format: clientID.toolName)
|
||||
parts := strings.SplitN(task.ExecutorID, ".", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("invalid MCP executor ID: %s (expected format: clientID.toolName)", task.ExecutorID)
|
||||
// Validate MCP-specific fields
|
||||
if task.MCPServer == "" || task.MCPTool == "" {
|
||||
return nil, fmt.Errorf("MCP task requires mcp_server and mcp_tool fields (executor_id: %s)", task.ExecutorID)
|
||||
}
|
||||
|
||||
clientID, toolName := parts[0], parts[1]
|
||||
|
||||
// Get MCP client
|
||||
client, err := mcp.Select(clientID)
|
||||
client, err := mcp.Select(task.MCPServer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MCP client not found: %s: %w", clientID, err)
|
||||
return nil, fmt.Errorf("MCP server not found: %s: %w", task.MCPServer, err)
|
||||
}
|
||||
|
||||
// Build arguments map from task.Args
|
||||
|
|
@ -281,9 +294,9 @@ func (r *Runner) ExecuteMCPTask(task *robottypes.Task, taskCtx *RunnerContext) (
|
|||
}
|
||||
|
||||
// Call MCP tool
|
||||
result, err := client.CallTool(r.ctx.Context, toolName, args)
|
||||
result, err := client.CallTool(r.ctx.Context, task.MCPTool, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MCP tool call failed: %w", err)
|
||||
return nil, fmt.Errorf("MCP tool call failed (%s.%s): %w", task.MCPServer, task.MCPTool, err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
|
|
@ -392,3 +405,56 @@ func (r *Runner) FormatPreviousResultsAsContext(results []robottypes.TaskResult)
|
|||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// validateMCPOutput performs simple structure validation for MCP task output
|
||||
// MCP tools return structured data - if execution succeeded, the result is valid
|
||||
// Only validates that output is non-empty and has expected structure
|
||||
// Does NOT perform semantic validation (that's for Agent tasks only)
|
||||
func (r *Runner) validateMCPOutput(task *robottypes.Task, output interface{}) *robottypes.ValidationResult {
|
||||
result := &robottypes.ValidationResult{
|
||||
Passed: true,
|
||||
Score: 1.0,
|
||||
Complete: true,
|
||||
}
|
||||
|
||||
// Check if output is nil or empty
|
||||
if output == nil {
|
||||
result.Passed = false
|
||||
result.Score = 0
|
||||
result.Complete = false
|
||||
result.Issues = append(result.Issues, "MCP tool returned nil output")
|
||||
return result
|
||||
}
|
||||
|
||||
// Check for empty output based on type
|
||||
switch o := output.(type) {
|
||||
case string:
|
||||
if strings.TrimSpace(o) == "" {
|
||||
result.Passed = false
|
||||
result.Score = 0
|
||||
result.Complete = false
|
||||
result.Issues = append(result.Issues, "MCP tool returned empty string")
|
||||
return result
|
||||
}
|
||||
case map[string]interface{}:
|
||||
if len(o) == 0 {
|
||||
result.Passed = false
|
||||
result.Score = 0
|
||||
result.Complete = false
|
||||
result.Issues = append(result.Issues, "MCP tool returned empty object")
|
||||
return result
|
||||
}
|
||||
case []interface{}:
|
||||
if len(o) == 0 {
|
||||
result.Passed = false
|
||||
result.Score = 0
|
||||
result.Complete = false
|
||||
result.Issues = append(result.Issues, "MCP tool returned empty array")
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// MCP execution succeeded with non-empty output - validation passed
|
||||
// No semantic validation needed for MCP tools
|
||||
return result
|
||||
}
|
||||
|
|
|
|||
|
|
@ -481,6 +481,105 @@ func TestRunnerExecuteNonAssistantTask(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MCP Output Validation Tests
|
||||
// ============================================================================
|
||||
|
||||
func TestRunnerValidateMCPOutput(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
ctx := types.NewContext(context.Background(), testAuth())
|
||||
|
||||
// Test that MCP tasks use simple structure validation, not semantic validation
|
||||
// This is tested indirectly through the validation result
|
||||
|
||||
t.Run("MCP validation passes with valid map output", func(t *testing.T) {
|
||||
robot := createRunnerTestRobot(t)
|
||||
config := standard.DefaultRunConfig()
|
||||
runner := standard.NewRunner(ctx, robot, config)
|
||||
|
||||
// Create a mock MCP task with validation rules
|
||||
// (normally these rules would trigger semantic validation for assistant tasks)
|
||||
task := &types.Task{
|
||||
ID: "task-mcp-test",
|
||||
ExecutorType: types.ExecutorMCP,
|
||||
ExecutorID: "test.tool",
|
||||
MCPServer: "test",
|
||||
MCPTool: "tool",
|
||||
// These semantic rules should be IGNORED for MCP tasks
|
||||
ExpectedOutput: "Image with file and content_type",
|
||||
ValidationRules: []string{
|
||||
"file field exists",
|
||||
"content_type is image/jpeg",
|
||||
},
|
||||
Status: types.TaskPending,
|
||||
}
|
||||
|
||||
// Simulate MCP output (normally would come from actual MCP call)
|
||||
output := map[string]interface{}{
|
||||
"file": "__yao.attachment://abc123",
|
||||
"content_type": "image/jpeg",
|
||||
}
|
||||
|
||||
// Test validateMCPOutput directly through reflection or mock
|
||||
// Since validateMCPOutput is private, we test the behavior indirectly:
|
||||
// MCP validation should only check for non-empty output, not semantic content
|
||||
|
||||
// The validation should pass because:
|
||||
// 1. Output is not nil
|
||||
// 2. Output is a non-empty map
|
||||
// (Semantic validation rules are NOT applied for MCP tasks)
|
||||
|
||||
t.Logf("MCP task configured with validation rules that should be ignored")
|
||||
t.Logf("Task ExpectedOutput: %s", task.ExpectedOutput)
|
||||
t.Logf("Task ValidationRules: %v", task.ValidationRules)
|
||||
t.Logf("MCP output: %v", output)
|
||||
|
||||
// Note: We can't directly call ExecuteWithRetry without an MCP server
|
||||
// This test documents the expected behavior
|
||||
_ = runner
|
||||
_ = task
|
||||
_ = output
|
||||
})
|
||||
|
||||
t.Run("MCP validation fails with nil output", func(t *testing.T) {
|
||||
// MCP validation should fail if output is nil
|
||||
t.Log("MCP validation should fail when output is nil")
|
||||
t.Log("Expected: Passed=false, Issues=['MCP tool returned nil output']")
|
||||
})
|
||||
|
||||
t.Run("MCP validation fails with empty string output", func(t *testing.T) {
|
||||
// MCP validation should fail if output is empty string
|
||||
t.Log("MCP validation should fail when output is empty string")
|
||||
t.Log("Expected: Passed=false, Issues=['MCP tool returned empty string']")
|
||||
})
|
||||
|
||||
t.Run("MCP validation fails with empty map output", func(t *testing.T) {
|
||||
// MCP validation should fail if output is empty map
|
||||
t.Log("MCP validation should fail when output is empty map")
|
||||
t.Log("Expected: Passed=false, Issues=['MCP tool returned empty object']")
|
||||
})
|
||||
|
||||
t.Run("MCP validation fails with empty array output", func(t *testing.T) {
|
||||
// MCP validation should fail if output is empty array
|
||||
t.Log("MCP validation should fail when output is empty array")
|
||||
t.Log("Expected: Passed=false, Issues=['MCP tool returned empty array']")
|
||||
})
|
||||
|
||||
t.Run("MCP validation passes with any non-empty output", func(t *testing.T) {
|
||||
// MCP validation should pass for any non-empty output
|
||||
// regardless of ExpectedOutput or ValidationRules
|
||||
t.Log("MCP validation should pass when output is non-empty")
|
||||
t.Log("Semantic validation (ExpectedOutput, ValidationRules) should NOT be applied")
|
||||
t.Log("Expected: Passed=true, Complete=true, Score=1.0")
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
|
|
|||
|
|
@ -173,6 +173,14 @@ func ParseTask(data map[string]interface{}, index int) (*robottypes.Task, error)
|
|||
copy(task.Args, args)
|
||||
}
|
||||
|
||||
// MCP-specific fields (required when executor_type is "mcp")
|
||||
if mcpServer, ok := data["mcp_server"].(string); ok {
|
||||
task.MCPServer = mcpServer
|
||||
}
|
||||
if mcpTool, ok := data["mcp_tool"].(string); ok {
|
||||
task.MCPTool = mcpTool
|
||||
}
|
||||
|
||||
// Optional: expected_output (for P3 validation)
|
||||
if expectedOutput, ok := data["expected_output"].(string); ok {
|
||||
task.ExpectedOutput = expectedOutput
|
||||
|
|
@ -324,6 +332,7 @@ func SortTasksByOrder(tasks []robottypes.Task) {
|
|||
// ValidateExecutorExists checks if the executor ID exists in available resources
|
||||
// This is an optional validation - tasks with unknown executors will still be created
|
||||
// but may fail during P3 execution
|
||||
// For MCP tasks, pass mcpServer as the second parameter (executorID is ignored for MCP)
|
||||
func ValidateExecutorExists(executorID string, executorType robottypes.ExecutorType, robot *robottypes.Robot) bool {
|
||||
if robot == nil || robot.Config == nil || robot.Config.Resources == nil {
|
||||
return true // Skip validation if no resources configured
|
||||
|
|
@ -339,6 +348,10 @@ func ValidateExecutorExists(executorID string, executorType robottypes.ExecutorT
|
|||
return false
|
||||
|
||||
case robottypes.ExecutorMCP:
|
||||
// For MCP, executorID can be either:
|
||||
// 1. The mcp_server value (new format)
|
||||
// 2. The combined mcp_server.mcp_tool format (for display)
|
||||
// We validate against mcp_server (the MCP server/client ID)
|
||||
for _, mcp := range robot.Config.Resources.MCP {
|
||||
if mcp.ID == executorID {
|
||||
return true
|
||||
|
|
@ -354,3 +367,18 @@ func ValidateExecutorExists(executorID string, executorType robottypes.ExecutorT
|
|||
|
||||
return false
|
||||
}
|
||||
|
||||
// ValidateMCPTask validates MCP task fields
|
||||
// Returns an error if mcp_server or mcp_tool is missing for MCP tasks
|
||||
func ValidateMCPTask(task *robottypes.Task) error {
|
||||
if task.ExecutorType != robottypes.ExecutorMCP {
|
||||
return nil
|
||||
}
|
||||
if task.MCPServer == "" {
|
||||
return fmt.Errorf("MCP task %s: mcp_server field is required", task.ID)
|
||||
}
|
||||
if task.MCPTool == "" {
|
||||
return fmt.Errorf("MCP task %s: mcp_tool field is required", task.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -614,9 +614,59 @@ func (r *RobotRecord) ToRobot() (*types.Robot, error) {
|
|||
robot.Config = config
|
||||
}
|
||||
|
||||
// Ensure Config exists for merging agents/mcp_servers
|
||||
if robot.Config == nil {
|
||||
robot.Config = &types.Config{}
|
||||
}
|
||||
if robot.Config.Resources == nil {
|
||||
robot.Config.Resources = &types.Resources{}
|
||||
}
|
||||
|
||||
// Merge agents from member table into Config.Resources.Agents
|
||||
if r.Agents != nil {
|
||||
agents := parseStringSlice(r.Agents)
|
||||
if len(agents) > 0 {
|
||||
robot.Config.Resources.Agents = agents
|
||||
}
|
||||
}
|
||||
|
||||
// Merge mcp_servers from member table into Config.Resources.MCP
|
||||
if r.MCPServers != nil {
|
||||
mcpServers := parseStringSlice(r.MCPServers)
|
||||
if len(mcpServers) > 0 {
|
||||
// Convert string slice to MCPConfig slice (each server ID becomes an MCPConfig)
|
||||
for _, serverID := range mcpServers {
|
||||
robot.Config.Resources.MCP = append(robot.Config.Resources.MCP, types.MCPConfig{
|
||||
ID: serverID,
|
||||
// Tools empty means all tools available
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return robot, nil
|
||||
}
|
||||
|
||||
// parseStringSlice converts interface{} to []string
|
||||
func parseStringSlice(v interface{}) []string {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case []string:
|
||||
return val
|
||||
case []interface{}:
|
||||
result := make([]string, 0, len(val))
|
||||
for _, item := range val {
|
||||
if s, ok := item.(string); ok {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FromRobot creates a RobotRecord from types.Robot
|
||||
func FromRobot(robot *types.Robot) *RobotRecord {
|
||||
record := &RobotRecord{
|
||||
|
|
|
|||
|
|
@ -224,9 +224,13 @@ type Task struct {
|
|||
|
||||
// Executor
|
||||
ExecutorType ExecutorType `json:"executor_type"`
|
||||
ExecutorID string `json:"executor_id"`
|
||||
ExecutorID string `json:"executor_id"` // unified ID: agent/assistant/process ID, or "mcp_server.mcp_tool" for MCP
|
||||
Args []any `json:"args,omitempty"`
|
||||
|
||||
// MCP-specific fields (required when executor_type is "mcp")
|
||||
MCPServer string `json:"mcp_server,omitempty"` // MCP server/client ID (e.g., "ark.image.text2img")
|
||||
MCPTool string `json:"mcp_tool,omitempty"` // MCP tool name (e.g., "generate")
|
||||
|
||||
// Validation (defined in P2, used in P3)
|
||||
// ExpectedOutput describes what the task should produce (for LLM semantic validation)
|
||||
ExpectedOutput string `json:"expected_output,omitempty"` // e.g., "JSON with sales_total, growth_rate fields"
|
||||
|
|
@ -407,9 +411,57 @@ func NewRobotFromMap(m map[string]interface{}) (*Robot, error) {
|
|||
robot.Config = config
|
||||
}
|
||||
|
||||
// Ensure Config exists for merging agents/mcp_servers
|
||||
if robot.Config == nil {
|
||||
robot.Config = &Config{}
|
||||
}
|
||||
if robot.Config.Resources == nil {
|
||||
robot.Config.Resources = &Resources{}
|
||||
}
|
||||
|
||||
// Merge agents from member table into Config.Resources.Agents
|
||||
if agentsData, ok := m["agents"]; ok && agentsData != nil {
|
||||
agents := getStringSlice(agentsData)
|
||||
if len(agents) > 0 {
|
||||
robot.Config.Resources.Agents = agents
|
||||
}
|
||||
}
|
||||
|
||||
// Merge mcp_servers from member table into Config.Resources.MCP
|
||||
if mcpData, ok := m["mcp_servers"]; ok && mcpData != nil {
|
||||
mcpServers := getStringSlice(mcpData)
|
||||
if len(mcpServers) > 0 {
|
||||
for _, serverID := range mcpServers {
|
||||
robot.Config.Resources.MCP = append(robot.Config.Resources.MCP, MCPConfig{
|
||||
ID: serverID,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return robot, nil
|
||||
}
|
||||
|
||||
// getStringSlice converts interface{} to []string
|
||||
func getStringSlice(v interface{}) []string {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case []string:
|
||||
return val
|
||||
case []interface{}:
|
||||
result := make([]string, 0, len(val))
|
||||
for _, item := range val {
|
||||
if s, ok := item.(string); ok {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getString safely gets a string value from map
|
||||
func getString(m map[string]interface{}, key string) string {
|
||||
if m == nil {
|
||||
|
|
|
|||
|
|
@ -1428,6 +1428,51 @@ Simulates user behavior for dynamic mode testing.
|
|||
yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent
|
||||
```
|
||||
|
||||
## Extract Command
|
||||
|
||||
Extract test results from output JSONL file to individual Markdown or JSON files for human review:
|
||||
|
||||
```bash
|
||||
# Extract to Markdown files (default)
|
||||
yao agent extract output-20260127104118.jsonl
|
||||
|
||||
# Specify output directory
|
||||
yao agent extract output.jsonl -o ./reports/
|
||||
|
||||
# Extract to JSON format
|
||||
yao agent extract output.jsonl --format json
|
||||
```
|
||||
|
||||
### Extract Command Options
|
||||
|
||||
| Flag | Description | Default |
|
||||
| ---------- | ---------------------------------------- | ---------- |
|
||||
| `-o` | Output directory | same as input |
|
||||
| `--format` | Output format: `markdown`, `json` | `markdown` |
|
||||
|
||||
### Output Format (Markdown)
|
||||
|
||||
Each test result is extracted to a separate file:
|
||||
|
||||
```markdown
|
||||
# T001-销售分析师-月末周五
|
||||
|
||||
**Status**: ✅ PASSED
|
||||
|
||||
**Duration**: 16743ms
|
||||
|
||||
## Input
|
||||
(Full input content in markdown code block)
|
||||
|
||||
## Output
|
||||
(Agent's response content)
|
||||
```
|
||||
|
||||
This is useful for:
|
||||
- Human review of agent outputs
|
||||
- Comparing results across test runs
|
||||
- Documentation and reporting
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Description |
|
||||
|
|
|
|||
189
agent/test/extract.go
Normal file
189
agent/test/extract.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
// ExtractOptions represents options for extracting test results
|
||||
type ExtractOptions struct {
|
||||
// InputFile is the path to the output JSONL file from test run
|
||||
InputFile string
|
||||
|
||||
// OutputDir is the directory to write extracted files (default: same as input file)
|
||||
OutputDir string
|
||||
|
||||
// Format is the output format: "markdown" (default), "json"
|
||||
Format string
|
||||
}
|
||||
|
||||
// Extractor extracts test results to individual files for review
|
||||
type Extractor struct {
|
||||
opts *ExtractOptions
|
||||
}
|
||||
|
||||
// NewExtractor creates a new extractor
|
||||
func NewExtractor(opts *ExtractOptions) *Extractor {
|
||||
if opts.Format == "" {
|
||||
opts.Format = "markdown"
|
||||
}
|
||||
if opts.OutputDir == "" {
|
||||
opts.OutputDir = filepath.Dir(opts.InputFile)
|
||||
}
|
||||
return &Extractor{opts: opts}
|
||||
}
|
||||
|
||||
// Extract reads the test output file and extracts results to individual files
|
||||
func (e *Extractor) Extract() ([]string, error) {
|
||||
// Read the JSONL file
|
||||
data, err := os.ReadFile(e.opts.InputFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read input file: %w", err)
|
||||
}
|
||||
|
||||
// Parse the JSON (the output file is a single JSON object, not JSONL)
|
||||
var report Report
|
||||
if err := jsoniter.Unmarshal(data, &report); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse test report: %w", err)
|
||||
}
|
||||
|
||||
// Create output directory if it doesn't exist
|
||||
if err := os.MkdirAll(e.opts.OutputDir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
|
||||
var extractedFiles []string
|
||||
|
||||
// Extract each result
|
||||
for _, result := range report.Results {
|
||||
var filename string
|
||||
var content string
|
||||
|
||||
switch e.opts.Format {
|
||||
case "markdown":
|
||||
filename = filepath.Join(e.opts.OutputDir, result.ID+".md")
|
||||
content = e.formatMarkdown(result)
|
||||
case "json":
|
||||
filename = filepath.Join(e.opts.OutputDir, result.ID+".json")
|
||||
jsonBytes, err := jsoniter.MarshalIndent(result, "", " ")
|
||||
if err != nil {
|
||||
return extractedFiles, fmt.Errorf("failed to marshal result %s: %w", result.ID, err)
|
||||
}
|
||||
content = string(jsonBytes)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported format: %s", e.opts.Format)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filename, []byte(content), 0644); err != nil {
|
||||
return extractedFiles, fmt.Errorf("failed to write file %s: %w", filename, err)
|
||||
}
|
||||
|
||||
extractedFiles = append(extractedFiles, filename)
|
||||
}
|
||||
|
||||
return extractedFiles, nil
|
||||
}
|
||||
|
||||
// formatMarkdown formats a single test result as Markdown
|
||||
func (e *Extractor) formatMarkdown(result *Result) string {
|
||||
var sb strings.Builder
|
||||
|
||||
// Title
|
||||
sb.WriteString(fmt.Sprintf("# %s\n\n", result.ID))
|
||||
|
||||
// Status badge
|
||||
switch result.Status {
|
||||
case StatusPassed:
|
||||
sb.WriteString("**Status**: ✅ PASSED\n\n")
|
||||
case StatusFailed:
|
||||
sb.WriteString("**Status**: ❌ FAILED\n\n")
|
||||
case StatusError:
|
||||
sb.WriteString("**Status**: ⚠️ ERROR\n\n")
|
||||
case StatusTimeout:
|
||||
sb.WriteString("**Status**: ⏱️ TIMEOUT\n\n")
|
||||
case StatusSkipped:
|
||||
sb.WriteString("**Status**: ⏭️ SKIPPED\n\n")
|
||||
}
|
||||
|
||||
// Duration
|
||||
sb.WriteString(fmt.Sprintf("**Duration**: %dms\n\n", result.DurationMs))
|
||||
|
||||
// Error (if any)
|
||||
if result.Error != "" {
|
||||
sb.WriteString("## Error\n\n")
|
||||
sb.WriteString("```\n")
|
||||
sb.WriteString(result.Error)
|
||||
sb.WriteString("\n```\n\n")
|
||||
}
|
||||
|
||||
// Input
|
||||
sb.WriteString("## Input\n\n")
|
||||
sb.WriteString("```markdown\n")
|
||||
sb.WriteString(formatInputAsString(result.Input))
|
||||
sb.WriteString("\n```\n\n")
|
||||
|
||||
// Output
|
||||
sb.WriteString("## Output\n\n")
|
||||
output := formatOutputAsString(result.Output)
|
||||
// Remove markdown code block wrapper if present
|
||||
output = strings.TrimPrefix(output, "```markdown\n")
|
||||
output = strings.TrimSuffix(output, "\n```")
|
||||
output = strings.TrimSuffix(output, "```")
|
||||
sb.WriteString(output)
|
||||
sb.WriteString("\n")
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// formatInputAsString converts input to string format
|
||||
func formatInputAsString(input interface{}) string {
|
||||
switch v := input.(type) {
|
||||
case string:
|
||||
return v
|
||||
case map[string]interface{}:
|
||||
// Single message format
|
||||
if content, ok := v["content"].(string); ok {
|
||||
return content
|
||||
}
|
||||
// Fallback to JSON
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||
return string(jsonBytes)
|
||||
case []interface{}:
|
||||
// Array of messages - extract content from last user message
|
||||
for i := len(v) - 1; i >= 0; i-- {
|
||||
if msg, ok := v[i].(map[string]interface{}); ok {
|
||||
if role, ok := msg["role"].(string); ok && role == "user" {
|
||||
if content, ok := msg["content"].(string); ok {
|
||||
return content
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback to JSON
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||
return string(jsonBytes)
|
||||
default:
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(input, "", " ")
|
||||
return string(jsonBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// formatOutputAsString converts output to string format
|
||||
func formatOutputAsString(output interface{}) string {
|
||||
switch v := output.(type) {
|
||||
case string:
|
||||
return v
|
||||
case map[string]interface{}, []interface{}:
|
||||
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||
return string(jsonBytes)
|
||||
default:
|
||||
if output == nil {
|
||||
return "(no output)"
|
||||
}
|
||||
return fmt.Sprintf("%v", output)
|
||||
}
|
||||
}
|
||||
|
|
@ -35,6 +35,11 @@ var langs = map[string]string{
|
|||
"Error: agent (-n) is required when using direct message input and not in an agent directory": "错误: 使用直接消息输入且不在智能体目录时需要指定 -n 参数",
|
||||
"Hint: Make sure you're in a Yao application directory or specify --app flag": "提示: 确保在 Yao 应用目录中或使用 --app 参数指定",
|
||||
"Error: invalid timeout format": "错误: 无效的超时格式",
|
||||
// Extract command
|
||||
"Extract test results to individual files for review": "提取测试结果到单独的文件供审查",
|
||||
"Extract test results from output JSONL file to individual Markdown or JSON files": "从输出 JSONL 文件中提取测试结果到单独的 Markdown 或 JSON 文件",
|
||||
"Output directory (default: same as input file)": "输出目录 (默认: 与输入文件相同)",
|
||||
"Output format: markdown, json": "输出格式: markdown, json",
|
||||
}
|
||||
|
||||
// L Language switch
|
||||
|
|
|
|||
87
cmd/agent/extract.go
Normal file
87
cmd/agent/extract.go
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/yaoapp/yao/agent/test"
|
||||
)
|
||||
|
||||
// Extract command flags
|
||||
var (
|
||||
extractOutput string
|
||||
extractFormat string
|
||||
)
|
||||
|
||||
// ExtractCmd is the agent extract command
|
||||
var ExtractCmd = &cobra.Command{
|
||||
Use: "extract <output-file.jsonl>",
|
||||
Short: L("Extract test results to individual files for review"),
|
||||
Long: L("Extract test results from output JSONL file to individual Markdown or JSON files"),
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
inputFile := args[0]
|
||||
|
||||
// Resolve absolute path
|
||||
absPath, err := filepath.Abs(inputFile)
|
||||
if err != nil {
|
||||
color.Red("Error: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(absPath); os.IsNotExist(err) {
|
||||
color.Red("Error: file not found: %s\n", absPath)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Build extract options
|
||||
opts := &test.ExtractOptions{
|
||||
InputFile: absPath,
|
||||
OutputDir: extractOutput,
|
||||
Format: extractFormat,
|
||||
}
|
||||
|
||||
// Create extractor and run
|
||||
extractor := test.NewExtractor(opts)
|
||||
files, err := extractor.Extract()
|
||||
if err != nil {
|
||||
color.Red("Error: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Print results
|
||||
fmt.Println()
|
||||
color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════")
|
||||
color.New(color.FgGreen, color.Bold).Println(" Extract Complete")
|
||||
color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════")
|
||||
fmt.Println()
|
||||
|
||||
for _, file := range files {
|
||||
color.New(color.FgGreen).Printf("✓ ")
|
||||
fmt.Printf("Written: %s\n", filepath.Base(file))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
color.New(color.FgWhite).Printf(" Total: ")
|
||||
color.New(color.FgCyan).Printf("%d files\n", len(files))
|
||||
|
||||
if extractOutput != "" {
|
||||
color.New(color.FgWhite).Printf(" Output: ")
|
||||
color.New(color.FgCyan).Printf("%s\n", extractOutput)
|
||||
} else {
|
||||
color.New(color.FgWhite).Printf(" Output: ")
|
||||
color.New(color.FgCyan).Printf("%s\n", filepath.Dir(absPath))
|
||||
}
|
||||
fmt.Println()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Extract command flags
|
||||
ExtractCmd.Flags().StringVarP(&extractOutput, "output", "o", "", L("Output directory (default: same as input file)"))
|
||||
ExtractCmd.Flags().StringVar(&extractFormat, "format", "markdown", L("Output format: markdown, json"))
|
||||
}
|
||||
|
|
@ -152,6 +152,7 @@ func init() {
|
|||
|
||||
// Agent
|
||||
agentCmd.AddCommand(agent.TestCmd)
|
||||
agentCmd.AddCommand(agent.ExtractCmd)
|
||||
|
||||
rootCmd.AddCommand(
|
||||
versionCmd,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue