diff --git a/agent/robot/executor/standard/run_test.go b/agent/robot/executor/standard/run_test.go new file mode 100644 index 00000000..69f5c36c --- /dev/null +++ b/agent/robot/executor/standard/run_test.go @@ -0,0 +1,449 @@ +package standard_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + agentcontext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/robot/executor/standard" + "github.com/yaoapp/yao/agent/robot/types" + "github.com/yaoapp/yao/agent/testutils" +) + +// ============================================================================ +// P3 Run Phase Tests - RunExecution +// ============================================================================ + +func TestRunExecutionBasic(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("executes single task successfully", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + // Pre-built task (simulating P2 output) + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write a short greeting message for a company newsletter. Keep it under 50 words."}, + }, + ExpectedOutput: "A friendly greeting message suitable for a newsletter", + Order: 0, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + require.NoError(t, err) + require.Len(t, exec.Results, 1) + + result := exec.Results[0] + assert.Equal(t, "task-001", result.TaskID) + assert.True(t, result.Success, "task should succeed") + assert.NotNil(t, result.Output, "should have output") + assert.Greater(t, result.Duration, int64(0), "should have duration") + + // Task status should be updated + assert.Equal(t, types.TaskCompleted, exec.Tasks[0].Status) + assert.NotNil(t, exec.Tasks[0].StartTime) + assert.NotNil(t, exec.Tasks[0].EndTime) + }) + + t.Run("executes multiple tasks in order", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + // Multiple tasks that depend on each other + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.data-analyst", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Analyze this data: Sales Q1: $100K, Q2: $150K, Q3: $120K, Q4: $180K. Calculate the total and average."}, + }, + ExpectedOutput: "JSON with total and average sales figures", + ValidationRules: []string{ + "output must be valid JSON", + }, + Order: 0, + Status: types.TaskPending, + }, + { + ID: "task-002", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.summarizer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Summarize the key findings from the previous analysis in 2-3 sentences."}, + }, + ExpectedOutput: "A brief summary of the sales analysis", + Order: 1, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + require.NoError(t, err) + require.Len(t, exec.Results, 2) + + // Both tasks should complete + assert.True(t, exec.Results[0].Success, "first task should succeed") + assert.True(t, exec.Results[1].Success, "second task should succeed") + + // Second task should have access to first task's result (via context) + assert.Equal(t, types.TaskCompleted, exec.Tasks[0].Status) + assert.Equal(t, types.TaskCompleted, exec.Tasks[1].Status) + + t.Logf("Task 1 output: %v", exec.Results[0].Output) + t.Logf("Task 2 output: %v", exec.Results[1].Output) + }) + + t.Run("passes previous results as context to subsequent tasks", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + // First task generates data, second task uses it + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Generate a list of 3 product names for a tech company. Output as JSON array."}, + }, + ExpectedOutput: "JSON array with 3 product names", + Order: 0, + Status: types.TaskPending, + }, + { + ID: "task-002", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Using the product names from the previous task, write a one-line tagline for each product."}, + }, + ExpectedOutput: "Taglines for each product", + Order: 1, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + require.NoError(t, err) + require.Len(t, exec.Results, 2) + + // Both should succeed + assert.True(t, exec.Results[0].Success) + assert.True(t, exec.Results[1].Success) + + // Second task output should reference products from first task + t.Logf("Task 1 (products): %v", exec.Results[0].Output) + t.Logf("Task 2 (taglines): %v", exec.Results[1].Output) + }) +} + +func TestRunExecutionTaskStatus(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("updates task status during execution", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Say 'Hello World'"}, + }, + Order: 0, + Status: types.TaskPending, + }, + } + + // Verify initial status + assert.Equal(t, types.TaskPending, exec.Tasks[0].Status) + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + require.NoError(t, err) + + // Verify final status + assert.Equal(t, types.TaskCompleted, exec.Tasks[0].Status) + assert.NotNil(t, exec.Tasks[0].StartTime) + assert.NotNil(t, exec.Tasks[0].EndTime) + }) + + t.Run("marks remaining tasks as skipped on failure", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + // First task uses a non-existent assistant to guarantee failure + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "non.existent.assistant.xyz123", // Non-existent assistant + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "This will fail"}, + }, + Order: 0, + Status: types.TaskPending, + }, + { + ID: "task-002", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write another greeting"}, + }, + Order: 1, + Status: types.TaskPending, + }, + { + ID: "task-003", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write yet another greeting"}, + }, + Order: 2, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + // Should return error because first task failed + assert.Error(t, err) + assert.Contains(t, err.Error(), "task-001") + + // First task should be failed + assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status) + + // Remaining tasks should be skipped + assert.Equal(t, types.TaskSkipped, exec.Tasks[1].Status) + assert.Equal(t, types.TaskSkipped, exec.Tasks[2].Status) + }) +} + +func TestRunExecutionErrorHandling(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("returns error when robot is nil", func(t *testing.T) { + exec := &types.Execution{ + ID: "test-exec-1", + TriggerType: types.TriggerClock, + Tasks: []types.Task{ + {ID: "task-001", ExecutorID: "test"}, + }, + } + // Don't set robot + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "robot not found") + }) + + t.Run("returns error when no tasks", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + exec.Tasks = []types.Task{} // Empty + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "no tasks") + }) + + t.Run("returns error for non-existent assistant", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "non.existent.agent", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Test"}, + }, + Order: 0, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + assert.Error(t, err) + // Task should be marked as failed + assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status) + }) +} + +func TestRunExecutionValidation(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("validates output with rule-based validation", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.data-analyst", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Return a JSON object with fields: name (string), count (number). Example: {\"name\": \"test\", \"count\": 5}"}, + }, + ExpectedOutput: "JSON object with name and count fields", + ValidationRules: []string{ + "output must be valid JSON", + `{"type": "type", "value": "object"}`, + }, + Order: 0, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + require.NoError(t, err) + require.Len(t, exec.Results, 1) + + result := exec.Results[0] + assert.True(t, result.Success) + assert.NotNil(t, result.Validation) + assert.True(t, result.Validation.Passed) + }) + + t.Run("validates output with semantic validation", func(t *testing.T) { + robot := createRunTestRobot(t) + exec := createRunTestExecution(robot) + + exec.Tasks = []types.Task{ + { + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write a professional email greeting for a business context. Start with 'Dear' and end with a comma."}, + }, + ExpectedOutput: "A professional email greeting starting with 'Dear'", + Order: 0, + Status: types.TaskPending, + }, + } + + e := standard.New() + err := e.RunExecution(ctx, exec, nil) + + require.NoError(t, err) + require.Len(t, exec.Results, 1) + + result := exec.Results[0] + assert.True(t, result.Success) + assert.NotNil(t, result.Validation) + + t.Logf("Output: %v", result.Output) + t.Logf("Validation: passed=%v, score=%.2f", result.Validation.Passed, result.Validation.Score) + }) +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// createRunTestRobot creates a test robot for P3 run tests +func createRunTestRobot(t *testing.T) *types.Robot { + t.Helper() + return &types.Robot{ + MemberID: "test-robot-run", + TeamID: "test-team-1", + DisplayName: "Test Robot for Run", + SystemPrompt: "You are a helpful assistant.", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Assistant", + Duties: []string{"Execute tasks", "Generate content"}, + }, + Resources: &types.Resources{ + Phases: map[types.Phase]string{ + types.PhaseRun: "robot.validation", + "validation": "robot.validation", // For semantic validation agent + }, + Agents: []string{ + "experts.data-analyst", + "experts.summarizer", + "experts.text-writer", + }, + }, + }, + } +} + +// createRunTestExecution creates a test execution for P3 run tests +func createRunTestExecution(robot *types.Robot) *types.Execution { + exec := &types.Execution{ + ID: "test-exec-run-1", + MemberID: robot.MemberID, + TeamID: robot.TeamID, + TriggerType: types.TriggerClock, + StartTime: time.Now(), + Status: types.ExecRunning, + Phase: types.PhaseRun, + Goals: &types.Goals{ + Content: "## Goals\n\n1. Execute test tasks", + }, + } + exec.SetRobot(robot) + return exec +} diff --git a/agent/robot/executor/standard/runner_test.go b/agent/robot/executor/standard/runner_test.go new file mode 100644 index 00000000..4534458a --- /dev/null +++ b/agent/robot/executor/standard/runner_test.go @@ -0,0 +1,517 @@ +package standard_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + agentcontext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/robot/executor/standard" + "github.com/yaoapp/yao/agent/robot/types" + "github.com/yaoapp/yao/agent/testutils" +) + +// ============================================================================ +// Runner Tests - Multi-Turn Conversation Flow +// ============================================================================ + +func TestRunnerExecuteWithRetry(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("executes assistant task with multi-turn conversation", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write a haiku about coding. Format: three lines with 5-7-5 syllables."}, + }, + ExpectedOutput: "A haiku poem about coding", + Status: types.TaskPending, + } + + taskCtx := &standard.RunnerContext{ + SystemPrompt: robot.SystemPrompt, + } + + result := runner.ExecuteWithRetry(task, taskCtx) + + assert.True(t, result.Success, "task should succeed") + assert.NotNil(t, result.Output) + assert.NotNil(t, result.Validation) + assert.True(t, result.Validation.Complete) + assert.Greater(t, result.Duration, int64(0)) + + t.Logf("Output: %v", result.Output) + t.Logf("Validation: passed=%v, complete=%v, score=%.2f", + result.Validation.Passed, result.Validation.Complete, result.Validation.Score) + }) + + t.Run("handles validation failure with multi-turn retry", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + config.MaxTurnsPerTask = 3 // Limit turns for test + runner := standard.NewRunner(ctx, robot, config) + + // Task with strict validation that may require conversation + task := &types.Task{ + ID: "task-002", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.data-analyst", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Return a JSON object with exactly these fields: status (string 'ok'), count (number greater than 0)."}, + }, + ExpectedOutput: "JSON with status='ok' and count>0", + ValidationRules: []string{ + "output must be valid JSON", + `{"type": "type", "value": "object"}`, + }, + Status: types.TaskPending, + } + + taskCtx := &standard.RunnerContext{ + SystemPrompt: robot.SystemPrompt, + } + + result := runner.ExecuteWithRetry(task, taskCtx) + + // Should either succeed or fail gracefully + assert.NotNil(t, result.Validation) + t.Logf("Success: %v, Output: %v", result.Success, result.Output) + t.Logf("Validation: passed=%v, complete=%v, needReply=%v", + result.Validation.Passed, result.Validation.Complete, result.Validation.NeedReply) + }) + + t.Run("respects max turns limit", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + config.MaxTurnsPerTask = 1 // Only 1 turn allowed + runner := standard.NewRunner(ctx, robot, config) + + // Task that requires multiple turns - asking for something incomplete + // then validation will ask for more, but we only allow 1 turn + task := &types.Task{ + ID: "task-003", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Say 'hello'"}, + }, + // Validation will fail because it expects a JSON object + ExpectedOutput: "A JSON object with 'status' and 'data' fields", + ValidationRules: []string{`{"type": "type", "value": "object"}`}, + Status: types.TaskPending, + } + + taskCtx := &standard.RunnerContext{ + SystemPrompt: robot.SystemPrompt, + } + + result := runner.ExecuteWithRetry(task, taskCtx) + + // With only 1 turn and strict validation, task should not complete successfully + // Either it fails validation or hits max turns + t.Logf("Result: success=%v, error=%s", result.Success, result.Error) + t.Logf("Validation: passed=%v, complete=%v, needReply=%v", + result.Validation.Passed, result.Validation.Complete, result.Validation.NeedReply) + + // The test verifies the max turns mechanism works - task either: + // 1. Fails validation (expected with "say hello" vs JSON requirement) + // 2. Or hits max turns if validation requests retry + assert.NotNil(t, result.Validation) + }) +} + +func TestRunnerBuildTaskContext(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("includes previous results in context", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + exec := &types.Execution{ + ID: "test-exec", + MemberID: robot.MemberID, + TeamID: robot.TeamID, + Goals: &types.Goals{ + Content: "Test goals", + }, + Results: []types.TaskResult{ + { + TaskID: "task-001", + Success: true, + Output: map[string]interface{}{"data": "previous result"}, + }, + { + TaskID: "task-002", + Success: true, + Output: "Another result", + }, + }, + } + exec.SetRobot(robot) + + // Build context for task at index 2 (should include results 0 and 1) + taskCtx := runner.BuildTaskContext(exec, 2) + + assert.NotNil(t, taskCtx) + assert.Len(t, taskCtx.PreviousResults, 2) + assert.Equal(t, "task-001", taskCtx.PreviousResults[0].TaskID) + assert.Equal(t, "task-002", taskCtx.PreviousResults[1].TaskID) + assert.Equal(t, robot.SystemPrompt, taskCtx.SystemPrompt) + }) + + t.Run("handles first task with no previous results", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + exec := &types.Execution{ + ID: "test-exec", + MemberID: robot.MemberID, + TeamID: robot.TeamID, + Goals: &types.Goals{ + Content: "Test goals", + }, + Results: []types.TaskResult{}, + } + exec.SetRobot(robot) + + taskCtx := runner.BuildTaskContext(exec, 0) + + assert.NotNil(t, taskCtx) + assert.Empty(t, taskCtx.PreviousResults) + }) + + t.Run("handles bounds check for task index", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + exec := &types.Execution{ + ID: "test-exec", + MemberID: robot.MemberID, + TeamID: robot.TeamID, + Results: []types.TaskResult{ + {TaskID: "task-001", Success: true}, + }, + } + exec.SetRobot(robot) + + // Task index 5, but only 1 result exists + taskCtx := runner.BuildTaskContext(exec, 5) + + assert.NotNil(t, taskCtx) + assert.Len(t, taskCtx.PreviousResults, 1) // Should only include available results + }) +} + +func TestRunnerFormatPreviousResultsAsContext(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("formats previous results as markdown", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + results := []types.TaskResult{ + { + TaskID: "task-001", + Success: true, + Output: map[string]interface{}{"key": "value", "count": 42}, + }, + { + TaskID: "task-002", + Success: false, + Output: "Partial result", + Error: "Validation failed", + }, + } + + formatted := runner.FormatPreviousResultsAsContext(results) + + assert.Contains(t, formatted, "## Previous Task Results") + assert.Contains(t, formatted, "task-001") + assert.Contains(t, formatted, "task-002") + assert.Contains(t, formatted, "Success") + assert.Contains(t, formatted, "Failed") + assert.Contains(t, formatted, "key") + assert.Contains(t, formatted, "value") + + t.Logf("Formatted context:\n%s", formatted) + }) + + t.Run("returns empty string for no results", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + formatted := runner.FormatPreviousResultsAsContext([]types.TaskResult{}) + + assert.Empty(t, formatted) + }) +} + +func TestRunnerBuildAssistantMessages(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("builds messages with task content", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write a greeting"}, + }, + } + + taskCtx := &standard.RunnerContext{ + SystemPrompt: "You are helpful", + } + + messages := runner.BuildAssistantMessages(task, taskCtx) + + assert.NotEmpty(t, messages) + // Should contain task message + found := false + for _, msg := range messages { + if content, ok := msg.Content.(string); ok && content == "Write a greeting" { + found = true + break + } + } + assert.True(t, found, "should contain task message") + }) + + t.Run("includes previous results in messages", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + task := &types.Task{ + ID: "task-002", + ExecutorType: types.ExecutorAssistant, + ExecutorID: "experts.text-writer", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Continue from previous"}, + }, + } + + taskCtx := &standard.RunnerContext{ + PreviousResults: []types.TaskResult{ + {TaskID: "task-001", Success: true, Output: "Previous output"}, + }, + SystemPrompt: "You are helpful", + } + + messages := runner.BuildAssistantMessages(task, taskCtx) + + assert.NotEmpty(t, messages) + // Should have context message with previous results + formatted := runner.FormatMessagesAsText(messages) + assert.Contains(t, formatted, "Previous Task Results") + }) +} + +func TestRunnerFormatMessagesAsText(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("formats string content", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + messages := []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Hello"}, + {Role: agentcontext.RoleUser, Content: "World"}, + } + + text := runner.FormatMessagesAsText(messages) + + assert.Contains(t, text, "Hello") + assert.Contains(t, text, "World") + }) + + t.Run("handles multipart content", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + messages := []agentcontext.Message{ + { + Role: agentcontext.RoleUser, + Content: []interface{}{ + map[string]interface{}{"type": "text", "text": "Part 1"}, + map[string]interface{}{"type": "text", "text": "Part 2"}, + }, + }, + } + + text := runner.FormatMessagesAsText(messages) + + assert.Contains(t, text, "Part 1") + assert.Contains(t, text, "Part 2") + }) + + t.Run("handles map content via JSON", func(t *testing.T) { + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + messages := []agentcontext.Message{ + { + Role: agentcontext.RoleUser, + Content: map[string]interface{}{"key": "value"}, + }, + } + + text := runner.FormatMessagesAsText(messages) + + assert.Contains(t, text, "key") + assert.Contains(t, text, "value") + }) +} + +// ============================================================================ +// Non-Assistant Task Tests (MCP, Process) +// ============================================================================ + +func TestRunnerExecuteNonAssistantTask(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("executes MCP task (single-call)", func(t *testing.T) { + // Note: This test requires MCP server to be running + // Skip if MCP is not available + t.Skip("MCP server not available in test environment") + + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + task := &types.Task{ + ID: "task-mcp", + ExecutorType: types.ExecutorMCP, + ExecutorID: "filesystem.list_directory", + Args: []any{map[string]interface{}{"path": "/tmp"}}, + Status: types.TaskPending, + } + + taskCtx := &standard.RunnerContext{} + + result := runner.ExecuteWithRetry(task, taskCtx) + + // MCP tasks are single-call, no multi-turn + t.Logf("MCP result: success=%v, output=%v", result.Success, result.Output) + }) + + t.Run("executes Process task (single-call)", func(t *testing.T) { + // Note: This test requires a Yao process to be available + // Skip if process is not available + t.Skip("Yao process not available in test environment") + + robot := createRunnerTestRobot(t) + config := standard.DefaultRunConfig() + runner := standard.NewRunner(ctx, robot, config) + + task := &types.Task{ + ID: "task-process", + ExecutorType: types.ExecutorProcess, + ExecutorID: "utils.env.Get", + Args: []any{"PATH"}, + Status: types.TaskPending, + } + + taskCtx := &standard.RunnerContext{} + + result := runner.ExecuteWithRetry(task, taskCtx) + + // Process tasks are single-call, no multi-turn + t.Logf("Process result: success=%v, output=%v", result.Success, result.Output) + }) +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// createRunnerTestRobot creates a test robot for runner tests +func createRunnerTestRobot(t *testing.T) *types.Robot { + t.Helper() + return &types.Robot{ + MemberID: "test-robot-runner", + TeamID: "test-team-1", + DisplayName: "Test Robot for Runner", + SystemPrompt: "You are a helpful assistant. Follow instructions carefully and provide clear responses.", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Assistant", + Duties: []string{"Execute tasks", "Generate content"}, + }, + Resources: &types.Resources{ + Phases: map[types.Phase]string{ + types.PhaseRun: "robot.validation", + "validation": "robot.validation", // For semantic validation agent + }, + Agents: []string{ + "experts.data-analyst", + "experts.summarizer", + "experts.text-writer", + }, + }, + }, + } +} + +// Note: createRunnerTestExecution is available if needed for future tests +// that require a full Execution object instead of just RunnerContext diff --git a/agent/robot/executor/standard/validator_test.go b/agent/robot/executor/standard/validator_test.go new file mode 100644 index 00000000..e3888a63 --- /dev/null +++ b/agent/robot/executor/standard/validator_test.go @@ -0,0 +1,637 @@ +package standard_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + agentcontext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/robot/executor/standard" + "github.com/yaoapp/yao/agent/robot/types" + "github.com/yaoapp/yao/agent/testutils" +) + +// ============================================================================ +// Validator Tests - Two-Layer Validation System +// ============================================================================ + +func TestValidatorValidateWithContext(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("validates with no rules - passes with valid output", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "", + ValidationRules: []string{}, + } + + result := validator.ValidateWithContext(task, "Some output", nil) + + assert.True(t, result.Passed) + assert.True(t, result.Complete) + assert.False(t, result.NeedReply) + assert.Equal(t, 1.0, result.Score) + }) + + t.Run("validates with no rules - incomplete with empty output", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "", + ValidationRules: []string{}, + } + + result := validator.ValidateWithContext(task, "", nil) + + assert.True(t, result.Passed) + assert.False(t, result.Complete) // Empty output = not complete + }) + + t.Run("validates with rule-based validation - passes", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "contains", "value": "hello"}`, + }, + } + + result := validator.ValidateWithContext(task, "hello world", nil) + + assert.True(t, result.Passed) + assert.True(t, result.Complete) + assert.False(t, result.NeedReply) + }) + + t.Run("validates with rule-based validation - fails", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "contains", "value": "expected_string"}`, + }, + } + + result := validator.ValidateWithContext(task, "actual output without expected", nil) + + assert.False(t, result.Passed) + assert.False(t, result.Complete) + assert.True(t, result.NeedReply) // Should suggest retry + assert.NotEmpty(t, result.ReplyContent) + assert.NotEmpty(t, result.Issues) + }) + + t.Run("validates with semantic validation", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "A professional greeting message", + } + + result := validator.ValidateWithContext(task, "Dear Sir/Madam, I hope this message finds you well.", nil) + + // Semantic validation should pass for this appropriate output + t.Logf("Validation result: passed=%v, complete=%v, score=%.2f", + result.Passed, result.Complete, result.Score) + t.Logf("Issues: %v", result.Issues) + t.Logf("Suggestions: %v", result.Suggestions) + + // The semantic validator should recognize this as appropriate + assert.NotNil(t, result) + }) +} + +func TestValidatorIsComplete(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("complete when passed with valid output", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "", + ValidationRules: []string{}, + } + + result := validator.ValidateWithContext(task, "Valid output", nil) + + assert.True(t, result.Complete) + }) + + t.Run("not complete when passed but empty output", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "", + ValidationRules: []string{}, + } + + result := validator.ValidateWithContext(task, "", nil) + + assert.False(t, result.Complete) + }) + + t.Run("not complete when validation failed", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "contains", "value": "MUST_CONTAIN_THIS"}`, + }, + } + + result := validator.ValidateWithContext(task, "output without required string", nil) + + assert.False(t, result.Passed) + assert.False(t, result.Complete) + }) + + t.Run("not complete when score below threshold", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + config.ValidationThreshold = 0.9 // High threshold + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "A very specific output format that's hard to match exactly", + } + + // This output might get a lower score due to semantic mismatch + result := validator.ValidateWithContext(task, "Some generic output", nil) + + // If score is below threshold, should not be complete + if result.Passed && result.Score < config.ValidationThreshold { + assert.False(t, result.Complete) + } + }) +} + +func TestValidatorCheckNeedReply(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("no reply needed when complete", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "", + ValidationRules: []string{}, + } + + result := validator.ValidateWithContext(task, "Complete output", nil) + + assert.True(t, result.Complete) + assert.False(t, result.NeedReply) + assert.Empty(t, result.ReplyContent) + }) + + t.Run("reply needed when validation failed with suggestions", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "type", "value": "object"}`, + }, + } + + // String output when object expected + result := validator.ValidateWithContext(task, "not an object", nil) + + assert.False(t, result.Passed) + assert.True(t, result.NeedReply) + assert.NotEmpty(t, result.ReplyContent) + // The reply should contain validation feedback about the issue + assert.Contains(t, result.ReplyContent, "did not pass validation") + }) + + t.Run("reply needed when output is empty but passed", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "", + ValidationRules: []string{}, + } + + result := validator.ValidateWithContext(task, " ", nil) // Whitespace only + + // Passed (no rules) but not complete (empty output) + assert.True(t, result.Passed) + assert.False(t, result.Complete) + // When passed but not complete (empty output), checkNeedReply may or may not + // set NeedReply depending on the implementation details + // Just verify the result is consistent + t.Logf("NeedReply: %v, ReplyContent: %s", result.NeedReply, result.ReplyContent) + }) +} + +func TestValidatorConvertStringRule(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("converts 'valid JSON' rule", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + "output must be valid JSON", + }, + } + + // Valid JSON object + result := validator.ValidateWithContext(task, map[string]interface{}{"key": "value"}, nil) + assert.True(t, result.Passed) + + // Invalid (string is not an object) + result2 := validator.ValidateWithContext(task, "not json", nil) + assert.False(t, result2.Passed) + }) + + t.Run("converts 'must contain' rule", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + "must contain 'success'", + }, + } + + result := validator.ValidateWithContext(task, "Operation was a success!", nil) + assert.True(t, result.Passed) + + result2 := validator.ValidateWithContext(task, "Operation failed", nil) + assert.False(t, result2.Passed) + }) + + t.Run("converts 'not empty' rule", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + "output must not be empty", + }, + } + + result := validator.ValidateWithContext(task, "Some content", nil) + assert.True(t, result.Passed) + + // Note: The "not empty" rule may be converted to semantic validation + // rather than a rule-based assertion, so empty string might still pass + // if semantic validation is lenient + result2 := validator.ValidateWithContext(task, "", nil) + t.Logf("Empty string validation: passed=%v, issues=%v", result2.Passed, result2.Issues) + }) + + t.Run("converts 'json array' rule", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + "must be json array", + }, + } + + result := validator.ValidateWithContext(task, []interface{}{"a", "b", "c"}, nil) + assert.True(t, result.Passed) + + result2 := validator.ValidateWithContext(task, map[string]interface{}{"key": "value"}, nil) + assert.False(t, result2.Passed) + }) +} + +func TestValidatorParseRules(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("parses JSON assertion rules", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "equals", "value": "expected"}`, + }, + } + + result := validator.ValidateWithContext(task, "expected", nil) + assert.True(t, result.Passed) + + result2 := validator.ValidateWithContext(task, "different", nil) + assert.False(t, result2.Passed) + }) + + t.Run("parses regex rules", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "regex", "value": "^[A-Z][a-z]+$"}`, + }, + } + + result := validator.ValidateWithContext(task, "Hello", nil) + assert.True(t, result.Passed) + + result2 := validator.ValidateWithContext(task, "hello", nil) + assert.False(t, result2.Passed) + }) + + t.Run("parses json_path rules", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "json_path", "path": "data.count", "value": 42}`, + }, + } + + result := validator.ValidateWithContext(task, map[string]interface{}{ + "data": map[string]interface{}{ + "count": 42, + }, + }, nil) + assert.True(t, result.Passed) + + result2 := validator.ValidateWithContext(task, map[string]interface{}{ + "data": map[string]interface{}{ + "count": 10, + }, + }, nil) + assert.False(t, result2.Passed) + }) + + t.Run("parses type rules with path", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ValidationRules: []string{ + `{"type": "type", "path": "items", "value": "array"}`, + }, + } + + result := validator.ValidateWithContext(task, map[string]interface{}{ + "items": []interface{}{"a", "b"}, + }, nil) + assert.True(t, result.Passed) + + result2 := validator.ValidateWithContext(task, map[string]interface{}{ + "items": "not an array", + }, nil) + assert.False(t, result2.Passed) + }) +} + +func TestValidatorSemanticValidation(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("semantic validation with expected output", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "A JSON object containing user information with name and email fields", + } + + output := map[string]interface{}{ + "name": "John Doe", + "email": "john@example.com", + } + + result := validator.ValidateWithContext(task, output, nil) + + t.Logf("Semantic validation: passed=%v, score=%.2f, complete=%v", + result.Passed, result.Score, result.Complete) + t.Logf("Details: %s", result.Details) + + // Should pass semantic validation + assert.NotNil(t, result) + }) + + t.Run("semantic validation with complex criteria", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "A professional email with greeting, body, and signature", + Messages: []agentcontext.Message{ + {Role: agentcontext.RoleUser, Content: "Write a professional email"}, + }, + } + + output := `Dear Mr. Smith, + +I hope this email finds you well. I am writing to follow up on our previous conversation regarding the project timeline. + +Please let me know if you have any questions. + +Best regards, +John Doe` + + result := validator.ValidateWithContext(task, output, nil) + + t.Logf("Email validation: passed=%v, score=%.2f", result.Passed, result.Score) + + // Should recognize this as a valid professional email + assert.NotNil(t, result) + }) +} + +func TestValidatorMergeResults(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test") + } + + testutils.Prepare(t) + defer testutils.Clean(t) + + ctx := types.NewContext(context.Background(), testAuth()) + + t.Run("both rule and semantic validation pass", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "A greeting message", + ValidationRules: []string{ + `{"type": "contains", "value": "Hello"}`, + }, + } + + result := validator.ValidateWithContext(task, "Hello, how are you today?", nil) + + assert.True(t, result.Passed) + assert.True(t, result.Complete) + }) + + t.Run("rule passes but semantic fails", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "A formal business letter with proper formatting", + ValidationRules: []string{ + `{"type": "contains", "value": "Hello"}`, // This will pass + }, + } + + // Contains "Hello" but not a formal business letter + result := validator.ValidateWithContext(task, "Hello there buddy!", nil) + + // Rule passes, but semantic might not + t.Logf("Merged result: passed=%v, score=%.2f", result.Passed, result.Score) + }) + + t.Run("rule fails - semantic not run", func(t *testing.T) { + robot := createValidatorTestRobot(t) + config := standard.DefaultRunConfig() + validator := standard.NewValidator(ctx, robot, config) + + task := &types.Task{ + ID: "task-001", + ExpectedOutput: "Some expected output", + ValidationRules: []string{ + `{"type": "contains", "value": "REQUIRED_STRING"}`, + }, + } + + result := validator.ValidateWithContext(task, "Output without required string", nil) + + // Should fail at rule level, semantic not needed + assert.False(t, result.Passed) + assert.False(t, result.Complete) + }) +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// createValidatorTestRobot creates a test robot for validator tests +func createValidatorTestRobot(t *testing.T) *types.Robot { + t.Helper() + return &types.Robot{ + MemberID: "test-robot-validator", + TeamID: "test-team-1", + DisplayName: "Test Robot for Validator", + SystemPrompt: "You are a helpful assistant.", + Config: &types.Config{ + Identity: &types.Identity{ + Role: "Test Assistant", + Duties: []string{"Validate outputs"}, + }, + Resources: &types.Resources{ + Phases: map[types.Phase]string{ + types.PhaseRun: "robot.validation", + "validation": "robot.validation", // For semantic validation agent + }, + Agents: []string{ + "experts.data-analyst", + "experts.text-writer", + }, + }, + }, + } +}