e0e68393a0
- Improved the multi-turn conversation handling in the `RunConfig`, adding parameters for better control over conversation execution. - Enhanced the `ValidationResult` structure to better accommodate multi-turn states, ensuring accurate tracking of conversation progress. - Updated the `ExecuteWithRetry` method to align with the new conversation flow, improving task execution reliability. - Revised the `Validator` logic to effectively manage conversation continuation based on validation outcomes. - Updated documentation and tests to ensure clarity and coverage of the new multi-turn capabilities and validation improvements.
517 lines
15 KiB
Go
517 lines
15 KiB
Go
package standard_test
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
agentcontext "github.com/yaoapp/yao/agent/context"
|
|
"github.com/yaoapp/yao/agent/robot/executor/standard"
|
|
"github.com/yaoapp/yao/agent/robot/types"
|
|
"github.com/yaoapp/yao/agent/testutils"
|
|
)
|
|
|
|
// ============================================================================
|
|
// Runner Tests - Multi-Turn Conversation Flow
|
|
// ============================================================================
|
|
|
|
func TestRunnerExecuteWithRetry(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
ctx := types.NewContext(context.Background(), testAuth())
|
|
|
|
t.Run("executes assistant task with multi-turn conversation", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
task := &types.Task{
|
|
ID: "task-001",
|
|
ExecutorType: types.ExecutorAssistant,
|
|
ExecutorID: "experts.text-writer",
|
|
Messages: []agentcontext.Message{
|
|
{Role: agentcontext.RoleUser, Content: "Write a haiku about coding. Format: three lines with 5-7-5 syllables."},
|
|
},
|
|
ExpectedOutput: "A haiku poem about coding",
|
|
Status: types.TaskPending,
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{
|
|
SystemPrompt: robot.SystemPrompt,
|
|
}
|
|
|
|
result := runner.ExecuteWithRetry(task, taskCtx)
|
|
|
|
assert.True(t, result.Success, "task should succeed")
|
|
assert.NotNil(t, result.Output)
|
|
assert.NotNil(t, result.Validation)
|
|
assert.True(t, result.Validation.Complete)
|
|
assert.Greater(t, result.Duration, int64(0))
|
|
|
|
t.Logf("Output: %v", result.Output)
|
|
t.Logf("Validation: passed=%v, complete=%v, score=%.2f",
|
|
result.Validation.Passed, result.Validation.Complete, result.Validation.Score)
|
|
})
|
|
|
|
t.Run("handles validation failure with multi-turn retry", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
config.MaxTurnsPerTask = 3 // Limit turns for test
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
// Task with strict validation that may require conversation
|
|
task := &types.Task{
|
|
ID: "task-002",
|
|
ExecutorType: types.ExecutorAssistant,
|
|
ExecutorID: "experts.data-analyst",
|
|
Messages: []agentcontext.Message{
|
|
{Role: agentcontext.RoleUser, Content: "Return a JSON object with exactly these fields: status (string 'ok'), count (number greater than 0)."},
|
|
},
|
|
ExpectedOutput: "JSON with status='ok' and count>0",
|
|
ValidationRules: []string{
|
|
"output must be valid JSON",
|
|
`{"type": "type", "value": "object"}`,
|
|
},
|
|
Status: types.TaskPending,
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{
|
|
SystemPrompt: robot.SystemPrompt,
|
|
}
|
|
|
|
result := runner.ExecuteWithRetry(task, taskCtx)
|
|
|
|
// Should either succeed or fail gracefully
|
|
assert.NotNil(t, result.Validation)
|
|
t.Logf("Success: %v, Output: %v", result.Success, result.Output)
|
|
t.Logf("Validation: passed=%v, complete=%v, needReply=%v",
|
|
result.Validation.Passed, result.Validation.Complete, result.Validation.NeedReply)
|
|
})
|
|
|
|
t.Run("respects max turns limit", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
config.MaxTurnsPerTask = 1 // Only 1 turn allowed
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
// Task that requires multiple turns - asking for something incomplete
|
|
// then validation will ask for more, but we only allow 1 turn
|
|
task := &types.Task{
|
|
ID: "task-003",
|
|
ExecutorType: types.ExecutorAssistant,
|
|
ExecutorID: "experts.text-writer",
|
|
Messages: []agentcontext.Message{
|
|
{Role: agentcontext.RoleUser, Content: "Say 'hello'"},
|
|
},
|
|
// Validation will fail because it expects a JSON object
|
|
ExpectedOutput: "A JSON object with 'status' and 'data' fields",
|
|
ValidationRules: []string{`{"type": "type", "value": "object"}`},
|
|
Status: types.TaskPending,
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{
|
|
SystemPrompt: robot.SystemPrompt,
|
|
}
|
|
|
|
result := runner.ExecuteWithRetry(task, taskCtx)
|
|
|
|
// With only 1 turn and strict validation, task should not complete successfully
|
|
// Either it fails validation or hits max turns
|
|
t.Logf("Result: success=%v, error=%s", result.Success, result.Error)
|
|
t.Logf("Validation: passed=%v, complete=%v, needReply=%v",
|
|
result.Validation.Passed, result.Validation.Complete, result.Validation.NeedReply)
|
|
|
|
// The test verifies the max turns mechanism works - task either:
|
|
// 1. Fails validation (expected with "say hello" vs JSON requirement)
|
|
// 2. Or hits max turns if validation requests retry
|
|
assert.NotNil(t, result.Validation)
|
|
})
|
|
}
|
|
|
|
func TestRunnerBuildTaskContext(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
ctx := types.NewContext(context.Background(), testAuth())
|
|
|
|
t.Run("includes previous results in context", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
exec := &types.Execution{
|
|
ID: "test-exec",
|
|
MemberID: robot.MemberID,
|
|
TeamID: robot.TeamID,
|
|
Goals: &types.Goals{
|
|
Content: "Test goals",
|
|
},
|
|
Results: []types.TaskResult{
|
|
{
|
|
TaskID: "task-001",
|
|
Success: true,
|
|
Output: map[string]interface{}{"data": "previous result"},
|
|
},
|
|
{
|
|
TaskID: "task-002",
|
|
Success: true,
|
|
Output: "Another result",
|
|
},
|
|
},
|
|
}
|
|
exec.SetRobot(robot)
|
|
|
|
// Build context for task at index 2 (should include results 0 and 1)
|
|
taskCtx := runner.BuildTaskContext(exec, 2)
|
|
|
|
assert.NotNil(t, taskCtx)
|
|
assert.Len(t, taskCtx.PreviousResults, 2)
|
|
assert.Equal(t, "task-001", taskCtx.PreviousResults[0].TaskID)
|
|
assert.Equal(t, "task-002", taskCtx.PreviousResults[1].TaskID)
|
|
assert.Equal(t, robot.SystemPrompt, taskCtx.SystemPrompt)
|
|
})
|
|
|
|
t.Run("handles first task with no previous results", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
exec := &types.Execution{
|
|
ID: "test-exec",
|
|
MemberID: robot.MemberID,
|
|
TeamID: robot.TeamID,
|
|
Goals: &types.Goals{
|
|
Content: "Test goals",
|
|
},
|
|
Results: []types.TaskResult{},
|
|
}
|
|
exec.SetRobot(robot)
|
|
|
|
taskCtx := runner.BuildTaskContext(exec, 0)
|
|
|
|
assert.NotNil(t, taskCtx)
|
|
assert.Empty(t, taskCtx.PreviousResults)
|
|
})
|
|
|
|
t.Run("handles bounds check for task index", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
exec := &types.Execution{
|
|
ID: "test-exec",
|
|
MemberID: robot.MemberID,
|
|
TeamID: robot.TeamID,
|
|
Results: []types.TaskResult{
|
|
{TaskID: "task-001", Success: true},
|
|
},
|
|
}
|
|
exec.SetRobot(robot)
|
|
|
|
// Task index 5, but only 1 result exists
|
|
taskCtx := runner.BuildTaskContext(exec, 5)
|
|
|
|
assert.NotNil(t, taskCtx)
|
|
assert.Len(t, taskCtx.PreviousResults, 1) // Should only include available results
|
|
})
|
|
}
|
|
|
|
func TestRunnerFormatPreviousResultsAsContext(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
ctx := types.NewContext(context.Background(), testAuth())
|
|
|
|
t.Run("formats previous results as markdown", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
results := []types.TaskResult{
|
|
{
|
|
TaskID: "task-001",
|
|
Success: true,
|
|
Output: map[string]interface{}{"key": "value", "count": 42},
|
|
},
|
|
{
|
|
TaskID: "task-002",
|
|
Success: false,
|
|
Output: "Partial result",
|
|
Error: "Validation failed",
|
|
},
|
|
}
|
|
|
|
formatted := runner.FormatPreviousResultsAsContext(results)
|
|
|
|
assert.Contains(t, formatted, "## Previous Task Results")
|
|
assert.Contains(t, formatted, "task-001")
|
|
assert.Contains(t, formatted, "task-002")
|
|
assert.Contains(t, formatted, "Success")
|
|
assert.Contains(t, formatted, "Failed")
|
|
assert.Contains(t, formatted, "key")
|
|
assert.Contains(t, formatted, "value")
|
|
|
|
t.Logf("Formatted context:\n%s", formatted)
|
|
})
|
|
|
|
t.Run("returns empty string for no results", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
formatted := runner.FormatPreviousResultsAsContext([]types.TaskResult{})
|
|
|
|
assert.Empty(t, formatted)
|
|
})
|
|
}
|
|
|
|
func TestRunnerBuildAssistantMessages(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
ctx := types.NewContext(context.Background(), testAuth())
|
|
|
|
t.Run("builds messages with task content", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
task := &types.Task{
|
|
ID: "task-001",
|
|
ExecutorType: types.ExecutorAssistant,
|
|
ExecutorID: "experts.text-writer",
|
|
Messages: []agentcontext.Message{
|
|
{Role: agentcontext.RoleUser, Content: "Write a greeting"},
|
|
},
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{
|
|
SystemPrompt: "You are helpful",
|
|
}
|
|
|
|
messages := runner.BuildAssistantMessages(task, taskCtx)
|
|
|
|
assert.NotEmpty(t, messages)
|
|
// Should contain task message
|
|
found := false
|
|
for _, msg := range messages {
|
|
if content, ok := msg.Content.(string); ok && content == "Write a greeting" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
assert.True(t, found, "should contain task message")
|
|
})
|
|
|
|
t.Run("includes previous results in messages", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
task := &types.Task{
|
|
ID: "task-002",
|
|
ExecutorType: types.ExecutorAssistant,
|
|
ExecutorID: "experts.text-writer",
|
|
Messages: []agentcontext.Message{
|
|
{Role: agentcontext.RoleUser, Content: "Continue from previous"},
|
|
},
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{
|
|
PreviousResults: []types.TaskResult{
|
|
{TaskID: "task-001", Success: true, Output: "Previous output"},
|
|
},
|
|
SystemPrompt: "You are helpful",
|
|
}
|
|
|
|
messages := runner.BuildAssistantMessages(task, taskCtx)
|
|
|
|
assert.NotEmpty(t, messages)
|
|
// Should have context message with previous results
|
|
formatted := runner.FormatMessagesAsText(messages)
|
|
assert.Contains(t, formatted, "Previous Task Results")
|
|
})
|
|
}
|
|
|
|
func TestRunnerFormatMessagesAsText(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
ctx := types.NewContext(context.Background(), testAuth())
|
|
|
|
t.Run("formats string content", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
messages := []agentcontext.Message{
|
|
{Role: agentcontext.RoleUser, Content: "Hello"},
|
|
{Role: agentcontext.RoleUser, Content: "World"},
|
|
}
|
|
|
|
text := runner.FormatMessagesAsText(messages)
|
|
|
|
assert.Contains(t, text, "Hello")
|
|
assert.Contains(t, text, "World")
|
|
})
|
|
|
|
t.Run("handles multipart content", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
messages := []agentcontext.Message{
|
|
{
|
|
Role: agentcontext.RoleUser,
|
|
Content: []interface{}{
|
|
map[string]interface{}{"type": "text", "text": "Part 1"},
|
|
map[string]interface{}{"type": "text", "text": "Part 2"},
|
|
},
|
|
},
|
|
}
|
|
|
|
text := runner.FormatMessagesAsText(messages)
|
|
|
|
assert.Contains(t, text, "Part 1")
|
|
assert.Contains(t, text, "Part 2")
|
|
})
|
|
|
|
t.Run("handles map content via JSON", func(t *testing.T) {
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
messages := []agentcontext.Message{
|
|
{
|
|
Role: agentcontext.RoleUser,
|
|
Content: map[string]interface{}{"key": "value"},
|
|
},
|
|
}
|
|
|
|
text := runner.FormatMessagesAsText(messages)
|
|
|
|
assert.Contains(t, text, "key")
|
|
assert.Contains(t, text, "value")
|
|
})
|
|
}
|
|
|
|
// ============================================================================
|
|
// Non-Assistant Task Tests (MCP, Process)
|
|
// ============================================================================
|
|
|
|
func TestRunnerExecuteNonAssistantTask(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
ctx := types.NewContext(context.Background(), testAuth())
|
|
|
|
t.Run("executes MCP task (single-call)", func(t *testing.T) {
|
|
// Note: This test requires MCP server to be running
|
|
// Skip if MCP is not available
|
|
t.Skip("MCP server not available in test environment")
|
|
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
task := &types.Task{
|
|
ID: "task-mcp",
|
|
ExecutorType: types.ExecutorMCP,
|
|
ExecutorID: "filesystem.list_directory",
|
|
Args: []any{map[string]interface{}{"path": "/tmp"}},
|
|
Status: types.TaskPending,
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{}
|
|
|
|
result := runner.ExecuteWithRetry(task, taskCtx)
|
|
|
|
// MCP tasks are single-call, no multi-turn
|
|
t.Logf("MCP result: success=%v, output=%v", result.Success, result.Output)
|
|
})
|
|
|
|
t.Run("executes Process task (single-call)", func(t *testing.T) {
|
|
// Note: This test requires a Yao process to be available
|
|
// Skip if process is not available
|
|
t.Skip("Yao process not available in test environment")
|
|
|
|
robot := createRunnerTestRobot(t)
|
|
config := standard.DefaultRunConfig()
|
|
runner := standard.NewRunner(ctx, robot, config)
|
|
|
|
task := &types.Task{
|
|
ID: "task-process",
|
|
ExecutorType: types.ExecutorProcess,
|
|
ExecutorID: "utils.env.Get",
|
|
Args: []any{"PATH"},
|
|
Status: types.TaskPending,
|
|
}
|
|
|
|
taskCtx := &standard.RunnerContext{}
|
|
|
|
result := runner.ExecuteWithRetry(task, taskCtx)
|
|
|
|
// Process tasks are single-call, no multi-turn
|
|
t.Logf("Process result: success=%v, output=%v", result.Success, result.Output)
|
|
})
|
|
}
|
|
|
|
// ============================================================================
|
|
// Helper Functions
|
|
// ============================================================================
|
|
|
|
// createRunnerTestRobot creates a test robot for runner tests
|
|
func createRunnerTestRobot(t *testing.T) *types.Robot {
|
|
t.Helper()
|
|
return &types.Robot{
|
|
MemberID: "test-robot-runner",
|
|
TeamID: "test-team-1",
|
|
DisplayName: "Test Robot for Runner",
|
|
SystemPrompt: "You are a helpful assistant. Follow instructions carefully and provide clear responses.",
|
|
Config: &types.Config{
|
|
Identity: &types.Identity{
|
|
Role: "Test Assistant",
|
|
Duties: []string{"Execute tasks", "Generate content"},
|
|
},
|
|
Resources: &types.Resources{
|
|
Phases: map[types.Phase]string{
|
|
types.PhaseRun: "robot.validation",
|
|
"validation": "robot.validation", // For semantic validation agent
|
|
},
|
|
Agents: []string{
|
|
"experts.data-analyst",
|
|
"experts.summarizer",
|
|
"experts.text-writer",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Note: createRunnerTestExecution is available if needed for future tests
|
|
// that require a full Execution object instead of just RunnerContext
|