From ac5e3e484f4e3ecc7986f5cd6917859c27b9928f Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 16 Jan 2026 10:30:37 +0800 Subject: [PATCH] Enhance Task Validation and Update TODO.md - Introduced a validation mechanism for task results, including a detailed validation structure with scores and issues. - Updated the input formatter to include validation results in the output, improving clarity on task success and validation status. - Enhanced test cases to cover the new validation fields and ensure comprehensive testing of task results. - Revised TODO.md to reflect the addition of validation features and the current status of the agent's development phases. --- agent/robot/TODO.md | 215 +++++++++++++++++++++-------- agent/robot/executor/input.go | 24 +++- agent/robot/executor/input_test.go | 30 ++-- agent/robot/executor/run.go | 13 +- agent/robot/types/robot.go | 60 ++++++-- agent/robot/types/robot_test.go | 85 ++++++++++-- 6 files changed, 327 insertions(+), 100 deletions(-) diff --git a/agent/robot/TODO.md b/agent/robot/TODO.md index d4d8de4b..8471cf64 100644 --- a/agent/robot/TODO.md +++ b/agent/robot/TODO.md @@ -428,85 +428,180 @@ Trigger → Manager → Cache → Dedup → Pool → Worker → Executor(stub) ## Phase 5: Test Scenario & Assistants Setup -**Goal:** Create a realistic test scenario with all required assistants. +**Goal:** Create realistic test scenarios with all required assistants. -**Test Scenario: Sales Analyst Robot** - -A Sales Analyst robot that: - -- Wakes up at 09:00 on weekdays -- Checks sales data and market news -- Generates daily goals based on findings -- Creates 2-3 actionable tasks -- (Future: executes tasks, delivers report, learns) - -Example flow: +**Architecture:** ``` -Clock: Monday 09:00 - ↓ -P0 Inspiration: - - Clock: Monday morning, start of week - - Data: 15 new orders (+20% vs last week) - - News: Competitor launched new product - ↓ -P1 Goals: - 1. [High] Analyze weekend sales spike - 2. [Normal] Review competitor product launch - 3. [Low] Update weekly forecast - ↓ -P2 Tasks: - Task 1: Query sales DB for weekend orders - Task 2: Search web for competitor news - Task 3: Generate forecast update +┌─────────────────────────────────────────────────────────────────────────────┐ +│ 6 Generic Phase Agents (P0-P5) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ inspiration │ goals │ tasks │ validation │ delivery │ learning │ +│ (P0) │ (P1) │ (P2) │ (P3) │ (P4) │ (P5) │ +└───────────────┴─────────┴─────────┴──────────────┴────────────┴─────────────┘ + ↓ P2 assigns tasks to +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Expert Agents (Task Executors) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ text-writer │ web-reader │ data-analyst │ summarizer │ ... │ +│ (Generate) │ (Fetch URL) │ (Analyze) │ (Summarize) │ │ +└───────────────┴──────────────┴────────────────┴──────────────┴─────────────┘ ``` -### 5.1 Test Assistant Directory Structure +**Test Strategy:** -Create `yao-dev-app/assistants/robot/` directory: +- Phase Agents (P0-P5) are **generic** and reusable across all robot types +- Expert Agents are **specialized** for specific tasks (text, web, data, etc.) +- Each P0-P5 test uses **different expert combinations** to cover real scenarios +- Tests use `interval: 1s` or `TriggerManual()` for easy triggering (no time dependency) + +### 5.1 Directory Structure ``` -assistants/robot/ -├── inspiration/ # P0: Inspiration Agent -│ ├── package.yao # Assistant config -│ └── prompts.yml # System prompt & templates -├── goals/ # P1: Goal Generation Agent -│ ├── package.yao -│ └── prompts.yml -├── tasks/ # P2: Task Planning Agent -│ ├── package.yao -│ └── prompts.yml -├── validation/ # P3: Validation Agent (future) -│ ├── package.yao -│ └── prompts.yml -├── delivery/ # P4: Delivery Agent (future) -│ ├── package.yao -│ └── prompts.yml -└── learning/ # P5: Learning Agent (future) - ├── package.yao - └── prompts.yml +yao-dev-app/assistants/ +├── robot/ # Generic Phase Agents +│ ├── inspiration/ # P0: Analyze clock context, generate insights +│ │ ├── package.yao +│ │ └── prompts.yml +│ ├── goals/ # P1: Generate prioritized goals +│ │ ├── package.yao +│ │ └── prompts.yml +│ ├── tasks/ # P2: Split goals into executable tasks +│ │ ├── package.yao +│ │ └── prompts.yml +│ ├── validation/ # P3: Validate task results +│ │ ├── package.yao +│ │ └── prompts.yml +│ ├── delivery/ # P4: Format and deliver results +│ │ ├── package.yao +│ │ └── prompts.yml +│ └── learning/ # P5: Summarize execution, extract insights +│ ├── package.yao +│ └── prompts.yml +│ +└── experts/ # Expert Agents (Task Executors) + ├── text-writer/ # Generate text content (reports, emails, summaries) + │ ├── package.yao + │ └── prompts.yml + ├── web-reader/ # Fetch and parse web page content + │ ├── package.yao + │ └── prompts.yml + ├── data-analyst/ # Analyze data, generate insights + │ ├── package.yao + │ └── prompts.yml + └── summarizer/ # Summarize long text into key points + ├── package.yao + └── prompts.yml ``` -### 5.2 Inspiration Assistant (P0) +### 5.2 Generic Phase Agents + +#### 5.2.1 Inspiration Agent (P0) - [ ] `robot/inspiration/package.yao` - config with model, temperature -- [ ] `robot/inspiration/prompts.yml` - system prompt for P0: - - Input: Clock context, robot identity, data sources +- [ ] `robot/inspiration/prompts.yml` - system prompt: + - Input: Clock context (time, day, markers), robot identity - Output: Markdown report with Summary, Highlights, Opportunities, Risks + - Style: Analytical, context-aware -### 5.3 Goals Assistant (P1) +#### 5.2.2 Goals Agent (P1) - [ ] `robot/goals/package.yao` - config -- [ ] `robot/goals/prompts.yml` - system prompt for P1: - - Input: Inspiration report, robot duties - - Output: Prioritized goals in markdown format +- [ ] `robot/goals/prompts.yml` - system prompt: + - Input: Inspiration report OR trigger input (human/event) + - Output: Prioritized goals in markdown (High/Normal/Low) + - Style: Strategic, actionable -### 5.4 Tasks Assistant (P2) +#### 5.2.3 Tasks Agent (P2) - [ ] `robot/tasks/package.yao` - config -- [ ] `robot/tasks/prompts.yml` - system prompt for P2: - - Input: Goals, available tools/agents - - Output: Structured task list (JSON) +- [ ] `robot/tasks/prompts.yml` - system prompt: + - Input: Goals, available expert agents list + - Output: Structured task list (JSON) with executor assignments + - Style: Detailed, executable + +#### 5.2.4 Validation Agent (P3) + +- [ ] `robot/validation/package.yao` - config +- [ ] `robot/validation/prompts.yml` - system prompt: + - Input: Task result, expected outcome + - Output: Validation result (pass/fail, issues, suggestions) + - Style: Critical, thorough + +#### 5.2.5 Delivery Agent (P4) + +- [ ] `robot/delivery/package.yao` - config +- [ ] `robot/delivery/prompts.yml` - system prompt: + - Input: Task results, delivery target (email, report, notification) + - Output: Formatted delivery content + - Style: Clear, professional + +#### 5.2.6 Learning Agent (P5) + +- [ ] `robot/learning/package.yao` - config +- [ ] `robot/learning/prompts.yml` - system prompt: + - Input: Full execution summary + - Output: Insights, patterns, improvement suggestions + - Style: Reflective, insightful + +### 5.3 Expert Agents (Task Executors) + +#### 5.3.1 Text Writer + +- [ ] `experts/text-writer/package.yao` - config +- [ ] `experts/text-writer/prompts.yml` - system prompt: + - Input: Topic, key points, style (formal/casual), length + - Output: Generated text content + - Use cases: Weekly reports, email drafts, summaries + +#### 5.3.2 Web Reader + +- [ ] `experts/web-reader/package.yao` - config +- [ ] `experts/web-reader/prompts.yml` - system prompt: + - Input: URL or topic to search + - Output: Extracted content, key information + - Use cases: News fetching, competitor monitoring, research + - Note: May use MCP tools for actual web access + +#### 5.3.3 Data Analyst + +- [ ] `experts/data-analyst/package.yao` - config +- [ ] `experts/data-analyst/prompts.yml` - system prompt: + - Input: Data description, analysis goal + - Output: Analysis report, trends, insights + - Use cases: Sales analysis, performance review + +#### 5.3.4 Summarizer + +- [ ] `experts/summarizer/package.yao` - config +- [ ] `experts/summarizer/prompts.yml` - system prompt: + - Input: Long text content + - Output: Concise summary with key points + - Use cases: Document summarization, meeting notes + +### 5.4 Test Scenarios + +Each phase test uses different expert combinations: + +| Test | Phase | Trigger | Expert Agents Used | Verification | +| ---- | ----- | ---------------- | ------------------------ | ----------------------------- | +| T1 | P0 | Clock (interval) | - | Clock → Inspiration report | +| T2 | P1 | Clock | - | Inspiration → Goals | +| T3 | P1 | Human | - | User input → Goals | +| T4 | P2 | Clock | text-writer, web-reader | Goals → Tasks with executors | +| T5 | P3 | Clock | text-writer | Task exec → Result validation | +| T6 | P3 | Human | summarizer | Task exec → Result validation | +| T7 | P4 | Clock | - | Results → Delivery format | +| T8 | P5 | Clock | - | Full execution → Insights | +| T9 | E2E | Clock | text-writer, summarizer | Full P0→P5 flow | +| T10 | E2E | Human | web-reader, data-analyst | Full P1→P5 flow | + +### 5.5 Test Data Setup + +- [ ] Create test robot config in `__yao.member` with: + - `trigger.clock.mode: interval`, `interval: 1s` (for fast testing) + - `resources.agents: [experts.text-writer, experts.web-reader, ...]` +- [ ] Create test trigger data for Human/Event scenarios --- diff --git a/agent/robot/executor/input.go b/agent/robot/executor/input.go index 541944ac..3d086523 100644 --- a/agent/robot/executor/input.go +++ b/agent/robot/executor/input.go @@ -273,6 +273,7 @@ func (f *InputFormatter) FormatTaskResults(results []types.TaskResult) string { successCount := 0 failCount := 0 + validatedCount := 0 for _, result := range results { if result.Success { @@ -280,6 +281,9 @@ func (f *InputFormatter) FormatTaskResults(results []types.TaskResult) string { } else { failCount++ } + if result.Validation != nil && result.Validation.Passed { + validatedCount++ + } sb.WriteString(fmt.Sprintf("### Task: %s\n\n", result.TaskID)) if result.Success { @@ -288,7 +292,21 @@ func (f *InputFormatter) FormatTaskResults(results []types.TaskResult) string { sb.WriteString("- **Status**: ✗ Failed\n") } sb.WriteString(fmt.Sprintf("- **Duration**: %dms\n", result.Duration)) - sb.WriteString(fmt.Sprintf("- **Validated**: %t\n", result.Validated)) + + // Validation result (P3) + if result.Validation != nil { + if result.Validation.Passed { + sb.WriteString(fmt.Sprintf("- **Validation**: ✓ Passed (score: %.2f)\n", result.Validation.Score)) + } else { + sb.WriteString("- **Validation**: ✗ Failed\n") + if len(result.Validation.Issues) > 0 { + sb.WriteString(" - Issues:\n") + for _, issue := range result.Validation.Issues { + sb.WriteString(fmt.Sprintf(" - %s\n", issue)) + } + } + } + } // Output if result.Output != nil { @@ -310,8 +328,8 @@ func (f *InputFormatter) FormatTaskResults(results []types.TaskResult) string { } // Summary - sb.WriteString(fmt.Sprintf("## Summary\n\n- Total: %d tasks\n- Success: %d\n- Failed: %d\n", - len(results), successCount, failCount)) + sb.WriteString(fmt.Sprintf("## Summary\n\n- Total: %d tasks\n- Success: %d\n- Failed: %d\n- Validated: %d\n", + len(results), successCount, failCount, validatedCount)) return sb.String() } diff --git a/agent/robot/executor/input_test.go b/agent/robot/executor/input_test.go index 2fe7ce80..2035187b 100644 --- a/agent/robot/executor/input_test.go +++ b/agent/robot/executor/input_test.go @@ -282,18 +282,24 @@ func TestInputFormatterFormatTaskResults(t *testing.T) { t.Run("formats task results with summary", func(t *testing.T) { results := []types.TaskResult{ { - TaskID: "task-1", - Success: true, - Duration: 150, - Validated: true, - Output: map[string]interface{}{"rows": 100}, + TaskID: "task-1", + Success: true, + Duration: 150, + Validation: &types.ValidationResult{ + Passed: true, + Score: 0.95, + }, + Output: map[string]interface{}{"rows": 100}, }, { - TaskID: "task-2", - Success: false, - Duration: 50, - Validated: false, - Error: "Connection timeout", + TaskID: "task-2", + Success: false, + Duration: 50, + Validation: &types.ValidationResult{ + Passed: false, + Issues: []string{"Connection timeout"}, + }, + Error: "Connection timeout", }, } @@ -303,14 +309,18 @@ func TestInputFormatterFormatTaskResults(t *testing.T) { assert.Contains(t, result, "### Task: task-1") assert.Contains(t, result, "✓ Success") assert.Contains(t, result, "150ms") + assert.Contains(t, result, "**Validation**: ✓ Passed") + assert.Contains(t, result, "score: 0.95") assert.Contains(t, result, "**Output**") assert.Contains(t, result, "### Task: task-2") assert.Contains(t, result, "✗ Failed") + assert.Contains(t, result, "**Validation**: ✗ Failed") assert.Contains(t, result, "Connection timeout") assert.Contains(t, result, "## Summary") assert.Contains(t, result, "Total: 2 tasks") assert.Contains(t, result, "Success: 1") assert.Contains(t, result, "Failed: 1") + assert.Contains(t, result, "Validated: 1") }) t.Run("returns message for empty results", func(t *testing.T) { diff --git a/agent/robot/executor/run.go b/agent/robot/executor/run.go index 82d55f3a..753f2443 100644 --- a/agent/robot/executor/run.go +++ b/agent/robot/executor/run.go @@ -66,11 +66,14 @@ func (e *Executor) RunExecution(_ *types.Context, exec *types.Execution, _ inter // Generate mock result with actual duration exec.Results[i] = types.TaskResult{ - TaskID: exec.Tasks[i].ID, - Success: true, - Output: fmt.Sprintf("Mock output for %s: Task completed successfully", exec.Tasks[i].ID), - Duration: endTime.Sub(startTime).Milliseconds(), - Validated: true, + TaskID: exec.Tasks[i].ID, + Success: true, + Output: fmt.Sprintf("Mock output for %s: Task completed successfully", exec.Tasks[i].ID), + Duration: endTime.Sub(startTime).Milliseconds(), + Validation: &types.ValidationResult{ + Passed: true, + Score: 1.0, + }, } } diff --git a/agent/robot/types/robot.go b/agent/robot/types/robot.go index fc8d67ea..68113ef0 100644 --- a/agent/robot/types/robot.go +++ b/agent/robot/types/robot.go @@ -172,7 +172,7 @@ type CurrentState struct { Progress string `json:"progress,omitempty"` // human-readable progress (e.g., "2/5 tasks") } -// Goals - P1 output (markdown for LLM) +// Goals - P1 output (markdown for LLM + structured metadata) // P1 Agent reads InspirationReport and generates goals as markdown // Example: // ## Goals @@ -186,6 +186,22 @@ type CurrentState struct { // - Reason: 3 pending leads from yesterday type Goals struct { Content string `json:"content"` // markdown text + + // Success criteria for P3 validation (semantic validation by LLM) + // Example: ["Report must include sales summary", "Growth rate must be calculated"] + SuccessCriteria []string `json:"success_criteria,omitempty"` + + // Delivery target for P4 (where to send results) + DeliveryTarget *DeliveryTarget `json:"delivery_target,omitempty"` +} + +// DeliveryTarget - where to deliver results (defined in P1, used in P4) +type DeliveryTarget struct { + Type DeliveryType `json:"type"` // email | webhook | report | notification + Recipients []string `json:"recipients,omitempty"` // email addresses, webhook URLs, user IDs + Format string `json:"format,omitempty"` // markdown | html | json | text + Template string `json:"template,omitempty"` // template name or inline template + Options map[string]interface{} `json:"options,omitempty"` // channel-specific options } // Task - planned task (structured, for execution) @@ -200,6 +216,12 @@ type Task struct { ExecutorID string `json:"executor_id"` Args []any `json:"args,omitempty"` + // Validation (defined in P2, used in P3) + // ExpectedOutput describes what the task should produce (for LLM semantic validation) + ExpectedOutput string `json:"expected_output,omitempty"` // e.g., "JSON with sales_total, growth_rate fields" + // ValidationRules are specific checks to perform (can be semantic or structural) + ValidationRules []string `json:"validation_rules,omitempty"` // e.g., ["output must be valid JSON", "sales_total > 0"] + // Runtime Status TaskStatus `json:"status"` Order int `json:"order"` // execution order (0-based) @@ -209,20 +231,34 @@ type Task struct { // TaskResult - task execution result type TaskResult struct { - TaskID string `json:"task_id"` - Success bool `json:"success"` - Output interface{} `json:"output,omitempty"` - Error string `json:"error,omitempty"` - Duration int64 `json:"duration_ms"` - Validated bool `json:"validated"` + TaskID string `json:"task_id"` + Success bool `json:"success"` + Output interface{} `json:"output,omitempty"` + Error string `json:"error,omitempty"` + Duration int64 `json:"duration_ms"` + + // Validation result (populated by P3) + Validation *ValidationResult `json:"validation,omitempty"` } -// DeliveryResult - delivery output +// ValidationResult - P3 semantic validation result +type ValidationResult struct { + Passed bool `json:"passed"` // overall validation passed + Score float64 `json:"score,omitempty"` // 0-1 confidence score + Issues []string `json:"issues,omitempty"` // what failed + Suggestions []string `json:"suggestions,omitempty"` // how to improve + Details string `json:"details,omitempty"` // detailed validation report (markdown) +} + +// DeliveryResult - P4 delivery output type DeliveryResult struct { - Type DeliveryType `json:"type"` - Success bool `json:"success"` - Details interface{} `json:"details,omitempty"` - Error string `json:"error,omitempty"` + Type DeliveryType `json:"type"` + Success bool `json:"success"` + Recipients []string `json:"recipients,omitempty"` // who received + Content string `json:"content,omitempty"` // formatted content that was delivered + Details interface{} `json:"details,omitempty"` // channel-specific response + Error string `json:"error,omitempty"` + SentAt *time.Time `json:"sent_at,omitempty"` } // LearningEntry - knowledge to save diff --git a/agent/robot/types/robot_test.go b/agent/robot/types/robot_test.go index 34ad9050..0328ce3d 100644 --- a/agent/robot/types/robot_test.go +++ b/agent/robot/types/robot_test.go @@ -2,6 +2,7 @@ package types_test import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/yaoapp/yao/agent/robot/types" @@ -397,6 +398,9 @@ func TestTaskStructure(t *testing.T) { ExecutorID: "assistant1", Status: types.TaskPending, Order: 0, + // P3 validation fields + ExpectedOutput: "JSON with sales_total and growth_rate fields", + ValidationRules: []string{"sales_total > 0", "growth_rate is a percentage"}, } assert.Equal(t, "task1", task.ID) @@ -406,46 +410,107 @@ func TestTaskStructure(t *testing.T) { assert.Equal(t, "assistant1", task.ExecutorID) assert.Equal(t, types.TaskPending, task.Status) assert.Equal(t, 0, task.Order) + // Validation fields + assert.Contains(t, task.ExpectedOutput, "sales_total") + assert.Len(t, task.ValidationRules, 2) } func TestGoalsStructure(t *testing.T) { goals := &types.Goals{ Content: "## Goals\n1. [High] Complete project\n2. [Normal] Review code", + SuccessCriteria: []string{ + "Project deliverables submitted", + "Code review completed with no blockers", + }, + DeliveryTarget: &types.DeliveryTarget{ + Type: types.DeliveryEmail, + Recipients: []string{"team@example.com"}, + Format: "markdown", + }, } assert.Contains(t, goals.Content, "Goals") assert.Contains(t, goals.Content, "Complete project") + assert.Len(t, goals.SuccessCriteria, 2) + assert.Contains(t, goals.SuccessCriteria[0], "deliverables") + assert.NotNil(t, goals.DeliveryTarget) + assert.Equal(t, types.DeliveryEmail, goals.DeliveryTarget.Type) } func TestTaskResultStructure(t *testing.T) { result := &types.TaskResult{ - TaskID: "task1", - Success: true, - Output: "Task completed successfully", - Duration: 1500, - Validated: true, + TaskID: "task1", + Success: true, + Output: "Task completed successfully", + Duration: 1500, + Validation: &types.ValidationResult{ + Passed: true, + Score: 0.98, + }, } assert.Equal(t, "task1", result.TaskID) assert.True(t, result.Success) assert.Equal(t, "Task completed successfully", result.Output) assert.Equal(t, int64(1500), result.Duration) - assert.True(t, result.Validated) + assert.NotNil(t, result.Validation) + assert.True(t, result.Validation.Passed) + assert.Equal(t, 0.98, result.Validation.Score) +} + +func TestValidationResultStructure(t *testing.T) { + validation := &types.ValidationResult{ + Passed: false, + Score: 0.45, + Issues: []string{"Missing required field: sales_total", "Growth rate is negative"}, + Suggestions: []string{"Add sales_total calculation", "Verify data source"}, + Details: "Detailed validation report...", + } + + assert.False(t, validation.Passed) + assert.Equal(t, 0.45, validation.Score) + assert.Len(t, validation.Issues, 2) + assert.Contains(t, validation.Issues[0], "sales_total") + assert.Len(t, validation.Suggestions, 2) } func TestDeliveryResultStructure(t *testing.T) { + sentAt := time.Now() delivery := &types.DeliveryResult{ - Type: types.DeliveryEmail, - Success: true, + Type: types.DeliveryEmail, + Success: true, + Recipients: []string{"user@example.com", "manager@example.com"}, + Content: "# Weekly Report\n\nSales increased by 20%...", Details: map[string]interface{}{ - "to": "user@example.com", - "subject": "Daily Report", + "message_id": "msg-12345", + "subject": "Daily Report", }, + SentAt: &sentAt, } assert.Equal(t, types.DeliveryEmail, delivery.Type) assert.True(t, delivery.Success) + assert.Len(t, delivery.Recipients, 2) + assert.Contains(t, delivery.Content, "Weekly Report") assert.NotNil(t, delivery.Details) + assert.NotNil(t, delivery.SentAt) +} + +func TestDeliveryTargetStructure(t *testing.T) { + target := &types.DeliveryTarget{ + Type: types.DeliveryEmail, + Recipients: []string{"team@example.com"}, + Format: "markdown", + Template: "weekly-report", + Options: map[string]interface{}{ + "cc": []string{"manager@example.com"}, + }, + } + + assert.Equal(t, types.DeliveryEmail, target.Type) + assert.Len(t, target.Recipients, 1) + assert.Equal(t, "markdown", target.Format) + assert.Equal(t, "weekly-report", target.Template) } func TestLearningEntryStructure(t *testing.T) {