From 2e2ce8bfad8190741e5b3216ad30ae141d13cfd1 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 14:18:27 +0800 Subject: [PATCH 01/17] Refactor LoadWithRoot Function for Enhanced Clarity and Consistency - Further streamlined the LoadWithRoot function by removing redundant checks and improving the overall readability of the path resolution logic. - Ensured consistent resolution of the absolute path for the configuration root, enhancing maintainability and clarity in the codebase. --- agent/test/DESIGN_V2.md | 846 ++++++++++++++++++++++++++++++++++++++++ agent/test/TODO_V2.md | 70 ++++ 2 files changed, 916 insertions(+) create mode 100644 agent/test/DESIGN_V2.md create mode 100644 agent/test/TODO_V2.md diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md new file mode 100644 index 00000000..41c50a09 --- /dev/null +++ b/agent/test/DESIGN_V2.md @@ -0,0 +1,846 @@ +# Agent Test Framework V2 Design + +## Overview + +This document describes the design for Agent Test Framework V2, which extends the existing testing capabilities with: + +- **Multi-turn conversations** - Test agents across multiple interaction rounds +- **Agent-driven testing** - Use agents to generate test cases and simulate user responses +- **Interactive testing** - Human-in-the-loop testing mode + +## Problem Statement + +Current single-turn testing cannot adequately test: + +1. **Conversational flows** - Agents that guide users through multi-step processes +2. **Confirmation dialogs** - Agents that ask for user confirmation before actions +3. **Clarification requests** - Agents that ask follow-up questions when input is ambiguous +4. **Stateful interactions** - Agents that maintain context across multiple turns + +## Design Goals + +1. **Unified format** - Single test case format that supports all modes +2. **Flexible execution** - Static, dynamic (simulator), and interactive modes +3. **Graceful degradation** - Skip tests when required input is unavailable +4. **CI/CD compatible** - Non-interactive mode for automated pipelines +5. **Agent-driven** - Both input generation and user simulation can be agent-powered + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ yao agent test │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ INPUT SOURCES (-i flag) │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ JSONL File │ │ Message │ │ Generator │ │ Interactive │ │ +│ │ ./test.jsonl│ │ "Hello..." │ │ agent:xxx │ │ (stdin) │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ │ +│ └────────────────┴────────────────┴────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ Test Case Parser │ │ +│ │ - Single-turn: {input, assertions} │ │ +│ │ - Multi-turn: {turns: [{input, assertions}, ...]} │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ Multi-Turn Executor │ │ +│ │ │ │ +│ │ ┌─────────┐ ┌─────────────────┐ ┌─────────────────┐ │ │ +│ │ │ Turn │───▶│ Target Agent │───▶│ Response │ │ │ +│ │ │ Input │ │ (being tested) │ │ + State │ │ │ +│ │ └─────────┘ └─────────────────┘ └────────┬────────┘ │ │ +│ │ ▲ │ │ │ +│ │ │ ▼ │ │ +│ │ │ ┌─────────────────────────────────────┐ │ │ +│ │ │ │ Awaiting Input Detection │ │ │ +│ │ │ │ - Explicit declaration │ │ │ +│ │ │ │ - Tool-based detection │ │ │ +│ │ │ │ - Content heuristics │ │ │ +│ │ │ └──────────────┬──────────────────────┘ │ │ +│ │ │ │ │ │ +│ │ │ ┌─────────┴─────────┐ │ │ +│ │ │ ▼ ▼ │ │ +│ │ │ Awaiting=YES Awaiting=NO │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ ▼ │ │ +│ │ NEXT INPUT ┌─────────┐ ┌─────────┐ │ │ +│ │ SOURCES: │ Get Next│ │Complete │ │ │ +│ │ │ Input │ │ Test │ │ │ +│ │ ┌──────────┐ └────┬────┘ └─────────┘ │ │ +│ │ │ Static │◀───────┤ │ │ +│ │ │ turns[] │ │ │ │ +│ │ └──────────┘ │ │ │ +│ │ ┌──────────┐ │ │ │ +│ │ │Simulator │◀───────┤ │ │ +│ │ │ Agent │ │ │ │ +│ │ └──────────┘ │ │ │ +│ │ ┌──────────┐ │ │ │ +│ │ │ Human │◀───────┤ │ │ +│ │ │ Input │ │ │ │ +│ │ └──────────┘ │ │ │ +│ │ ┌──────────┐ │ │ │ +│ │ │ SKIP │◀───────┘ │ │ +│ │ │ (no src) │ │ │ +│ │ └──────────┘ │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ Assertions │ │ +│ │ - Per-turn assertions │ │ +│ │ - Final assertions │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ Reporter │ │ +│ │ - Console output │ │ +│ │ - JSONL output │ │ +│ │ - Custom reporter agent │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +## Core Challenge: Detecting "Awaiting Input" State + +The key challenge is determining when an agent is waiting for user input vs. when it has completed its task. + +### Detection Strategies + +#### Strategy 1: Explicit Declaration (Recommended) + +Agent explicitly declares its state in the response: + +```json +{ + "content": "What is the expense amount?", + "awaiting_input": true, + "input_hint": "Enter amount, e.g., $3500" +} +``` + +#### Strategy 2: Finish Reason Analysis + +Use the LLM's `finish_reason` to determine state: + +- `stop` - Agent completed normally (may or may not need input) +- `tool_calls` - Agent is executing tools (not awaiting input) +- `length` - Response truncated (not awaiting input) + +#### Strategy 3: Content Heuristics + +Analyze response content for question patterns: + +```go +func looksLikeQuestion(content string) bool { + patterns := []string{ + `\?$`, // Ends with question mark + `(?i)^(what|how|when|where|which|who|please|could you)`, + `(?i)(confirm|verify|proceed|continue)\?`, + } + for _, pattern := range patterns { + if regexp.MustCompile(pattern).MatchString(content) { + return true + } + } + return false +} +``` + +#### Strategy 4: Tool-Based Detection + +Certain tools indicate awaiting input: + +```go +func toolRequiresInput(toolCall ToolCall) bool { + confirmationTools := []string{ + "request_confirmation", + "ask_user", + "get_user_input", + } + return contains(confirmationTools, toolCall.Name) +} +``` + +### Recommended Approach: Hybrid Detection + +Combine multiple strategies with priority: + +```go +func IsAwaitingInput(result *TurnResult) (awaiting bool, reason string) { + // Priority 1: Explicit declaration + if result.AwaitingInput { + return true, "agent_declared" + } + + // Priority 2: Tool-based detection + for _, tc := range result.ToolCalls { + if toolRequiresInput(tc) { + return true, "tool_requires_confirmation" + } + } + + // Priority 3: Content heuristics + if looksLikeQuestion(result.Content) { + return true, "content_is_question" + } + + return false, "completed" +} +``` + +## Test Case Format + +### Single-Turn (Existing) + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assertions": [ + { + "type": "contains", + "value": "Hi" + } + ] +} +``` + +### Multi-Turn (New) + +```jsonl +{ + "id": "T001", + "name": "Expense Reimbursement Flow", + "type": "multi_turn", + "turns": [ + { + "input": "I want to submit an expense report", + "assertions": [ + { + "type": "contains", + "value": "type of expense" + } + ] + }, + { + "input": "Business travel to Beijing, flight $2000, hotel $1500", + "assertions": [ + { + "type": "tool_called", + "name": "create_expense" + } + ] + }, + { + "input": "Yes, confirm", + "assertions": [ + { + "type": "contains", + "value": "submitted" + } + ] + } + ], + "simulator": { + "use": "agents:workers.test.user-simulator", + "persona": "New employee unfamiliar with expense process", + "goal": "Submit a $3500 travel expense", + "max_turns": 10 + }, + "interactive": { + "enabled": false, + "timeout": "5m" + }, + "on_missing_input": "skip", + "final_assertions": [ + { + "type": "json_path", + "path": "$.expense.status", + "value": "submitted" + } + ] +} +``` + +### Field Descriptions + +| Field | Type | Required | Description | +| --------------------- | ------ | -------- | ------------------------------------------------ | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `type` | string | No | `"single_turn"` (default) or `"multi_turn"` | +| `turns` | array | No | Static turn definitions | +| `turns[].input` | string | Yes | User input for this turn | +| `turns[].assertions` | array | No | Assertions for this turn's response | +| `simulator` | object | No | Dynamic input generator configuration | +| `simulator.use` | string | Yes | Simulator reference: `agents:id` or `scripts:id` | +| `simulator.persona` | string | No | User persona description | +| `simulator.goal` | string | No | What the simulated user wants to achieve | +| `simulator.max_turns` | int | No | Maximum turns before timeout (default: 20) | +| `interactive` | object | No | Interactive mode configuration | +| `interactive.enabled` | bool | No | Enable human input (default: false) | +| `interactive.timeout` | string | No | Timeout for human input (default: "5m") | +| `on_missing_input` | string | No | `"skip"`, `"fail"`, or `"end"` (default: "skip") | +| `final_assertions` | array | No | Assertions after conversation completes | + +## Execution Modes + +### Mode 1: Static Turns + +Uses predefined `turns` array. Best for deterministic flows. + +``` +Turn 1: Send turns[0].input → Assert turns[0].assertions +Turn 2: Send turns[1].input → Assert turns[1].assertions +... +``` + +### Mode 2: Dynamic Simulator + +Uses an agent to simulate user responses. Best for complex/variable flows. + +``` +Turn 1: Send initial input → Get response +Turn 2: Simulator generates input based on response → Get response +... +Until: Goal achieved OR max_turns reached +``` + +### Mode 3: Interactive + +Prompts human for input when agent awaits. Best for debugging/exploration. + +``` +Turn 1: Send input → Get response +Turn 2: [Agent awaiting] → Prompt human → Get response +... +``` + +### Mode 4: Skip (Default Fallback) + +When agent awaits input but no source available, skip with explanation. + +### Mode Priority + +When multiple input sources are configured, they are used in this order: + +1. **Static turns** - If `turns[n+1]` exists, use it +2. **Simulator** - If no more static turns but simulator configured, use it +3. **Interactive** - If `--interactive` flag and no simulator, prompt human +4. **Skip/Fail/End** - Based on `on_missing_input` setting + +This allows hybrid testing: define some turns statically, then let simulator handle the rest. + +```jsonl +{ + "turns": [ + { + "input": "Start expense report" + }, + { + "input": "Travel expense, $500" + } + ], + "simulator": { + "use": "agents:workers.test.user-sim", + "goal": "Complete the expense submission" + } +} +``` + +In this example: + +- Turn 1-2: Use static inputs +- Turn 3+: Simulator generates inputs until goal achieved + +## Execution Flow + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Multi-Turn Test Execution │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ START: Get initial input │ +│ ├─ From turns[0].input if defined │ +│ └─ From test.input (single-turn compat) │ +│ ↓ │ +│ LOOP: │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ 1. Send input to Agent │ │ +│ │ ↓ │ │ +│ │ 2. Get Agent response │ │ +│ │ ↓ │ │ +│ │ 3. Execute turn assertions (if defined) │ │ +│ │ ↓ │ │ +│ │ 4. Check: Is Agent awaiting input? │ │ +│ │ │ │ │ +│ │ ├─ NO → Exit loop (conversation complete) │ │ +│ │ │ │ │ +│ │ └─ YES → Get next input: │ │ +│ │ │ │ │ +│ │ ├─ turns[n+1] exists? │ │ +│ │ │ → Use static input │ │ +│ │ │ │ │ +│ │ ├─ simulator configured? │ │ +│ │ │ → Call simulator agent │ │ +│ │ │ │ │ +│ │ ├─ interactive enabled? │ │ +│ │ │ → Prompt for human input │ │ +│ │ │ │ │ +│ │ └─ None available? │ │ +│ │ → Handle per on_missing_input: │ │ +│ │ skip: SKIP test │ │ +│ │ fail: FAIL test │ │ +│ │ end: Exit loop normally │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ ↓ │ +│ END: Execute final_assertions │ +│ Report result │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Command Line Interface + +### Flags Reference + +| Flag | Long | Description | +| ---- | --------------- | -------------------------------------------------------- | +| `-i` | `--input` | Input source: file path, message, or `type:id` reference | +| `-n` | `--name` | Target agent ID (the agent being tested) | +| `-o` | `--output` | Output file path for results | +| `-c` | `--connector` | Override connector for the target agent | +| `-v` | `--verbose` | Verbose output showing all turns | +| | `--interactive` | Enable human input when agent awaits | +| | `--simulator` | Default simulator: `agents:id` or `scripts:id` | +| | `--timeout` | Timeout per test case (default: 5m) | +| | `--parallel` | Number of parallel test cases | +| | `--fail-fast` | Stop on first failure | +| | `--dry-run` | Generate/parse tests without running | + +### Input Sources (`-i` flag) + +The `-i` flag supports multiple input sources with unified `type:id` format: + +```bash +# 1. File path (default, no prefix needed) +yao agent test -i ./tests/multi-turn.jsonl + +# 2. Direct message (no prefix, auto-detected as non-file) +yao agent test -i "Hello, how are you?" -n assistants.chat + +# 3. Agent-generated test cases +yao agent test -i agents:workers.test.generator -n assistants.expense + +# 4. Script-generated test cases +yao agent test -i scripts:tests.generate -n assistants.expense + +# 5. With parameters (query string style) +yao agent test -i "agents:workers.test.generator?count=10&focus=edge-cases" -n assistants.expense +``` + +### Input Type Prefixes + +| Prefix | Description | Example | +| ---------- | --------------------------- | -------------------------- | +| (none) | File path or direct message | `./tests.jsonl`, `"Hello"` | +| `agents:` | Agent generates test cases | `agents:workers.test.gen` | +| `scripts:` | Script generates test cases | `scripts:tests.generate` | + +### Input Format + +``` +[prefix:][?param1=value1¶m2=value2] +``` + +- `prefix` - Input type: `agents:` or `scripts:` (optional, default is file/message) +- `id` - Agent ID or script ID +- `?params` - Query parameters passed to generator + +#### Generator Agent Interface + +```typescript +// Input to generator agent +interface GeneratorInput { + target_agent: string; // Agent being tested (from -n flag) + target_description?: string; // Agent's description/purpose + target_tools?: Tool[]; // Agent's available tools + count?: number; // Number of test cases to generate + focus?: string; // Focus area: "happy-path", "edge-cases", "errors" + complexity?: string; // "simple", "medium", "complex" +} + +// Output from generator agent +interface GeneratorOutput { + cases: TestCase[]; // Generated test cases +} +``` + +#### Example Generator Prompt + +``` +You are a test case generator for AI agents. + +Target Agent: {{target_agent}} +Description: {{target_description}} +Available Tools: {{target_tools}} + +Generate {{count}} test cases with focus on: {{focus}} + +For each test case, provide: +- id: Unique identifier +- name: Descriptive name +- input: User message or turns array for multi-turn +- assertions: Expected behaviors to verify + +Output as JSON array of test cases. +``` + +### Complete Examples + +```bash +# Basic: Run tests from file +yao agent test -i ./tests/expense.jsonl + +# With target agent specified (required for message/agent input) +yao agent test -i "Help me file an expense" -n assistants.expense + +# Agent generates tests, then runs them +yao agent test \ + -i "agents:workers.test.generator?count=20&focus=edge-cases" \ + -n assistants.expense + +# Script generates tests +yao agent test \ + -i "scripts:tests.expense.generate?scenario=approval-flow" \ + -n assistants.expense + +# Fully dynamic: Agent generates tests + Agent simulates user responses +yao agent test \ + -i "agents:workers.test.generator?count=10" \ + -n assistants.expense \ + --simulator agents:workers.test.user-simulator + +# Generate tests only, save to file (dry-run) +yao agent test \ + -i "agents:workers.test.generator?count=50" \ + -n assistants.expense \ + -o ./tests/generated.jsonl \ + --dry-run + +# Interactive mode: human provides input when agent awaits +yao agent test -i ./tests/multi-turn.jsonl --interactive + +# CI/CD mode: skip tests requiring human input +yao agent test -i ./tests/multi-turn.jsonl --skip-interactive + +# Fail instead of skip when input unavailable +yao agent test -i ./tests/multi-turn.jsonl --on-missing-input=fail + +# Verbose output showing all turns +yao agent test -i ./tests/multi-turn.jsonl -v +``` + +## Output Format + +### Console Output + +``` +═══════════════════════════════════════════════════════════════ + Agent Test (Multi-Turn) +═══════════════════════════════════════════════════════════════ +ℹ Agent: assistants.expense +ℹ Input: ./tests/expense-flow.jsonl (5 test cases) + +─────────────────────────────────────────────────────────────── + Running Tests +─────────────────────────────────────────────────────────────── + +► [T001] Expense Reimbursement Flow (3 turns) + ├─ Turn 1: "I want to submit an expense" → PASSED (2.1s) + │ Agent: "What type of expense would you like to submit?" + │ ✓ contains "type of expense" + │ + ├─ Turn 2: "Business travel, $3500" → PASSED (3.2s) + │ Agent: [tool: create_expense({amount: 3500, type: "travel"})] + │ ✓ tool_called "create_expense" + │ + ├─ Turn 3: "Yes, confirm" → PASSED (1.8s) + │ Agent: "Expense submitted. Reference: EXP-2025-001" + │ ✓ contains "submitted" + │ + └─ Final Assertions: PASSED + ✓ $.expense.status = "submitted" + +► [T002] Large Expense Approval + ├─ Turn 1: "Submit $100,000 equipment purchase" → PASSED (2.0s) + │ Agent: "This requires manager approval. Please provide PO number." + │ + ├─ Turn 2: SKIPPED + │ Reason: Agent awaiting input, no next turn defined + │ Agent asked: "Please provide PO number" + │ Hint: Add more turns, use --interactive, or configure simulator + │ + └─ Result: SKIPPED + +► [T003] Dynamic Expense Flow (simulator: workers.test.user-sim) + ├─ Turn 1: [Initial] "Help me file an expense" → PASSED (2.1s) + ├─ Turn 2: [Simulated] "It's for client dinner, $250" → PASSED (2.8s) + ├─ Turn 3: [Simulated] "Yesterday evening" → PASSED (2.2s) + ├─ Turn 4: [Simulated] "Confirm" → PASSED (1.9s) + │ Goal achieved: Expense submitted + │ + └─ Final Assertions: PASSED + +─────────────────────────────────────────────────────────────── + Summary +─────────────────────────────────────────────────────────────── + Total: 3 tests + Passed: 2 + Failed: 0 + Skipped: 1 + + Total turns: 10 + Avg turns/test: 3.3 + Total time: 18.1s +``` + +### JSONL Output + +```jsonl +{ + "id": "T001", + "name": "Expense Reimbursement Flow", + "status": "passed", + "turns": [ + { + "turn": 1, + "input": "I want to submit an expense", + "input_source": "static", + "output": "What type of expense would you like to submit?", + "awaiting_input": true, + "assertions": [ + { + "type": "contains", + "value": "type of expense", + "passed": true + } + ], + "duration_ms": 2100 + }, + { + "turn": 2, + "input": "Business travel, $3500", + "input_source": "static", + "output": "", + "tool_calls": [ + { + "name": "create_expense", + "args": { + "amount": 3500 + } + } + ], + "awaiting_input": true, + "assertions": [ + { + "type": "tool_called", + "name": "create_expense", + "passed": true + } + ], + "duration_ms": 3200 + }, + { + "turn": 3, + "input": "Yes, confirm", + "input_source": "static", + "output": "Expense submitted. Reference: EXP-2025-001", + "awaiting_input": false, + "assertions": [ + { + "type": "contains", + "value": "submitted", + "passed": true + } + ], + "duration_ms": 1800 + } + ], + "final_assertions": [ + { + "type": "json_path", + "path": "$.expense.status", + "value": "submitted", + "passed": true + } + ], + "total_turns": 3, + "duration_ms": 7100 +} +``` + +## User Simulator Agent + +### Interface + +The simulator agent receives conversation context and generates the next user input: + +```typescript +// Input to simulator +interface SimulatorInput { + persona: string; // User persona description + goal: string; // What user wants to achieve + conversation: Message[]; // Conversation history + last_response: string; // Agent's last response + turn_number: number; // Current turn (1-based) + max_turns: number; // Maximum allowed turns +} + +// Output from simulator +interface SimulatorOutput { + input: string; // Generated user input + goal_achieved: boolean; // Whether goal is complete + reasoning?: string; // Why this input was chosen +} +``` + +### Example Simulator Prompt + +``` +You are simulating a user with the following characteristics: + +Persona: {{persona}} +Goal: {{goal}} + +Current conversation: +{{conversation}} + +The agent just responded: +"{{last_response}}" + +Generate the next user message to continue toward the goal. +If the goal has been achieved, set goal_achieved to true. + +Respond in JSON format: +{ + "input": "your response as the user", + "goal_achieved": true/false, + "reasoning": "brief explanation" +} +``` + +## Backward Compatibility + +Existing single-turn tests continue to work unchanged: + +```jsonl +// This still works +{"id": "T001", "input": "Hello", "assertions": [...]} + +// Equivalent to +{"id": "T001", "type": "single_turn", "turns": [{"input": "Hello", "assertions": [...]}]} +``` + +## Error Handling + +### Turn-Level Errors + +| Error Type | Behavior | Output | +| ---------------- | -------------------- | -------------------------------- | +| Agent timeout | Mark turn as FAILED | `error: "timeout after 30s"` | +| Agent error | Mark turn as FAILED | `error: "agent error: ..."` | +| Assertion failed | Mark turn as FAILED | `assertion_errors: [...]` | +| Simulator error | Mark turn as SKIPPED | `skip_reason: "simulator error"` | + +### Test-Level Errors + +| Error Type | Behavior | Output | +| ---------------------------- | -------------------- | ---------------------------------- | +| No initial input | Mark test as FAILED | `error: "no initial input"` | +| Max turns exceeded | Mark test as FAILED | `error: "max turns (20) exceeded"` | +| All turns passed | Mark test as PASSED | `status: "passed"` | +| Any turn failed | Mark test as FAILED | `status: "failed"` | +| Skipped due to missing input | Mark test as SKIPPED | `status: "skipped"` | + +## Context and State + +### Conversation Context + +Each multi-turn test maintains a conversation context: + +```go +type ConversationContext struct { + SessionID string // Unique session for this test + Messages []Message // Full conversation history + ToolResults map[string]any // Results from tool calls + Variables map[string]any // Custom variables set during test + TurnCount int // Current turn number +} +``` + +### Context Passing to Simulator + +The simulator receives full context to generate appropriate responses: + +```json +{ + "persona": "New employee", + "goal": "Submit expense report", + "context": { + "session_id": "test-session-001", + "messages": [...], + "tool_results": { + "create_expense": {"id": "EXP-001", "status": "pending"} + }, + "turn_count": 3 + }, + "last_response": "Please confirm the expense details..." +} +``` + +## Attachments in Multi-Turn + +Multi-turn tests support attachments at the turn level: + +```jsonl +{ + "id": "T001", + "name": "Receipt Upload Flow", + "turns": [ + { + "input": "I want to submit an expense with receipt", + "attachments": [ + { + "type": "image", + "source": "file://./tests/fixtures/receipt.jpg" + } + ] + }, + { + "input": "The amount is $150" + } + ] +} +``` + +## Open Questions + +1. **Session Management**: How to handle session state across turns? Use existing session or create new per-test? + +2. **Timeout Strategy**: Per-turn timeout vs. total test timeout? + +3. **Parallel Execution**: Can multi-turn tests run in parallel, or must they be sequential? + +4. **Retry Logic**: If a turn fails, retry just that turn or restart entire conversation? + +5. **Snapshot Testing**: Should we support "golden file" comparison for conversation flows? diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md new file mode 100644 index 00000000..ff7c2058 --- /dev/null +++ b/agent/test/TODO_V2.md @@ -0,0 +1,70 @@ +# Agent Test Framework V2 - Implementation TODO + +## Phase 1: Static Multi-Turn + +- [ ] Extend test case parser for `turns` array +- [ ] Implement turn-by-turn execution +- [ ] Implement conversation context management +- [ ] Add per-turn assertions +- [ ] Support attachments at turn level +- [ ] Implement awaiting input detection (heuristics) +- [ ] Add `on_missing_input` handling (`skip`, `fail`, `end`) +- [ ] Implement mode priority (static → simulator → interactive → skip) +- [ ] Update console output for multi-turn display +- [ ] Update JSONL output format for turns + +## Phase 2: Agent-Driven Input + +- [ ] Parse `agents:` prefix in `-i` flag +- [ ] Parse `scripts:` prefix in `-i` flag +- [ ] Define generator agent interface (GeneratorInput/GeneratorOutput) +- [ ] Implement generator invocation +- [ ] Support query parameters (`?count=10&focus=...`) +- [ ] Pass target agent metadata to generator (description, tools) +- [ ] Add `--dry-run` flag to save generated cases without running +- [ ] Create example generator agent + +## Phase 3: Dynamic Simulator + +- [ ] Define simulator agent interface (SimulatorInput/SimulatorOutput) +- [ ] Implement simulator invocation with full context +- [ ] Pass conversation history and tool results to simulator +- [ ] Add goal completion detection +- [ ] Add max_turns limit and timeout +- [ ] Support hybrid mode (static turns + simulator fallback) +- [ ] Create example simulator agent + +## Phase 4: Interactive Mode + +- [ ] Add `--interactive` flag +- [ ] Implement terminal input prompt with context display +- [ ] Add input timeout handling +- [ ] Support input history/editing +- [ ] Add `--skip-interactive` for CI/CD mode + +## Phase 5: Enhanced Detection + +- [ ] Add `awaiting_input` field to agent response schema +- [ ] Implement tool-based detection (confirmation tools) +- [ ] Add configurable detection rules +- [ ] Support custom detection via script/agent + +## Phase 6: Error Handling & Reporting + +- [ ] Implement turn-level error handling +- [ ] Implement test-level error aggregation +- [ ] Add detailed error messages with hints +- [ ] Support custom reporter agent + +## Open Questions + +1. **Session Management**: How to handle session state across turns? Use existing session or create new per-test? + +2. **Timeout Strategy**: Per-turn timeout vs. total test timeout? + +3. **Parallel Execution**: Can multi-turn tests run in parallel, or must they be sequential? + +4. **Retry Logic**: If a turn fails, retry just that turn or restart entire conversation? + +5. **Snapshot Testing**: Should we support "golden file" comparison for conversation flows? + From e4179448501232cfbf8b2d0393ffc1b0e5eda3e4 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 17:26:58 +0800 Subject: [PATCH 02/17] Enhance Test Framework with Standard Agent Interface and Options Support - Introduced a comprehensive standard agent interface for agent-driven features, including generator, simulator, and validator modes. - Added support for `context.Options` to pass parameters in test cases, allowing for flexible configuration of agent behavior. - Updated test case format to include options at both the test and per-turn levels, enhancing customization and control over agent interactions. - Expanded documentation to detail the usage of options in various agent modes, improving clarity for developers and users. --- agent/test/DESIGN_V2.md | 310 +++++++++++++++++++++++++++++++++++++--- agent/test/TODO_V2.md | 41 ++++-- 2 files changed, 317 insertions(+), 34 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 41c50a09..da13502f 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -197,6 +197,247 @@ func IsAwaitingInput(result *TurnResult) (awaiting bool, reason string) { } ``` +## Standard Agent Interface + +All agent-driven features (generator, simulator, validator) use standard Yao Agent interfaces. + +### Using `context.Options` + +The test framework uses `context.Options` to pass parameters, aligned with `Assistant.Stream()`: + +```go +// context.Options - standard Yao Agent options +type Options struct { + Skip *Skip `json:"skip,omitempty"` // Skip history, trace, etc. + Connector string `json:"connector,omitempty"` // LLM connector to use + Search any `json:"search,omitempty"` // Search behavior control + Mode string `json:"mode,omitempty"` // Agent mode (chat, etc.) + Metadata map[string]any `json:"metadata,omitempty"` // Custom metadata +} +``` + +### Test Framework Usage + +```go +// Prepare options for agent invocation +options := &context.Options{ + Skip: &context.Skip{ + History: true, // Don't save test messages to history + Trace: false, // Keep trace for debugging + }, + Metadata: map[string]any{ + "test_mode": "validator", // "generator" | "simulator" | "validator" + "test_id": "T001", + "criteria": "Response should be helpful", + // ... other custom params from test config + }, +} + +// Prepare context +ctx := context.New(parent, authorized, chatID) +ctx.Referer = "agent-test" + +// Call agent with standard Stream API +assistant := agent.Get(agentID) +response, err := assistant.Stream(ctx, messages, options) +``` + +### Test Case Options Field + +Test cases can specify options to pass to the target agent or helper agents: + +```jsonl +{ + "id": "T001", + "input": "Hello", + "options": { + "connector": "openai-gpt4", + "skip": { + "history": true + }, + "metadata": { + "scenario": "edge-case" + } + } +} +``` + +### Generator Mode + +Used when `-i agents:xxx` is specified to generate test cases: + +```go +// Framework calls generator agent +options := &context.Options{ + Skip: &context.Skip{History: true}, + Metadata: map[string]any{ + "test_mode": "generator", + "target_agent": "assistants.expense", + "target_description": "Expense reimbursement assistant", + "target_tools": []string{"create_expense", "get_policy"}, + // From query params: agents:xxx?count=10&focus=edge-cases + "count": 10, + "focus": "edge-cases", + "complexity": "medium", + }, +} +``` + +Expected structured output: + +```json +{ + "cases": [ + {"id": "G001", "input": "...", "assertions": [...]}, + {"id": "G002", "input": "...", "assertions": [...]} + ] +} +``` + +### Simulator Mode + +Used when test case has `simulator` config: + +```go +// Framework calls simulator agent for next user input +options := &context.Options{ + Skip: &context.Skip{History: true}, + Metadata: map[string]any{ + "test_mode": "simulator", + "test_id": "T001", + // From simulator.metadata in test case + "persona": "New employee", + "goal": "Submit expense report", + // Runtime context + "turn_number": 3, + "max_turns": 10, + "tool_results": map[string]any{...}, + }, +} + +// Messages include full conversation history +messages := conversationHistory +``` + +Expected structured output: + +```json +{ + "input": "The amount is $500", + "goal_achieved": false, + "reasoning": "Providing requested amount info" +} +``` + +### Validator Mode + +Used when assertion has `type: "agent"`: + +```go +// Framework calls validator agent +options := &context.Options{ + Skip: &context.Skip{History: true}, + Metadata: map[string]any{ + "test_mode": "validator", + "test_id": "T001", + // From assertion.metadata + "criteria": "Response should be helpful", + "expected_intent": "answer question", + "tone": "professional", + }, +} + +// Messages include agent response to validate +messages := []context.Message{ + {Role: "user", Content: "Original user question"}, + {Role: "assistant", Content: "Agent's response to validate"}, +} +``` + +Expected structured output: + +```json +{ + "passed": true, + "score": 0.95, + "reason": "Response is helpful and addresses the question", + "suggestions": [] +} +``` + +## Assertion Types + +### Static Assertions (Existing) + +| Type | Description | Example | +| ------------- | ---------------------- | ---------------------------------------------------------- | +| `contains` | Response contains text | `{"type": "contains", "value": "success"}` | +| `equals` | Exact match | `{"type": "equals", "value": "OK"}` | +| `regex` | Regex pattern match | `{"type": "regex", "pattern": "order-\\d+"}` | +| `json_path` | JSONPath value check | `{"type": "json_path", "path": "$.status", "value": "ok"}` | +| `tool_called` | Tool was invoked | `{"type": "tool_called", "name": "create_expense"}` | +| `type` | Value type check | `{"type": "type", "path": "$.count", "value": "number"}` | + +### Agent-Driven Assertions (New) + +For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `context.Options`: + +```jsonl +{ + "type": "agent", + "use": "agents:workers.test.validator", + "options": { + "connector": "openai-gpt4", + "metadata": { + "criteria": "Response should be helpful and answer the user's question", + "expected_intent": "provide expense submission guidance", + "tone": "professional and friendly" + } + } +} +``` + +### Script Assertions + +For custom validation logic: + +```jsonl +{ + "type": "script", + "use": "scripts:tests.validate-expense", + "options": { + "metadata": { + "min_amount": 100, + "max_amount": 10000 + } + } +} +``` + +### Combined Assertions + +Mix static and agent-driven assertions: + +```jsonl +{ + "assertions": [ + { + "type": "tool_called", + "name": "create_expense" + }, + { + "type": "agent", + "use": "agents:workers.test.validator", + "options": { + "metadata": { + "criteria": "Confirmation message should include expense amount and be polite" + } + } + } + ] +} +``` + ## Test Case Format ### Single-Turn (Existing) @@ -221,6 +462,15 @@ func IsAwaitingInput(result *TurnResult) (awaiting bool, reason string) { "id": "T001", "name": "Expense Reimbursement Flow", "type": "multi_turn", + "options": { + "connector": "openai-gpt4", + "skip": { + "history": true + }, + "metadata": { + "test_scenario": "happy-path" + } + }, "turns": [ { "input": "I want to submit an expense report", @@ -252,9 +502,13 @@ func IsAwaitingInput(result *TurnResult) (awaiting bool, reason string) { ], "simulator": { "use": "agents:workers.test.user-simulator", - "persona": "New employee unfamiliar with expense process", - "goal": "Submit a $3500 travel expense", - "max_turns": 10 + "options": { + "metadata": { + "persona": "New employee unfamiliar with expense process", + "goal": "Submit a $3500 travel expense", + "max_turns": 10 + } + } }, "interactive": { "enabled": false, @@ -278,14 +532,19 @@ func IsAwaitingInput(result *TurnResult) (awaiting bool, reason string) { | `id` | string | Yes | Unique test identifier | | `name` | string | No | Human-readable test name | | `type` | string | No | `"single_turn"` (default) or `"multi_turn"` | +| `options` | object | No | `context.Options` passed to target agent | +| `options.connector` | string | No | LLM connector to use | +| `options.skip` | object | No | Skip config (history, trace, etc.) | +| `options.search` | any | No | Search behavior control | +| `options.mode` | string | No | Agent mode | +| `options.metadata` | object | No | Custom metadata passed to agent | | `turns` | array | No | Static turn definitions | | `turns[].input` | string | Yes | User input for this turn | | `turns[].assertions` | array | No | Assertions for this turn's response | +| `turns[].options` | object | No | Per-turn options override | | `simulator` | object | No | Dynamic input generator configuration | | `simulator.use` | string | Yes | Simulator reference: `agents:id` or `scripts:id` | -| `simulator.persona` | string | No | User persona description | -| `simulator.goal` | string | No | What the simulated user wants to achieve | -| `simulator.max_turns` | int | No | Maximum turns before timeout (default: 20) | +| `simulator.options` | object | No | `context.Options` passed to simulator agent | | `interactive` | object | No | Interactive mode configuration | | `interactive.enabled` | bool | No | Enable human input (default: false) | | `interactive.timeout` | string | No | Timeout for human input (default: "5m") | @@ -352,7 +611,9 @@ This allows hybrid testing: define some turns statically, then let simulator han ], "simulator": { "use": "agents:workers.test.user-sim", - "goal": "Complete the expense submission" + "metadata": { + "goal": "Complete the expense submission" + } } } ``` @@ -790,21 +1051,28 @@ type ConversationContext struct { ### Context Passing to Simulator -The simulator receives full context to generate appropriate responses: +The simulator receives context via standard Yao Agent Context metadata: -```json -{ - "persona": "New employee", - "goal": "Submit expense report", - "context": { - "session_id": "test-session-001", - "messages": [...], - "tool_results": { - "create_expense": {"id": "EXP-001", "status": "pending"} - }, - "turn_count": 3 - }, - "last_response": "Please confirm the expense details..." +```go +// Framework prepares context for simulator +ctx.Metadata = map[string]any{ + "test_mode": "simulator", + "test_id": "T001", + // From simulator config + "persona": "New employee", + "goal": "Submit expense report", + // Runtime context + "session_id": "test-session-001", + "turn_count": 3, + "tool_results": map[string]any{...}, +} + +// Messages include conversation history +messages := []Message{ + {Role: "user", Content: "I want to submit an expense"}, + {Role: "assistant", Content: "What type of expense?"}, + {Role: "user", Content: "Travel expense"}, + {Role: "assistant", Content: "Please confirm the details..."}, } ``` diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index ff7c2058..2119ccd5 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -3,7 +3,9 @@ ## Phase 1: Static Multi-Turn - [ ] Extend test case parser for `turns` array -- [ ] Implement turn-by-turn execution +- [ ] Add `options` field support (aligned with `context.Options`) +- [ ] Support test-level `options` and per-turn `options` override +- [ ] Implement turn-by-turn execution with options passing - [ ] Implement conversation context management - [ ] Add per-turn assertions - [ ] Support attachments at turn level @@ -17,22 +19,24 @@ - [ ] Parse `agents:` prefix in `-i` flag - [ ] Parse `scripts:` prefix in `-i` flag -- [ ] Define generator agent interface (GeneratorInput/GeneratorOutput) -- [ ] Implement generator invocation -- [ ] Support query parameters (`?count=10&focus=...`) -- [ ] Pass target agent metadata to generator (description, tools) +- [ ] Use standard `context.Options` for all agent invocations +- [ ] Pass `test_mode: "generator"` in `options.metadata` +- [ ] Pass target agent info (description, tools) in `options.metadata` +- [ ] Support query parameters (`?count=10&focus=...`) → merged into `options.metadata` - [ ] Add `--dry-run` flag to save generated cases without running -- [ ] Create example generator agent +- [ ] Create example generator agent with prompt template ## Phase 3: Dynamic Simulator -- [ ] Define simulator agent interface (SimulatorInput/SimulatorOutput) -- [ ] Implement simulator invocation with full context -- [ ] Pass conversation history and tool results to simulator -- [ ] Add goal completion detection -- [ ] Add max_turns limit and timeout +- [ ] Implement simulator invocation via `Assistant.Stream()` with `context.Options` +- [ ] Pass `test_mode: "simulator"` in `options.metadata` +- [ ] Pass persona, goal, turn_count in `options.metadata` +- [ ] Pass conversation history as messages +- [ ] Pass tool results in `options.metadata` +- [ ] Add goal completion detection (`goal_achieved` in response) +- [ ] Add max_turns limit and timeout (from `options.metadata`) - [ ] Support hybrid mode (static turns + simulator fallback) -- [ ] Create example simulator agent +- [ ] Create example simulator agent with prompt template ## Phase 4: Interactive Mode @@ -49,7 +53,18 @@ - [ ] Add configurable detection rules - [ ] Support custom detection via script/agent -## Phase 6: Error Handling & Reporting +## Phase 6: Agent-Driven Assertions + +- [ ] Add `agent` assertion type to assertion parser +- [ ] Support `options` field in assertion (aligned with `context.Options`) +- [ ] Implement validator agent invocation via `Assistant.Stream()` +- [ ] Pass `test_mode: "validator"` in `options.metadata` +- [ ] Pass conversation context and criteria in `options.metadata` +- [ ] Support score-based pass/fail threshold (configurable in `options.metadata`) +- [ ] Add `suggestions` to assertion error output +- [ ] Create example validator agent with prompt template + +## Phase 7: Error Handling & Reporting - [ ] Implement turn-level error handling - [ ] Implement test-level error aggregation From d7b84bdf364ebb1e946d775722a5537fce8675ad Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 17:35:30 +0800 Subject: [PATCH 03/17] Enhance Documentation and Add JSONL Support for Agent Assertions - Updated the DESIGN_V2.md file to clarify the usage of agent-driven assertions in JSONL test cases, including detailed examples and API specifications. - Introduced a new section on script testing with agent assertions, outlining the implementation and usage of the `t.assert.Agent()` method. - Modified the TODO_V2.md file to reflect the addition of JSONL support for agent assertions and outlined tasks for further development in this area. --- agent/test/DESIGN_V2.md | 112 ++++++++++++++++++++++++++++++++++++++-- agent/test/TODO_V2.md | 11 ++++ 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index da13502f..4ad6fbc3 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -385,7 +385,7 @@ For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `c ```jsonl { "type": "agent", - "use": "agents:workers.test.validator", + "use": "workers.test.validator", "options": { "connector": "openai-gpt4", "metadata": { @@ -397,9 +397,9 @@ For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `c } ``` -### Script Assertions +### Script Assertions (in JSONL) -For custom validation logic: +For custom validation logic in JSONL test cases: ```jsonl { @@ -427,7 +427,7 @@ Mix static and agent-driven assertions: }, { "type": "agent", - "use": "agents:workers.test.validator", + "use": "workers.test.validator", "options": { "metadata": { "criteria": "Confirmation message should include expense amount and be polite" @@ -438,6 +438,110 @@ Mix static and agent-driven assertions: } ``` +## Script Testing with Agent Assertions + +Script tests can also use Agent-driven assertions via the `t.assert.Agent()` method. + +### API + +```typescript +// t.assert.Agent(response, agentID, options?) -> ValidatorResult +// agentID: Direct agent ID without prefix, e.g., "workers.test.validator" +interface ValidatorResult { + passed: boolean; + score?: number; + reason: string; + suggestions?: string[]; +} +``` + +### Usage in Script Tests + +```typescript +// tests/expense_test.ts +export function TestExpenseResponse(t: TestingT, ctx: Context) { + // Call the agent being tested + const response = Process("agents.expense.Stream", ctx, [ + { role: "user", content: "How do I submit an expense?" }, + ]); + + // Static assertions + t.assert.NotNil(response); + t.assert.Contains(response.content, "expense"); + + // Agent-driven assertion - automatically fails test if validation fails + t.assert.Agent(response.content, "workers.test.validator", { + metadata: { + criteria: + "Response should explain the expense submission process clearly", + expected_topics: ["receipt", "approval", "deadline"], + tone: "helpful", + }, + }); +} +``` + +### With Conversation Context + +```typescript +export function TestMultiTurnExpense(t: TestingT, ctx: Context) { + const messages = [ + { role: "user", content: "I need to submit a travel expense" }, + { role: "assistant", content: "I'd be happy to help..." }, + { role: "user", content: "It's for a flight to Beijing, $2000" }, + ]; + + const response = Process("agents.expense.Stream", ctx, messages); + + // Agent assertion with conversation context + // Automatically fails test and logs suggestions if validation fails + t.assert.Agent(response.content, "workers.test.validator", { + metadata: { + criteria: + "Response should confirm the expense details and ask for receipt", + conversation: messages, + }, + }); +} +``` + +### Implementation + +The `t.assert.Agent()` method internally: + +1. Prepares `context.Options` with `test_mode: "validator"` +2. Calls the validator agent via `Assistant.Stream()` +3. Parses structured output (JSON) +4. Returns `ValidatorResult` + +```go +// In script_assert.go +func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + // Parse arguments: response, agentID, options + response := args[0].String() + agentID := args[1].String() // Direct agent ID, e.g., "workers.test.validator" + options := parseOptions(args[2]) + + // Prepare validator options + validatorOpts := &context.Options{ + Skip: &context.Skip{History: true}, + Metadata: map[string]any{ + "test_mode": "validator", + ...options.Metadata, + }, + } + + // Call validator agent + assistant, _ := agent.Get(agentID) + result, _ := assistant.Stream(agentCtx, messages, validatorOpts) + + // Parse and return result + return toJsValue(parseValidatorResult(result)) + }) +} +``` + ## Test Case Format ### Single-Turn (Existing) diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 2119ccd5..137c1e8f 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -55,6 +55,7 @@ ## Phase 6: Agent-Driven Assertions +### In JSONL Test Cases - [ ] Add `agent` assertion type to assertion parser - [ ] Support `options` field in assertion (aligned with `context.Options`) - [ ] Implement validator agent invocation via `Assistant.Stream()` @@ -62,7 +63,17 @@ - [ ] Pass conversation context and criteria in `options.metadata` - [ ] Support score-based pass/fail threshold (configurable in `options.metadata`) - [ ] Add `suggestions` to assertion error output + +### In Script Tests +- [ ] Add `t.assert.Agent(response, agentID, options?)` method +- [ ] `agentID` is direct ID (e.g., `workers.test.validator`), no prefix needed +- [ ] Invoke validator agent with context +- [ ] Return `ValidatorResult` object to JavaScript +- [ ] Support passing conversation history in options + +### Shared - [ ] Create example validator agent with prompt template +- [ ] Document `ValidatorResult` interface ## Phase 7: Error Handling & Reporting From 38dd455cc5b2d97331fb6c6d95d6f5faf4ab68cd Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 17:43:34 +0800 Subject: [PATCH 04/17] Update DESIGN_V2.md and TODO_V2.md for Agent Test Framework - Corrected references in DESIGN_V2.md to ensure consistent usage of agent identifiers, including updates to input sources and simulator configurations. - Enhanced the documentation in TODO_V2.md with a summary of format rules for agent testing, clarifying the usage of prefixes for various contexts and options. - Added tasks related to the dynamic simulator implementation and metadata handling to guide future development efforts. --- agent/test/DESIGN_V2.md | 92 +++++++++++++++++++++-------------------- agent/test/TODO_V2.md | 15 ++++++- 2 files changed, 60 insertions(+), 47 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 4ad6fbc3..55f22f24 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -35,7 +35,7 @@ Current single-turn testing cannot adequately test: │ INPUT SOURCES (-i flag) │ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ │ JSONL File │ │ Message │ │ Generator │ │ Interactive │ │ -│ │ ./test.jsonl│ │ "Hello..." │ │ agent:xxx │ │ (stdin) │ │ +│ │ ./test.jsonl│ │ "Hello..." │ │ agents:xxx │ │ (stdin) │ │ │ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ │ │ │ │ │ │ │ └────────────────┴────────────────┴────────────────┘ │ @@ -305,7 +305,7 @@ options := &context.Options{ Metadata: map[string]any{ "test_mode": "simulator", "test_id": "T001", - // From simulator.metadata in test case + // From simulator.options.metadata in test case "persona": "New employee", "goal": "Submit expense report", // Runtime context @@ -385,7 +385,7 @@ For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `c ```jsonl { "type": "agent", - "use": "workers.test.validator", + "use": "agents:workers.test.validator", "options": { "connector": "openai-gpt4", "metadata": { @@ -427,7 +427,7 @@ Mix static and agent-driven assertions: }, { "type": "agent", - "use": "workers.test.validator", + "use": "agents:workers.test.validator", "options": { "metadata": { "criteria": "Confirmation message should include expense amount and be polite" @@ -605,7 +605,7 @@ func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context } ], "simulator": { - "use": "agents:workers.test.user-simulator", + "use": "workers.test.user-simulator", "options": { "metadata": { "persona": "New employee unfamiliar with expense process", @@ -631,29 +631,29 @@ func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context ### Field Descriptions -| Field | Type | Required | Description | -| --------------------- | ------ | -------- | ------------------------------------------------ | -| `id` | string | Yes | Unique test identifier | -| `name` | string | No | Human-readable test name | -| `type` | string | No | `"single_turn"` (default) or `"multi_turn"` | -| `options` | object | No | `context.Options` passed to target agent | -| `options.connector` | string | No | LLM connector to use | -| `options.skip` | object | No | Skip config (history, trace, etc.) | -| `options.search` | any | No | Search behavior control | -| `options.mode` | string | No | Agent mode | -| `options.metadata` | object | No | Custom metadata passed to agent | -| `turns` | array | No | Static turn definitions | -| `turns[].input` | string | Yes | User input for this turn | -| `turns[].assertions` | array | No | Assertions for this turn's response | -| `turns[].options` | object | No | Per-turn options override | -| `simulator` | object | No | Dynamic input generator configuration | -| `simulator.use` | string | Yes | Simulator reference: `agents:id` or `scripts:id` | -| `simulator.options` | object | No | `context.Options` passed to simulator agent | -| `interactive` | object | No | Interactive mode configuration | -| `interactive.enabled` | bool | No | Enable human input (default: false) | -| `interactive.timeout` | string | No | Timeout for human input (default: "5m") | -| `on_missing_input` | string | No | `"skip"`, `"fail"`, or `"end"` (default: "skip") | -| `final_assertions` | array | No | Assertions after conversation completes | +| Field | Type | Required | Description | +| --------------------- | ------ | -------- | -------------------------------------------------- | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `type` | string | No | `"single_turn"` (default) or `"multi_turn"` | +| `options` | object | No | `context.Options` passed to target agent | +| `options.connector` | string | No | LLM connector to use | +| `options.skip` | object | No | Skip config (history, trace, etc.) | +| `options.search` | any | No | Search behavior control | +| `options.mode` | string | No | Agent mode | +| `options.metadata` | object | No | Custom metadata passed to agent | +| `turns` | array | No | Static turn definitions | +| `turns[].input` | string | Yes | User input for this turn | +| `turns[].assertions` | array | No | Assertions for this turn's response | +| `turns[].options` | object | No | Per-turn options override | +| `simulator` | object | No | Dynamic input generator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (e.g., `workers.test.user-sim`) | +| `simulator.options` | object | No | `context.Options` passed to simulator agent | +| `interactive` | object | No | Interactive mode configuration | +| `interactive.enabled` | bool | No | Enable human input (default: false) | +| `interactive.timeout` | string | No | Timeout for human input (default: "5m") | +| `on_missing_input` | string | No | `"skip"`, `"fail"`, or `"end"` (default: "skip") | +| `final_assertions` | array | No | Assertions after conversation completes | ## Execution Modes @@ -714,9 +714,11 @@ This allows hybrid testing: define some turns statically, then let simulator han } ], "simulator": { - "use": "agents:workers.test.user-sim", - "metadata": { - "goal": "Complete the expense submission" + "use": "workers.test.user-sim", + "options": { + "metadata": { + "goal": "Complete the expense submission" + } } } } @@ -778,19 +780,19 @@ In this example: ### Flags Reference -| Flag | Long | Description | -| ---- | --------------- | -------------------------------------------------------- | -| `-i` | `--input` | Input source: file path, message, or `type:id` reference | -| `-n` | `--name` | Target agent ID (the agent being tested) | -| `-o` | `--output` | Output file path for results | -| `-c` | `--connector` | Override connector for the target agent | -| `-v` | `--verbose` | Verbose output showing all turns | -| | `--interactive` | Enable human input when agent awaits | -| | `--simulator` | Default simulator: `agents:id` or `scripts:id` | -| | `--timeout` | Timeout per test case (default: 5m) | -| | `--parallel` | Number of parallel test cases | -| | `--fail-fast` | Stop on first failure | -| | `--dry-run` | Generate/parse tests without running | +| Flag | Long | Description | +| ---- | --------------- | ---------------------------------------------------------- | +| `-i` | `--input` | Input source: file path, message, or `type:id` reference | +| `-n` | `--name` | Target agent ID (the agent being tested) | +| `-o` | `--output` | Output file path for results | +| `-c` | `--connector` | Override connector for the target agent | +| `-v` | `--verbose` | Verbose output showing all turns | +| | `--interactive` | Enable human input when agent awaits | +| | `--simulator` | Default simulator agent ID (e.g., `workers.test.user-sim`) | +| | `--timeout` | Timeout per test case (default: 5m) | +| | `--parallel` | Number of parallel test cases | +| | `--fail-fast` | Stop on first failure | +| | `--dry-run` | Generate/parse tests without running | ### Input Sources (`-i` flag) @@ -893,7 +895,7 @@ yao agent test \ yao agent test \ -i "agents:workers.test.generator?count=10" \ -n assistants.expense \ - --simulator agents:workers.test.user-simulator + --simulator workers.test.user-simulator # Generate tests only, save to file (dry-run) yao agent test \ diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 137c1e8f..51eac659 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -1,5 +1,15 @@ # Agent Test Framework V2 - Implementation TODO +## Format Rules Summary + +| Context | Format | Example | +|---------|--------|---------| +| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | +| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-sim"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-sim` | +| `t.assert.Agent()` | No prefix (method is explicit) | `t.assert.Agent(resp, "workers.test.validator", {...})` | + ## Phase 1: Static Multi-Turn - [ ] Extend test case parser for `turns` array @@ -29,12 +39,13 @@ ## Phase 3: Dynamic Simulator - [ ] Implement simulator invocation via `Assistant.Stream()` with `context.Options` +- [ ] `simulator.use` is direct agent ID (no prefix needed) - [ ] Pass `test_mode: "simulator"` in `options.metadata` -- [ ] Pass persona, goal, turn_count in `options.metadata` +- [ ] Pass persona, goal, turn_count from `simulator.options.metadata` - [ ] Pass conversation history as messages - [ ] Pass tool results in `options.metadata` - [ ] Add goal completion detection (`goal_achieved` in response) -- [ ] Add max_turns limit and timeout (from `options.metadata`) +- [ ] Add max_turns limit and timeout - [ ] Support hybrid mode (static turns + simulator fallback) - [ ] Create example simulator agent with prompt template From d58aa12d4d82442fa0ee5cd66c95103997a052ed Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 17:57:26 +0800 Subject: [PATCH 05/17] Enhance DESIGN_V2.md and TODO_V2.md for Multi-Turn Testing - Added detailed sections on Static and Dynamic modes in DESIGN_V2.md, outlining their characteristics and execution flows for multi-turn testing. - Introduced a Quick Reference table for format rules, clarifying the usage of flags and assertions in test cases. - Updated TODO_V2.md to reflect tasks for implementing mode support, including the addition of checkpoints and handling of order constraints in dynamic testing. - Improved documentation for error handling in both static and dynamic modes, ensuring clarity on expected behaviors during test execution. --- agent/test/DESIGN_V2.md | 568 ++++++++++++++++++++++++++-------------- agent/test/TODO_V2.md | 23 +- 2 files changed, 389 insertions(+), 202 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 55f22f24..b31b1c6d 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -8,6 +8,16 @@ This document describes the design for Agent Test Framework V2, which extends th - **Agent-driven testing** - Use agents to generate test cases and simulate user responses - **Interactive testing** - Human-in-the-loop testing mode +## Quick Reference: Format Rules + +| Context | Format | Example | +| --------------------- | ------------------------ | ------------------------------------------------------- | +| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | +| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-sim"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-sim` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | + ## Problem Statement Current single-turn testing cannot adequately test: @@ -51,51 +61,36 @@ Current single-turn testing cannot adequately test: │ ┌─────────────────────────────────────────────────────────────────┐ │ │ │ Multi-Turn Executor │ │ │ │ │ │ -│ │ ┌─────────┐ ┌─────────────────┐ ┌─────────────────┐ │ │ -│ │ │ Turn │───▶│ Target Agent │───▶│ Response │ │ │ -│ │ │ Input │ │ (being tested) │ │ + State │ │ │ -│ │ └─────────┘ └─────────────────┘ └────────┬────────┘ │ │ -│ │ ▲ │ │ │ -│ │ │ ▼ │ │ -│ │ │ ┌─────────────────────────────────────┐ │ │ -│ │ │ │ Awaiting Input Detection │ │ │ -│ │ │ │ - Explicit declaration │ │ │ -│ │ │ │ - Tool-based detection │ │ │ -│ │ │ │ - Content heuristics │ │ │ -│ │ │ └──────────────┬──────────────────────┘ │ │ -│ │ │ │ │ │ -│ │ │ ┌─────────┴─────────┐ │ │ -│ │ │ ▼ ▼ │ │ -│ │ │ Awaiting=YES Awaiting=NO │ │ -│ │ │ │ │ │ │ -│ │ │ ▼ ▼ │ │ -│ │ NEXT INPUT ┌─────────┐ ┌─────────┐ │ │ -│ │ SOURCES: │ Get Next│ │Complete │ │ │ -│ │ │ Input │ │ Test │ │ │ -│ │ ┌──────────┐ └────┬────┘ └─────────┘ │ │ -│ │ │ Static │◀───────┤ │ │ -│ │ │ turns[] │ │ │ │ -│ │ └──────────┘ │ │ │ -│ │ ┌──────────┐ │ │ │ -│ │ │Simulator │◀───────┤ │ │ -│ │ │ Agent │ │ │ │ -│ │ └──────────┘ │ │ │ -│ │ ┌──────────┐ │ │ │ -│ │ │ Human │◀───────┤ │ │ -│ │ │ Input │ │ │ │ -│ │ └──────────┘ │ │ │ -│ │ ┌──────────┐ │ │ │ -│ │ │ SKIP │◀───────┘ │ │ -│ │ │ (no src) │ │ │ -│ │ └──────────┘ │ │ +│ │ ┌─────────────────────────────────────────────────────────┐ │ │ +│ │ │ MODE SELECTION (based on test case fields) │ │ │ +│ │ │ │ │ │ +│ │ │ Has `turns`? ─────────────────▶ STATIC MODE │ │ │ +│ │ │ Has `simulator` + `checkpoints`? ──▶ DYNAMIC MODE │ │ │ +│ │ │ Neither? ─────────────────────▶ SINGLE-TURN (legacy) │ │ │ +│ │ └─────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ │ +│ │ ┌───────────────┴───────────────┐ │ │ +│ │ ▼ ▼ │ │ +│ │ ┌─────────────────────┐ ┌─────────────────────────┐ │ │ +│ │ │ STATIC MODE │ │ DYNAMIC MODE │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ FOR each turn: │ │ LOOP until terminated: │ │ │ +│ │ │ 1. Send input │ │ 1. Simulator → input │ │ │ +│ │ │ 2. Get response │ │ 2. Send to Agent │ │ │ +│ │ │ 3. Run assertions │ │ 3. Check checkpoints │ │ │ +│ │ │ 4. Continue/Fail │ │ 4. Check termination │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ All passed → PASS │ │ All checkpoints → PASS │ │ │ +│ │ │ Any failed → FAIL │ │ Timeout/Missing → FAIL │ │ │ +│ │ └─────────────────────┘ └─────────────────────────┘ │ │ │ │ │ │ │ └─────────────────────────────────────────────────────────────────┘ │ │ │ │ │ ▼ │ │ ┌─────────────────────────────────────────────────────────────────┐ │ │ │ Assertions │ │ -│ │ - Per-turn assertions │ │ -│ │ - Final assertions │ │ +│ │ - Static: Per-turn assertions │ │ +│ │ - Dynamic: Checkpoint assertions (order-independent) │ │ │ └─────────────────────────────────────────────────────────────────┘ │ │ │ │ │ ▼ │ @@ -559,20 +554,19 @@ func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context } ``` -### Multi-Turn (New) +### Multi-Turn: Static Mode + +For **deterministic flows** where you know the exact conversation sequence: ```jsonl { "id": "T001", - "name": "Expense Reimbursement Flow", - "type": "multi_turn", + "name": "Expense Reimbursement - Happy Path", + "mode": "static", "options": { "connector": "openai-gpt4", "skip": { "history": true - }, - "metadata": { - "test_scenario": "happy-path" } }, "turns": [ @@ -586,7 +580,7 @@ func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context ] }, { - "input": "Business travel to Beijing, flight $2000, hotel $1500", + "input": "Business travel to Beijing, $3500", "assertions": [ { "type": "tool_called", @@ -603,175 +597,363 @@ func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context } ] } - ], + ] +} +``` + +**Characteristics:** + +- Fixed number of turns +- Each turn has specific input and assertions +- Test fails if any turn assertion fails +- Best for regression testing known flows + +### Multi-Turn: Dynamic Mode (Checkpoints) + +For **coverage testing** where you care about functionality, not exact sequence: + +```jsonl +{ + "id": "T002", + "name": "Expense Submission Coverage", + "mode": "dynamic", "simulator": { "use": "workers.test.user-simulator", "options": { "metadata": { "persona": "New employee unfamiliar with expense process", - "goal": "Submit a $3500 travel expense", - "max_turns": 10 + "goal": "Submit a $3500 travel expense" } } }, - "interactive": { - "enabled": false, - "timeout": "5m" - }, - "on_missing_input": "skip", - "final_assertions": [ + "checkpoints": [ { - "type": "json_path", - "path": "$.expense.status", - "value": "submitted" + "id": "ask_type", + "description": "Agent asks for expense type", + "assertion": { + "type": "contains", + "value": "type" + } + }, + { + "id": "call_create", + "description": "Agent calls create_expense tool", + "assertion": { + "type": "tool_called", + "name": "create_expense" + } + }, + { + "id": "confirm_submit", + "description": "Agent confirms submission", + "assertion": { + "type": "contains", + "value": "submitted" + } + } + ], + "max_turns": 10, + "timeout": "2m" +} +``` + +**Characteristics:** + +- Simulator drives the conversation +- Checkpoints are verified across all turns (order-independent by default) +- Test passes when ALL checkpoints are reached +- Test fails if max_turns/timeout reached before all checkpoints +- Best for functional coverage testing + +### Checkpoints with Order Constraints + +When checkpoints must occur in a specific order: + +```jsonl +{ + "checkpoints": [ + { + "id": "ask_type", + "description": "Agent asks for expense type", + "assertion": { + "type": "contains", + "value": "type" + } + }, + { + "id": "call_create", + "description": "Agent calls create_expense", + "after": [ + "ask_type" + ], + "assertion": { + "type": "tool_called", + "name": "create_expense" + } + }, + { + "id": "confirm_submit", + "description": "Agent confirms submission", + "after": [ + "call_create" + ], + "assertion": { + "type": "contains", + "value": "submitted" + } } ] } ``` -### Field Descriptions +### Checkpoints with Agent Validation -| Field | Type | Required | Description | -| --------------------- | ------ | -------- | -------------------------------------------------- | -| `id` | string | Yes | Unique test identifier | -| `name` | string | No | Human-readable test name | -| `type` | string | No | `"single_turn"` (default) or `"multi_turn"` | -| `options` | object | No | `context.Options` passed to target agent | -| `options.connector` | string | No | LLM connector to use | -| `options.skip` | object | No | Skip config (history, trace, etc.) | -| `options.search` | any | No | Search behavior control | -| `options.mode` | string | No | Agent mode | -| `options.metadata` | object | No | Custom metadata passed to agent | -| `turns` | array | No | Static turn definitions | -| `turns[].input` | string | Yes | User input for this turn | -| `turns[].assertions` | array | No | Assertions for this turn's response | -| `turns[].options` | object | No | Per-turn options override | -| `simulator` | object | No | Dynamic input generator configuration | -| `simulator.use` | string | Yes | Simulator agent ID (e.g., `workers.test.user-sim`) | -| `simulator.options` | object | No | `context.Options` passed to simulator agent | -| `interactive` | object | No | Interactive mode configuration | -| `interactive.enabled` | bool | No | Enable human input (default: false) | -| `interactive.timeout` | string | No | Timeout for human input (default: "5m") | -| `on_missing_input` | string | No | `"skip"`, `"fail"`, or `"end"` (default: "skip") | -| `final_assertions` | array | No | Assertions after conversation completes | - -## Execution Modes - -### Mode 1: Static Turns - -Uses predefined `turns` array. Best for deterministic flows. - -``` -Turn 1: Send turns[0].input → Assert turns[0].assertions -Turn 2: Send turns[1].input → Assert turns[1].assertions -... -``` - -### Mode 2: Dynamic Simulator - -Uses an agent to simulate user responses. Best for complex/variable flows. - -``` -Turn 1: Send initial input → Get response -Turn 2: Simulator generates input based on response → Get response -... -Until: Goal achieved OR max_turns reached -``` - -### Mode 3: Interactive - -Prompts human for input when agent awaits. Best for debugging/exploration. - -``` -Turn 1: Send input → Get response -Turn 2: [Agent awaiting] → Prompt human → Get response -... -``` - -### Mode 4: Skip (Default Fallback) - -When agent awaits input but no source available, skip with explanation. - -### Mode Priority - -When multiple input sources are configured, they are used in this order: - -1. **Static turns** - If `turns[n+1]` exists, use it -2. **Simulator** - If no more static turns but simulator configured, use it -3. **Interactive** - If `--interactive` flag and no simulator, prompt human -4. **Skip/Fail/End** - Based on `on_missing_input` setting - -This allows hybrid testing: define some turns statically, then let simulator handle the rest. +Use Agent-driven assertions for semantic validation: ```jsonl { - "turns": [ + "checkpoints": [ { - "input": "Start expense report" + "id": "helpful_guidance", + "description": "Agent provides helpful expense guidance", + "assertion": { + "type": "agent", + "use": "agents:workers.test.validator", + "options": { + "metadata": { + "criteria": "Response explains expense process clearly and professionally" + } + } + } }, { - "input": "Travel expense, $500" - } - ], - "simulator": { - "use": "workers.test.user-sim", - "options": { - "metadata": { - "goal": "Complete the expense submission" + "id": "tool_called", + "description": "Agent creates expense record", + "assertion": { + "type": "tool_called", + "name": "create_expense" } } - } + ] } ``` -In this example: +### Dynamic Mode Termination -- Turn 1-2: Use static inputs -- Turn 3+: Simulator generates inputs until goal achieved +| Condition | Result | Description | +| ------------------------------------ | ---------- | -------------------------- | +| All checkpoints reached | ✅ PASSED | All functionality verified | +| Agent completes, checkpoints missing | ❌ FAILED | Missing coverage | +| max_turns exceeded | ❌ FAILED | Timeout - flow too long | +| timeout exceeded | ❌ FAILED | Time limit reached | +| Checkpoint assertion fails | ❌ FAILED | Functionality broken | +| Simulator error | ⚠️ SKIPPED | Cannot continue | + +### Field Descriptions + +| Field | Type | Required | Description | +| --------------------------- | ------ | ------------- | ------------------------------------------ | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `mode` | string | No | `"static"` (default) or `"dynamic"` | +| `options` | object | No | `context.Options` passed to target agent | +| **Static Mode Fields** | +| `turns` | array | Yes (static) | Static turn definitions | +| `turns[].input` | string | Yes | User input for this turn | +| `turns[].assertions` | array | No | Assertions for this turn's response | +| `turns[].options` | object | No | Per-turn options override | +| **Dynamic Mode Fields** | +| `simulator` | object | Yes (dynamic) | User simulator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | +| `simulator.options` | object | No | `context.Options` passed to simulator | +| `checkpoints` | array | Yes (dynamic) | Functionality checkpoints to verify | +| `checkpoints[].id` | string | Yes | Unique checkpoint identifier | +| `checkpoints[].description` | string | No | Human-readable description | +| `checkpoints[].assertion` | object | Yes | Assertion to verify | +| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | +| `max_turns` | int | No | Maximum turns before timeout (default: 20) | +| `timeout` | string | No | Maximum time (default: "5m") | +| **Shared Fields** | +| `interactive` | object | No | Interactive mode configuration | +| `interactive.enabled` | bool | No | Enable human input (default: false) | +| `interactive.timeout` | string | No | Timeout for human input (default: "5m") | + +## Execution Modes + +### Static Mode + +Uses predefined `turns` array. Best for **regression testing** known flows. + +``` +┌─────────────────────────────────────────────────────────┐ +│ Static Mode Flow │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ FOR each turn in turns[]: │ +│ 1. Send turn.input to Agent │ +│ 2. Get Agent response │ +│ 3. Run turn.assertions │ +│ ├─ PASS → Continue to next turn │ +│ └─ FAIL → Test FAILED, stop │ +│ │ +│ All turns completed → Test PASSED │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +### Dynamic Mode (Checkpoints) + +Uses simulator + checkpoints. Best for **coverage testing** functionality. + +``` +┌─────────────────────────────────────────────────────────┐ +│ Dynamic Mode Flow │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ Initialize: pending_checkpoints = all checkpoints │ +│ │ +│ LOOP (until terminated): │ +│ 1. Simulator generates user input │ +│ 2. Send input to Agent │ +│ 3. Get Agent response │ +│ 4. Check response against pending_checkpoints │ +│ └─ If matched → Move to reached_checkpoints │ +│ 5. Check termination conditions: │ +│ ├─ All checkpoints reached → PASSED │ +│ ├─ Agent completed, missing checkpoints → FAILED │ +│ ├─ max_turns exceeded → FAILED │ +│ └─ timeout exceeded → FAILED │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +### Interactive Mode + +For debugging, human can provide input when agent awaits: + +```bash +# Enable with --interactive flag +yao agent test -i ./tests.jsonl --interactive +``` + +In interactive mode: + +- Static mode: Human can override any turn input +- Dynamic mode: Human can replace simulator for specific turns + +### Mode Selection + +| Has `turns`? | Has `simulator` + `checkpoints`? | Mode | +| ------------ | -------------------------------- | ----------------------- | +| Yes | No | Static | +| No | Yes | Dynamic | +| Yes | Yes | ❌ Invalid (choose one) | +| No | No | Single-turn (legacy) | + +### Example: Static vs Dynamic + +**Same feature, different testing approaches:** + +```jsonl +// Static Mode - Exact sequence testing +{ + "id": "expense-static", + "mode": "static", + "turns": [ + {"input": "Submit expense", "assertions": [{"type": "contains", "value": "type"}]}, + {"input": "Travel, $500", "assertions": [{"type": "tool_called", "name": "create_expense"}]}, + {"input": "Confirm", "assertions": [{"type": "contains", "value": "submitted"}]} + ] +} + +// Dynamic Mode - Coverage testing +{ + "id": "expense-dynamic", + "mode": "dynamic", + "simulator": {"use": "workers.test.user-sim", "options": {"metadata": {"goal": "Submit $500 expense"}}}, + "checkpoints": [ + {"id": "ask", "assertion": {"type": "contains", "value": "type"}}, + {"id": "create", "assertion": {"type": "tool_called", "name": "create_expense"}}, + {"id": "done", "assertion": {"type": "contains", "value": "submitted"}} + ], + "max_turns": 10 +} +``` ## Execution Flow +### Static Mode Flow + ``` ┌─────────────────────────────────────────────────────────────────┐ -│ Multi-Turn Test Execution │ +│ Static Mode Execution │ ├─────────────────────────────────────────────────────────────────┤ │ │ -│ START: Get initial input │ -│ ├─ From turns[0].input if defined │ -│ └─ From test.input (single-turn compat) │ +│ INITIALIZE: │ +│ - Load turns[] from test case │ +│ - Set current_turn = 0 │ +│ ↓ │ +│ FOR each turn in turns[]: │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ 1. Get input from turns[current_turn].input │ │ +│ │ ↓ │ │ +│ │ 2. Send input to Agent │ │ +│ │ ↓ │ │ +│ │ 3. Get Agent response │ │ +│ │ ↓ │ │ +│ │ 4. Run turns[current_turn].assertions │ │ +│ │ │ │ │ +│ │ ├─ PASS → Continue to next turn │ │ +│ │ └─ FAIL → Test FAILED, stop │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ ↓ │ +│ All turns completed → Test PASSED │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Dynamic Mode Flow + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Dynamic Mode Execution │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ INITIALIZE: │ +│ - pending_checkpoints = all checkpoints │ +│ - reached_checkpoints = [] │ +│ - turn_count = 0 │ +│ - start_time = now() │ │ ↓ │ │ LOOP: │ │ ┌─────────────────────────────────────────────────────────┐ │ -│ │ 1. Send input to Agent │ │ +│ │ 1. Call Simulator Agent → Get user input │ │ +│ │ (pass: persona, goal, conversation history) │ │ │ │ ↓ │ │ -│ │ 2. Get Agent response │ │ +│ │ 2. Send input to Target Agent │ │ │ │ ↓ │ │ -│ │ 3. Execute turn assertions (if defined) │ │ +│ │ 3. Get Agent response │ │ │ │ ↓ │ │ -│ │ 4. Check: Is Agent awaiting input? │ │ -│ │ │ │ │ -│ │ ├─ NO → Exit loop (conversation complete) │ │ -│ │ │ │ │ -│ │ └─ YES → Get next input: │ │ -│ │ │ │ │ -│ │ ├─ turns[n+1] exists? │ │ -│ │ │ → Use static input │ │ -│ │ │ │ │ -│ │ ├─ simulator configured? │ │ -│ │ │ → Call simulator agent │ │ -│ │ │ │ │ -│ │ ├─ interactive enabled? │ │ -│ │ │ → Prompt for human input │ │ -│ │ │ │ │ -│ │ └─ None available? │ │ -│ │ → Handle per on_missing_input: │ │ -│ │ skip: SKIP test │ │ -│ │ fail: FAIL test │ │ -│ │ end: Exit loop normally │ │ +│ │ 4. Check response against pending_checkpoints │ │ +│ │ FOR each pending checkpoint: │ │ +│ │ - Run checkpoint.assertion │ │ +│ │ - If PASS and `after` satisfied → move to reached │ │ +│ │ ↓ │ │ +│ │ 5. Check termination conditions: │ │ +│ │ ├─ pending_checkpoints empty? │ │ +│ │ │ → Test PASSED ✅ │ │ +│ │ │ │ │ +│ │ ├─ Agent completed (not awaiting)? │ │ +│ │ │ → Test FAILED ❌ (missing checkpoints) │ │ +│ │ │ │ │ +│ │ ├─ turn_count >= max_turns? │ │ +│ │ │ → Test FAILED ❌ (turn limit) │ │ +│ │ │ │ │ +│ │ ├─ now() - start_time > timeout? │ │ +│ │ │ → Test FAILED ❌ (timeout) │ │ +│ │ │ │ │ +│ │ └─ Otherwise → Continue loop │ │ │ └─────────────────────────────────────────────────────────┘ │ -│ ↓ │ -│ END: Execute final_assertions │ -│ Report result │ │ │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -1111,33 +1293,35 @@ Respond in JSON format: Existing single-turn tests continue to work unchanged: ```jsonl -// This still works +// This still works (single-turn, legacy format) {"id": "T001", "input": "Hello", "assertions": [...]} -// Equivalent to -{"id": "T001", "type": "single_turn", "turns": [{"input": "Hello", "assertions": [...]}]} +// Static mode with one turn (equivalent) +{"id": "T001", "mode": "static", "turns": [{"input": "Hello", "assertions": [...]}]} ``` ## Error Handling -### Turn-Level Errors +### Static Mode Errors -| Error Type | Behavior | Output | -| ---------------- | -------------------- | -------------------------------- | -| Agent timeout | Mark turn as FAILED | `error: "timeout after 30s"` | -| Agent error | Mark turn as FAILED | `error: "agent error: ..."` | -| Assertion failed | Mark turn as FAILED | `assertion_errors: [...]` | -| Simulator error | Mark turn as SKIPPED | `skip_reason: "simulator error"` | +| Error Type | Behavior | Output | +| ---------------- | ------------------- | ---------------------------- | +| Agent timeout | Mark turn as FAILED | `error: "timeout after 30s"` | +| Agent error | Mark turn as FAILED | `error: "agent error: ..."` | +| Assertion failed | Mark turn as FAILED | `assertion_errors: [...]` | +| All turns passed | Test PASSED | `status: "passed"` | +| Any turn failed | Test FAILED | `status: "failed"` | -### Test-Level Errors +### Dynamic Mode Errors -| Error Type | Behavior | Output | -| ---------------------------- | -------------------- | ---------------------------------- | -| No initial input | Mark test as FAILED | `error: "no initial input"` | -| Max turns exceeded | Mark test as FAILED | `error: "max turns (20) exceeded"` | -| All turns passed | Mark test as PASSED | `status: "passed"` | -| Any turn failed | Mark test as FAILED | `status: "failed"` | -| Skipped due to missing input | Mark test as SKIPPED | `status: "skipped"` | +| Error Type | Behavior | Output | +| --------------------------- | ----------- | ----------------------------------- | +| All checkpoints reached | Test PASSED | `status: "passed"` | +| Checkpoints missing | Test FAILED | `error: "missing checkpoints: ..."` | +| Max turns exceeded | Test FAILED | `error: "max turns (20) exceeded"` | +| Timeout exceeded | Test FAILED | `error: "timeout after 5m"` | +| Simulator error | Test FAILED | `error: "simulator error: ..."` | +| Checkpoint assertion failed | Test FAILED | `error: "checkpoint X failed"` | ## Context and State diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 51eac659..c865d9b0 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -10,8 +10,9 @@ | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-sim` | | `t.assert.Agent()` | No prefix (method is explicit) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -## Phase 1: Static Multi-Turn +## Phase 1: Static Mode +- [ ] Add `mode` field to test case parser (`static` | `dynamic`) - [ ] Extend test case parser for `turns` array - [ ] Add `options` field support (aligned with `context.Options`) - [ ] Support test-level `options` and per-turn `options` override @@ -19,9 +20,6 @@ - [ ] Implement conversation context management - [ ] Add per-turn assertions - [ ] Support attachments at turn level -- [ ] Implement awaiting input detection (heuristics) -- [ ] Add `on_missing_input` handling (`skip`, `fail`, `end`) -- [ ] Implement mode priority (static → simulator → interactive → skip) - [ ] Update console output for multi-turn display - [ ] Update JSONL output format for turns @@ -36,17 +34,22 @@ - [ ] Add `--dry-run` flag to save generated cases without running - [ ] Create example generator agent with prompt template -## Phase 3: Dynamic Simulator +## Phase 3: Dynamic Mode (Checkpoints) -- [ ] Implement simulator invocation via `Assistant.Stream()` with `context.Options` +- [ ] Add `checkpoints` array to test case parser +- [ ] Implement checkpoint matching against agent responses +- [ ] Support `after` field for order constraints +- [ ] Track pending/reached checkpoints during execution +- [ ] Implement termination conditions: + - [ ] All checkpoints reached → PASSED + - [ ] Agent completed, missing checkpoints → FAILED + - [ ] max_turns exceeded → FAILED + - [ ] timeout exceeded → FAILED +- [ ] Implement simulator invocation via `Assistant.Stream()` - [ ] `simulator.use` is direct agent ID (no prefix needed) - [ ] Pass `test_mode: "simulator"` in `options.metadata` - [ ] Pass persona, goal, turn_count from `simulator.options.metadata` - [ ] Pass conversation history as messages -- [ ] Pass tool results in `options.metadata` -- [ ] Add goal completion detection (`goal_achieved` in response) -- [ ] Add max_turns limit and timeout -- [ ] Support hybrid mode (static turns + simulator fallback) - [ ] Create example simulator agent with prompt template ## Phase 4: Interactive Mode From d03668e8769da8d88825480610fcf477a033cc27 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 18:10:52 +0800 Subject: [PATCH 06/17] Enhance DESIGN_V2.md and TODO_V2.md for Message History Support - Updated DESIGN_V2.md to introduce message history support in the Agent Test Framework, allowing tests to simulate multi-turn conversations without complex state management. - Revised the test case format to include a `messages` field, enabling the passing of full conversation history directly to the agent. - Enhanced TODO_V2.md to outline tasks for implementing message history support, including updates to the test case parser and output formats. - Improved documentation on agent-driven assertions and error handling to reflect the new capabilities and ensure clarity for developers. --- agent/test/DESIGN_V2.md | 1619 ++++++++++++--------------------------- agent/test/TODO_V2.md | 77 +- 2 files changed, 503 insertions(+), 1193 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index b31b1c6d..3b3f498c 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -4,9 +4,9 @@ This document describes the design for Agent Test Framework V2, which extends the existing testing capabilities with: -- **Multi-turn conversations** - Test agents across multiple interaction rounds -- **Agent-driven testing** - Use agents to generate test cases and simulate user responses -- **Interactive testing** - Human-in-the-loop testing mode +- **Message history support** - Test agents with conversation context via `messages[]` +- **Agent-driven testing** - Use agents to generate test cases and validate responses +- **Dynamic testing** - Simulator-driven testing with checkpoint validation ## Quick Reference: Format Rules @@ -14,26 +14,17 @@ This document describes the design for Agent Test Framework V2, which extends th | --------------------- | ------------------------ | ------------------------------------------------------- | | `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | | JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | -| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-sim"` | -| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-sim` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -## Problem Statement - -Current single-turn testing cannot adequately test: - -1. **Conversational flows** - Agents that guide users through multi-step processes -2. **Confirmation dialogs** - Agents that ask for user confirmation before actions -3. **Clarification requests** - Agents that ask follow-up questions when input is ambiguous -4. **Stateful interactions** - Agents that maintain context across multiple turns - ## Design Goals -1. **Unified format** - Single test case format that supports all modes -2. **Flexible execution** - Static, dynamic (simulator), and interactive modes -3. **Graceful degradation** - Skip tests when required input is unavailable -4. **CI/CD compatible** - Non-interactive mode for automated pipelines -5. **Agent-driven** - Both input generation and user simulation can be agent-powered +1. **Simple** - Single-turn with optional message history, no complex multi-turn state +2. **Stateless** - Each test is independent, no session management needed +3. **Parallel** - Tests can run in parallel since they don't share state +4. **Flexible** - Support both static (messages) and dynamic (simulator) testing +5. **Agent-driven** - Input generation, simulation, and validation can all be agent-powered ## Architecture Overview @@ -43,503 +34,70 @@ Current single-turn testing cannot adequately test: ├─────────────────────────────────────────────────────────────────────────┤ │ │ │ INPUT SOURCES (-i flag) │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ JSONL File │ │ Message │ │ Generator │ │ Interactive │ │ -│ │ ./test.jsonl│ │ "Hello..." │ │ agents:xxx │ │ (stdin) │ │ -│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ -│ │ │ │ │ │ -│ └────────────────┴────────────────┴────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌─────────────────────────────────────────────────────────────────┐ │ -│ │ Test Case Parser │ │ -│ │ - Single-turn: {input, assertions} │ │ -│ │ - Multi-turn: {turns: [{input, assertions}, ...]} │ │ -│ └─────────────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌─────────────────────────────────────────────────────────────────┐ │ -│ │ Multi-Turn Executor │ │ -│ │ │ │ -│ │ ┌─────────────────────────────────────────────────────────┐ │ │ -│ │ │ MODE SELECTION (based on test case fields) │ │ │ -│ │ │ │ │ │ -│ │ │ Has `turns`? ─────────────────▶ STATIC MODE │ │ │ -│ │ │ Has `simulator` + `checkpoints`? ──▶ DYNAMIC MODE │ │ │ -│ │ │ Neither? ─────────────────────▶ SINGLE-TURN (legacy) │ │ │ -│ │ └─────────────────────────────────────────────────────────┘ │ │ -│ │ │ │ │ -│ │ ┌───────────────┴───────────────┐ │ │ -│ │ ▼ ▼ │ │ -│ │ ┌─────────────────────┐ ┌─────────────────────────┐ │ │ -│ │ │ STATIC MODE │ │ DYNAMIC MODE │ │ │ -│ │ │ │ │ │ │ │ -│ │ │ FOR each turn: │ │ LOOP until terminated: │ │ │ -│ │ │ 1. Send input │ │ 1. Simulator → input │ │ │ -│ │ │ 2. Get response │ │ 2. Send to Agent │ │ │ -│ │ │ 3. Run assertions │ │ 3. Check checkpoints │ │ │ -│ │ │ 4. Continue/Fail │ │ 4. Check termination │ │ │ -│ │ │ │ │ │ │ │ -│ │ │ All passed → PASS │ │ All checkpoints → PASS │ │ │ -│ │ │ Any failed → FAIL │ │ Timeout/Missing → FAIL │ │ │ -│ │ └─────────────────────┘ └─────────────────────────┘ │ │ -│ │ │ │ -│ └─────────────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌─────────────────────────────────────────────────────────────────┐ │ -│ │ Assertions │ │ -│ │ - Static: Per-turn assertions │ │ -│ │ - Dynamic: Checkpoint assertions (order-independent) │ │ -│ └─────────────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌─────────────────────────────────────────────────────────────────┐ │ -│ │ Reporter │ │ -│ │ - Console output │ │ -│ │ - JSONL output │ │ -│ │ - Custom reporter agent │ │ -│ └─────────────────────────────────────────────────────────────────┘ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ JSONL File │ │ Message │ │ Generator │ │ +│ │ ./test.jsonl│ │ "Hello..." │ │ agents:xxx │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +│ └────────────────┴────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌───────────────────────────────────────────────────────────────────┐ │ +│ │ Test Case Parser │ │ +│ │ │ │ +│ │ Standard Mode: {input: "...", messages: [...], assertions} │ │ +│ │ Dynamic Mode: {simulator: {...}, checkpoints: [...]} │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌───────────────┴───────────────┐ │ +│ ▼ ▼ │ +│ ┌───────────────────┐ ┌───────────────────────┐ │ +│ │ STANDARD MODE │ │ DYNAMIC MODE │ │ +│ │ │ │ │ │ +│ │ 1. Build messages │ │ LOOP: │ │ +│ │ 2. Call Agent │ │ 1. Simulator→input │ │ +│ │ 3. Run assertions │ │ 2. Call Agent │ │ +│ │ │ │ 3. Check checkpoints │ │ +│ │ → PASS/FAIL │ │ 4. Until done │ │ +│ └───────────────────┘ └───────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌───────────────────────────────────────────────────────────────────┐ │ +│ │ Reporter │ │ +│ │ - Console output │ │ +│ │ - JSONL output │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ │ │ └─────────────────────────────────────────────────────────────────────────┘ ``` -## Core Challenge: Detecting "Awaiting Input" State +## Test Modes -The key challenge is determining when an agent is waiting for user input vs. when it has completed its task. +### Standard Mode (Default) -### Detection Strategies +Single call to agent with optional message history. **No multi-turn state management needed.** -#### Strategy 1: Explicit Declaration (Recommended) +| Field | Type | Description | +| ------------ | ------ | ------------------------------------------------ | +| `input` | string | Simple text input (shorthand for single message) | +| `messages` | array | Full message history (overrides `input`) | +| `assertions` | array | Assertions to validate response | +| `options` | object | `context.Options` passed to agent | -Agent explicitly declares its state in the response: +### Dynamic Mode -```json -{ - "content": "What is the expense amount?", - "awaiting_input": true, - "input_hint": "Enter amount, e.g., $3500" -} -``` +Simulator-driven testing with checkpoint validation. -#### Strategy 2: Finish Reason Analysis - -Use the LLM's `finish_reason` to determine state: - -- `stop` - Agent completed normally (may or may not need input) -- `tool_calls` - Agent is executing tools (not awaiting input) -- `length` - Response truncated (not awaiting input) - -#### Strategy 3: Content Heuristics - -Analyze response content for question patterns: - -```go -func looksLikeQuestion(content string) bool { - patterns := []string{ - `\?$`, // Ends with question mark - `(?i)^(what|how|when|where|which|who|please|could you)`, - `(?i)(confirm|verify|proceed|continue)\?`, - } - for _, pattern := range patterns { - if regexp.MustCompile(pattern).MatchString(content) { - return true - } - } - return false -} -``` - -#### Strategy 4: Tool-Based Detection - -Certain tools indicate awaiting input: - -```go -func toolRequiresInput(toolCall ToolCall) bool { - confirmationTools := []string{ - "request_confirmation", - "ask_user", - "get_user_input", - } - return contains(confirmationTools, toolCall.Name) -} -``` - -### Recommended Approach: Hybrid Detection - -Combine multiple strategies with priority: - -```go -func IsAwaitingInput(result *TurnResult) (awaiting bool, reason string) { - // Priority 1: Explicit declaration - if result.AwaitingInput { - return true, "agent_declared" - } - - // Priority 2: Tool-based detection - for _, tc := range result.ToolCalls { - if toolRequiresInput(tc) { - return true, "tool_requires_confirmation" - } - } - - // Priority 3: Content heuristics - if looksLikeQuestion(result.Content) { - return true, "content_is_question" - } - - return false, "completed" -} -``` - -## Standard Agent Interface - -All agent-driven features (generator, simulator, validator) use standard Yao Agent interfaces. - -### Using `context.Options` - -The test framework uses `context.Options` to pass parameters, aligned with `Assistant.Stream()`: - -```go -// context.Options - standard Yao Agent options -type Options struct { - Skip *Skip `json:"skip,omitempty"` // Skip history, trace, etc. - Connector string `json:"connector,omitempty"` // LLM connector to use - Search any `json:"search,omitempty"` // Search behavior control - Mode string `json:"mode,omitempty"` // Agent mode (chat, etc.) - Metadata map[string]any `json:"metadata,omitempty"` // Custom metadata -} -``` - -### Test Framework Usage - -```go -// Prepare options for agent invocation -options := &context.Options{ - Skip: &context.Skip{ - History: true, // Don't save test messages to history - Trace: false, // Keep trace for debugging - }, - Metadata: map[string]any{ - "test_mode": "validator", // "generator" | "simulator" | "validator" - "test_id": "T001", - "criteria": "Response should be helpful", - // ... other custom params from test config - }, -} - -// Prepare context -ctx := context.New(parent, authorized, chatID) -ctx.Referer = "agent-test" - -// Call agent with standard Stream API -assistant := agent.Get(agentID) -response, err := assistant.Stream(ctx, messages, options) -``` - -### Test Case Options Field - -Test cases can specify options to pass to the target agent or helper agents: - -```jsonl -{ - "id": "T001", - "input": "Hello", - "options": { - "connector": "openai-gpt4", - "skip": { - "history": true - }, - "metadata": { - "scenario": "edge-case" - } - } -} -``` - -### Generator Mode - -Used when `-i agents:xxx` is specified to generate test cases: - -```go -// Framework calls generator agent -options := &context.Options{ - Skip: &context.Skip{History: true}, - Metadata: map[string]any{ - "test_mode": "generator", - "target_agent": "assistants.expense", - "target_description": "Expense reimbursement assistant", - "target_tools": []string{"create_expense", "get_policy"}, - // From query params: agents:xxx?count=10&focus=edge-cases - "count": 10, - "focus": "edge-cases", - "complexity": "medium", - }, -} -``` - -Expected structured output: - -```json -{ - "cases": [ - {"id": "G001", "input": "...", "assertions": [...]}, - {"id": "G002", "input": "...", "assertions": [...]} - ] -} -``` - -### Simulator Mode - -Used when test case has `simulator` config: - -```go -// Framework calls simulator agent for next user input -options := &context.Options{ - Skip: &context.Skip{History: true}, - Metadata: map[string]any{ - "test_mode": "simulator", - "test_id": "T001", - // From simulator.options.metadata in test case - "persona": "New employee", - "goal": "Submit expense report", - // Runtime context - "turn_number": 3, - "max_turns": 10, - "tool_results": map[string]any{...}, - }, -} - -// Messages include full conversation history -messages := conversationHistory -``` - -Expected structured output: - -```json -{ - "input": "The amount is $500", - "goal_achieved": false, - "reasoning": "Providing requested amount info" -} -``` - -### Validator Mode - -Used when assertion has `type: "agent"`: - -```go -// Framework calls validator agent -options := &context.Options{ - Skip: &context.Skip{History: true}, - Metadata: map[string]any{ - "test_mode": "validator", - "test_id": "T001", - // From assertion.metadata - "criteria": "Response should be helpful", - "expected_intent": "answer question", - "tone": "professional", - }, -} - -// Messages include agent response to validate -messages := []context.Message{ - {Role: "user", Content: "Original user question"}, - {Role: "assistant", Content: "Agent's response to validate"}, -} -``` - -Expected structured output: - -```json -{ - "passed": true, - "score": 0.95, - "reason": "Response is helpful and addresses the question", - "suggestions": [] -} -``` - -## Assertion Types - -### Static Assertions (Existing) - -| Type | Description | Example | -| ------------- | ---------------------- | ---------------------------------------------------------- | -| `contains` | Response contains text | `{"type": "contains", "value": "success"}` | -| `equals` | Exact match | `{"type": "equals", "value": "OK"}` | -| `regex` | Regex pattern match | `{"type": "regex", "pattern": "order-\\d+"}` | -| `json_path` | JSONPath value check | `{"type": "json_path", "path": "$.status", "value": "ok"}` | -| `tool_called` | Tool was invoked | `{"type": "tool_called", "name": "create_expense"}` | -| `type` | Value type check | `{"type": "type", "path": "$.count", "value": "number"}` | - -### Agent-Driven Assertions (New) - -For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `context.Options`: - -```jsonl -{ - "type": "agent", - "use": "agents:workers.test.validator", - "options": { - "connector": "openai-gpt4", - "metadata": { - "criteria": "Response should be helpful and answer the user's question", - "expected_intent": "provide expense submission guidance", - "tone": "professional and friendly" - } - } -} -``` - -### Script Assertions (in JSONL) - -For custom validation logic in JSONL test cases: - -```jsonl -{ - "type": "script", - "use": "scripts:tests.validate-expense", - "options": { - "metadata": { - "min_amount": 100, - "max_amount": 10000 - } - } -} -``` - -### Combined Assertions - -Mix static and agent-driven assertions: - -```jsonl -{ - "assertions": [ - { - "type": "tool_called", - "name": "create_expense" - }, - { - "type": "agent", - "use": "agents:workers.test.validator", - "options": { - "metadata": { - "criteria": "Confirmation message should include expense amount and be polite" - } - } - } - ] -} -``` - -## Script Testing with Agent Assertions - -Script tests can also use Agent-driven assertions via the `t.assert.Agent()` method. - -### API - -```typescript -// t.assert.Agent(response, agentID, options?) -> ValidatorResult -// agentID: Direct agent ID without prefix, e.g., "workers.test.validator" -interface ValidatorResult { - passed: boolean; - score?: number; - reason: string; - suggestions?: string[]; -} -``` - -### Usage in Script Tests - -```typescript -// tests/expense_test.ts -export function TestExpenseResponse(t: TestingT, ctx: Context) { - // Call the agent being tested - const response = Process("agents.expense.Stream", ctx, [ - { role: "user", content: "How do I submit an expense?" }, - ]); - - // Static assertions - t.assert.NotNil(response); - t.assert.Contains(response.content, "expense"); - - // Agent-driven assertion - automatically fails test if validation fails - t.assert.Agent(response.content, "workers.test.validator", { - metadata: { - criteria: - "Response should explain the expense submission process clearly", - expected_topics: ["receipt", "approval", "deadline"], - tone: "helpful", - }, - }); -} -``` - -### With Conversation Context - -```typescript -export function TestMultiTurnExpense(t: TestingT, ctx: Context) { - const messages = [ - { role: "user", content: "I need to submit a travel expense" }, - { role: "assistant", content: "I'd be happy to help..." }, - { role: "user", content: "It's for a flight to Beijing, $2000" }, - ]; - - const response = Process("agents.expense.Stream", ctx, messages); - - // Agent assertion with conversation context - // Automatically fails test and logs suggestions if validation fails - t.assert.Agent(response.content, "workers.test.validator", { - metadata: { - criteria: - "Response should confirm the expense details and ask for receipt", - conversation: messages, - }, - }); -} -``` - -### Implementation - -The `t.assert.Agent()` method internally: - -1. Prepares `context.Options` with `test_mode: "validator"` -2. Calls the validator agent via `Assistant.Stream()` -3. Parses structured output (JSON) -4. Returns `ValidatorResult` - -```go -// In script_assert.go -func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context) *v8go.FunctionTemplate { - return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { - // Parse arguments: response, agentID, options - response := args[0].String() - agentID := args[1].String() // Direct agent ID, e.g., "workers.test.validator" - options := parseOptions(args[2]) - - // Prepare validator options - validatorOpts := &context.Options{ - Skip: &context.Skip{History: true}, - Metadata: map[string]any{ - "test_mode": "validator", - ...options.Metadata, - }, - } - - // Call validator agent - assistant, _ := agent.Get(agentID) - result, _ := assistant.Stream(agentCtx, messages, validatorOpts) - - // Parse and return result - return toJsValue(parseValidatorResult(result)) - }) -} -``` +| Field | Type | Description | +| ------------- | ------ | -------------------------------- | +| `simulator` | object | Simulator agent configuration | +| `checkpoints` | array | Functional checkpoints to verify | +| `max_turns` | int | Maximum turns before timeout | +| `timeout` | string | Maximum time (e.g., "5m") | ## Test Case Format -### Single-Turn (Existing) +### Simple Input (Existing) ```jsonl { @@ -554,69 +112,132 @@ func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context } ``` -### Multi-Turn: Static Mode +### With Message History (New) -For **deterministic flows** where you know the exact conversation sequence: +Test agent with conversation context - simulates multi-turn without complex state: ```jsonl { - "id": "T001", - "name": "Expense Reimbursement - Happy Path", - "mode": "static", - "options": { - "connector": "openai-gpt4", - "skip": { - "history": true + "id": "T002", + "name": "Expense submission - final confirmation", + "messages": [ + { + "role": "user", + "content": "I want to submit an expense" + }, + { + "role": "assistant", + "content": "What type of expense would you like to submit?" + }, + { + "role": "user", + "content": "Business travel to Beijing, $3500" + }, + { + "role": "assistant", + "content": "I'll create an expense for business travel, $3500. Please confirm." + }, + { + "role": "user", + "content": "Yes, confirm" } - }, - "turns": [ + ], + "assertions": [ { - "input": "I want to submit an expense report", - "assertions": [ - { - "type": "contains", - "value": "type of expense" - } - ] + "type": "contains", + "value": "submitted" }, { - "input": "Business travel to Beijing, $3500", - "assertions": [ - { - "type": "tool_called", - "name": "create_expense" - } - ] - }, - { - "input": "Yes, confirm", - "assertions": [ - { - "type": "contains", - "value": "submitted" - } - ] + "type": "tool_called", + "name": "create_expense" } ] } ``` -**Characteristics:** +**Key insight**: Instead of executing 3 turns sequentially, we pass the full conversation history. The agent sees the context and responds to the last message. This is: -- Fixed number of turns -- Each turn has specific input and assertions -- Test fails if any turn assertion fails -- Best for regression testing known flows +- **Simpler** - No turn-by-turn execution, no session state +- **Faster** - Single API call instead of multiple +- **Parallelizable** - Each test is independent +- **Debuggable** - Clear input/output for each test -### Multi-Turn: Dynamic Mode (Checkpoints) +### Testing Different Points in a Conversation -For **coverage testing** where you care about functionality, not exact sequence: +To test agent behavior at different conversation stages, create separate test cases: + +```jsonl +// Test 1: First turn - agent should ask for expense type +{ + "id": "expense-turn1", + "messages": [ + {"role": "user", "content": "I want to submit an expense"} + ], + "assertions": [{"type": "contains", "value": "type"}] +} + +// Test 2: Second turn - agent should create expense +{ + "id": "expense-turn2", + "messages": [ + {"role": "user", "content": "I want to submit an expense"}, + {"role": "assistant", "content": "What type of expense would you like to submit?"}, + {"role": "user", "content": "Business travel, $3500"} + ], + "assertions": [{"type": "tool_called", "name": "create_expense"}] +} + +// Test 3: Final turn - agent should confirm submission +{ + "id": "expense-turn3", + "messages": [ + {"role": "user", "content": "I want to submit an expense"}, + {"role": "assistant", "content": "What type of expense?"}, + {"role": "user", "content": "Business travel, $3500"}, + {"role": "assistant", "content": "Confirm $3500 expense?"}, + {"role": "user", "content": "Yes"} + ], + "assertions": [{"type": "contains", "value": "submitted"}] +} +``` + +### With Attachments ```jsonl { - "id": "T002", + "id": "T003", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What's in this receipt?" + }, + { + "type": "image", + "source": "file://./fixtures/receipt.jpg" + } + ] + } + ], + "assertions": [ + { + "type": "contains", + "value": "amount" + } + ] +} +``` + +### Dynamic Mode (Simulator + Checkpoints) + +For coverage testing where conversation flow is unpredictable: + +```jsonl +{ + "id": "T004", "name": "Expense Submission Coverage", - "mode": "dynamic", "simulator": { "use": "workers.test.user-simulator", "options": { @@ -626,51 +247,6 @@ For **coverage testing** where you care about functionality, not exact sequence: } } }, - "checkpoints": [ - { - "id": "ask_type", - "description": "Agent asks for expense type", - "assertion": { - "type": "contains", - "value": "type" - } - }, - { - "id": "call_create", - "description": "Agent calls create_expense tool", - "assertion": { - "type": "tool_called", - "name": "create_expense" - } - }, - { - "id": "confirm_submit", - "description": "Agent confirms submission", - "assertion": { - "type": "contains", - "value": "submitted" - } - } - ], - "max_turns": 10, - "timeout": "2m" -} -``` - -**Characteristics:** - -- Simulator drives the conversation -- Checkpoints are verified across all turns (order-independent by default) -- Test passes when ALL checkpoints are reached -- Test fails if max_turns/timeout reached before all checkpoints -- Best for functional coverage testing - -### Checkpoints with Order Constraints - -When checkpoints must occur in a specific order: - -```jsonl -{ "checkpoints": [ { "id": "ask_type", @@ -692,7 +268,7 @@ When checkpoints must occur in a specific order: } }, { - "id": "confirm_submit", + "id": "confirm", "description": "Agent confirms submission", "after": [ "call_create" @@ -702,401 +278,263 @@ When checkpoints must occur in a specific order: "value": "submitted" } } - ] -} -``` - -### Checkpoints with Agent Validation - -Use Agent-driven assertions for semantic validation: - -```jsonl -{ - "checkpoints": [ - { - "id": "helpful_guidance", - "description": "Agent provides helpful expense guidance", - "assertion": { - "type": "agent", - "use": "agents:workers.test.validator", - "options": { - "metadata": { - "criteria": "Response explains expense process clearly and professionally" - } - } - } - }, - { - "id": "tool_called", - "description": "Agent creates expense record", - "assertion": { - "type": "tool_called", - "name": "create_expense" - } - } - ] -} -``` - -### Dynamic Mode Termination - -| Condition | Result | Description | -| ------------------------------------ | ---------- | -------------------------- | -| All checkpoints reached | ✅ PASSED | All functionality verified | -| Agent completes, checkpoints missing | ❌ FAILED | Missing coverage | -| max_turns exceeded | ❌ FAILED | Timeout - flow too long | -| timeout exceeded | ❌ FAILED | Time limit reached | -| Checkpoint assertion fails | ❌ FAILED | Functionality broken | -| Simulator error | ⚠️ SKIPPED | Cannot continue | - -### Field Descriptions - -| Field | Type | Required | Description | -| --------------------------- | ------ | ------------- | ------------------------------------------ | -| `id` | string | Yes | Unique test identifier | -| `name` | string | No | Human-readable test name | -| `mode` | string | No | `"static"` (default) or `"dynamic"` | -| `options` | object | No | `context.Options` passed to target agent | -| **Static Mode Fields** | -| `turns` | array | Yes (static) | Static turn definitions | -| `turns[].input` | string | Yes | User input for this turn | -| `turns[].assertions` | array | No | Assertions for this turn's response | -| `turns[].options` | object | No | Per-turn options override | -| **Dynamic Mode Fields** | -| `simulator` | object | Yes (dynamic) | User simulator configuration | -| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | -| `simulator.options` | object | No | `context.Options` passed to simulator | -| `checkpoints` | array | Yes (dynamic) | Functionality checkpoints to verify | -| `checkpoints[].id` | string | Yes | Unique checkpoint identifier | -| `checkpoints[].description` | string | No | Human-readable description | -| `checkpoints[].assertion` | object | Yes | Assertion to verify | -| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | -| `max_turns` | int | No | Maximum turns before timeout (default: 20) | -| `timeout` | string | No | Maximum time (default: "5m") | -| **Shared Fields** | -| `interactive` | object | No | Interactive mode configuration | -| `interactive.enabled` | bool | No | Enable human input (default: false) | -| `interactive.timeout` | string | No | Timeout for human input (default: "5m") | - -## Execution Modes - -### Static Mode - -Uses predefined `turns` array. Best for **regression testing** known flows. - -``` -┌─────────────────────────────────────────────────────────┐ -│ Static Mode Flow │ -├─────────────────────────────────────────────────────────┤ -│ │ -│ FOR each turn in turns[]: │ -│ 1. Send turn.input to Agent │ -│ 2. Get Agent response │ -│ 3. Run turn.assertions │ -│ ├─ PASS → Continue to next turn │ -│ └─ FAIL → Test FAILED, stop │ -│ │ -│ All turns completed → Test PASSED │ -│ │ -└─────────────────────────────────────────────────────────┘ -``` - -### Dynamic Mode (Checkpoints) - -Uses simulator + checkpoints. Best for **coverage testing** functionality. - -``` -┌─────────────────────────────────────────────────────────┐ -│ Dynamic Mode Flow │ -├─────────────────────────────────────────────────────────┤ -│ │ -│ Initialize: pending_checkpoints = all checkpoints │ -│ │ -│ LOOP (until terminated): │ -│ 1. Simulator generates user input │ -│ 2. Send input to Agent │ -│ 3. Get Agent response │ -│ 4. Check response against pending_checkpoints │ -│ └─ If matched → Move to reached_checkpoints │ -│ 5. Check termination conditions: │ -│ ├─ All checkpoints reached → PASSED │ -│ ├─ Agent completed, missing checkpoints → FAILED │ -│ ├─ max_turns exceeded → FAILED │ -│ └─ timeout exceeded → FAILED │ -│ │ -└─────────────────────────────────────────────────────────┘ -``` - -### Interactive Mode - -For debugging, human can provide input when agent awaits: - -```bash -# Enable with --interactive flag -yao agent test -i ./tests.jsonl --interactive -``` - -In interactive mode: - -- Static mode: Human can override any turn input -- Dynamic mode: Human can replace simulator for specific turns - -### Mode Selection - -| Has `turns`? | Has `simulator` + `checkpoints`? | Mode | -| ------------ | -------------------------------- | ----------------------- | -| Yes | No | Static | -| No | Yes | Dynamic | -| Yes | Yes | ❌ Invalid (choose one) | -| No | No | Single-turn (legacy) | - -### Example: Static vs Dynamic - -**Same feature, different testing approaches:** - -```jsonl -// Static Mode - Exact sequence testing -{ - "id": "expense-static", - "mode": "static", - "turns": [ - {"input": "Submit expense", "assertions": [{"type": "contains", "value": "type"}]}, - {"input": "Travel, $500", "assertions": [{"type": "tool_called", "name": "create_expense"}]}, - {"input": "Confirm", "assertions": [{"type": "contains", "value": "submitted"}]} - ] -} - -// Dynamic Mode - Coverage testing -{ - "id": "expense-dynamic", - "mode": "dynamic", - "simulator": {"use": "workers.test.user-sim", "options": {"metadata": {"goal": "Submit $500 expense"}}}, - "checkpoints": [ - {"id": "ask", "assertion": {"type": "contains", "value": "type"}}, - {"id": "create", "assertion": {"type": "tool_called", "name": "create_expense"}}, - {"id": "done", "assertion": {"type": "contains", "value": "submitted"}} ], - "max_turns": 10 + "max_turns": 10, + "timeout": "2m" } ``` +## Field Descriptions + +### Standard Mode Fields + +| Field | Type | Required | Description | +| ------------ | ------ | -------- | --------------------------------- | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `input` | string | No\* | Simple text input | +| `messages` | array | No\* | Full message history | +| `assertions` | array | No | Assertions to validate response | +| `options` | object | No | `context.Options` passed to agent | + +\*Either `input` or `messages` required + +### Dynamic Mode Fields + +| Field | Type | Required | Description | +| --------------------------- | ------ | -------- | ------------------------------------------ | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `simulator` | object | Yes | User simulator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | +| `simulator.options` | object | No | `context.Options` passed to simulator | +| `checkpoints` | array | Yes | Functionality checkpoints to verify | +| `checkpoints[].id` | string | Yes | Unique checkpoint identifier | +| `checkpoints[].description` | string | No | Human-readable description | +| `checkpoints[].assertion` | object | Yes | Assertion to verify | +| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | +| `max_turns` | int | No | Maximum turns before timeout (default: 20) | +| `timeout` | string | No | Maximum time (default: "5m") | +| `options` | object | No | `context.Options` passed to target agent | + ## Execution Flow -### Static Mode Flow +### Standard Mode ``` ┌─────────────────────────────────────────────────────────────────┐ -│ Static Mode Execution │ +│ Standard Mode Execution │ ├─────────────────────────────────────────────────────────────────┤ │ │ -│ INITIALIZE: │ -│ - Load turns[] from test case │ -│ - Set current_turn = 0 │ +│ 1. Parse test case │ +│ ├─ Has `messages`? → Use as-is │ +│ └─ Has `input`? → Convert to [{role: "user", content: input}]│ │ ↓ │ -│ FOR each turn in turns[]: │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ 1. Get input from turns[current_turn].input │ │ -│ │ ↓ │ │ -│ │ 2. Send input to Agent │ │ -│ │ ↓ │ │ -│ │ 3. Get Agent response │ │ -│ │ ↓ │ │ -│ │ 4. Run turns[current_turn].assertions │ │ -│ │ │ │ │ -│ │ ├─ PASS → Continue to next turn │ │ -│ │ └─ FAIL → Test FAILED, stop │ │ -│ └─────────────────────────────────────────────────────────┘ │ +│ 2. Call Agent.Stream(ctx, messages, options) │ │ ↓ │ -│ All turns completed → Test PASSED │ +│ 3. Run assertions against response │ +│ ├─ All PASS → Test PASSED ✅ │ +│ └─ Any FAIL → Test FAILED ❌ │ │ │ └─────────────────────────────────────────────────────────────────┘ ``` -### Dynamic Mode Flow +### Dynamic Mode ``` ┌─────────────────────────────────────────────────────────────────┐ │ Dynamic Mode Execution │ ├─────────────────────────────────────────────────────────────────┤ │ │ -│ INITIALIZE: │ +│ Initialize: │ │ - pending_checkpoints = all checkpoints │ -│ - reached_checkpoints = [] │ +│ - messages = [] │ │ - turn_count = 0 │ -│ - start_time = now() │ │ ↓ │ │ LOOP: │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ 1. Call Simulator Agent → Get user input │ │ -│ │ (pass: persona, goal, conversation history) │ │ -│ │ ↓ │ │ -│ │ 2. Send input to Target Agent │ │ -│ │ ↓ │ │ -│ │ 3. Get Agent response │ │ -│ │ ↓ │ │ -│ │ 4. Check response against pending_checkpoints │ │ -│ │ FOR each pending checkpoint: │ │ -│ │ - Run checkpoint.assertion │ │ -│ │ - If PASS and `after` satisfied → move to reached │ │ -│ │ ↓ │ │ -│ │ 5. Check termination conditions: │ │ -│ │ ├─ pending_checkpoints empty? │ │ -│ │ │ → Test PASSED ✅ │ │ -│ │ │ │ │ -│ │ ├─ Agent completed (not awaiting)? │ │ -│ │ │ → Test FAILED ❌ (missing checkpoints) │ │ -│ │ │ │ │ -│ │ ├─ turn_count >= max_turns? │ │ -│ │ │ → Test FAILED ❌ (turn limit) │ │ -│ │ │ │ │ -│ │ ├─ now() - start_time > timeout? │ │ -│ │ │ → Test FAILED ❌ (timeout) │ │ -│ │ │ │ │ -│ │ └─ Otherwise → Continue loop │ │ -│ └─────────────────────────────────────────────────────────┘ │ +│ 1. Call Simulator → get user input │ +│ 2. Append user message to messages │ +│ 3. Call Agent.Stream(ctx, messages, options) │ +│ 4. Append assistant response to messages │ +│ 5. Check response against pending_checkpoints │ +│ └─ If matched (and `after` satisfied) → move to reached │ +│ 6. Check termination: │ +│ ├─ All checkpoints reached → PASSED ✅ │ +│ ├─ Simulator signals goal_achieved → FAILED ❌ │ +│ ├─ turn_count >= max_turns → FAILED ❌ │ +│ └─ timeout exceeded → FAILED ❌ │ │ │ └─────────────────────────────────────────────────────────────────┘ ``` +## Assertion Types + +### Static Assertions + +| Type | Description | Example | +| ------------- | ---------------------- | ---------------------------------------------------------- | +| `contains` | Response contains text | `{"type": "contains", "value": "success"}` | +| `equals` | Exact match | `{"type": "equals", "value": "OK"}` | +| `regex` | Regex pattern match | `{"type": "regex", "pattern": "order-\\d+"}` | +| `json_path` | JSONPath value check | `{"type": "json_path", "path": "$.status", "value": "ok"}` | +| `tool_called` | Tool was invoked | `{"type": "tool_called", "name": "create_expense"}` | +| `type` | Value type check | `{"type": "type", "path": "$.count", "value": "number"}` | + +### Agent-Driven Assertions + +For semantic or fuzzy validation: + +```jsonl +{ + "type": "agent", + "use": "agents:workers.test.validator", + "options": { + "metadata": { + "criteria": "Response should be helpful and answer the user's question", + "tone": "professional and friendly" + } + } +} +``` + +### Script Assertions + +For custom validation logic: + +```jsonl +{ + "type": "script", + "use": "scripts:tests.validate-expense", + "options": { + "metadata": { + "min_amount": 100, + "max_amount": 10000 + } + } +} +``` + +## Script Testing with Agent Assertions + +Script tests can use Agent-driven assertions via `t.assert.Agent()`: + +```typescript +export function TestExpenseResponse(t: TestingT, ctx: Context) { + const messages = [ + { role: "user", content: "I want to submit an expense" }, + { role: "assistant", content: "What type of expense?" }, + { role: "user", content: "Travel, $500" }, + ]; + + const response = Process("agents.expense.Stream", ctx, messages); + + // Static assertion + t.assert.Contains(response.content, "confirm"); + + // Agent-driven assertion + t.assert.Agent(response.content, "workers.test.validator", { + metadata: { + criteria: "Response should ask for confirmation before creating expense", + conversation: messages, + }, + }); +} +``` + +## Standard Agent Interface + +All agent-driven features use `context.Options`: + +```go +type Options struct { + Skip *Skip `json:"skip,omitempty"` + Connector string `json:"connector,omitempty"` + Search any `json:"search,omitempty"` + Mode string `json:"mode,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} +``` + +### Generator Agent + +Called when `-i agents:xxx` is used: + +```go +options := &context.Options{ + Metadata: map[string]any{ + "test_mode": "generator", + "target_agent": "assistants.expense", + "count": 10, + "focus": "edge-cases", + }, +} +``` + +### Simulator Agent + +Called in dynamic mode to generate user input: + +```go +options := &context.Options{ + Metadata: map[string]any{ + "test_mode": "simulator", + "persona": "New employee", + "goal": "Submit expense", + "turn_number": 3, + }, +} +``` + +### Validator Agent + +Called for agent-driven assertions: + +```go +options := &context.Options{ + Metadata: map[string]any{ + "test_mode": "validator", + "criteria": "Response should be helpful", + }, +} +``` + ## Command Line Interface ### Flags Reference -| Flag | Long | Description | -| ---- | --------------- | ---------------------------------------------------------- | -| `-i` | `--input` | Input source: file path, message, or `type:id` reference | -| `-n` | `--name` | Target agent ID (the agent being tested) | -| `-o` | `--output` | Output file path for results | -| `-c` | `--connector` | Override connector for the target agent | -| `-v` | `--verbose` | Verbose output showing all turns | -| | `--interactive` | Enable human input when agent awaits | -| | `--simulator` | Default simulator agent ID (e.g., `workers.test.user-sim`) | -| | `--timeout` | Timeout per test case (default: 5m) | -| | `--parallel` | Number of parallel test cases | -| | `--fail-fast` | Stop on first failure | -| | `--dry-run` | Generate/parse tests without running | +| Flag | Long | Description | +| ---- | ------------- | -------------------------------------------------------- | +| `-i` | `--input` | Input source: file path, message, or `type:id` reference | +| `-n` | `--name` | Target agent ID (the agent being tested) | +| `-o` | `--output` | Output file path for results | +| `-c` | `--connector` | Override connector for the target agent | +| `-v` | `--verbose` | Verbose output | +| | `--simulator` | Default simulator agent ID | +| | `--timeout` | Timeout per test case (default: 5m) | +| | `--parallel` | Number of parallel test cases | +| | `--fail-fast` | Stop on first failure | +| | `--dry-run` | Generate/parse tests without running | -### Input Sources (`-i` flag) - -The `-i` flag supports multiple input sources with unified `type:id` format: +### Examples ```bash -# 1. File path (default, no prefix needed) -yao agent test -i ./tests/multi-turn.jsonl - -# 2. Direct message (no prefix, auto-detected as non-file) +# Simple test yao agent test -i "Hello, how are you?" -n assistants.chat -# 3. Agent-generated test cases -yao agent test -i agents:workers.test.generator -n assistants.expense - -# 4. Script-generated test cases -yao agent test -i scripts:tests.generate -n assistants.expense - -# 5. With parameters (query string style) -yao agent test -i "agents:workers.test.generator?count=10&focus=edge-cases" -n assistants.expense -``` - -### Input Type Prefixes - -| Prefix | Description | Example | -| ---------- | --------------------------- | -------------------------- | -| (none) | File path or direct message | `./tests.jsonl`, `"Hello"` | -| `agents:` | Agent generates test cases | `agents:workers.test.gen` | -| `scripts:` | Script generates test cases | `scripts:tests.generate` | - -### Input Format - -``` -[prefix:][?param1=value1¶m2=value2] -``` - -- `prefix` - Input type: `agents:` or `scripts:` (optional, default is file/message) -- `id` - Agent ID or script ID -- `?params` - Query parameters passed to generator - -#### Generator Agent Interface - -```typescript -// Input to generator agent -interface GeneratorInput { - target_agent: string; // Agent being tested (from -n flag) - target_description?: string; // Agent's description/purpose - target_tools?: Tool[]; // Agent's available tools - count?: number; // Number of test cases to generate - focus?: string; // Focus area: "happy-path", "edge-cases", "errors" - complexity?: string; // "simple", "medium", "complex" -} - -// Output from generator agent -interface GeneratorOutput { - cases: TestCase[]; // Generated test cases -} -``` - -#### Example Generator Prompt - -``` -You are a test case generator for AI agents. - -Target Agent: {{target_agent}} -Description: {{target_description}} -Available Tools: {{target_tools}} - -Generate {{count}} test cases with focus on: {{focus}} - -For each test case, provide: -- id: Unique identifier -- name: Descriptive name -- input: User message or turns array for multi-turn -- assertions: Expected behaviors to verify - -Output as JSON array of test cases. -``` - -### Complete Examples - -```bash -# Basic: Run tests from file +# From JSONL file yao agent test -i ./tests/expense.jsonl -# With target agent specified (required for message/agent input) -yao agent test -i "Help me file an expense" -n assistants.expense +# Agent-generated tests +yao agent test -i "agents:workers.test.generator?count=10" -n assistants.expense -# Agent generates tests, then runs them -yao agent test \ - -i "agents:workers.test.generator?count=20&focus=edge-cases" \ - -n assistants.expense +# With simulator for dynamic mode +yao agent test -i ./tests/dynamic.jsonl --simulator workers.test.user-simulator -# Script generates tests -yao agent test \ - -i "scripts:tests.expense.generate?scenario=approval-flow" \ - -n assistants.expense +# Parallel execution +yao agent test -i ./tests/expense.jsonl --parallel 5 -# Fully dynamic: Agent generates tests + Agent simulates user responses -yao agent test \ - -i "agents:workers.test.generator?count=10" \ - -n assistants.expense \ - --simulator workers.test.user-simulator - -# Generate tests only, save to file (dry-run) -yao agent test \ - -i "agents:workers.test.generator?count=50" \ - -n assistants.expense \ - -o ./tests/generated.jsonl \ - --dry-run - -# Interactive mode: human provides input when agent awaits -yao agent test -i ./tests/multi-turn.jsonl --interactive - -# CI/CD mode: skip tests requiring human input -yao agent test -i ./tests/multi-turn.jsonl --skip-interactive - -# Fail instead of skip when input unavailable -yao agent test -i ./tests/multi-turn.jsonl --on-missing-input=fail - -# Verbose output showing all turns -yao agent test -i ./tests/multi-turn.jsonl -v +# Verbose output +yao agent test -i ./tests/expense.jsonl -v ``` ## Output Format @@ -1105,136 +543,96 @@ yao agent test -i ./tests/multi-turn.jsonl -v ``` ═══════════════════════════════════════════════════════════════ - Agent Test (Multi-Turn) + Agent Test ═══════════════════════════════════════════════════════════════ ℹ Agent: assistants.expense -ℹ Input: ./tests/expense-flow.jsonl (5 test cases) +ℹ Input: ./tests/expense.jsonl (3 test cases) ─────────────────────────────────────────────────────────────── Running Tests ─────────────────────────────────────────────────────────────── -► [T001] Expense Reimbursement Flow (3 turns) - ├─ Turn 1: "I want to submit an expense" → PASSED (2.1s) - │ Agent: "What type of expense would you like to submit?" - │ ✓ contains "type of expense" - │ - ├─ Turn 2: "Business travel, $3500" → PASSED (3.2s) - │ Agent: [tool: create_expense({amount: 3500, type: "travel"})] - │ ✓ tool_called "create_expense" - │ - ├─ Turn 3: "Yes, confirm" → PASSED (1.8s) - │ Agent: "Expense submitted. Reference: EXP-2025-001" - │ ✓ contains "submitted" - │ - └─ Final Assertions: PASSED - ✓ $.expense.status = "submitted" +✓ [expense-turn1] First turn - ask type (1.2s) + Messages: 1, Assertions: 1/1 passed -► [T002] Large Expense Approval - ├─ Turn 1: "Submit $100,000 equipment purchase" → PASSED (2.0s) - │ Agent: "This requires manager approval. Please provide PO number." - │ - ├─ Turn 2: SKIPPED - │ Reason: Agent awaiting input, no next turn defined - │ Agent asked: "Please provide PO number" - │ Hint: Add more turns, use --interactive, or configure simulator - │ - └─ Result: SKIPPED +✓ [expense-turn2] Second turn - create expense (2.1s) + Messages: 3, Assertions: 1/1 passed -► [T003] Dynamic Expense Flow (simulator: workers.test.user-sim) - ├─ Turn 1: [Initial] "Help me file an expense" → PASSED (2.1s) - ├─ Turn 2: [Simulated] "It's for client dinner, $250" → PASSED (2.8s) - ├─ Turn 3: [Simulated] "Yesterday evening" → PASSED (2.2s) - ├─ Turn 4: [Simulated] "Confirm" → PASSED (1.9s) - │ Goal achieved: Expense submitted - │ - └─ Final Assertions: PASSED +✓ [expense-turn3] Final turn - confirm (1.8s) + Messages: 5, Assertions: 1/1 passed ─────────────────────────────────────────────────────────────── Summary ─────────────────────────────────────────────────────────────── Total: 3 tests - Passed: 2 + Passed: 3 Failed: 0 - Skipped: 1 - - Total turns: 10 - Avg turns/test: 3.3 - Total time: 18.1s + Time: 5.1s ``` ### JSONL Output ```jsonl { - "id": "T001", - "name": "Expense Reimbursement Flow", + "id": "expense-turn3", + "name": "Final turn - confirm", "status": "passed", - "turns": [ + "messages_count": 5, + "response": "Expense submitted. Reference: EXP-2025-001", + "assertions": [ { - "turn": 1, - "input": "I want to submit an expense", - "input_source": "static", - "output": "What type of expense would you like to submit?", - "awaiting_input": true, - "assertions": [ - { - "type": "contains", - "value": "type of expense", - "passed": true - } - ], - "duration_ms": 2100 - }, - { - "turn": 2, - "input": "Business travel, $3500", - "input_source": "static", - "output": "", - "tool_calls": [ - { - "name": "create_expense", - "args": { - "amount": 3500 - } - } - ], - "awaiting_input": true, - "assertions": [ - { - "type": "tool_called", - "name": "create_expense", - "passed": true - } - ], - "duration_ms": 3200 - }, - { - "turn": 3, - "input": "Yes, confirm", - "input_source": "static", - "output": "Expense submitted. Reference: EXP-2025-001", - "awaiting_input": false, - "assertions": [ - { - "type": "contains", - "value": "submitted", - "passed": true - } - ], - "duration_ms": 1800 - } - ], - "final_assertions": [ - { - "type": "json_path", - "path": "$.expense.status", + "type": "contains", "value": "submitted", "passed": true } ], + "duration_ms": 1800 +} +``` + +## Dynamic Mode Output + +```jsonl +{ + "id": "expense-dynamic", + "name": "Expense Coverage Test", + "status": "passed", + "turns": [ + { + "turn": 1, + "input": "Help me file an expense", + "output": "What type?" + }, + { + "turn": 2, + "input": "Client dinner, $250", + "output": "Confirm?" + }, + { + "turn": 3, + "input": "Yes", + "output": "Submitted!" + } + ], + "checkpoints": [ + { + "id": "ask_type", + "reached_at_turn": 1, + "passed": true + }, + { + "id": "call_create", + "reached_at_turn": 2, + "passed": true + }, + { + "id": "confirm", + "reached_at_turn": 3, + "passed": true + } + ], "total_turns": 3, - "duration_ms": 7100 + "duration_ms": 6800 } ``` @@ -1242,28 +640,23 @@ yao agent test -i ./tests/multi-turn.jsonl -v ### Interface -The simulator agent receives conversation context and generates the next user input: - ```typescript -// Input to simulator interface SimulatorInput { - persona: string; // User persona description - goal: string; // What user wants to achieve - conversation: Message[]; // Conversation history - last_response: string; // Agent's last response - turn_number: number; // Current turn (1-based) - max_turns: number; // Maximum allowed turns + persona: string; + goal: string; + conversation: Message[]; + turn_number: number; + max_turns: number; } -// Output from simulator interface SimulatorOutput { - input: string; // Generated user input - goal_achieved: boolean; // Whether goal is complete - reasoning?: string; // Why this input was chosen + input: string; + goal_achieved: boolean; + reasoning?: string; } ``` -### Example Simulator Prompt +### Example Prompt ``` You are simulating a user with the following characteristics: @@ -1274,9 +667,6 @@ Goal: {{goal}} Current conversation: {{conversation}} -The agent just responded: -"{{last_response}}" - Generate the next user message to continue toward the goal. If the goal has been achieved, set goal_achieved to true. @@ -1290,27 +680,25 @@ Respond in JSON format: ## Backward Compatibility -Existing single-turn tests continue to work unchanged: +Existing single-turn tests work unchanged: ```jsonl -// This still works (single-turn, legacy format) +// This still works {"id": "T001", "input": "Hello", "assertions": [...]} -// Static mode with one turn (equivalent) -{"id": "T001", "mode": "static", "turns": [{"input": "Hello", "assertions": [...]}]} +// Equivalent to +{"id": "T001", "messages": [{"role": "user", "content": "Hello"}], "assertions": [...]} ``` ## Error Handling -### Static Mode Errors +### Standard Mode Errors -| Error Type | Behavior | Output | -| ---------------- | ------------------- | ---------------------------- | -| Agent timeout | Mark turn as FAILED | `error: "timeout after 30s"` | -| Agent error | Mark turn as FAILED | `error: "agent error: ..."` | -| Assertion failed | Mark turn as FAILED | `assertion_errors: [...]` | -| All turns passed | Test PASSED | `status: "passed"` | -| Any turn failed | Test FAILED | `status: "failed"` | +| Error Type | Behavior | Output | +| ---------------- | ----------- | ---------------------------- | +| Agent timeout | Test FAILED | `error: "timeout after 30s"` | +| Agent error | Test FAILED | `error: "agent error: ..."` | +| Assertion failed | Test FAILED | `assertion_errors: [...]` | ### Dynamic Mode Errors @@ -1323,82 +711,21 @@ Existing single-turn tests continue to work unchanged: | Simulator error | Test FAILED | `error: "simulator error: ..."` | | Checkpoint assertion failed | Test FAILED | `error: "checkpoint X failed"` | -## Context and State +## Comparison: Old vs New Design -### Conversation Context - -Each multi-turn test maintains a conversation context: - -```go -type ConversationContext struct { - SessionID string // Unique session for this test - Messages []Message // Full conversation history - ToolResults map[string]any // Results from tool calls - Variables map[string]any // Custom variables set during test - TurnCount int // Current turn number -} -``` - -### Context Passing to Simulator - -The simulator receives context via standard Yao Agent Context metadata: - -```go -// Framework prepares context for simulator -ctx.Metadata = map[string]any{ - "test_mode": "simulator", - "test_id": "T001", - // From simulator config - "persona": "New employee", - "goal": "Submit expense report", - // Runtime context - "session_id": "test-session-001", - "turn_count": 3, - "tool_results": map[string]any{...}, -} - -// Messages include conversation history -messages := []Message{ - {Role: "user", Content: "I want to submit an expense"}, - {Role: "assistant", Content: "What type of expense?"}, - {Role: "user", Content: "Travel expense"}, - {Role: "assistant", Content: "Please confirm the details..."}, -} -``` - -## Attachments in Multi-Turn - -Multi-turn tests support attachments at the turn level: - -```jsonl -{ - "id": "T001", - "name": "Receipt Upload Flow", - "turns": [ - { - "input": "I want to submit an expense with receipt", - "attachments": [ - { - "type": "image", - "source": "file://./tests/fixtures/receipt.jpg" - } - ] - }, - { - "input": "The amount is $150" - } - ] -} -``` +| Aspect | Old (Static Mode) | New (Messages) | +| ------------------ | ------------------------- | --------------------------- | +| Multi-turn testing | Sequential turn execution | Pass message history | +| State management | Session state per test | Stateless | +| Parallelization | Sequential within test | Fully parallel | +| Implementation | Complex turn loop | Single agent call | +| Debugging | Need to trace turns | Clear input/output per test | +| Flexibility | Coupled turns | Independent tests | ## Open Questions -1. **Session Management**: How to handle session state across turns? Use existing session or create new per-test? +1. **Message Generation**: Should we provide a helper to generate message history from a script? -2. **Timeout Strategy**: Per-turn timeout vs. total test timeout? +2. **Snapshot Testing**: Should we support "golden file" comparison for responses? -3. **Parallel Execution**: Can multi-turn tests run in parallel, or must they be sequential? - -4. **Retry Logic**: If a turn fails, retry just that turn or restart entire conversation? - -5. **Snapshot Testing**: Should we support "golden file" comparison for conversation flows? +3. **Retry Logic**: If a test fails, should we support automatic retry? diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index c865d9b0..946e3804 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -2,32 +2,30 @@ ## Format Rules Summary -| Context | Format | Example | -|---------|--------|---------| -| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | -| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | -| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-sim"` | -| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-sim` | -| `t.assert.Agent()` | No prefix (method is explicit) | `t.assert.Agent(resp, "workers.test.validator", {...})` | +| Context | Format | Example | +| --------------------- | ------------------------ | ------------------------------------------------------- | +| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | +| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -## Phase 1: Static Mode +## Phase 1: Message History Support -- [ ] Add `mode` field to test case parser (`static` | `dynamic`) -- [ ] Extend test case parser for `turns` array +- [ ] Add `messages` field to test case parser +- [ ] Support both `input` (string) and `messages` (array) fields +- [ ] Convert `input` to `messages` format internally +- [ ] Pass messages directly to `Agent.Stream()` - [ ] Add `options` field support (aligned with `context.Options`) -- [ ] Support test-level `options` and per-turn `options` override -- [ ] Implement turn-by-turn execution with options passing -- [ ] Implement conversation context management -- [ ] Add per-turn assertions -- [ ] Support attachments at turn level -- [ ] Update console output for multi-turn display -- [ ] Update JSONL output format for turns +- [ ] Support attachments in message content parts +- [ ] Update console output to show message count +- [ ] Update JSONL output format ## Phase 2: Agent-Driven Input - [ ] Parse `agents:` prefix in `-i` flag - [ ] Parse `scripts:` prefix in `-i` flag -- [ ] Use standard `context.Options` for all agent invocations +- [ ] Use standard `context.Options` for generator invocation - [ ] Pass `test_mode: "generator"` in `options.metadata` - [ ] Pass target agent info (description, tools) in `options.metadata` - [ ] Support query parameters (`?count=10&focus=...`) → merged into `options.metadata` @@ -37,12 +35,13 @@ ## Phase 3: Dynamic Mode (Checkpoints) - [ ] Add `checkpoints` array to test case parser +- [ ] Add `simulator` field to test case parser - [ ] Implement checkpoint matching against agent responses - [ ] Support `after` field for order constraints - [ ] Track pending/reached checkpoints during execution - [ ] Implement termination conditions: - [ ] All checkpoints reached → PASSED - - [ ] Agent completed, missing checkpoints → FAILED + - [ ] Simulator signals goal_achieved but checkpoints missing → FAILED - [ ] max_turns exceeded → FAILED - [ ] timeout exceeded → FAILED - [ ] Implement simulator invocation via `Assistant.Stream()` @@ -52,24 +51,10 @@ - [ ] Pass conversation history as messages - [ ] Create example simulator agent with prompt template -## Phase 4: Interactive Mode - -- [ ] Add `--interactive` flag -- [ ] Implement terminal input prompt with context display -- [ ] Add input timeout handling -- [ ] Support input history/editing -- [ ] Add `--skip-interactive` for CI/CD mode - -## Phase 5: Enhanced Detection - -- [ ] Add `awaiting_input` field to agent response schema -- [ ] Implement tool-based detection (confirmation tools) -- [ ] Add configurable detection rules -- [ ] Support custom detection via script/agent - -## Phase 6: Agent-Driven Assertions +## Phase 4: Agent-Driven Assertions ### In JSONL Test Cases + - [ ] Add `agent` assertion type to assertion parser - [ ] Support `options` field in assertion (aligned with `context.Options`) - [ ] Implement validator agent invocation via `Assistant.Stream()` @@ -79,6 +64,7 @@ - [ ] Add `suggestions` to assertion error output ### In Script Tests + - [ ] Add `t.assert.Agent(response, agentID, options?)` method - [ ] `agentID` is direct ID (e.g., `workers.test.validator`), no prefix needed - [ ] Invoke validator agent with context @@ -86,25 +72,22 @@ - [ ] Support passing conversation history in options ### Shared + - [ ] Create example validator agent with prompt template - [ ] Document `ValidatorResult` interface -## Phase 7: Error Handling & Reporting +## Phase 5: Error Handling & Reporting -- [ ] Implement turn-level error handling -- [ ] Implement test-level error aggregation +- [ ] Implement test-level error handling - [ ] Add detailed error messages with hints -- [ ] Support custom reporter agent +- [ ] Support `--parallel` flag for concurrent test execution +- [ ] Support `--fail-fast` flag to stop on first failure +- [ ] Add verbose mode (`-v`) for detailed output ## Open Questions -1. **Session Management**: How to handle session state across turns? Use existing session or create new per-test? +1. **Message Generation**: Should we provide a helper to generate message history from a script? -2. **Timeout Strategy**: Per-turn timeout vs. total test timeout? - -3. **Parallel Execution**: Can multi-turn tests run in parallel, or must they be sequential? - -4. **Retry Logic**: If a turn fails, retry just that turn or restart entire conversation? - -5. **Snapshot Testing**: Should we support "golden file" comparison for conversation flows? +2. **Snapshot Testing**: Should we support "golden file" comparison for responses? +3. **Retry Logic**: If a test fails, should we support automatic retry? From c34cae62e80d94c306c8b3e1e4ff61af372c5035 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 18:16:26 +0800 Subject: [PATCH 07/17] Update DESIGN_V2.md and TODO_V2.md for Input Field Enhancements - Revised DESIGN_V2.md to clarify the `input` field's capabilities, allowing for string, single message, or message array formats for conversation context. - Updated the test case format to reflect the new `input` structure, ensuring compatibility with existing single-turn tests. - Enhanced TODO_V2.md to indicate the completion of message history support and outline remaining tasks, including options field support and JSONL output format updates. - Improved documentation to ensure clarity on the new input handling and its implications for agent-driven testing. --- agent/test/DESIGN_V2.md | 81 +++++++++++++++++++++-------------------- agent/test/TODO_V2.md | 20 ++++++---- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 3b3f498c..5648e0f3 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -4,7 +4,7 @@ This document describes the design for Agent Test Framework V2, which extends the existing testing capabilities with: -- **Message history support** - Test agents with conversation context via `messages[]` +- **Message history support** - Test agents with conversation context via `input` array (already implemented) - **Agent-driven testing** - Use agents to generate test cases and validate responses - **Dynamic testing** - Simulator-driven testing with checkpoint validation @@ -45,7 +45,7 @@ This document describes the design for Agent Test Framework V2, which extends th │ ┌───────────────────────────────────────────────────────────────────┐ │ │ │ Test Case Parser │ │ │ │ │ │ -│ │ Standard Mode: {input: "...", messages: [...], assertions} │ │ +│ │ Standard Mode: {input: "..." | [...], assertions} │ │ │ │ Dynamic Mode: {simulator: {...}, checkpoints: [...]} │ │ │ └───────────────────────────────────────────────────────────────────┘ │ │ │ │ @@ -77,12 +77,11 @@ This document describes the design for Agent Test Framework V2, which extends th Single call to agent with optional message history. **No multi-turn state management needed.** -| Field | Type | Description | -| ------------ | ------ | ------------------------------------------------ | -| `input` | string | Simple text input (shorthand for single message) | -| `messages` | array | Full message history (overrides `input`) | -| `assertions` | array | Assertions to validate response | -| `options` | object | `context.Options` passed to agent | +| Field | Type | Description | +| ------------ | ------------------------------ | --------------------------------------------- | +| `input` | string \| Message \| Message[] | Text, single message, or conversation history | +| `assertions` | array | Assertions to validate response | +| `options` | object | `context.Options` passed to agent | ### Dynamic Mode @@ -112,15 +111,15 @@ Simulator-driven testing with checkpoint validation. } ``` -### With Message History (New) +### With Message History (Existing) -Test agent with conversation context - simulates multi-turn without complex state: +The `input` field already supports message arrays for conversation context: ```jsonl { "id": "T002", "name": "Expense submission - final confirmation", - "messages": [ + "input": [ { "role": "user", "content": "I want to submit an expense" @@ -170,16 +169,14 @@ To test agent behavior at different conversation stages, create separate test ca // Test 1: First turn - agent should ask for expense type { "id": "expense-turn1", - "messages": [ - {"role": "user", "content": "I want to submit an expense"} - ], + "input": [{"role": "user", "content": "I want to submit an expense"}], "assertions": [{"type": "contains", "value": "type"}] } // Test 2: Second turn - agent should create expense { "id": "expense-turn2", - "messages": [ + "input": [ {"role": "user", "content": "I want to submit an expense"}, {"role": "assistant", "content": "What type of expense would you like to submit?"}, {"role": "user", "content": "Business travel, $3500"} @@ -190,7 +187,7 @@ To test agent behavior at different conversation stages, create separate test ca // Test 3: Final turn - agent should confirm submission { "id": "expense-turn3", - "messages": [ + "input": [ {"role": "user", "content": "I want to submit an expense"}, {"role": "assistant", "content": "What type of expense?"}, {"role": "user", "content": "Business travel, $3500"}, @@ -206,7 +203,7 @@ To test agent behavior at different conversation stages, create separate test ca ```jsonl { "id": "T003", - "messages": [ + "input": [ { "role": "user", "content": [ @@ -288,16 +285,19 @@ For coverage testing where conversation flow is unpredictable: ### Standard Mode Fields -| Field | Type | Required | Description | -| ------------ | ------ | -------- | --------------------------------- | -| `id` | string | Yes | Unique test identifier | -| `name` | string | No | Human-readable test name | -| `input` | string | No\* | Simple text input | -| `messages` | array | No\* | Full message history | -| `assertions` | array | No | Assertions to validate response | -| `options` | object | No | `context.Options` passed to agent | +| Field | Type | Required | Description | +| ------------ | ------------------------------ | -------- | ------------------------------------------------- | +| `id` | string | Yes | Unique test identifier | +| `name` | string | No | Human-readable test name | +| `input` | string \| Message \| Message[] | Yes | Input: text, single message, or message array | +| `assertions` | array | No | Assertions to validate response (alias: `assert`) | +| `options` | object | No | `context.Options` passed to agent | -\*Either `input` or `messages` required +**Note**: The `input` field supports three formats: + +- `string`: Simple text (converted to `[{role: "user", content: "..."}]`) +- `object`: Single message `{role: "...", content: "..."}` +- `array`: Message history `[{role: "user", ...}, {role: "assistant", ...}, ...]` ### Dynamic Mode Fields @@ -327,8 +327,8 @@ For coverage testing where conversation flow is unpredictable: ├─────────────────────────────────────────────────────────────────┤ │ │ │ 1. Parse test case │ -│ ├─ Has `messages`? → Use as-is │ -│ └─ Has `input`? → Convert to [{role: "user", content: input}]│ +│ ├─ `input` is array? → Use as messages │ +│ └─ `input` is string? → Convert to [{role: "user", content}] │ │ ↓ │ │ 2. Call Agent.Stream(ctx, messages, options) │ │ ↓ │ @@ -683,11 +683,11 @@ Respond in JSON format: Existing single-turn tests work unchanged: ```jsonl -// This still works +// Simple string input {"id": "T001", "input": "Hello", "assertions": [...]} -// Equivalent to -{"id": "T001", "messages": [{"role": "user", "content": "Hello"}], "assertions": [...]} +// Equivalent to array format +{"id": "T001", "input": [{"role": "user", "content": "Hello"}], "assertions": [...]} ``` ## Error Handling @@ -711,16 +711,17 @@ Existing single-turn tests work unchanged: | Simulator error | Test FAILED | `error: "simulator error: ..."` | | Checkpoint assertion failed | Test FAILED | `error: "checkpoint X failed"` | -## Comparison: Old vs New Design +## Current Implementation Status -| Aspect | Old (Static Mode) | New (Messages) | -| ------------------ | ------------------------- | --------------------------- | -| Multi-turn testing | Sequential turn execution | Pass message history | -| State management | Session state per test | Stateless | -| Parallelization | Sequential within test | Fully parallel | -| Implementation | Complex turn loop | Single agent call | -| Debugging | Need to trace turns | Clear input/output per test | -| Flexibility | Coupled turns | Independent tests | +| Feature | Status | Notes | +| ----------------------- | ---------- | ---------------------------------------- | +| Simple text input | ✅ Done | `input: "Hello"` | +| Message history | ✅ Done | `input: [{role, content}, ...]` | +| File attachments | ✅ Done | `file://` protocol in content parts | +| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | +| Agent-driven assertions | 🔲 Planned | `type: "agent"` with validator agent | +| Dynamic mode | 🔲 Planned | Simulator + Checkpoints | +| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | ## Open Questions diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 946e3804..f7b395d7 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -10,16 +10,20 @@ | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -## Phase 1: Message History Support +## Phase 1: Message History Support ✅ (Already Implemented) -- [ ] Add `messages` field to test case parser -- [ ] Support both `input` (string) and `messages` (array) fields -- [ ] Convert `input` to `messages` format internally -- [ ] Pass messages directly to `Agent.Stream()` -- [ ] Add `options` field support (aligned with `context.Options`) -- [ ] Support attachments in message content parts +The `input` field already supports: +- `string`: Simple text input +- `object`: Single message `{role, content}` +- `array`: Message history `[{role, content}, ...]` + +See `input.go` → `ParseInputWithOptions()` for implementation. + +**Remaining tasks:** +- [ ] Add `options` field support (aligned with `context.Options`) - partially done via `CaseOptions` +- [x] Support attachments in message content parts (file:// protocol) - [ ] Update console output to show message count -- [ ] Update JSONL output format +- [ ] Update JSONL output format with `messages_count` ## Phase 2: Agent-Driven Input From a5e974dc07d171bb747df43adb9a7f53419998e5 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 18:58:31 +0800 Subject: [PATCH 08/17] Update DESIGN_V2.md and TODO_V2.md for Output Format Enhancements - Revised DESIGN_V2.md to clarify output formats for console and JSON, including detailed descriptions for standard, dynamic, and parallel modes. - Updated the console output sections to provide clearer examples and summaries of test results, enhancing readability and usability. - Modified TODO_V2.md to reflect the change from JSONL to JSON output format for message counts, ensuring consistency in documentation. - Improved overall documentation to support better understanding of output handling in the Agent Test Framework. --- agent/test/DESIGN_V2.md | 278 ++++++++++++++++++++++++++++++---------- agent/test/TODO_V2.md | 2 +- 2 files changed, 212 insertions(+), 68 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 5648e0f3..7754d3ef 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -65,7 +65,7 @@ This document describes the design for Agent Test Framework V2, which extends th │ ┌───────────────────────────────────────────────────────────────────┐ │ │ │ Reporter │ │ │ │ - Console output │ │ -│ │ - JSONL output │ │ +│ │ - JSON file output │ │ │ └───────────────────────────────────────────────────────────────────┘ │ │ │ └─────────────────────────────────────────────────────────────────────────┘ @@ -539,100 +539,244 @@ yao agent test -i ./tests/expense.jsonl -v ## Output Format -### Console Output +### Console Output (Standard Mode) + +Standard mode shows each test case as a single line with input preview: ``` ═══════════════════════════════════════════════════════════════ Agent Test ═══════════════════════════════════════════════════════════════ -ℹ Agent: assistants.expense -ℹ Input: ./tests/expense.jsonl (3 test cases) +ℹ Agent: workers.system.keyword +ℹ Connector: deepseek.v3 +ℹ Input: ./tests/inputs.jsonl (42 test cases) +ℹ Timeout: 5m0s ─────────────────────────────────────────────────────────────── Running Tests ─────────────────────────────────────────────────────────────── - -✓ [expense-turn1] First turn - ask type (1.2s) - Messages: 1, Assertions: 1/1 passed - -✓ [expense-turn2] Second turn - create expense (2.1s) - Messages: 3, Assertions: 1/1 passed - -✓ [expense-turn3] Final turn - confirm (1.8s) - Messages: 5, Assertions: 1/1 passed +► [T001] 人工智能和机器学习正在改变我们�... PASSED (2.7s) +► [T002] The rapid development of cloud computing has re... PASSED (3.0s) +► [T003] 区块链技术是一种分布式账本技术�... PASSED (2.7s) +... ─────────────────────────────────────────────────────────────── Summary ─────────────────────────────────────────────────────────────── - Total: 3 tests - Passed: 3 - Failed: 0 - Time: 5.1s + Agent: workers.system.keyword + Connector: deepseek.v3 + Total: 42 + Passed: 42 + Failed: 0 + Pass Rate: 100.0% + Duration: 1.8m + + Output: ./tests/output-20251225185335.jsonl + +═══════════════════════════════════════════════════════════════ + ✨ ALL TESTS PASSED ✨ +═══════════════════════════════════════════════════════════════ ``` -### JSONL Output +### Console Output (Dynamic Mode) -```jsonl +Dynamic mode shows each test case as a tree with turns and checkpoints: + +``` +═══════════════════════════════════════════════════════════════ + Agent Test (Dynamic Mode) +═══════════════════════════════════════════════════════════════ +ℹ Agent: assistants.expense +ℹ Connector: openai.gpt4 +ℹ Input: ./tests/dynamic.jsonl (2 test cases) +ℹ Simulator: workers.test.user-simulator + +─────────────────────────────────────────────────────────────── + Running Tests +─────────────────────────────────────────────────────────────── +► [T001] Expense Submission Coverage + ├─ Turn 1: "Help me file an expense" → "What type of expense?" + │ └─ ✓ checkpoint: ask_type + ├─ Turn 2: "Client dinner, $250" → "I'll create... Please confirm." + │ └─ ✓ checkpoint: call_create (tool: create_expense) + └─ Turn 3: "Yes, confirm" → "Expense submitted! Reference: EXP-001" + └─ ✓ checkpoint: confirm + PASSED (6.8s) - 3 turns, 3/3 checkpoints + +► [T002] Expense with Attachment + ├─ Turn 1: "Submit receipt" + [receipt.jpg] → "What type?" + │ └─ ✓ checkpoint: ask_type + ├─ Turn 2: "Business lunch" → "Amount from receipt: $85.50. Confirm?" + │ └─ ✓ checkpoint: extract_amount + └─ Turn 3: "Yes" → "Submitted! Reference: EXP-002" + └─ ✓ checkpoint: confirm + PASSED (8.2s) - 3 turns, 3/3 checkpoints + +─────────────────────────────────────────────────────────────── + Summary +─────────────────────────────────────────────────────────────── + Agent: assistants.expense + Connector: openai.gpt4 + Simulator: workers.test.user-simulator + Total: 2 + Passed: 2 + Failed: 0 + Pass Rate: 100.0% + Duration: 15.0s + + Output: ./tests/output-20251225190000.jsonl + +═══════════════════════════════════════════════════════════════ + ✨ ALL TESTS PASSED ✨ +═══════════════════════════════════════════════════════════════ +``` + +### Console Output (Parallel Mode) + +When `--parallel N` is enabled, tests run concurrently. Output is buffered and displayed as complete test trees: + +``` +═══════════════════════════════════════════════════════════════ + Agent Test (Parallel: 5) +═══════════════════════════════════════════════════════════════ +ℹ Agent: assistants.expense +ℹ Input: ./tests/dynamic.jsonl (10 test cases) +ℹ Parallel: 5 concurrent + +─────────────────────────────────────────────────────────────── + Running Tests (5 parallel) +─────────────────────────────────────────────────────────────── +► [T003] Quick approval flow + ├─ Turn 1: "Approve expense EXP-001" → "Approved!" + └─ ✓ checkpoint: approved + PASSED (1.2s) - 1 turn, 1/1 checkpoints + +► [T001] Expense Submission Coverage + ├─ Turn 1: "Help me file an expense" → "What type?" + │ └─ ✓ checkpoint: ask_type + ├─ Turn 2: "Client dinner, $250" → "Confirm?" + │ └─ ✓ checkpoint: call_create + └─ Turn 3: "Yes" → "Submitted!" + └─ ✓ checkpoint: confirm + PASSED (6.8s) - 3 turns, 3/3 checkpoints + +► [T002] Expense with Attachment + ├─ Turn 1: "Submit receipt" + [receipt.jpg] → "What type?" + ... + PASSED (8.2s) - 3 turns, 3/3 checkpoints + +[Progress: 3/10 completed, 5 running...] + +► [T004] Rejection flow + ... + PASSED (4.5s) - 2 turns, 2/2 checkpoints + +─────────────────────────────────────────────────────────────── + Summary +─────────────────────────────────────────────────────────────── + Total: 10 + Passed: 10 + Failed: 0 + Pass Rate: 100.0% + Duration: 25.3s (effective: 2.5s/test with 5 parallel) + +═══════════════════════════════════════════════════════════════ + ✨ ALL TESTS PASSED ✨ +═══════════════════════════════════════════════════════════════ +``` + +**Note**: In parallel mode, test results appear in completion order (not input order). Each test's output is buffered and displayed as a complete tree to maintain readability. + +### JSON Output (Standard Mode) + +Output file is a JSON object with `summary`, `environment`, `results`, and `metadata`: + +```json { - "id": "expense-turn3", - "name": "Final turn - confirm", - "status": "passed", - "messages_count": 5, - "response": "Expense submitted. Reference: EXP-2025-001", - "assertions": [ + "summary": { + "total": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "errors": 0, + "timeouts": 0, + "duration_ms": 5100, + "agent_id": "assistants.expense", + "agent_path": "/path/to/expense" + }, + "environment": { + "user_id": "test-user", + "team_id": "test-team", + "locale": "en-us" + }, + "results": [ { - "type": "contains", - "value": "submitted", - "passed": true + "id": "expense-turn1", + "status": "passed", + "input": [{ "role": "user", "content": "I want to submit an expense" }], + "output": "What type of expense would you like to submit?", + "duration_ms": 1200 + }, + { + "id": "expense-turn2", + "status": "passed", + "input": [ + { "role": "user", "content": "I want to submit an expense" }, + { "role": "assistant", "content": "What type?" }, + { "role": "user", "content": "Business travel, $3500" } + ], + "output": "Confirm $3500 expense?", + "duration_ms": 2100 } ], - "duration_ms": 1800 + "metadata": { + "started_at": "2025-12-25T10:00:00Z", + "completed_at": "2025-12-25T10:00:05Z", + "input_file": "./tests/expense.jsonl" + } } ``` -## Dynamic Mode Output +### JSON Output (Dynamic Mode) -```jsonl +Dynamic mode adds `turns` and `checkpoints` to each result: + +```json { - "id": "expense-dynamic", - "name": "Expense Coverage Test", - "status": "passed", - "turns": [ + "summary": { + "total": 1, + "passed": 1, + "failed": 0, + "duration_ms": 6800, + "agent_id": "assistants.expense" + }, + "results": [ { - "turn": 1, - "input": "Help me file an expense", - "output": "What type?" - }, - { - "turn": 2, - "input": "Client dinner, $250", - "output": "Confirm?" - }, - { - "turn": 3, - "input": "Yes", - "output": "Submitted!" + "id": "expense-dynamic", + "name": "Expense Coverage Test", + "status": "passed", + "turns": [ + { + "turn": 1, + "input": "Help me file an expense", + "output": "What type?" + }, + { "turn": 2, "input": "Client dinner, $250", "output": "Confirm?" }, + { "turn": 3, "input": "Yes", "output": "Submitted!" } + ], + "checkpoints": [ + { "id": "ask_type", "reached_at_turn": 1, "passed": true }, + { "id": "call_create", "reached_at_turn": 2, "passed": true }, + { "id": "confirm", "reached_at_turn": 3, "passed": true } + ], + "total_turns": 3, + "duration_ms": 6800 } ], - "checkpoints": [ - { - "id": "ask_type", - "reached_at_turn": 1, - "passed": true - }, - { - "id": "call_create", - "reached_at_turn": 2, - "passed": true - }, - { - "id": "confirm", - "reached_at_turn": 3, - "passed": true - } - ], - "total_turns": 3, - "duration_ms": 6800 + "metadata": { + "started_at": "2025-12-25T10:00:00Z", + "completed_at": "2025-12-25T10:00:07Z" + } } ``` diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index f7b395d7..7e6b5933 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -23,7 +23,7 @@ See `input.go` → `ParseInputWithOptions()` for implementation. - [ ] Add `options` field support (aligned with `context.Options`) - partially done via `CaseOptions` - [x] Support attachments in message content parts (file:// protocol) - [ ] Update console output to show message count -- [ ] Update JSONL output format with `messages_count` +- [ ] Update JSON output format with `messages_count` ## Phase 2: Agent-Driven Input From 2ec28a1710e6bbad59fb50e7452ce3400533921f Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 08:58:33 +0800 Subject: [PATCH 09/17] Enhance DESIGN_V2.md and TODO_V2.md with Before/After Script Support - Updated DESIGN_V2.md to introduce support for `before` and `after` scripts in JSONL test cases, detailing their usage and execution order. - Added examples for defining and utilizing before/after functions, including global initialization and cleanup processes. - Revised TODO_V2.md to outline tasks for implementing before/after script functionality, ensuring clarity on remaining development efforts. - Improved overall documentation to facilitate understanding of the new scripting capabilities in the Agent Test Framework. --- agent/test/DESIGN_V2.md | 155 ++++++++++++++++++++++++++++++++++++---- agent/test/TODO_V2.md | 33 ++++++--- 2 files changed, 168 insertions(+), 20 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 7754d3ef..08b48215 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -17,6 +17,8 @@ This document describes the design for Agent Test Framework V2, which extends th | JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | +| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | +| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | ## Design Goals @@ -292,6 +294,8 @@ For coverage testing where conversation flow is unpredictable: | `input` | string \| Message \| Message[] | Yes | Input: text, single message, or message array | | `assertions` | array | No | Assertions to validate response (alias: `assert`) | | `options` | object | No | `context.Options` passed to agent | +| `before` | string | No | Before script (e.g., `scripts:tests.env.Before`) | +| `after` | string | No | After script (e.g., `scripts:tests.env.After`) | **Note**: The `input` field supports three formats: @@ -316,6 +320,131 @@ For coverage testing where conversation flow is unpredictable: | `max_turns` | int | No | Maximum turns before timeout (default: 20) | | `timeout` | string | No | Maximum time (default: "5m") | | `options` | object | No | `context.Options` passed to target agent | +| `before` | string | No | Before script function | +| `after` | string | No | After script function | + +## Before and After Scripts + +JSONL test cases can reference `*_test.ts` scripts for environment preparation: + +### Script Location + +Scripts are located in the agent's `tests/` directory: + +``` +assistants/expense/ +├── agent.yml +├── package.yao +└── tests/ + ├── inputs.jsonl # Test cases + ├── env_test.ts # Before/after functions + └── fixtures/ + └── receipt.jpg +``` + +### Script Interface + +```typescript +// tests/env_test.ts + +// Before function - called before test case runs +// Returns context data that will be passed to After +export function Before(ctx: Context, testCase: TestCase): BeforeResult { + // Prepare database + const userId = Process("models.user.Create", { + name: "Test User", + email: "test@example.com", + }); + + // Prepare knowledge base + Process("knowledge.expense.Index", { + documents: [{ title: "Policy", content: "Max expense $5000" }], + }); + + return { + data: { userId, testId: testCase.id }, + }; +} + +// After function - called after test case completes (pass or fail) +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + beforeData: any +) { + // Clean up database + if (beforeData?.userId) { + Process("models.user.Delete", beforeData.userId); + } + + // Clean up knowledge base + Process("knowledge.expense.Clear"); +} + +// Global before - called once before all test cases +export function BeforeAll(ctx: Context, testCases: TestCase[]): BeforeResult { + // One-time initialization + Process("models.migrate"); + return { data: { initialized: true } }; +} + +// Global after - called once after all test cases +export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { + // Final cleanup + Process("models.cleanup"); +} +``` + +### Test Case with Before/After + +```jsonl +{ + "id": "T001", + "name": "Submit expense with user context", + "before": "scripts:tests.env.Before", + "after": "scripts:tests.env.After", + "input": "Submit a $500 travel expense", + "assertions": [ + { + "type": "tool_called", + "name": "create_expense" + } + ] +} +``` + +### Global Before/After via CLI + +```bash +# Run with global before/after +yao agent test -i ./tests/inputs.jsonl \ + --before scripts:tests.env.BeforeAll \ + --after scripts:tests.env.AfterAll +``` + +### Execution Order + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Test Execution with Before/After │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ 1. BeforeAll() - Global initialization (once) │ +│ ↓ │ +│ FOR EACH test case: │ +│ 2. Before() - Per-test initialization │ +│ ↓ │ +│ 3. Run test (call agent, check assertions) │ +│ ↓ │ +│ 4. After() - Per-test cleanup (always runs) │ +│ ↓ │ +│ 5. AfterAll() - Global cleanup (once) │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +**Note**: Script tests (`*_test.ts`) don't need before/after fields since they can call functions directly within the test. ## Execution Flow @@ -502,18 +631,20 @@ options := &context.Options{ ### Flags Reference -| Flag | Long | Description | -| ---- | ------------- | -------------------------------------------------------- | -| `-i` | `--input` | Input source: file path, message, or `type:id` reference | -| `-n` | `--name` | Target agent ID (the agent being tested) | -| `-o` | `--output` | Output file path for results | -| `-c` | `--connector` | Override connector for the target agent | -| `-v` | `--verbose` | Verbose output | -| | `--simulator` | Default simulator agent ID | -| | `--timeout` | Timeout per test case (default: 5m) | -| | `--parallel` | Number of parallel test cases | -| | `--fail-fast` | Stop on first failure | -| | `--dry-run` | Generate/parse tests without running | +| Flag | Long | Description | +| ---- | ------------- | ---------------------------------------------------------- | +| `-i` | `--input` | Input source: file path, message, or `type:id` reference | +| `-n` | `--name` | Target agent ID (the agent being tested) | +| `-o` | `--output` | Output file path for results | +| `-c` | `--connector` | Override connector for the target agent | +| `-v` | `--verbose` | Verbose output | +| | `--simulator` | Default simulator agent ID | +| | `--before` | Global before script (e.g., `scripts:tests.env.BeforeAll`) | +| | `--after` | Global after script (e.g., `scripts:tests.env.AfterAll`) | +| | `--timeout` | Timeout per test case (default: 5m) | +| | `--parallel` | Number of parallel test cases | +| | `--fail-fast` | Stop on first failure | +| | `--dry-run` | Generate/parse tests without running | ### Examples diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 7e6b5933..6309ed87 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -2,13 +2,15 @@ ## Format Rules Summary -| Context | Format | Example | -| --------------------- | ------------------------ | ------------------------------------------------------- | -| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | -| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | -| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | -| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | -| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | +| Context | Format | Example | +| -------------------- | ------------------------ | ------------------------------------------------------- | +| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | +| JSONL assertion `use`| Prefix required | `"use": "agents:workers.test.validator"` | +| JSONL `simulator.use`| No prefix (agent only) | `"use": "workers.test.user-simulator"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | +| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | +| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | ## Phase 1: Message History Support ✅ (Already Implemented) @@ -80,7 +82,22 @@ See `input.go` → `ParseInputWithOptions()` for implementation. - [ ] Create example validator agent with prompt template - [ ] Document `ValidatorResult` interface -## Phase 5: Error Handling & Reporting +## Phase 5: Before and After Scripts + +- [ ] Add `before` field to test case parser +- [ ] Add `after` field to test case parser +- [ ] Add `--before` CLI flag for global before +- [ ] Add `--after` CLI flag for global after +- [ ] Parse `scripts:` prefix in before/after fields +- [ ] Implement script function invocation via `Process()` +- [ ] Pass `TestCase` object to before function +- [ ] Pass `TestResult` and before data to after function +- [ ] Ensure after runs even if test fails +- [ ] Support `BeforeAll()` for one-time initialization +- [ ] Support `AfterAll()` for final cleanup +- [ ] Create example before/after script + +## Phase 6: Error Handling & Reporting - [ ] Implement test-level error handling - [ ] Add detailed error messages with hints From 45c01f9c0410231c2ca8b3f64b1b5ba3d03b1663 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 09:10:27 +0800 Subject: [PATCH 10/17] Update TODO_V2.md and .gitignore for New Test Plan Documentation - Added `UPGRADE_PLAN.md` to .gitignore to exclude the new upgrade plan documentation from version control. - Revised TODO_V2.md to include a detailed implementation plan for the Agent Test Framework, outlining phases for before/after scripts, agent-driven assertions, and dynamic mode features. - Improved overall documentation clarity to facilitate understanding of upcoming enhancements and tasks within the framework. --- .gitignore | 1 + agent/test/TODO_V2.md | 155 +++++++++++++++++------------------------- 2 files changed, 65 insertions(+), 91 deletions(-) diff --git a/.gitignore b/.gitignore index d11f893d..d1b23f38 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,4 @@ agent/assistant/hook/*.test.md agent/search/TODO.md agent/search/job-logs.txt agent/test/MULTI_TURN_DESIGN.md +agent/test/UPGRADE_PLAN.md diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 6309ed87..8b66e7ff 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -1,114 +1,87 @@ -# Agent Test Framework V2 - Implementation TODO +# Agent Test Framework V2 - TODO -## Format Rules Summary +> 详细实施计划见 [UPGRADE_PLAN.md](./UPGRADE_PLAN.md) -| Context | Format | Example | -| -------------------- | ------------------------ | ------------------------------------------------------- | -| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | -| JSONL assertion `use`| Prefix required | `"use": "agents:workers.test.validator"` | -| JSONL `simulator.use`| No prefix (agent only) | `"use": "workers.test.user-simulator"` | -| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | -| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | -| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | +## Format Rules -## Phase 1: Message History Support ✅ (Already Implemented) +| Context | Format | Example | +| --------------------- | ------------------------ | ---------------------------------------------- | +| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | +| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | +| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.val")` | +| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | +| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | -The `input` field already supports: -- `string`: Simple text input -- `object`: Single message `{role, content}` -- `array`: Message history `[{role, content}, ...]` +## Phase 1: Before/After Scripts -See `input.go` → `ParseInputWithOptions()` for implementation. +**新增文件**: `script_hooks.go` -**Remaining tasks:** -- [ ] Add `options` field support (aligned with `context.Options`) - partially done via `CaseOptions` -- [x] Support attachments in message content parts (file:// protocol) -- [ ] Update console output to show message count -- [ ] Update JSON output format with `messages_count` +- [ ] `types.go`: 添加 `Before`, `After` 字段到 `Case` +- [ ] `types.go`: 添加 `BeforeAll`, `AfterAll` 字段到 `Options` +- [ ] `script_hooks.go`: 实现 `HookExecutor` +- [ ] `script_hooks.go`: 解析 `scripts:` 前缀 +- [ ] `runner.go`: 集成 before/after 到 `runSingleTest` +- [ ] `runner.go`: 集成 beforeAll/afterAll 到 `RunTests` +- [ ] `cmd/agent/agent.go`: 添加 `--before`, `--after` flags +- [ ] 创建示例脚本 `tests/env_test.ts` -## Phase 2: Agent-Driven Input +## Phase 2: Agent-Driven Assertions -- [ ] Parse `agents:` prefix in `-i` flag -- [ ] Parse `scripts:` prefix in `-i` flag -- [ ] Use standard `context.Options` for generator invocation -- [ ] Pass `test_mode: "generator"` in `options.metadata` -- [ ] Pass target agent info (description, tools) in `options.metadata` -- [ ] Support query parameters (`?count=10&focus=...`) → merged into `options.metadata` -- [ ] Add `--dry-run` flag to save generated cases without running -- [ ] Create example generator agent with prompt template +**修改文件**: `assert.go`, `script_assert.go` -## Phase 3: Dynamic Mode (Checkpoints) +- [ ] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion` +- [ ] `assert.go`: 实现 `assertAgent` 方法 +- [ ] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型 +- [ ] `script_assert.go`: 添加 `AssertAgent` 方法到 `TestingT` +- [ ] 创建示例 validator agent -- [ ] Add `checkpoints` array to test case parser -- [ ] Add `simulator` field to test case parser -- [ ] Implement checkpoint matching against agent responses -- [ ] Support `after` field for order constraints -- [ ] Track pending/reached checkpoints during execution -- [ ] Implement termination conditions: - - [ ] All checkpoints reached → PASSED - - [ ] Simulator signals goal_achieved but checkpoints missing → FAILED - - [ ] max_turns exceeded → FAILED - - [ ] timeout exceeded → FAILED -- [ ] Implement simulator invocation via `Assistant.Stream()` -- [ ] `simulator.use` is direct agent ID (no prefix needed) -- [ ] Pass `test_mode: "simulator"` in `options.metadata` -- [ ] Pass persona, goal, turn_count from `simulator.options.metadata` -- [ ] Pass conversation history as messages -- [ ] Create example simulator agent with prompt template +## Phase 3: Dynamic Mode (Simulator + Checkpoints) -## Phase 4: Agent-Driven Assertions +**新增文件**: `dynamic_runner.go`, `dynamic_types.go` -### In JSONL Test Cases +- [ ] `types.go`: 添加 `Simulator`, `Checkpoints` 字段到 `Case` +- [ ] `dynamic_types.go`: 定义 `Checkpoint`, `DynamicResult` 等类型 +- [ ] `dynamic_runner.go`: 实现 `DynamicRunner` +- [ ] `dynamic_runner.go`: 实现 checkpoint 匹配逻辑 +- [ ] `dynamic_runner.go`: 实现终止条件判断 +- [ ] `runner.go`: 在 `runSingleTest` 判断并调用动态模式 +- [ ] 创建示例 simulator agent -- [ ] Add `agent` assertion type to assertion parser -- [ ] Support `options` field in assertion (aligned with `context.Options`) -- [ ] Implement validator agent invocation via `Assistant.Stream()` -- [ ] Pass `test_mode: "validator"` in `options.metadata` -- [ ] Pass conversation context and criteria in `options.metadata` -- [ ] Support score-based pass/fail threshold (configurable in `options.metadata`) -- [ ] Add `suggestions` to assertion error output +## Phase 4: Agent-Driven Input -### In Script Tests +**新增文件**: `input_source.go` -- [ ] Add `t.assert.Agent(response, agentID, options?)` method -- [ ] `agentID` is direct ID (e.g., `workers.test.validator`), no prefix needed -- [ ] Invoke validator agent with context -- [ ] Return `ValidatorResult` object to JavaScript -- [ ] Support passing conversation history in options +- [ ] `input_source.go`: 实现 `ParseInputSource` +- [ ] `input_source.go`: 实现 `GenerateTestCases` +- [ ] `loader.go`: 添加 `LoadFromAgent` 方法 +- [ ] `loader.go`: 添加 `LoadFromScript` 方法 +- [ ] `runner.go`: 在 `RunTests` 支持不同输入源 +- [ ] `cmd/agent/agent.go`: 添加 `--dry-run` flag +- [ ] 创建示例 generator agent -### Shared +## Phase 5: Console Output Optimization -- [ ] Create example validator agent with prompt template -- [ ] Document `ValidatorResult` interface +**修改文件**: `output.go` -## Phase 5: Before and After Scripts +- [ ] `output.go`: 添加 `DynamicTestStart` 方法 +- [ ] `output.go`: 添加 `DynamicTurn` 方法 +- [ ] `output.go`: 添加 `DynamicTestResult` 方法 +- [ ] `output.go`: 添加 `ParallelResults` 方法 +- [ ] 测试并行模式输出效果 -- [ ] Add `before` field to test case parser -- [ ] Add `after` field to test case parser -- [ ] Add `--before` CLI flag for global before -- [ ] Add `--after` CLI flag for global after -- [ ] Parse `scripts:` prefix in before/after fields -- [ ] Implement script function invocation via `Process()` -- [ ] Pass `TestCase` object to before function -- [ ] Pass `TestResult` and before data to after function -- [ ] Ensure after runs even if test fails -- [ ] Support `BeforeAll()` for one-time initialization -- [ ] Support `AfterAll()` for final cleanup -- [ ] Create example before/after script +## Already Implemented ✅ -## Phase 6: Error Handling & Reporting - -- [ ] Implement test-level error handling -- [ ] Add detailed error messages with hints -- [ ] Support `--parallel` flag for concurrent test execution -- [ ] Support `--fail-fast` flag to stop on first failure -- [ ] Add verbose mode (`-v`) for detailed output +- [x] Message history support (`input` as array) +- [x] File attachments (`file://` protocol) +- [x] `--parallel` flag +- [x] `--fail-fast` flag +- [x] `-v` verbose mode +- [x] Script testing (`*_test.ts`) ## Open Questions -1. **Message Generation**: Should we provide a helper to generate message history from a script? - -2. **Snapshot Testing**: Should we support "golden file" comparison for responses? - -3. **Retry Logic**: If a test fails, should we support automatic retry? +1. **Message Generation**: 是否提供 helper 从脚本生成 message history? +2. **Snapshot Testing**: 是否支持 "golden file" 对比? +3. **Retry Logic**: 测试失败是否支持自动重试? From 5351128f89655d4d33ab752a1fb3b4417dbe6052 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 09:44:56 +0800 Subject: [PATCH 11/17] Refactor Before/After Script Integration in Agent Test Framework - Updated DESIGN_V2.md to reflect changes in the handling of before/after scripts, removing the `scripts:` prefix and clarifying their usage in JSONL test cases. - Enhanced runner.go to integrate global before/after hooks, ensuring they execute correctly before and after test cases. - Revised types.go to include new fields for before/after scripts in test case and options structures. - Improved TODO_V2.md to track the implementation progress of before/after script functionality and related tasks. - Added utility function LoadAgentTestScripts to facilitate loading of test scripts from the agent's src directory. --- agent/test/DESIGN_V2.md | 30 +- agent/test/TODO_V2.md | 25 +- agent/test/runner.go | 94 +++-- agent/test/script_hooks.go | 592 ++++++++++++++++++++++++++++++++ agent/test/script_hooks_test.go | 244 +++++++++++++ agent/test/types.go | 16 + cmd/agent/test.go | 6 + test/utils.go | 67 ++++ 8 files changed, 1026 insertions(+), 48 deletions(-) create mode 100644 agent/test/script_hooks.go create mode 100644 agent/test/script_hooks_test.go diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 08b48215..be1a2b9a 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -17,8 +17,8 @@ This document describes the design for Agent Test Framework V2, which extends th | JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.validator", {...})` | -| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | -| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | ## Design Goals @@ -294,8 +294,8 @@ For coverage testing where conversation flow is unpredictable: | `input` | string \| Message \| Message[] | Yes | Input: text, single message, or message array | | `assertions` | array | No | Assertions to validate response (alias: `assert`) | | `options` | object | No | `context.Options` passed to agent | -| `before` | string | No | Before script (e.g., `scripts:tests.env.Before`) | -| `after` | string | No | After script (e.g., `scripts:tests.env.After`) | +| `before` | string | No | Before script (e.g., `env_test.Before`) | +| `after` | string | No | After script (e.g., `env_test.After`) | **Note**: The `input` field supports three formats: @@ -329,15 +329,17 @@ JSONL test cases can reference `*_test.ts` scripts for environment preparation: ### Script Location -Scripts are located in the agent's `tests/` directory: +Scripts are located in the agent's `src/` directory (as `*_test.ts` files): ``` assistants/expense/ -├── agent.yml ├── package.yao +├── prompts.yml +├── src/ +│ ├── index.ts # Main agent script +│ └── env_test.ts # Before/after functions └── tests/ ├── inputs.jsonl # Test cases - ├── env_test.ts # Before/after functions └── fixtures/ └── receipt.jpg ``` @@ -345,7 +347,7 @@ assistants/expense/ ### Script Interface ```typescript -// tests/env_test.ts +// src/env_test.ts // Before function - called before test case runs // Returns context data that will be passed to After @@ -402,8 +404,8 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { { "id": "T001", "name": "Submit expense with user context", - "before": "scripts:tests.env.Before", - "after": "scripts:tests.env.After", + "before": "env_test.Before", + "after": "env_test.After", "input": "Submit a $500 travel expense", "assertions": [ { @@ -419,8 +421,8 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { ```bash # Run with global before/after yao agent test -i ./tests/inputs.jsonl \ - --before scripts:tests.env.BeforeAll \ - --after scripts:tests.env.AfterAll + --before env_test.BeforeAll \ + --after env_test.AfterAll ``` ### Execution Order @@ -639,8 +641,8 @@ options := &context.Options{ | `-c` | `--connector` | Override connector for the target agent | | `-v` | `--verbose` | Verbose output | | | `--simulator` | Default simulator agent ID | -| | `--before` | Global before script (e.g., `scripts:tests.env.BeforeAll`) | -| | `--after` | Global after script (e.g., `scripts:tests.env.AfterAll`) | +| | `--before` | Global before script (e.g., `env_test.BeforeAll`) | +| | `--after` | Global after script (e.g., `env_test.AfterAll`) | | | `--timeout` | Timeout per test case (default: 5m) | | | `--parallel` | Number of parallel test cases | | | `--fail-fast` | Stop on first failure | diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 8b66e7ff..aa1fb513 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -11,21 +11,23 @@ | JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | | `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | | `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.val")` | -| JSONL `before/after` | `scripts:` prefix | `"before": "scripts:tests.env.Before"` | -| `--before/--after` | `scripts:` prefix | `--before scripts:tests.env.BeforeAll` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | -## Phase 1: Before/After Scripts +## Phase 1: Before/After Scripts ✅ **新增文件**: `script_hooks.go` -- [ ] `types.go`: 添加 `Before`, `After` 字段到 `Case` -- [ ] `types.go`: 添加 `BeforeAll`, `AfterAll` 字段到 `Options` -- [ ] `script_hooks.go`: 实现 `HookExecutor` -- [ ] `script_hooks.go`: 解析 `scripts:` 前缀 -- [ ] `runner.go`: 集成 before/after 到 `runSingleTest` -- [ ] `runner.go`: 集成 beforeAll/afterAll 到 `RunTests` -- [ ] `cmd/agent/agent.go`: 添加 `--before`, `--after` flags -- [ ] 创建示例脚本 `tests/env_test.ts` +- [x] `types.go`: 添加 `Before`, `After` 字段到 `Case` +- [x] `types.go`: 添加 `BeforeAll`, `AfterAll` 字段到 `Options` +- [x] `script_hooks.go`: 实现 `HookExecutor` +- [x] `script_hooks.go`: 通过 V8 直接执行 `*_test.ts` 脚本 +- [x] `runner.go`: 集成 before/after 到 `runSingleTest` +- [x] `runner.go`: 集成 beforeAll/afterAll 到 `RunTests` +- [x] `cmd/agent/test.go`: 添加 `--before`, `--after` flags +- [x] `test/utils.go`: 添加 `LoadAgentTestScripts()` 通用函数 +- [x] 创建示例脚本 `assistants/tests/hooks-test/src/env_test.ts` +- [x] 创建单元测试 `script_hooks_test.go` (黑盒测试) ## Phase 2: Agent-Driven Assertions @@ -79,6 +81,7 @@ - [x] `--fail-fast` flag - [x] `-v` verbose mode - [x] Script testing (`*_test.ts`) +- [x] Before/After hooks (Phase 1) ## Open Questions diff --git a/agent/test/runner.go b/agent/test/runner.go index 32658fdb..be0f2d04 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -1,7 +1,6 @@ package test import ( - "bufio" stdContext "context" "fmt" "os" @@ -16,19 +15,22 @@ import ( // Executor executes test cases against an agent type Executor struct { - opts *Options - output *OutputWriter - resolver Resolver - loader Loader + opts *Options + output *OutputWriter + resolver Resolver + loader Loader + hookExecutor *HookExecutor + agentPath string // Path to the agent being tested } // NewRunner creates a new test runner func NewRunner(opts *Options) *Executor { return &Executor{ - opts: opts, - output: NewOutputWriter(opts.Verbose), - resolver: NewResolver(), - loader: NewLoader(), + opts: opts, + output: NewOutputWriter(opts.Verbose), + resolver: NewResolver(), + loader: NewLoader(), + hookExecutor: NewHookExecutor(opts.Verbose), } } @@ -161,6 +163,7 @@ func (r *Executor) RunTests() (*Report, error) { } r.output.Info("Agent: %s", agentInfo.ID) + r.agentPath = agentInfo.Path // Store agent path for hook execution if r.opts.Connector != "" { r.output.Info("Connector: %s (override)", r.opts.Connector) } else if agentInfo.Connector != "" { @@ -222,6 +225,27 @@ func (r *Executor) RunTests() (*Report, error) { }, } + // Execute global BeforeAll if specified + var globalBeforeData interface{} + if r.opts.BeforeAll != "" { + r.output.Info("BeforeAll: %s", r.opts.BeforeAll) + var err error + globalBeforeData, err = r.hookExecutor.ExecuteBeforeAll(r.opts.BeforeAll, activeTests, agentInfo.Path) + if err != nil { + return nil, fmt.Errorf("beforeAll script failed: %w", err) + } + } + + // Ensure AfterAll runs even if tests fail + defer func() { + if r.opts.AfterAll != "" { + r.output.Info("AfterAll: %s", r.opts.AfterAll) + if err := r.hookExecutor.ExecuteAfterAll(r.opts.AfterAll, report.Results, globalBeforeData, agentInfo.Path); err != nil { + r.output.Warning("afterAll script failed: %s", err.Error()) + } + } + }() + // Run tests r.output.SubHeader("Running Tests") @@ -322,6 +346,31 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str Options: tc.Options, } + // Execute before script if specified + var beforeData interface{} + if tc.Before != "" { + var err error + beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath) + if err != nil { + result.Status = StatusError + result.Error = fmt.Sprintf("before script failed: %s", err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + r.output.TestResult(result.Status, time.Since(startTime)) + r.output.TestError(result.Error) + // Note: after script is NOT called when before fails + return result + } + } + + // Ensure after script runs even if test fails (but only if before succeeded) + defer func() { + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) + } + } + }() + // Parse input to messages with file loading support // BaseDir is derived from the input file directory inputOpts := r.getInputOptions() @@ -395,6 +444,19 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str return result } +// isBeforeError checks if the error message indicates a before script failure +func isBeforeError(errMsg string) bool { + return len(errMsg) > 0 && errMsg[:min(len(errMsg), 20)] == "before script failed" +} + +// min returns the minimum of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + // runStabilityTests runs each test case multiple times for stability analysis func (r *Executor) runStabilityTests(ast *assistant.Assistant, testCases []*Case, agentID string) []*StabilityResult { results := make([]*StabilityResult, 0, len(testCases)) @@ -499,20 +561,6 @@ func (r *Executor) writeOutput(report *Report) error { return reporter.Write(report, file) } -// writeJSONLine writes a JSON line to the writer -func writeJSONLine(writer *bufio.Writer, data interface{}) error { - line, err := jsoniter.Marshal(data) - if err != nil { - return err - } - _, err = writer.Write(line) - if err != nil { - return err - } - _, err = writer.WriteString("\n") - return err -} - // buildContextOptions builds context.Options from test case and runner options // Priority: test case options > runner options > defaults func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options { diff --git a/agent/test/script_hooks.go b/agent/test/script_hooks.go new file mode 100644 index 00000000..ea9349e1 --- /dev/null +++ b/agent/test/script_hooks.go @@ -0,0 +1,592 @@ +package test + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/yaoapp/gou/application" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/gou/runtime/v8/bridge" + "github.com/yaoapp/yao/agent/context" + "rogchap.com/v8go" +) + +// HookExecutor executes before/after scripts from *_test.ts files +// Scripts are loaded via V8 and executed directly, not via Process() +type HookExecutor struct { + verbose bool + output *OutputWriter + loadedDirs map[string]bool // Track which directories have been loaded + agentContext *context.Context +} + +// NewHookExecutor creates a new hook executor +func NewHookExecutor(verbose bool) *HookExecutor { + return &HookExecutor{ + verbose: verbose, + output: NewOutputWriter(verbose), + loadedDirs: make(map[string]bool), + } +} + +// SetAgentContext sets the agent context for script execution +func (h *HookExecutor) SetAgentContext(ctx *context.Context) { + h.agentContext = ctx +} + +// HookRef represents a parsed hook reference +// Format: "src/env_test.ts:Before" or just "Before" (uses default test file) +type HookRef struct { + ScriptFile string // e.g., "env_test.ts" + Function string // e.g., "Before" +} + +// ParseHookRef parses a hook reference string +// Formats: +// - "Before" -> uses first *_test.ts file found +// - "env_test.Before" -> uses src/env_test.ts +// - "src/env_test.Before" -> uses src/env_test.ts +func ParseHookRef(ref string) (*HookRef, error) { + if ref == "" { + return nil, fmt.Errorf("empty hook reference") + } + + // Split by last dot to get function name + lastDot := strings.LastIndex(ref, ".") + if lastDot == -1 { + // Just function name, will use default test file + return &HookRef{ + ScriptFile: "", // Will be resolved later + Function: ref, + }, nil + } + + scriptPart := ref[:lastDot] + funcName := ref[lastDot+1:] + + // Normalize script file name + scriptFile := scriptPart + if !strings.HasSuffix(scriptFile, "_test") { + scriptFile += "_test" + } + scriptFile += ".ts" + + // Remove "src/" prefix if present + scriptFile = strings.TrimPrefix(scriptFile, "src/") + + return &HookRef{ + ScriptFile: scriptFile, + Function: funcName, + }, nil +} + +// LoadTestScripts loads all *_test.ts scripts from the agent's src directory +// Returns the script IDs that were loaded +func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { + srcDir := filepath.Join(agentPath, "src") + + // Check if already loaded + if h.loadedDirs[srcDir] { + return nil, nil + } + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + return nil, err + } + if !exists { + return nil, nil // No src directory, not an error + } + + var loadedScripts []string + exts := []string{"*_test.ts", "*_test.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Only load *_test.ts/js files + base := filepath.Base(file) + if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") { + return nil + } + + // Generate script ID + scriptID := generateHookScriptID(file, srcDir) + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + if h.verbose { + h.output.Warning("Failed to load hook script %s: %v", base, err) + } + return nil // Continue loading other scripts + } + + loadedScripts = append(loadedScripts, scriptID) + if h.verbose { + h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID) + } + + return nil + }, exts...) + + if err != nil { + return nil, fmt.Errorf("failed to walk src directory: %w", err) + } + + h.loadedDirs[srcDir] = true + return loadedScripts, nil +} + +// generateHookScriptID generates a script ID for hook scripts +// Example: assistants/test/src/env_test.ts -> hook.env_test +func generateHookScriptID(filePath string, srcDir string) string { + filePath = filepath.ToSlash(filePath) + srcDir = filepath.ToSlash(srcDir) + + relPath := strings.TrimPrefix(filePath, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + + return "hook." + strings.ReplaceAll(relPath, "/", ".") +} + +// FindTestScript finds a loaded test script by pattern +// If scriptFile is empty, returns the first *_test script found +func (h *HookExecutor) FindTestScript(scriptFile string) (*v8.Script, string, error) { + if scriptFile != "" { + // Look for specific script + scriptID := "hook." + strings.TrimSuffix(scriptFile, ".ts") + scriptID = strings.TrimSuffix(scriptID, ".js") + + if script, ok := v8.Scripts[scriptID]; ok { + return script, scriptID, nil + } + return nil, "", fmt.Errorf("hook script not found: %s (id: %s)", scriptFile, scriptID) + } + + // Find first *_test script + for id, script := range v8.Scripts { + if strings.HasPrefix(id, "hook.") && strings.Contains(id, "_test") { + return script, id, nil + } + } + + return nil, "", fmt.Errorf("no hook test script found") +} + +// ExecuteBefore executes a Before function from a test script +func (h *HookExecutor) ExecuteBefore(ref string, testCase *Case, agentPath string) (interface{}, error) { + hookRef, err := ParseHookRef(ref) + if err != nil { + return nil, err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return nil, fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return nil, err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute the function + return h.executeHookFunction(script, hookRef.Function, testCase, nil, nil) +} + +// ExecuteAfter executes an After function from a test script +func (h *HookExecutor) ExecuteAfter(ref string, testCase *Case, result *Result, beforeData interface{}, agentPath string) error { + hookRef, err := ParseHookRef(ref) + if err != nil { + return err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute the function + _, err = h.executeHookFunction(script, hookRef.Function, testCase, result, beforeData) + return err +} + +// ExecuteBeforeAll executes a BeforeAll function +func (h *HookExecutor) ExecuteBeforeAll(ref string, testCases []*Case, agentPath string) (interface{}, error) { + hookRef, err := ParseHookRef(ref) + if err != nil { + return nil, err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return nil, fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return nil, err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute with test cases array + return h.executeHookFunctionWithCases(script, hookRef.Function, testCases) +} + +// ExecuteAfterAll executes an AfterAll function +func (h *HookExecutor) ExecuteAfterAll(ref string, results []*Result, beforeData interface{}, agentPath string) error { + hookRef, err := ParseHookRef(ref) + if err != nil { + return err + } + + // Ensure scripts are loaded + if _, err := h.LoadTestScripts(agentPath); err != nil { + return fmt.Errorf("failed to load test scripts: %w", err) + } + + // Find the script + script, scriptID, err := h.FindTestScript(hookRef.ScriptFile) + if err != nil { + return err + } + + if h.verbose { + h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID) + } + + // Execute with results array + _, err = h.executeHookFunctionWithResults(script, hookRef.Function, results, beforeData) + return err +} + +// executeHookFunction executes a hook function with test case context +func (h *HookExecutor) executeHookFunction(script *v8.Script, funcName string, testCase *Case, result *Result, beforeData interface{}) (interface{}, error) { + // Create script context + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + // Set share data + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + // Get the function + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Build arguments + args, err := h.buildHookArgs(v8ctx, testCase, result, beforeData) + if err != nil { + return nil, err + } + + // Convert to v8go.Valuer slice for Call + valuerArgs := make([]v8go.Valuer, len(args)) + for i, arg := range args { + valuerArgs[i] = arg + } + + // Call the function + jsResult, err := fn.Call(global, valuerArgs...) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + // Convert result to Go value + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + // Extract data field if present + if resultMap, ok := goResult.(map[string]interface{}); ok { + if data, exists := resultMap["data"]; exists { + return data, nil + } + } + + return goResult, nil +} + +// executeHookFunctionWithCases executes BeforeAll with test cases array +func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName string, testCases []*Case) (interface{}, error) { + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Convert test cases to JS array + casesJS, err := h.testCasesToJS(v8ctx, testCases) + if err != nil { + return nil, err + } + + jsResult, err := fn.Call(global, casesJS) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + if resultMap, ok := goResult.(map[string]interface{}); ok { + if data, exists := resultMap["data"]; exists { + return data, nil + } + } + + return goResult, nil +} + +// executeHookFunctionWithResults executes AfterAll with results array +func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcName string, results []*Result, beforeData interface{}) (interface{}, error) { + scriptCtx, err := script.NewContext("", nil) + if err != nil { + return nil, fmt.Errorf("failed to create script context: %w", err) + } + defer scriptCtx.Close() + + v8ctx := scriptCtx.Context + + if err := h.setShareData(v8ctx); err != nil { + return nil, err + } + + global := v8ctx.Global() + fnValue, err := global.Get(funcName) + if err != nil { + return nil, fmt.Errorf("failed to get function %s: %w", funcName, err) + } + + if fnValue.IsUndefined() || fnValue.IsNull() { + return nil, fmt.Errorf("function %s not defined", funcName) + } + + if !fnValue.IsFunction() { + return nil, fmt.Errorf("%s is not a function", funcName) + } + + fn, err := fnValue.AsFunction() + if err != nil { + return nil, fmt.Errorf("failed to convert to function: %w", err) + } + + // Convert results to JS array + resultsJS, err := h.resultsToJS(v8ctx, results) + if err != nil { + return nil, err + } + + // Convert beforeData to JS + beforeDataJS, err := bridge.JsValue(v8ctx, beforeData) + if err != nil { + return nil, fmt.Errorf("failed to convert beforeData: %w", err) + } + + jsResult, err := fn.Call(global, resultsJS, beforeDataJS) + if err != nil { + return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) + } + + if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() { + return nil, nil + } + + goResult, err := bridge.GoValue(jsResult, v8ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + + return goResult, nil +} + +// setShareData sets the share data for script execution +func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error { + var authorized map[string]interface{} + if h.agentContext != nil && h.agentContext.Authorized != nil { + authorized = h.agentContext.Authorized.AuthorizedToMap() + } + + return bridge.SetShareData(v8ctx, v8ctx.Global(), &bridge.Share{ + Sid: "", + Root: false, + Global: nil, + Authorized: authorized, + }) +} + +// buildHookArgs builds the arguments for a hook function call +func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) { + var args []*v8go.Value + + // Arg 1: testCase + if testCase != nil { + tcMap := map[string]interface{}{ + "id": testCase.ID, + "input": testCase.Input, + } + if testCase.Metadata != nil { + tcMap["metadata"] = testCase.Metadata + } + if testCase.Assert != nil { + tcMap["assert"] = testCase.Assert + } + + tcJS, err := bridge.JsValue(v8ctx, tcMap) + if err != nil { + return nil, fmt.Errorf("failed to convert testCase: %w", err) + } + args = append(args, tcJS) + } + + // Arg 2: result (for After) + if result != nil { + resultMap := map[string]interface{}{ + "id": result.ID, + "status": string(result.Status), + "duration_ms": result.DurationMs, + } + if result.Output != nil { + resultMap["output"] = result.Output + } + if result.Error != "" { + resultMap["error"] = result.Error + } + + resultJS, err := bridge.JsValue(v8ctx, resultMap) + if err != nil { + return nil, fmt.Errorf("failed to convert result: %w", err) + } + args = append(args, resultJS) + } + + // Arg 3: beforeData (for After) + if beforeData != nil { + beforeDataJS, err := bridge.JsValue(v8ctx, beforeData) + if err != nil { + return nil, fmt.Errorf("failed to convert beforeData: %w", err) + } + args = append(args, beforeDataJS) + } + + return args, nil +} + +// testCasesToJS converts test cases to a JS array +func (h *HookExecutor) testCasesToJS(v8ctx *v8go.Context, testCases []*Case) (*v8go.Value, error) { + cases := make([]map[string]interface{}, len(testCases)) + for i, tc := range testCases { + cases[i] = map[string]interface{}{ + "id": tc.ID, + "input": tc.Input, + } + if tc.Metadata != nil { + cases[i]["metadata"] = tc.Metadata + } + } + + return bridge.JsValue(v8ctx, cases) +} + +// resultsToJS converts results to a JS array +func (h *HookExecutor) resultsToJS(v8ctx *v8go.Context, results []*Result) (*v8go.Value, error) { + resultMaps := make([]map[string]interface{}, len(results)) + for i, r := range results { + resultMaps[i] = map[string]interface{}{ + "id": r.ID, + "status": string(r.Status), + "duration_ms": r.DurationMs, + } + if r.Output != nil { + resultMaps[i]["output"] = r.Output + } + if r.Error != "" { + resultMaps[i]["error"] = r.Error + } + } + + return bridge.JsValue(v8ctx, resultMaps) +} diff --git a/agent/test/script_hooks_test.go b/agent/test/script_hooks_test.go new file mode 100644 index 00000000..8720c463 --- /dev/null +++ b/agent/test/script_hooks_test.go @@ -0,0 +1,244 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v8 "github.com/yaoapp/gou/runtime/v8" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +const hooksTestAgent = "assistants/tests/hooks-test" + +func TestParseHookRef(t *testing.T) { + tests := []struct { + name string + input string + wantFile string + wantFunc string + expectErr bool + }{ + { + name: "function only", + input: "Before", + wantFile: "", + wantFunc: "Before", + }, + { + name: "with script file", + input: "env_test.Before", + wantFile: "env_test.ts", + wantFunc: "Before", + }, + { + name: "with src prefix", + input: "src/env_test.Before", + wantFile: "env_test.ts", + wantFunc: "Before", + }, + { + name: "nested path", + input: "setup/db_test.Before", + wantFile: "setup/db_test.ts", + wantFunc: "Before", + }, + { + name: "empty string", + input: "", + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := agenttest.ParseHookRef(tt.input) + if tt.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantFile, ref.ScriptFile) + assert.Equal(t, tt.wantFunc, ref.Function) + }) + } +} + +func TestHookExecutorLoadTestScripts(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts using the utility function + scripts := test.LoadAgentTestScripts(t, hooksTestAgent) + + assert.NotEmpty(t, scripts, "Should load at least one test script") + + // Verify the script was loaded into V8 + found := false + for _, scriptID := range scripts { + if _, ok := v8.Scripts[scriptID]; ok { + found = true + t.Logf("Loaded script: %s", scriptID) + break + } + } + assert.True(t, found, "At least one script should be loaded into V8") +} + +func TestHookExecutorExecuteBefore(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello World", + } + + // Execute Before hook + beforeData, err := executor.ExecuteBefore("env_test.Before", testCase, hooksTestAgent) + assert.NoError(t, err) + assert.NotNil(t, beforeData) + + // Verify returned data + dataMap, ok := beforeData.(map[string]interface{}) + assert.True(t, ok, "beforeData should be a map") + assert.Equal(t, "TEST001", dataMap["test_id"]) + assert.NotEmpty(t, dataMap["mock_user_id"]) + assert.NotEmpty(t, dataMap["mock_session_id"]) +} + +func TestHookExecutorExecuteAfter(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST002", + Input: "Test input", + } + + result := &agenttest.Result{ + ID: "TEST002", + Status: agenttest.StatusPassed, + DurationMs: 100, + } + + beforeData := map[string]interface{}{ + "test_id": "TEST002", + "mock_user_id": "user_TEST002_12345", + "mock_session_id": "session_12345", + } + + // Execute After hook + err := executor.ExecuteAfter("env_test.After", testCase, result, beforeData, hooksTestAgent) + assert.NoError(t, err) +} + +func TestHookExecutorExecuteBeforeAll(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCases := []*agenttest.Case{ + {ID: "T001", Input: "Test 1"}, + {ID: "T002", Input: "Test 2"}, + {ID: "T003", Input: "Test 3"}, + } + + // Execute BeforeAll hook + globalData, err := executor.ExecuteBeforeAll("env_test.BeforeAll", testCases, hooksTestAgent) + assert.NoError(t, err) + assert.NotNil(t, globalData) + + // Verify returned data + dataMap, ok := globalData.(map[string]interface{}) + assert.True(t, ok, "globalData should be a map") + assert.NotEmpty(t, dataMap["suite_id"]) + assert.Equal(t, float64(3), dataMap["test_count"]) // JSON numbers are float64 +} + +func TestHookExecutorExecuteAfterAll(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + results := []*agenttest.Result{ + {ID: "T001", Status: agenttest.StatusPassed, DurationMs: 100}, + {ID: "T002", Status: agenttest.StatusFailed, DurationMs: 200, Error: "assertion failed"}, + {ID: "T003", Status: agenttest.StatusPassed, DurationMs: 150}, + } + + globalData := map[string]interface{}{ + "suite_id": "suite_12345", + "test_count": 3, + } + + // Execute AfterAll hook + err := executor.ExecuteAfterAll("env_test.AfterAll", results, globalData, hooksTestAgent) + assert.NoError(t, err) +} + +func TestHookExecutorFunctionNotFound(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello", + } + + // Try to execute non-existent function + _, err := executor.ExecuteBefore("env_test.NonExistent", testCase, hooksTestAgent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not defined") +} + +func TestHookExecutorScriptNotFound(t *testing.T) { + // Prepare test environment + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent test scripts + test.LoadAgentTestScripts(t, hooksTestAgent) + + executor := agenttest.NewHookExecutor(true) + + testCase := &agenttest.Case{ + ID: "TEST001", + Input: "Hello", + } + + // Try to execute from non-existent script + _, err := executor.ExecuteBefore("nonexistent_test.Before", testCase, hooksTestAgent) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} diff --git a/agent/test/types.go b/agent/test/types.go index b40ab457..f74a0307 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -144,6 +144,14 @@ type Options struct { // Only tests matching the pattern will be executed // Example: "TestSystem" matches TestSystemReady, TestSystemError, etc. Run string `json:"run,omitempty"` + + // BeforeAll is the global before script (e.g., "scripts:tests.env.BeforeAll") + // Called once before all test cases + BeforeAll string `json:"before_all,omitempty"` + + // AfterAll is the global after script (e.g., "scripts:tests.env.AfterAll") + // Called once after all test cases + AfterAll string `json:"after_all,omitempty"` } // ContextConfig represents custom context configuration from JSON file @@ -375,6 +383,14 @@ type Case struct { // Timeout overrides the default timeout for this test case // Format: "30s", "1m", "2m30s" Timeout string `json:"timeout,omitempty"` + + // Before script function (e.g., "scripts:tests.env.Before") + // Called before the test case runs, returns data passed to After + Before string `json:"before,omitempty"` + + // After script function (e.g., "scripts:tests.env.After") + // Called after the test case completes (pass or fail) + After string `json:"after,omitempty"` } // CaseOptions represents per-test-case context options diff --git a/cmd/agent/test.go b/cmd/agent/test.go index 7af0ef03..f48384f0 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -33,6 +33,8 @@ var ( testParallel int testVerbose bool testFailFast bool + testBefore string // --before flag for global BeforeAll hook + testAfter string // --after flag for global AfterAll hook ) // TestCmd is the agent test command @@ -157,6 +159,8 @@ var TestCmd = &cobra.Command{ Parallel: testParallel, Verbose: testVerbose, FailFast: testFailFast, + BeforeAll: testBefore, + AfterAll: testAfter, } // Merge with defaults @@ -244,6 +248,8 @@ func init() { TestCmd.Flags().IntVar(&testParallel, "parallel", 1, L("Number of parallel test cases")) TestCmd.Flags().BoolVarP(&testVerbose, "verbose", "v", false, L("Verbose output")) TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure")) + TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)")) + TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)")) // Mark input as required TestCmd.MarkFlagRequired("input") diff --git a/test/utils.go b/test/utils.go index 624bdb59..b7d06d5b 100644 --- a/test/utils.go +++ b/test/utils.go @@ -800,3 +800,70 @@ func GuardBearerJWT(c *gin.Context) { claims := helper.JwtValidate(tokenString) c.Set("__sid", claims.SID) } + +// LoadAgentTestScripts loads all *_test.ts/js scripts from an agent's src directory. +// This is useful for testing agent hooks (before/after scripts) and other agent-specific test scripts. +// +// Usage: +// +// test.Prepare(t, config.Conf) +// defer test.Clean() +// scripts := test.LoadAgentTestScripts(t, "assistants/tests/hooks-test") +// +// Parameters: +// - t: testing.T instance +// - agentRelPath: relative path to agent directory from app root (e.g., "assistants/tests/hooks-test") +// +// Returns: +// - []string: list of loaded script IDs (e.g., ["hook.env_test"]) +func LoadAgentTestScripts(t *testing.T, agentRelPath string) []string { + srcDir := filepath.Join(agentRelPath, "src") + + // Check if src directory exists + exists, err := application.App.Exists(srcDir) + if err != nil { + t.Fatalf("Failed to check src directory: %v", err) + } + if !exists { + t.Logf("No src directory found at %s, skipping", srcDir) + return nil + } + + var loadedScripts []string + exts := []string{"*_test.ts", "*_test.js"} + + err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + if isdir { + return nil + } + + // Only load *_test.ts/js files + base := filepath.Base(file) + if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") { + return nil + } + + // Generate script ID: hook.{relative_path_without_ext} + // e.g., assistants/tests/hooks-test/src/env_test.ts -> hook.env_test + relPath := strings.TrimPrefix(file, srcDir+"/") + relPath = strings.TrimPrefix(relPath, "/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + scriptID := "hook." + strings.ReplaceAll(relPath, "/", ".") + + // Load the script + _, err := v8.Load(file, scriptID) + if err != nil { + t.Logf("Warning: Failed to load hook script %s: %v", base, err) + return nil // Continue loading other scripts + } + + loadedScripts = append(loadedScripts, scriptID) + return nil + }, exts...) + + if err != nil { + t.Fatalf("Failed to walk src directory: %v", err) + } + + return loadedScripts +} From 88610da75bf829ad64716a31c5b59a348d0fbc10 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 10:16:40 +0800 Subject: [PATCH 12/17] Implement Agent-Driven Assertions in Test Framework - Added support for agent-driven assertions in the Asserter, allowing validation of responses using specified agents. - Introduced the `Use` and `Options` fields in the Assertion struct to facilitate agent configuration. - Enhanced the `evaluateAssertion` method to handle assertions of type "agent". - Implemented the `assertAgent` method to manage agent interactions and validation logic. - Updated `script_assert.go` to include the `assertAgentMethod` for JavaScript API integration. - Revised documentation in DESIGN_V2.md and TODO_V2.md to reflect the new agent-driven assertion capabilities and implementation status. --- agent/test/DESIGN_V2.md | 19 +-- agent/test/TODO_V2.md | 15 +- agent/test/assert.go | 217 ++++++++++++++++++++----- agent/test/assert_agent_test.go | 275 ++++++++++++++++++++++++++++++++ agent/test/script_assert.go | 67 ++++++++ agent/test/types.go | 18 +++ 6 files changed, 558 insertions(+), 53 deletions(-) create mode 100644 agent/test/assert_agent_test.go diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index be1a2b9a..9b956550 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -990,15 +990,16 @@ Existing single-turn tests work unchanged: ## Current Implementation Status -| Feature | Status | Notes | -| ----------------------- | ---------- | ---------------------------------------- | -| Simple text input | ✅ Done | `input: "Hello"` | -| Message history | ✅ Done | `input: [{role, content}, ...]` | -| File attachments | ✅ Done | `file://` protocol in content parts | -| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | -| Agent-driven assertions | 🔲 Planned | `type: "agent"` with validator agent | -| Dynamic mode | 🔲 Planned | Simulator + Checkpoints | -| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | +| Feature | Status | Notes | +| ----------------------- | ---------- | -------------------------------------------------- | +| Simple text input | ✅ Done | `input: "Hello"` | +| Message history | ✅ Done | `input: [{role, content}, ...]` | +| File attachments | ✅ Done | `file://` protocol in content parts | +| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | +| Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | +| Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | +| Dynamic mode | 🔲 Planned | Simulator + Checkpoints | +| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | ## Open Questions diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index aa1fb513..8179ffc4 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -29,15 +29,17 @@ - [x] 创建示例脚本 `assistants/tests/hooks-test/src/env_test.ts` - [x] 创建单元测试 `script_hooks_test.go` (黑盒测试) -## Phase 2: Agent-Driven Assertions +## Phase 2: Agent-Driven Assertions ✅ **修改文件**: `assert.go`, `script_assert.go` -- [ ] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion` -- [ ] `assert.go`: 实现 `assertAgent` 方法 -- [ ] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型 -- [ ] `script_assert.go`: 添加 `AssertAgent` 方法到 `TestingT` -- [ ] 创建示例 validator agent +- [x] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion` +- [x] `assert.go`: 实现 `assertAgent` 方法 +- [x] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型 +- [x] `assert.go`: 使用 `goutext.ExtractJSON` 容错解析 LLM 响应 +- [x] `script_assert.go`: 添加 `assertAgentMethod` 到 `newAssertObject` +- [x] 创建示例 validator agent (`assistants/tests/validator-agent`) +- [x] 创建单元测试 `assert_agent_test.go` (JSONL 断言 + JSAPI 断言) ## Phase 3: Dynamic Mode (Simulator + Checkpoints) @@ -82,6 +84,7 @@ - [x] `-v` verbose mode - [x] Script testing (`*_test.ts`) - [x] Before/After hooks (Phase 1) +- [x] Agent-driven assertions (Phase 2) ## Open Questions diff --git a/agent/test/assert.go b/agent/test/assert.go index fc3c97e7..505f0e62 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -9,6 +9,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/process" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" ) // Asserter handles test assertions @@ -115,6 +118,9 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion { if s, ok := m["script"].(string); ok { assertion.Script = s } + if u, ok := m["use"].(string); ok { + assertion.Use = u + } if msg, ok := m["message"].(string); ok { assertion.Message = msg } @@ -122,6 +128,17 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion { assertion.Negate = n } + // Parse options for agent assertions + if opts, ok := m["options"].(map[string]interface{}); ok { + assertion.Options = &AssertionOptions{} + if c, ok := opts["connector"].(string); ok { + assertion.Options.Connector = c + } + if meta, ok := opts["metadata"].(map[string]interface{}); ok { + assertion.Options.Metadata = meta + } + } + return assertion } @@ -147,6 +164,8 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa result = a.assertType(assertion, output) case "script": result = a.assertScript(assertion, output, input) + case "agent": + result = a.assertAgent(assertion, output, input) default: result.Passed = false result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type) @@ -229,17 +248,14 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass var jsonData interface{} switch v := output.(type) { case string: - // Try to parse as JSON - if err := jsoniter.Unmarshal([]byte(v), &jsonData); err != nil { - // Try to extract JSON from markdown code blocks - extracted := extractJSONFromText(v) - if extracted != nil { - jsonData = extracted - } else { - result.Passed = false - result.Message = fmt.Sprintf("output is not valid JSON: %s", err.Error()) - return result - } + // Use gou/text to extract JSON (handles markdown, auto-repair, etc.) + extracted := goutext.ExtractJSON(v) + if extracted != nil { + jsonData = extracted + } else { + result.Passed = false + result.Message = fmt.Sprintf("output is not valid JSON: %s", v) + return result } case map[string]interface{}, []interface{}: jsonData = v @@ -478,6 +494,158 @@ func (a *Asserter) getType(v interface{}) string { } } +// assertAgent uses an agent to validate the output +func (a *Asserter) assertAgent(assertion *Assertion, output, input interface{}) *AssertionResult { + result := &AssertionResult{ + Assertion: assertion, + Actual: output, + } + + // Parse use field: "agents:tests.validator-agent" + if !strings.HasPrefix(assertion.Use, "agents:") { + result.Passed = false + result.Message = "agent assertion requires 'use' field with 'agents:' prefix" + return result + } + + agentID := strings.TrimPrefix(assertion.Use, "agents:") + + // Get assistant + ast, err := assistant.Get(agentID) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("failed to get validator agent: %s", err.Error()) + return result + } + + // Build validation request + validationInput := map[string]interface{}{ + "output": output, + "input": input, + } + + // Add criteria from Value field + if assertion.Value != nil { + validationInput["criteria"] = assertion.Value + } + + // Add metadata from options + if assertion.Options != nil && assertion.Options.Metadata != nil { + for k, v := range assertion.Options.Metadata { + validationInput[k] = v + } + } + + // Build context options - skip history and trace for validator + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "validator", + }, + } + if assertion.Options != nil && assertion.Options.Connector != "" { + opts.Connector = assertion.Options.Connector + } + + // Create context and call agent + env := NewEnvironment("", "") + ctx := NewTestContext("validator", agentID, env) + defer ctx.Release() + + // Convert validation input to JSON string for the message + inputJSON, err := json.Marshal(validationInput) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error()) + return result + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + response, err := ast.Stream(ctx, messages, opts) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("validator agent error: %s", err.Error()) + return result + } + + // Parse response + return a.parseValidatorResponse(response, result) +} + +// parseValidatorResponse parses the validator agent's response +func (a *Asserter) parseValidatorResponse(response *context.Response, result *AssertionResult) *AssertionResult { + output := extractValidatorOutput(response) + + // Expected format: { "passed": bool, "reason": string, "score": float, "suggestions": [] } + if outputMap, ok := output.(map[string]interface{}); ok { + if passed, ok := outputMap["passed"].(bool); ok { + result.Passed = passed + } else { + result.Passed = false + result.Message = "validator response missing 'passed' field" + return result + } + if reason, ok := outputMap["reason"].(string); ok { + result.Message = reason + } + // Store score and suggestions in expected field for reference + result.Expected = outputMap + } else { + result.Passed = false + result.Message = "validator agent returned invalid response format" + } + + return result +} + +// extractValidatorOutput extracts the output from a validator response +func extractValidatorOutput(response *context.Response) interface{} { + if response == nil || response.Completion == nil { + return nil + } + + // Get content from completion + content := response.Completion.Content + if content == nil { + return nil + } + + // Try to get text content + var text string + switch v := content.(type) { + case string: + text = v + default: + // Try to marshal and use as-is + data, err := json.Marshal(content) + if err != nil { + return nil + } + text = string(data) + } + + if text == "" { + return nil + } + + // Use gou/text to extract JSON (handles markdown code blocks, auto-repair, etc.) + result := goutext.ExtractJSON(text) + if result != nil { + return result + } + + // Return raw text if extraction fails + return text +} + // assertScript runs a custom assertion script func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) *AssertionResult { result := &AssertionResult{ @@ -559,30 +727,3 @@ func (a *Asserter) toString(v interface{}) string { return string(b) } } - -// extractJSONFromText tries to extract JSON from text (e.g., markdown code blocks) -func extractJSONFromText(text string) interface{} { - // Try to find JSON in code blocks - patterns := []string{ - "```json\n", - "```\n", - } - - for _, start := range patterns { - if idx := strings.Index(text, start); idx >= 0 { - text = text[idx+len(start):] - if endIdx := strings.Index(text, "```"); endIdx >= 0 { - text = text[:endIdx] - } - break - } - } - - // Try to parse - var result interface{} - if err := jsoniter.Unmarshal([]byte(strings.TrimSpace(text)), &result); err == nil { - return result - } - - return nil -} diff --git a/agent/test/assert_agent_test.go b/agent/test/assert_agent_test.go new file mode 100644 index 00000000..20dbc4c9 --- /dev/null +++ b/agent/test/assert_agent_test.go @@ -0,0 +1,275 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" + "rogchap.com/v8go" +) + +func TestAsserter_AgentAssertion(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + tests := []struct { + name string + tc *agenttest.Case + output interface{} + expected bool + skipMsg string + }{ + { + name: "agent assertion - pass", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be a greeting", + }, + }, + output: "Hello! How can I help you today?", + expected: true, + }, + { + name: "agent assertion - fail", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should provide a detailed technical answer", + }, + }, + output: "I don't know.", + expected: false, + }, + { + name: "agent assertion - missing prefix", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "tests.validator-agent", // Missing agents: prefix + "value": "Should pass", + }, + }, + output: "Hello", + expected: false, // Should fail due to missing prefix + }, + { + name: "agent assertion - with metadata", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response is helpful", + "options": map[string]interface{}{ + "metadata": map[string]interface{}{ + "context": "customer support", + }, + }, + }, + }, + output: "I'd be happy to help you with your order. Let me look that up for you.", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.skipMsg != "" { + t.Skip(tt.skipMsg) + } + + passed, errMsg := asserter.Validate(tt.tc, tt.output) + if passed != tt.expected { + t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg) + } + }) + } +} + +func TestAsserter_AgentAssertion_InvalidAgent(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + tc := &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:nonexistent.agent", + "value": "Should fail", + }, + } + + passed, errMsg := asserter.Validate(tc, "Hello") + assert.False(t, passed, "Should fail for nonexistent agent") + assert.Contains(t, errMsg, "failed to get validator agent", "Error should mention agent loading failure") +} + +func TestAsserter_MapToAssertion_WithUseAndOptions(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + // Test that mapToAssertion correctly parses use and options fields + tc := &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "criteria here", + "options": map[string]interface{}{ + "connector": "gpt-4o", + "metadata": map[string]interface{}{ + "key": "value", + }, + }, + }, + } + + // Validate triggers parseAssertions internally + // We just verify it doesn't panic and processes correctly + _, _ = asserter.Validate(tc, "test output") + // If we get here without panic, the parsing worked +} + +// TestTestingT_AssertAgent tests the JSAPI t.assert.Agent() method +func TestTestingT_AssertAgent(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + tests := []struct { + name string + script string + shouldFail bool + }{ + { + name: "JSAPI agent assertion - pass", + script: ` + function test(t) { + var response = "Hello! How can I help you today?"; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be a friendly greeting" + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - JSON response", + script: ` + function test(t) { + var response = { + status: "success", + data: { user: "john", email: "john@example.com" }, + message: "User created successfully" + }; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be a successful API response with user data" + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - with metadata", + script: ` + function test(t) { + var response = "I'd be happy to help you with your order."; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response is helpful and professional", + metadata: { context: "customer support" } + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - fail case", + script: ` + function test(t) { + var response = "I don't know."; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should provide a detailed technical explanation" + }); + } + test(__test_t); + `, + shouldFail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create TestingT + testingT := agenttest.NewTestingT(tt.name) + + // Create V8 isolate and context + iso := v8go.NewIsolate() + defer iso.Dispose() + + v8ctx := v8go.NewContext(iso) + defer v8ctx.Close() + + // Create testing object + testObj, err := agenttest.NewTestingTObject(v8ctx, testingT) + if err != nil { + t.Fatalf("Failed to create testing object: %v", err) + } + + // Set testing object as global + global := v8ctx.Global() + global.Set("__test_t", testObj) + + // Run the test script + _, err = v8ctx.RunScript(tt.script, "test.js") + + // Check results + if tt.shouldFail { + assert.True(t, testingT.Failed(), "Test should have failed") + } else { + if err != nil { + t.Errorf("Script execution error: %v", err) + } + assert.False(t, testingT.Failed(), "Test should have passed, errors: %v", testingT.Errors()) + } + }) + } +} + +// Ensure v8 is used (for script loading) +var _ = v8.Scripts diff --git a/agent/test/script_assert.go b/agent/test/script_assert.go index c0cc0245..c2c22ffd 100644 --- a/agent/test/script_assert.go +++ b/agent/test/script_assert.go @@ -261,6 +261,9 @@ func newAssertObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) { // JSON path assertion assertObj.Set("JSONPath", assertJSONPathMethod(iso, t)) + // Agent-driven assertion + assertObj.Set("Agent", assertAgentMethod(iso, t)) + // Create instance instance, err := assertObj.NewInstance(v8ctx) if err != nil { @@ -924,6 +927,70 @@ func assertJSONPathMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate }) } +// assertAgentMethod implements assert.Agent(response, agentID, options?) +// Uses a validator agent to check the response +// agentID is the direct agent ID (no "agents:" prefix needed) +func assertAgentMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 2 { + t.fail("Agent requires response and agentID arguments", &ScriptAssertionInfo{Type: "Agent"}) + return v8go.Undefined(iso) + } + + response, _ := bridge.GoValue(args[0], v8ctx) + agentID := args[1].String() + + // Get options if provided + var options map[string]interface{} + if len(args) > 2 && args[2].IsObject() { + optVal, _ := bridge.GoValue(args[2], v8ctx) + options, _ = optVal.(map[string]interface{}) + } + + // Build assertion with agents: prefix + assertion := &Assertion{ + Type: "agent", + Use: "agents:" + agentID, + } + + // Extract criteria and metadata from options + if options != nil { + if criteria, ok := options["criteria"]; ok { + assertion.Value = criteria + } + if metadata, ok := options["metadata"].(map[string]interface{}); ok { + assertion.Options = &AssertionOptions{Metadata: metadata} + } + if connector, ok := options["connector"].(string); ok { + if assertion.Options == nil { + assertion.Options = &AssertionOptions{} + } + assertion.Options.Connector = connector + } + } + + // Use the asserter to validate + asserter := &Asserter{} + result := asserter.assertAgent(assertion, response, nil) + + if !result.Passed { + msg := result.Message + if msg == "" { + msg = "agent assertion failed" + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Agent", + Actual: response, + Message: msg, + }) + } + + return v8go.Undefined(iso) + }) +} + // Helper functions // deepEqual performs deep equality comparison diff --git a/agent/test/types.go b/agent/test/types.go index f74a0307..75387879 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -436,6 +436,7 @@ type Assertion struct { // - "script": run a custom assertion script // - "type": check output type (string, object, array, number, boolean) // - "schema": validate against JSON schema + // - "agent": use an agent to validate the response Type string `json:"type"` // Value is the expected value or pattern (depends on type) @@ -448,6 +449,14 @@ type Assertion struct { // The script receives (output, input, expected) and returns {pass: bool, message: string} Script string `json:"script,omitempty"` + // Use specifies the agent/script for validation + // For agent assertions: "agents:tests.validator-agent" (with prefix) + // For script assertions: "scripts:tests.validate" (with prefix) + Use string `json:"use,omitempty"` + + // Options for agent-driven assertions (aligned with context.Options) + Options *AssertionOptions `json:"options,omitempty"` + // Message is a custom failure message Message string `json:"message,omitempty"` @@ -455,6 +464,15 @@ type Assertion struct { Negate bool `json:"negate,omitempty"` } +// AssertionOptions for agent-driven assertions +type AssertionOptions struct { + // Connector overrides the agent's default connector + Connector string `json:"connector,omitempty"` + + // Metadata contains custom data passed to the validator agent + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + // AssertionResult represents the result of an assertion type AssertionResult struct { // Passed indicates whether the assertion passed From 6ea982f027bf70de378fbf324595d42c4a096c35 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 10:53:11 +0800 Subject: [PATCH 13/17] Implement Agent-Driven Input and Dry-Run Mode in Test Framework - Added support for generating test cases using agents and scripts through the new `LoadFromAgent` and `LoadFromScript` methods in the loader. - Enhanced the `RunTests` method to handle different input sources, including agent-driven and script-based test case generation. - Introduced a `--dry-run` flag to allow users to preview generated test cases without executing them. - Updated relevant documentation in DESIGN_V2.md and TODO_V2.md to reflect the new features and implementation status. --- agent/test/DESIGN_V2.md | 3 +- agent/test/TODO_V2.md | 50 +++- agent/test/input_source.go | 392 ++++++++++++++++++++++++++ agent/test/input_source_test.go | 181 ++++++++++++ agent/test/interfaces.go | 6 + agent/test/loader.go | 33 +++ agent/test/runner.go | 117 +++++++- agent/test/runner_integration_test.go | 280 ++++++++++++++++++ agent/test/types.go | 4 + cmd/agent/test.go | 3 + 10 files changed, 1050 insertions(+), 19 deletions(-) create mode 100644 agent/test/input_source.go create mode 100644 agent/test/input_source_test.go create mode 100644 agent/test/runner_integration_test.go diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 9b956550..08ed791a 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -998,8 +998,9 @@ Existing single-turn tests work unchanged: | Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | | Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | | Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | +| Agent-driven input | ✅ Done | `-i agents:xxx` for test generation | +| Dry-run mode | ✅ Done | `--dry-run` to preview generated tests | | Dynamic mode | 🔲 Planned | Simulator + Checkpoints | -| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | ## Open Questions diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 8179ffc4..7ba60499 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -41,29 +41,53 @@ - [x] 创建示例 validator agent (`assistants/tests/validator-agent`) - [x] 创建单元测试 `assert_agent_test.go` (JSONL 断言 + JSAPI 断言) -## Phase 3: Dynamic Mode (Simulator + Checkpoints) +## Phase 3: Agent-Driven Input ✅ + +**新增文件**: `input_source.go` + +> 用 Agent 生成测试用例，生成后使用标准模式执行。相对简单。 + +**准备工作**: + +- [x] 创建 generator agent (`yao-dev-app/assistants/tests/generator-agent`) +- [x] 编写 generator agent 的 prompts.yml + +**实现**: + +- [x] `input_source.go`: 实现 `ParseInputSource` +- [x] `input_source.go`: 实现 `GenerateTestCases` +- [x] `loader.go`: 添加 `LoadFromAgent` 方法 +- [x] `loader.go`: 添加 `LoadFromScript` 方法 +- [x] `runner.go`: 在 `RunTests` 支持不同输入源 +- [x] `cmd/agent/test.go`: 添加 `--dry-run` flag + +**测试**: + +- [x] 创建单元测试 `input_source_test.go` + +## Phase 4: Dynamic Mode (Simulator + Checkpoints) **新增文件**: `dynamic_runner.go`, `dynamic_types.go` +> 运行时使用 Simulator Agent 动态生成对话，需要多轮循环和 checkpoint 匹配。依赖 Phase 3 的 Agent 调用经验。 + +**准备工作**: + +- [ ] 创建 simulator agent (`yao-dev-app/assistants/tests/simulator-agent`) +- [ ] 编写 simulator agent 的 prompts.yml (模拟用户行为) + +**实现**: + - [ ] `types.go`: 添加 `Simulator`, `Checkpoints` 字段到 `Case` - [ ] `dynamic_types.go`: 定义 `Checkpoint`, `DynamicResult` 等类型 - [ ] `dynamic_runner.go`: 实现 `DynamicRunner` - [ ] `dynamic_runner.go`: 实现 checkpoint 匹配逻辑 - [ ] `dynamic_runner.go`: 实现终止条件判断 - [ ] `runner.go`: 在 `runSingleTest` 判断并调用动态模式 -- [ ] 创建示例 simulator agent -## Phase 4: Agent-Driven Input +**测试**: -**新增文件**: `input_source.go` - -- [ ] `input_source.go`: 实现 `ParseInputSource` -- [ ] `input_source.go`: 实现 `GenerateTestCases` -- [ ] `loader.go`: 添加 `LoadFromAgent` 方法 -- [ ] `loader.go`: 添加 `LoadFromScript` 方法 -- [ ] `runner.go`: 在 `RunTests` 支持不同输入源 -- [ ] `cmd/agent/agent.go`: 添加 `--dry-run` flag -- [ ] 创建示例 generator agent +- [ ] 创建单元测试 `dynamic_runner_test.go` ## Phase 5: Console Output Optimization @@ -85,6 +109,8 @@ - [x] Script testing (`*_test.ts`) - [x] Before/After hooks (Phase 1) - [x] Agent-driven assertions (Phase 2) +- [x] Agent-driven input (Phase 3) +- [x] `--dry-run` flag ## Open Questions diff --git a/agent/test/input_source.go b/agent/test/input_source.go new file mode 100644 index 00000000..0b905f80 --- /dev/null +++ b/agent/test/input_source.go @@ -0,0 +1,392 @@ +package test + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + jsoniter "github.com/json-iterator/go" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" +) + +// InputSourceType represents the type of input source +type InputSourceType string + +const ( + // InputSourceFile indicates input from a JSONL file + InputSourceFile InputSourceType = "file" + // InputSourceMessage indicates input from a direct message string + InputSourceMessage InputSourceType = "message" + // InputSourceScript indicates script test mode + InputSourceScript InputSourceType = "script" + // InputSourceAgent indicates input generated by an agent + InputSourceAgent InputSourceType = "agent" +) + +// InputSource represents a parsed input source +type InputSource struct { + Type InputSourceType // file, message, script, agent + Value string // path, message, script ref, or agent ID + Params map[string]interface{} // query parameters (for agent source) +} + +// ParseInputSource parses the -i flag value into an InputSource +// Supported formats: +// - "agents:workers.test.generator" - Agent-generated test cases +// - "agents:workers.test.generator?count=10&focus=edge-cases" - With parameters +// - "scripts.tests.gen" - Script-generated test cases +// - "./tests/inputs.jsonl" - JSONL file +// - "Hello, how are you?" - Direct message +func ParseInputSource(input string) *InputSource { + // Check for agents: prefix + if strings.HasPrefix(input, "agents:") { + return parseAgentSource(strings.TrimPrefix(input, "agents:")) + } + + // Check for scripts: prefix (for generator scripts) + if strings.HasPrefix(input, "scripts:") { + return &InputSource{ + Type: InputSourceScript, + Value: strings.TrimPrefix(input, "scripts:"), + } + } + + // Check for script test mode (scripts.xxx format without prefix) + if strings.HasPrefix(input, "scripts.") { + return &InputSource{ + Type: InputSourceScript, + Value: input, + } + } + + // Check for file extension + if strings.HasSuffix(input, ".jsonl") || strings.HasSuffix(input, ".json") { + return &InputSource{ + Type: InputSourceFile, + Value: input, + } + } + + // Check if it looks like a file path + if strings.Contains(input, "/") || strings.Contains(input, "\\") { + return &InputSource{ + Type: InputSourceFile, + Value: input, + } + } + + // Default to message + return &InputSource{ + Type: InputSourceMessage, + Value: input, + } +} + +// parseAgentSource parses an agent source string with optional query parameters +// Format: "agent.id" or "agent.id?count=10&focus=edge-cases" +func parseAgentSource(input string) *InputSource { + source := &InputSource{ + Type: InputSourceAgent, + Params: make(map[string]interface{}), + } + + // Check for query parameters + if idx := strings.Index(input, "?"); idx >= 0 { + source.Value = input[:idx] + queryStr := input[idx+1:] + + // Parse query parameters + values, err := url.ParseQuery(queryStr) + if err == nil { + for key, vals := range values { + if len(vals) > 0 { + // Try to parse as number + if num, err := strconv.Atoi(vals[0]); err == nil { + source.Params[key] = num + } else if num, err := strconv.ParseFloat(vals[0], 64); err == nil { + source.Params[key] = num + } else if vals[0] == "true" { + source.Params[key] = true + } else if vals[0] == "false" { + source.Params[key] = false + } else { + source.Params[key] = vals[0] + } + } + } + } + } else { + source.Value = input + } + + return source +} + +// GeneratorInput represents the input sent to a generator agent +type GeneratorInput struct { + TargetAgent *TargetAgentInfo `json:"target_agent"` + Count int `json:"count,omitempty"` + Focus string `json:"focus,omitempty"` + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// TargetAgentInfo contains information about the agent being tested +type TargetAgentInfo struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Tools []map[string]interface{} `json:"tools,omitempty"` +} + +// GenerateTestCases generates test cases using a generator agent +func GenerateTestCases(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { + // Get generator assistant + ast, err := assistant.Get(agentID) + if err != nil { + return nil, fmt.Errorf("failed to get generator agent %s: %w", agentID, err) + } + + // Build generation request + genInput := &GeneratorInput{ + TargetAgent: targetInfo, + Count: 5, // Default count + } + + // Apply parameters + if params != nil { + if count, ok := params["count"].(int); ok { + genInput.Count = count + } + if focus, ok := params["focus"].(string); ok { + genInput.Focus = focus + } + // Store extra parameters + genInput.Extra = make(map[string]interface{}) + for k, v := range params { + if k != "count" && k != "focus" { + genInput.Extra[k] = v + } + } + } + + // Create context + env := NewEnvironment("", "") + ctx := NewTestContext("generator", agentID, env) + defer ctx.Release() + + // Build options - skip history and trace for efficiency + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "generator", + }, + } + + // Build message + inputJSON, err := jsoniter.Marshal(genInput) + if err != nil { + return nil, fmt.Errorf("failed to marshal generator input: %w", err) + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + // Call generator agent + response, err := ast.Stream(ctx, messages, opts) + if err != nil { + return nil, fmt.Errorf("generator agent error: %w", err) + } + + // Extract and parse response + return parseGeneratedCases(response) +} + +// parseGeneratedCases parses the generator agent's response into test cases +func parseGeneratedCases(response *context.Response) ([]*Case, error) { + if response == nil || response.Completion == nil { + return nil, fmt.Errorf("empty response from generator agent") + } + + // Extract content + content := response.Completion.Content + if content == nil { + return nil, fmt.Errorf("no content in generator response") + } + + // Convert content to string + var text string + switch v := content.(type) { + case string: + text = v + default: + data, err := jsoniter.Marshal(content) + if err != nil { + return nil, fmt.Errorf("failed to marshal content: %w", err) + } + text = string(data) + } + + // Use goutext.ExtractJSON for fault-tolerant parsing + parsed := goutext.ExtractJSON(text) + if parsed == nil { + return nil, fmt.Errorf("failed to parse generator response as JSON: %s", truncateOutput(text, 200)) + } + + // Convert to []*Case + return convertToCases(parsed) +} + +// convertToCases converts parsed JSON to test cases +func convertToCases(parsed interface{}) ([]*Case, error) { + // Handle array of cases + arr, ok := parsed.([]interface{}) + if !ok { + // Maybe it's a single case wrapped in an object + if obj, ok := parsed.(map[string]interface{}); ok { + if cases, ok := obj["cases"].([]interface{}); ok { + arr = cases + } else if testCases, ok := obj["test_cases"].([]interface{}); ok { + arr = testCases + } else { + // Single case + arr = []interface{}{obj} + } + } else { + return nil, fmt.Errorf("expected array of test cases, got %T", parsed) + } + } + + cases := make([]*Case, 0, len(arr)) + for i, item := range arr { + caseMap, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("test case %d is not an object", i) + } + + tc, err := mapToCase(caseMap) + if err != nil { + return nil, fmt.Errorf("failed to parse test case %d: %w", i, err) + } + + cases = append(cases, tc) + } + + return cases, nil +} + +// mapToCase converts a map to a Case struct +func mapToCase(m map[string]interface{}) (*Case, error) { + tc := &Case{} + + // Required: id + if id, ok := m["id"].(string); ok { + tc.ID = id + } else { + return nil, fmt.Errorf("missing required field 'id'") + } + + // Required: input + if input, ok := m["input"]; ok { + tc.Input = input + } else { + return nil, fmt.Errorf("missing required field 'input'") + } + + // Optional: assertions/assert + if assertions, ok := m["assertions"]; ok { + tc.Assert = assertions + } else if assert, ok := m["assert"]; ok { + tc.Assert = assert + } + + // Optional: options - convert map to CaseOptions + if options, ok := m["options"].(map[string]interface{}); ok { + tc.Options = mapToCaseOptions(options) + } + + // Optional: before/after + if before, ok := m["before"].(string); ok { + tc.Before = before + } + if after, ok := m["after"].(string); ok { + tc.After = after + } + + // Optional: timeout + if timeout, ok := m["timeout"].(string); ok { + tc.Timeout = timeout + } + + return tc, nil +} + +// ToInputMode converts InputSourceType to InputMode for backward compatibility +func (s *InputSource) ToInputMode() InputMode { + switch s.Type { + case InputSourceFile: + return InputModeFile + case InputSourceMessage: + return InputModeMessage + case InputSourceScript: + return InputModeScript + case InputSourceAgent: + // Agent source generates cases, then runs in file mode + return InputModeFile + default: + return InputModeMessage + } +} + +// mapToCaseOptions converts a map to CaseOptions +func mapToCaseOptions(m map[string]interface{}) *CaseOptions { + opts := &CaseOptions{} + + if connector, ok := m["connector"].(string); ok { + opts.Connector = connector + } + + if mode, ok := m["mode"].(string); ok { + opts.Mode = mode + } + + if disableGlobalPrompts, ok := m["disable_global_prompts"].(bool); ok { + opts.DisableGlobalPrompts = disableGlobalPrompts + } + + if search, ok := m["search"].(bool); ok { + opts.Search = &search + } + + if metadata, ok := m["metadata"].(map[string]interface{}); ok { + opts.Metadata = metadata + } + + if skip, ok := m["skip"].(map[string]interface{}); ok { + opts.Skip = &CaseSkipOptions{} + if history, ok := skip["history"].(bool); ok { + opts.Skip.History = history + } + if trace, ok := skip["trace"].(bool); ok { + opts.Skip.Trace = trace + } + if output, ok := skip["output"].(bool); ok { + opts.Skip.Output = output + } + if keyword, ok := skip["keyword"].(bool); ok { + opts.Skip.Keyword = keyword + } + if searchSkip, ok := skip["search"].(bool); ok { + opts.Skip.Search = searchSkip + } + } + + return opts +} diff --git a/agent/test/input_source_test.go b/agent/test/input_source_test.go new file mode 100644 index 00000000..7e89a9ae --- /dev/null +++ b/agent/test/input_source_test.go @@ -0,0 +1,181 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestParseInputSource(t *testing.T) { + tests := []struct { + name string + input string + wantType agenttest.InputSourceType + wantValue string + wantParams map[string]interface{} + }{ + { + name: "JSONL file", + input: "./tests/inputs.jsonl", + wantType: agenttest.InputSourceFile, + wantValue: "./tests/inputs.jsonl", + }, + { + name: "JSON file", + input: "./tests/inputs.json", + wantType: agenttest.InputSourceFile, + wantValue: "./tests/inputs.json", + }, + { + name: "direct message", + input: "Hello, how are you?", + wantType: agenttest.InputSourceMessage, + wantValue: "Hello, how are you?", + }, + { + name: "agent source simple", + input: "agents:tests.generator-agent", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + }, + { + name: "agent source with params", + input: "agents:tests.generator-agent?count=10&focus=edge-cases", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + wantParams: map[string]interface{}{ + "count": 10, + "focus": "edge-cases", + }, + }, + { + name: "agent source with boolean param", + input: "agents:tests.generator-agent?verbose=true", + wantType: agenttest.InputSourceAgent, + wantValue: "tests.generator-agent", + wantParams: map[string]interface{}{ + "verbose": true, + }, + }, + { + name: "script source with prefix", + input: "scripts:tests.gen.Generate", + wantType: agenttest.InputSourceScript, + wantValue: "tests.gen.Generate", + }, + { + name: "script test mode", + input: "scripts.tests.gen", + wantType: agenttest.InputSourceScript, + wantValue: "scripts.tests.gen", + }, + { + name: "path with separator", + input: "/path/to/inputs.jsonl", + wantType: agenttest.InputSourceFile, + wantValue: "/path/to/inputs.jsonl", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := agenttest.ParseInputSource(tt.input) + + assert.Equal(t, tt.wantType, source.Type, "Type mismatch") + assert.Equal(t, tt.wantValue, source.Value, "Value mismatch") + + if tt.wantParams != nil { + for k, v := range tt.wantParams { + assert.Equal(t, v, source.Params[k], "Param %s mismatch", k) + } + } + }) + } +} + +func TestInputSource_ToInputMode(t *testing.T) { + tests := []struct { + name string + source *agenttest.InputSource + wantMode agenttest.InputMode + }{ + { + name: "file source", + source: &agenttest.InputSource{Type: agenttest.InputSourceFile}, + wantMode: agenttest.InputModeFile, + }, + { + name: "message source", + source: &agenttest.InputSource{Type: agenttest.InputSourceMessage}, + wantMode: agenttest.InputModeMessage, + }, + { + name: "script source", + source: &agenttest.InputSource{Type: agenttest.InputSourceScript}, + wantMode: agenttest.InputModeScript, + }, + { + name: "agent source", + source: &agenttest.InputSource{Type: agenttest.InputSourceAgent}, + wantMode: agenttest.InputModeFile, // Agent generates cases, then runs in file mode + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode := tt.source.ToInputMode() + assert.Equal(t, tt.wantMode, mode) + }) + } +} + +func TestGenerateTestCases(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + // Test generating test cases from the generator agent + targetInfo := &agenttest.TargetAgentInfo{ + ID: "tests.next", + Description: "A simple test agent for greeting", + } + + params := map[string]interface{}{ + "count": 3, + "focus": "happy-path", + } + + cases, err := agenttest.GenerateTestCases("tests.generator-agent", targetInfo, params) + if err != nil { + t.Fatalf("Failed to generate test cases: %v", err) + } + + // Verify we got some test cases + assert.NotEmpty(t, cases, "Should generate at least one test case") + + // Verify each case has required fields + for _, tc := range cases { + assert.NotEmpty(t, tc.ID, "Test case should have ID") + assert.NotNil(t, tc.Input, "Test case should have Input") + } + + t.Logf("Generated %d test cases", len(cases)) + for _, tc := range cases { + t.Logf(" - %s", tc.ID) + } +} + +func TestMapToCaseOptions(t *testing.T) { + // Test that options map is correctly converted + source := agenttest.ParseInputSource("agents:test?count=5") + assert.Equal(t, 5, source.Params["count"]) +} diff --git a/agent/test/interfaces.go b/agent/test/interfaces.go index a5996010..d0083d2e 100644 --- a/agent/test/interfaces.go +++ b/agent/test/interfaces.go @@ -43,6 +43,12 @@ type Loader interface { // LoadFile loads test cases from a JSONL file LoadFile(path string) ([]*Case, error) + + // LoadFromAgent generates test cases using a generator agent + LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) + + // LoadFromScript generates test cases using a script + LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) } // Resolver is the interface for resolving agent information diff --git a/agent/test/loader.go b/agent/test/loader.go index 49de1a77..52092b68 100644 --- a/agent/test/loader.go +++ b/agent/test/loader.go @@ -8,6 +8,7 @@ import ( "time" jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/process" ) // JSONLLoader loads test cases from JSONL files @@ -143,3 +144,35 @@ func FilterByIDs(cases []*Case, ids []string) []*Case { return idSet[tc.ID] }) } + +// LoadFromAgent generates test cases using a generator agent +func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { + return GenerateTestCases(agentID, targetInfo, params) +} + +// LoadFromScript generates test cases using a script +// scriptRef format: "module.FunctionName" (e.g., "tests.gen.Generate") +func (l *JSONLLoader) LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) { + // Parse script reference + parts := strings.Split(scriptRef, ".") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid script reference format: %s (expected 'module.Function')", scriptRef) + } + + // Build process name: scripts.module.Function + processName := "scripts." + scriptRef + + // Execute via process + p, err := process.Of(processName, targetInfo) + if err != nil { + return nil, fmt.Errorf("failed to create process %s: %w", processName, err) + } + + result, err := p.Exec() + if err != nil { + return nil, fmt.Errorf("script execution failed: %w", err) + } + + // Parse result as test cases + return convertToCases(result) +} diff --git a/agent/test/runner.go b/agent/test/runner.go index be0f2d04..7afbee70 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -5,6 +5,8 @@ import ( "fmt" "os" "path/filepath" + "reflect" + "strings" "sync" "time" @@ -170,15 +172,58 @@ func (r *Executor) RunTests() (*Report, error) { r.output.Info("Connector: %s", agentInfo.Connector) } - // Load test cases + // Load test cases based on input source var testCases []*Case + inputSource := ParseInputSource(r.opts.Input) - // File mode - load from JSONL - testCases, err = r.loader.LoadFile(r.opts.Input) - if err != nil { - return nil, fmt.Errorf("failed to load test cases: %w", err) + switch inputSource.Type { + case InputSourceAgent: + // Generate test cases using agent + r.output.Info("Generating test cases from agent: %s", inputSource.Value) + targetInfo := &TargetAgentInfo{ + ID: agentInfo.ID, + Description: agentInfo.Description, + } + testCases, err = r.loader.LoadFromAgent(inputSource.Value, targetInfo, inputSource.Params) + if err != nil { + return nil, fmt.Errorf("failed to generate test cases: %w", err) + } + r.output.Info("Generated: %d test cases", len(testCases)) + + case InputSourceScript: + // Generate test cases using script (if it's a generator script, not test script) + // Note: scripts. prefix without "scripts:" is handled by RunScriptTests + if strings.HasPrefix(r.opts.Input, "scripts:") { + scriptRef := strings.TrimPrefix(r.opts.Input, "scripts:") + r.output.Info("Generating test cases from script: %s", scriptRef) + targetInfo := &TargetAgentInfo{ + ID: agentInfo.ID, + Description: agentInfo.Description, + } + testCases, err = r.loader.LoadFromScript(scriptRef, targetInfo) + if err != nil { + return nil, fmt.Errorf("failed to generate test cases from script: %w", err) + } + r.output.Info("Generated: %d test cases", len(testCases)) + } else { + // This is a test script (scripts.xxx format), handled by RunScriptTests + return nil, fmt.Errorf("script test mode should be handled by RunScriptTests") + } + + default: + // File mode - load from JSONL + testCases, err = r.loader.LoadFile(r.opts.Input) + if err != nil { + return nil, fmt.Errorf("failed to load test cases: %w", err) + } + r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) + } + + // Handle dry-run mode - just output the generated test cases + if r.opts.DryRun { + r.output.Info("Dry-run mode: outputting generated test cases") + return r.outputDryRun(testCases, agentInfo) } - r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases)) // Filter skipped tests activeTests := FilterSkipped(testCases) @@ -641,6 +686,12 @@ func isEmptyValue(v interface{}) bool { return true } + // Use reflection to check for typed nil (e.g., *NextHookResponse(nil)) + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Ptr && rv.IsNil() { + return true + } + switch val := v.(type) { case string: return val == "" @@ -648,6 +699,12 @@ func isEmptyValue(v interface{}) bool { return len(val) == 0 case []interface{}: return len(val) == 0 + case *context.NextHookResponse: + // Check if NextHookResponse is effectively empty + if val == nil { + return true + } + return val.Data == nil && val.Delegate == nil } return false @@ -681,3 +738,51 @@ func (r *Executor) getInputOptions() *InputOptions { return opts } + +// outputDryRun outputs generated test cases without running them +func (r *Executor) outputDryRun(testCases []*Case, agentInfo *AgentInfo) (*Report, error) { + r.output.Info("Generated Test Cases:") + + // Output each test case as JSONL + for _, tc := range testCases { + data, err := jsoniter.Marshal(tc) + if err != nil { + r.output.Warning("Failed to marshal test case %s: %s", tc.ID, err.Error()) + continue + } + fmt.Println(string(data)) + } + + // Write to output file if specified + if r.opts.OutputFile != "" { + file, err := os.Create(r.opts.OutputFile) + if err != nil { + return nil, fmt.Errorf("failed to create output file: %w", err) + } + defer file.Close() + + for _, tc := range testCases { + data, err := jsoniter.Marshal(tc) + if err != nil { + continue + } + file.WriteString(string(data) + "\n") + } + + r.output.Info("Output written to: %s", r.opts.OutputFile) + } + + // Return a minimal report + connector := r.opts.Connector + if connector == "" { + connector = agentInfo.Connector + } + + return &Report{ + Summary: &Summary{ + Total: len(testCases), + AgentID: agentInfo.ID, + Connector: connector, + }, + }, nil +} diff --git a/agent/test/runner_integration_test.go b/agent/test/runner_integration_test.go new file mode 100644 index 00000000..aae1c398 --- /dev/null +++ b/agent/test/runner_integration_test.go @@ -0,0 +1,280 @@ +package test_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestRunner_AgentDrivenInput tests the complete flow: +// 1. Use generator-agent to generate test cases +// 2. Run the generated tests against simple-greeting agent +func TestRunner_AgentDrivenInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with agent-driven input + opts := &agenttest.Options{ + Input: "agents:tests.generator-agent?count=3", + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeFile, // Will be overridden by ParseInputSource + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + require.NotNil(t, report.Summary, "Summary should not be nil") + + // Verify report + assert.Greater(t, report.Summary.Total, 0, "Should have at least one test case") + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_AgentDrivenInput_DryRun tests dry-run mode +func TestRunner_AgentDrivenInput_DryRun(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with dry-run mode + opts := &agenttest.Options{ + Input: "agents:tests.generator-agent?count=2", + AgentID: "tests.simple-greeting", + DryRun: true, + Verbose: true, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Dry-run should not return error") + require.NotNil(t, report, "Report should not be nil") + + // In dry-run mode, tests are generated but not executed + // So Passed and Failed should both be 0, but Total should have the count + assert.Greater(t, report.Summary.Total, 0, "Should have generated test cases") + + t.Logf("Generated %d test cases in dry-run mode", report.Summary.Total) +} + +// TestRunner_FileInput tests loading test cases from JSONL file +func TestRunner_FileInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with test cases + // Use case-insensitive contains for robustness + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + testCases := `{"id": "greeting-hello", "input": "Hello", "assert": {"type": "regex", "value": "(?i)hello"}} +{"id": "greeting-hi", "input": "Hi there", "assert": {"type": "regex", "value": "(?i)(hi|hello)"}} +{"id": "greeting-morning", "input": "Good morning", "assert": {"type": "regex", "value": "(?i)(hello|morning|good)"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests from file + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Verify report + assert.Equal(t, 3, report.Summary.Total, "Should have 3 test cases") + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Check results for debugging + if report.Results != nil { + for _, r := range report.Results { + t.Logf(" [%s] Status: %s, Output: %v", r.ID, r.Status, r.Output) + } + } +} + +// TestRunner_DirectMessage tests direct message mode +func TestRunner_DirectMessage(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Test with direct message + opts := &agenttest.Options{ + Input: "Hello, how are you?", + AgentID: "tests.simple-greeting", + Verbose: true, + InputMode: agenttest.InputModeMessage, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Direct message mode returns a minimal report + assert.Equal(t, 1, report.Summary.Total, "Should have 1 test case") + assert.Equal(t, 1, report.Summary.Passed, "Direct message should pass") +} + +// TestRunner_WithBeforeAfter tests before/after hooks +func TestRunner_WithBeforeAfter(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with test cases that use hooks + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + // Note: hooks-test agent has env_test.ts with Before/After functions + testCases := `{"id": "hook-test-1", "input": "Hello", "assert": {"type": "contains", "value": "hello"}, "before": "env_test.Before", "after": "env_test.After"}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests with hooks (using hooks-test agent which has the hook scripts) + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.hooks-test", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_Parallel tests parallel execution +func TestRunner_Parallel(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with multiple test cases + // Use regex for case-insensitive matching + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + testCases := `{"id": "parallel-1", "input": "Hello", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}} +{"id": "parallel-2", "input": "Hi", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}} +{"id": "parallel-3", "input": "Hey", "assert": {"type": "regex", "value": "(?i)(hello|hi|hey)"}} +{"id": "parallel-4", "input": "Good day", "assert": {"type": "regex", "value": "(?i)(hello|good|day)"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests in parallel + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + Parallel: 2, // Run 2 tests in parallel + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + assert.Equal(t, 4, report.Summary.Total, "Should have 4 test cases") + t.Logf("Total: %d, Passed: %d, Failed: %d (parallel: 2)", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestRunner_FailFast tests fail-fast behavior +func TestRunner_FailFast(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with a failing test first + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "inputs.jsonl") + + // First test will fail (expects "impossible" which won't be in response) + testCases := `{"id": "fail-first", "input": "Hello", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_12345"}} +{"id": "should-skip", "input": "Hi", "assert": {"type": "contains", "value": "hi"}}` + + err = os.WriteFile(inputFile, []byte(testCases), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run tests with fail-fast + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.simple-greeting", + FailFast: true, + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error (fail-fast is not an error)") + require.NotNil(t, report, "Report should not be nil") + + // With fail-fast, only the first test should run + assert.Equal(t, 1, report.Summary.Failed, "First test should fail") + // The second test might not run due to fail-fast + t.Logf("Total: %d, Passed: %d, Failed: %d (fail-fast enabled)", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} diff --git a/agent/test/types.go b/agent/test/types.go index 75387879..4d78561d 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -152,6 +152,10 @@ type Options struct { // AfterAll is the global after script (e.g., "scripts:tests.env.AfterAll") // Called once after all test cases AfterAll string `json:"after_all,omitempty"` + + // DryRun generates test cases without running them + // Useful for previewing agent-generated test cases + DryRun bool `json:"dry_run,omitempty"` } // ContextConfig represents custom context configuration from JSON file diff --git a/cmd/agent/test.go b/cmd/agent/test.go index f48384f0..7447c463 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -35,6 +35,7 @@ var ( testFailFast bool testBefore string // --before flag for global BeforeAll hook testAfter string // --after flag for global AfterAll hook + testDryRun bool // --dry-run flag for generating tests without running ) // TestCmd is the agent test command @@ -161,6 +162,7 @@ var TestCmd = &cobra.Command{ FailFast: testFailFast, BeforeAll: testBefore, AfterAll: testAfter, + DryRun: testDryRun, } // Merge with defaults @@ -250,6 +252,7 @@ func init() { TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure")) TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)")) TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)")) + TestCmd.Flags().BoolVar(&testDryRun, "dry-run", false, L("Generate test cases without running them")) // Mark input as required TestCmd.MarkFlagRequired("input") From 8de302d4e8758229ae9a7f0bb35aebaf8ac54705 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 11:17:54 +0800 Subject: [PATCH 14/17] Implement Dynamic Testing Features in Agent Test Framework - Introduced dynamic testing capabilities, allowing for multi-turn conversations with checkpoints through the new `DynamicRunner`. - Enhanced the `runSingleTest` method to support dynamic mode, including the execution of before/after scripts and detailed output for dynamic test results. - Added new output methods in `output.go` for dynamic test start, turns, checkpoints, and results, improving console feedback during testing. - Updated `DESIGN_V2.md` and `TODO_V2.md` to reflect the new dynamic mode features, including simulator configurations and checkpoint definitions. - Revised the `Case` struct in `types.go` to include fields for dynamic testing, such as `Simulator`, `Checkpoints`, and `MaxTurns`. --- agent/test/DESIGN_V2.md | 36 ++- agent/test/TODO_V2.md | 35 ++- agent/test/dynamic_integration_test.go | 250 +++++++++++++++ agent/test/dynamic_runner.go | 413 +++++++++++++++++++++++++ agent/test/dynamic_runner_test.go | 319 +++++++++++++++++++ agent/test/dynamic_types.go | 159 ++++++++++ agent/test/output.go | 39 +++ agent/test/runner.go | 57 ++++ agent/test/types.go | 58 ++++ cmd/agent/test.go | 3 + 10 files changed, 1338 insertions(+), 31 deletions(-) create mode 100644 agent/test/dynamic_integration_test.go create mode 100644 agent/test/dynamic_runner.go create mode 100644 agent/test/dynamic_runner_test.go create mode 100644 agent/test/dynamic_types.go diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 08ed791a..98fd99e1 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -633,20 +633,25 @@ options := &context.Options{ ### Flags Reference -| Flag | Long | Description | -| ---- | ------------- | ---------------------------------------------------------- | -| `-i` | `--input` | Input source: file path, message, or `type:id` reference | -| `-n` | `--name` | Target agent ID (the agent being tested) | -| `-o` | `--output` | Output file path for results | -| `-c` | `--connector` | Override connector for the target agent | -| `-v` | `--verbose` | Verbose output | -| | `--simulator` | Default simulator agent ID | -| | `--before` | Global before script (e.g., `env_test.BeforeAll`) | -| | `--after` | Global after script (e.g., `env_test.AfterAll`) | -| | `--timeout` | Timeout per test case (default: 5m) | -| | `--parallel` | Number of parallel test cases | -| | `--fail-fast` | Stop on first failure | -| | `--dry-run` | Generate/parse tests without running | +| Flag | Long | Description | +| ---- | ------------- | ------------------------------------------------------------ | +| `-i` | `--input` | Input source: file path, message, or `agents:`/`scripts:` ID | +| `-n` | `--name` | Target agent ID (the agent being tested) | +| `-o` | `--output` | Output file path for results | +| `-c` | `--connector` | Override connector for the target agent | +| `-u` | `--user` | Test user ID (default: test-user) | +| `-t` | `--team` | Test team ID (default: test-team) | +| `-v` | `--verbose` | Verbose output | +| | `--ctx` | Path to context JSON file for custom authorization | +| | `--simulator` | Default simulator agent ID for dynamic mode | +| | `--before` | Global before script (e.g., `env_test.BeforeAll`) | +| | `--after` | Global after script (e.g., `env_test.AfterAll`) | +| | `--timeout` | Timeout per test case (default: 5m) | +| | `--parallel` | Number of parallel test cases | +| | `--runs` | Number of runs for stability analysis | +| | `--run` | Regex pattern to filter which tests to run | +| | `--fail-fast` | Stop on first failure | +| | `--dry-run` | Generate/parse tests without running | ### Examples @@ -1000,7 +1005,8 @@ Existing single-turn tests work unchanged: | Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | | Agent-driven input | ✅ Done | `-i agents:xxx` for test generation | | Dry-run mode | ✅ Done | `--dry-run` to preview generated tests | -| Dynamic mode | 🔲 Planned | Simulator + Checkpoints | +| Dynamic mode | ✅ Done | Simulator + Checkpoints | +| Console output | ✅ Done | Dynamic mode tree output, checkpoint display | ## Open Questions diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 7ba60499..51cd45bd 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -65,7 +65,7 @@ - [x] 创建单元测试 `input_source_test.go` -## Phase 4: Dynamic Mode (Simulator + Checkpoints) +## Phase 4: Dynamic Mode (Simulator + Checkpoints) ✅ **新增文件**: `dynamic_runner.go`, `dynamic_types.go` @@ -73,31 +73,31 @@ **准备工作**: -- [ ] 创建 simulator agent (`yao-dev-app/assistants/tests/simulator-agent`) -- [ ] 编写 simulator agent 的 prompts.yml (模拟用户行为) +- [x] 创建 simulator agent (`yao-dev-app/assistants/tests/simulator-agent`) +- [x] 编写 simulator agent 的 prompts.yml (模拟用户行为) **实现**: -- [ ] `types.go`: 添加 `Simulator`, `Checkpoints` 字段到 `Case` -- [ ] `dynamic_types.go`: 定义 `Checkpoint`, `DynamicResult` 等类型 -- [ ] `dynamic_runner.go`: 实现 `DynamicRunner` -- [ ] `dynamic_runner.go`: 实现 checkpoint 匹配逻辑 -- [ ] `dynamic_runner.go`: 实现终止条件判断 -- [ ] `runner.go`: 在 `runSingleTest` 判断并调用动态模式 +- [x] `types.go`: 添加 `Simulator`, `Checkpoints` 字段到 `Case` +- [x] `dynamic_types.go`: 定义 `Checkpoint`, `DynamicResult` 等类型 +- [x] `dynamic_runner.go`: 实现 `DynamicRunner` +- [x] `dynamic_runner.go`: 实现 checkpoint 匹配逻辑 +- [x] `dynamic_runner.go`: 实现终止条件判断 +- [x] `runner.go`: 在 `runSingleTest` 判断并调用动态模式 **测试**: -- [ ] 创建单元测试 `dynamic_runner_test.go` +- [x] 创建单元测试 `dynamic_runner_test.go` -## Phase 5: Console Output Optimization +## Phase 5: Console Output Optimization ✅ **修改文件**: `output.go` -- [ ] `output.go`: 添加 `DynamicTestStart` 方法 -- [ ] `output.go`: 添加 `DynamicTurn` 方法 -- [ ] `output.go`: 添加 `DynamicTestResult` 方法 -- [ ] `output.go`: 添加 `ParallelResults` 方法 -- [ ] 测试并行模式输出效果 +- [x] `output.go`: 添加 `DynamicTestStart` 方法 +- [x] `output.go`: 添加 `DynamicTurn` 方法 +- [x] `output.go`: 添加 `DynamicCheckpoint` 方法 +- [x] `output.go`: 添加 `DynamicTestResult` 方法 +- [x] 动态模式输出效果已验证 ## Already Implemented ✅ @@ -111,6 +111,9 @@ - [x] Agent-driven assertions (Phase 2) - [x] Agent-driven input (Phase 3) - [x] `--dry-run` flag +- [x] Dynamic mode (Phase 4) +- [x] `--simulator` flag +- [x] Console output optimization (Phase 5) ## Open Questions diff --git a/agent/test/dynamic_integration_test.go b/agent/test/dynamic_integration_test.go new file mode 100644 index 00000000..b37c7554 --- /dev/null +++ b/agent/test/dynamic_integration_test.go @@ -0,0 +1,250 @@ +package test_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestDynamicRunner_CoffeeOrder tests a complete dynamic mode flow: +// Simulator acts as a customer ordering coffee, agent handles the order +func TestDynamicRunner_CoffeeOrder(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a temporary JSONL file with a dynamic test case + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Dynamic test case: customer ordering coffee (JSONL must be single line) + testCase := `{"id": "coffee-order-flow", "name": "Complete Coffee Order", "input": "Hi, I would like to order a coffee please", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "A customer who wants to order a medium latte with oat milk", "goal": "Successfully complete a coffee order"}}}, "checkpoints": [{"id": "greeting", "description": "Agent greets and asks for order", "assert": {"type": "regex", "value": "(?i)(order|like|help)"}}, {"id": "ask_size", "description": "Agent asks for size", "after": ["greeting"], "assert": {"type": "regex", "value": "(?i)size"}}, {"id": "confirm_order", "description": "Agent confirms the order", "after": ["ask_size"], "assert": {"type": "regex", "value": "(?i)confirm"}}], "max_turns": 8}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run dynamic test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + require.NotNil(t, report.Summary, "Summary should not be nil") + + // Log results + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Check results + if len(report.Results) > 0 { + result := report.Results[0] + t.Logf("Test [%s] Status: %s", result.ID, result.Status) + + // Check metadata for dynamic mode info + if result.Metadata != nil { + if mode, ok := result.Metadata["mode"].(string); ok { + assert.Equal(t, "dynamic", mode, "Should be dynamic mode") + } + if turns, ok := result.Metadata["total_turns"].(int); ok { + t.Logf("Total turns: %d", turns) + } + } + + if result.Error != "" { + t.Logf("Error: %s", result.Error) + } + } +} + +// TestDynamicRunner_WithInitialInput tests dynamic mode with initial user input +func TestDynamicRunner_WithInitialInput(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + // Create a test case with initial input + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Start with user's first message (JSONL must be single line) + testCase := `{"id": "coffee-with-initial", "name": "Coffee Order with Initial Message", "input": "Hi, I want to order a coffee", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Customer ordering a large cappuccino", "goal": "Complete the coffee order"}}}, "checkpoints": [{"id": "acknowledge", "description": "Agent acknowledges the order request", "assert": {"type": "regex", "value": "(?i)(coffee|order|help)"}}, {"id": "ask_details", "description": "Agent asks for more details", "after": ["acknowledge"], "assert": {"type": "regex", "value": "(?i)(size|type|what)"}}], "max_turns": 5}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) +} + +// TestDynamicRunner_OptionalCheckpoint tests optional checkpoint behavior +func TestDynamicRunner_OptionalCheckpoint(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Test with one required and one optional checkpoint (JSONL must be single line) + testCase := `{"id": "optional-checkpoint-test", "name": "Test with Optional Checkpoint", "input": "Hello", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Simple customer", "goal": "Get a greeting response"}}}, "checkpoints": [{"id": "greeting_response", "description": "Agent responds with greeting", "assert": {"type": "regex", "value": "(?i)(hello|hi|help)"}}, {"id": "special_offer", "description": "Agent mentions special offer (optional)", "required": false, "assert": {"type": "contains", "value": "special offer"}}], "max_turns": 3}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Test should pass even if optional checkpoint is not reached + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // If the required checkpoint is reached, the test should pass + if len(report.Results) > 0 && report.Results[0].Metadata != nil { + if checkpoints, ok := report.Results[0].Metadata["checkpoints"].(map[string]*agenttest.CheckpointResult); ok { + for id, cp := range checkpoints { + t.Logf("Checkpoint [%s]: reached=%v, required=%v", id, cp.Reached, cp.Required) + } + } + } +} + +// TestDynamicRunner_MaxTurnsExceeded tests behavior when max turns is exceeded +func TestDynamicRunner_MaxTurnsExceeded(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Test case with impossible checkpoint and low max_turns (JSONL must be single line) + testCase := `{"id": "max-turns-test", "name": "Test Max Turns Exceeded", "input": "Hello", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Persistent customer", "goal": "Keep talking"}}}, "checkpoints": [{"id": "impossible", "description": "This checkpoint will never be reached", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_NEVER_APPEARS_12345"}}], "max_turns": 2}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + // Test should fail due to max turns exceeded + assert.Equal(t, 1, report.Summary.Failed, "Test should fail") + + if len(report.Results) > 0 { + result := report.Results[0] + assert.Equal(t, agenttest.StatusFailed, result.Status, "Status should be failed") + assert.Contains(t, result.Error, "max turns", "Error should mention max turns") + t.Logf("Error (expected): %s", result.Error) + } +} + +// TestDynamicRunner_CheckpointOrdering tests that checkpoint ordering is enforced +func TestDynamicRunner_CheckpointOrderingEnforced(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agents + err := agent.Load(config.Conf) + require.NoError(t, err, "Failed to load agents") + + tmpDir := t.TempDir() + inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl") + + // Test case with ordered checkpoints (JSONL must be single line) + testCase := `{"id": "ordered-checkpoints", "name": "Test Checkpoint Ordering", "input": "I want to order coffee", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Customer ordering step by step", "goal": "Complete coffee order following the flow"}}}, "checkpoints": [{"id": "step1_greeting", "description": "Agent greets", "assert": {"type": "regex", "value": "(?i)(hello|hi|help|order)"}}, {"id": "step2_size", "description": "Agent asks about size", "after": ["step1_greeting"], "assert": {"type": "regex", "value": "(?i)size"}}, {"id": "step3_confirm", "description": "Agent confirms", "after": ["step2_size"], "assert": {"type": "regex", "value": "(?i)confirm"}}], "max_turns": 10}` + + err = os.WriteFile(inputFile, []byte(testCase), 0644) + require.NoError(t, err, "Failed to write test file") + + // Run test + opts := &agenttest.Options{ + Input: inputFile, + AgentID: "tests.dynamic-test-agent", + Verbose: true, + InputMode: agenttest.InputModeFile, + } + opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions()) + + runner := agenttest.NewRunner(opts) + report, err := runner.Run() + + require.NoError(t, err, "Runner should not return error") + require.NotNil(t, report, "Report should not be nil") + + t.Logf("Total: %d, Passed: %d, Failed: %d", + report.Summary.Total, report.Summary.Passed, report.Summary.Failed) + + // Log checkpoint order + if len(report.Results) > 0 && report.Results[0].Metadata != nil { + if checkpoints, ok := report.Results[0].Metadata["checkpoints"].(map[string]*agenttest.CheckpointResult); ok { + for id, cp := range checkpoints { + t.Logf("Checkpoint [%s]: reached=%v, at_turn=%d", id, cp.Reached, cp.ReachedAtTurn) + } + } + } +} diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go new file mode 100644 index 00000000..2b6020df --- /dev/null +++ b/agent/test/dynamic_runner.go @@ -0,0 +1,413 @@ +package test + +import ( + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" +) + +// DynamicRunner handles dynamic (simulator-driven) test execution +type DynamicRunner struct { + opts *Options + output *OutputWriter + asserter *Asserter +} + +// NewDynamicRunner creates a new dynamic runner +func NewDynamicRunner(opts *Options) *DynamicRunner { + return &DynamicRunner{ + opts: opts, + output: NewOutputWriter(opts.Verbose), + asserter: NewAsserter(), + } +} + +// RunDynamic executes a dynamic test case +func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID string) *DynamicResult { + startTime := time.Now() + + result := &DynamicResult{ + ID: tc.ID, + Turns: make([]*TurnResult, 0), + Checkpoints: make(map[string]*CheckpointResult), + } + + // Initialize checkpoints + for _, cp := range tc.Checkpoints { + result.Checkpoints[cp.ID] = &CheckpointResult{ + ID: cp.ID, + Reached: false, + Required: cp.IsRequired(), + } + } + + // Get simulator agent + simAST, err := assistant.Get(tc.Simulator.Use) + if err != nil { + result.Status = StatusError + result.Error = fmt.Sprintf("failed to get simulator agent: %s", err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + return result + } + + // Get configuration + maxTurns := tc.GetMaxTurns() + timeout := tc.GetTimeout(r.opts.Timeout) + + // Build simulator metadata + simMetadata := make(map[string]interface{}) + if tc.Simulator.Options != nil && tc.Simulator.Options.Metadata != nil { + for k, v := range tc.Simulator.Options.Metadata { + simMetadata[k] = v + } + } + + // Conversation history + messages := make([]context.Message, 0) + + // Get initial input if provided + initialMessages, err := tc.GetMessages() + if err == nil && len(initialMessages) > 0 { + messages = append(messages, initialMessages...) + } + + // Output dynamic test start + if r.opts.Verbose { + r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns) + } + + // Conversation loop + for turn := 1; turn <= maxTurns; turn++ { + turnStart := time.Now() + turnResult := &TurnResult{Turn: turn} + + // Check timeout + if time.Since(startTime) > timeout { + result.Status = StatusTimeout + result.Error = fmt.Sprintf("timeout after %s", timeout) + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn - 1 + return result + } + + // For turns after the first, get input from simulator + if turn > 1 || len(messages) == 0 { + simInput := r.buildSimulatorInput(tc, messages, result, turn, maxTurns, simMetadata) + simOutput, err := r.callSimulator(simAST, tc, simInput) + if err != nil { + turnResult.Error = fmt.Sprintf("simulator error: %s", err.Error()) + result.Turns = append(result.Turns, turnResult) + result.Status = StatusError + result.Error = turnResult.Error + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn + return result + } + + // Check if goal achieved + if simOutput.GoalAchieved { + if r.opts.Verbose { + r.output.Info(" Turn %d: Simulator signaled goal achieved", turn) + } + + // Check if all required checkpoints reached + if r.allRequiredCheckpointsReached(result) { + result.Status = StatusPassed + } else { + result.Status = StatusFailed + result.Error = "simulator signaled goal achieved but not all required checkpoints reached" + } + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn - 1 + return result + } + + // Add user message + userMessage := context.Message{ + Role: context.RoleUser, + Content: simOutput.Message, + } + messages = append(messages, userMessage) + turnResult.Input = simOutput.Message + + if r.opts.Verbose { + r.output.Info(" Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50)) + } + } else { + // Use initial input for first turn + if len(messages) > 0 { + lastMsg := messages[len(messages)-1] + turnResult.Input = lastMsg.Content + if r.opts.Verbose { + r.output.Info(" Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50)) + } + } + } + + // Call target agent + ctx := NewTestContextFromOptions( + fmt.Sprintf("dynamic-%s-%d", tc.ID, turn), + agentID, + r.opts, + tc, + ) + + opts := buildContextOptions(tc, r.opts) + response, err := ast.Stream(ctx, messages, opts) + ctx.Release() + + if err != nil { + turnResult.Error = err.Error() + turnResult.DurationMs = time.Since(turnStart).Milliseconds() + result.Turns = append(result.Turns, turnResult) + result.Status = StatusError + result.Error = fmt.Sprintf("agent error at turn %d: %s", turn, err.Error()) + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn + return result + } + + // Extract output + output := extractOutput(response) + turnResult.Output = output + turnResult.DurationMs = time.Since(turnStart).Milliseconds() + + if r.opts.Verbose { + r.output.Info(" Turn %d: Agent: %s", turn, truncateOutput(output, 50)) + } + + // Add assistant response to messages + messages = append(messages, context.Message{ + Role: context.RoleAssistant, + Content: output, + }) + + // Check checkpoints against this response + reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result) + turnResult.CheckpointsReached = reachedIDs + + if r.opts.Verbose && len(reachedIDs) > 0 { + for _, id := range reachedIDs { + r.output.Info(" ✓ checkpoint: %s", id) + } + } + + result.Turns = append(result.Turns, turnResult) + + // Check if all required checkpoints reached + if r.allRequiredCheckpointsReached(result) { + result.Status = StatusPassed + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = turn + return result + } + } + + // Max turns exceeded + result.Status = StatusFailed + result.Error = fmt.Sprintf("max turns (%d) exceeded without reaching all checkpoints", maxTurns) + result.DurationMs = time.Since(startTime).Milliseconds() + result.TotalTurns = maxTurns + return result +} + +// buildSimulatorInput builds the input for the simulator agent +func (r *DynamicRunner) buildSimulatorInput( + tc *Case, + messages []context.Message, + result *DynamicResult, + turn, maxTurns int, + metadata map[string]interface{}, +) *SimulatorInput { + input := &SimulatorInput{ + Conversation: messages, + TurnNumber: turn, + MaxTurns: maxTurns, + } + + // Extract persona and goal from metadata + if persona, ok := metadata["persona"].(string); ok { + input.Persona = persona + } + if goal, ok := metadata["goal"].(string); ok { + input.Goal = goal + } + + // Build checkpoint lists + input.CheckpointsReached = make([]string, 0) + input.CheckpointsPending = make([]string, 0) + for id, cp := range result.Checkpoints { + if cp.Reached { + input.CheckpointsReached = append(input.CheckpointsReached, id) + } else { + input.CheckpointsPending = append(input.CheckpointsPending, id) + } + } + + // Store extra metadata + input.Extra = make(map[string]interface{}) + for k, v := range metadata { + if k != "persona" && k != "goal" { + input.Extra[k] = v + } + } + + return input +} + +// callSimulator calls the simulator agent and parses the response +func (r *DynamicRunner) callSimulator(simAST *assistant.Assistant, tc *Case, input *SimulatorInput) (*SimulatorOutput, error) { + // Create context + env := NewEnvironment("", "") + ctx := NewTestContext("simulator", tc.Simulator.Use, env) + defer ctx.Release() + + // Build options - skip history and trace + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "simulator", + }, + } + + // Override connector if specified + if tc.Simulator.Options != nil && tc.Simulator.Options.Connector != "" { + opts.Connector = tc.Simulator.Options.Connector + } + + // Build message + inputJSON, err := jsoniter.Marshal(input) + if err != nil { + return nil, fmt.Errorf("failed to marshal simulator input: %w", err) + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + // Call simulator + response, err := simAST.Stream(ctx, messages, opts) + if err != nil { + return nil, fmt.Errorf("simulator agent error: %w", err) + } + + // Parse response + return r.parseSimulatorResponse(response) +} + +// parseSimulatorResponse parses the simulator agent's response +func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*SimulatorOutput, error) { + if response == nil || response.Completion == nil { + return nil, fmt.Errorf("empty response from simulator") + } + + // Extract content + content := response.Completion.Content + if content == nil { + return nil, fmt.Errorf("no content in simulator response") + } + + // Convert to string + var text string + switch v := content.(type) { + case string: + text = v + default: + data, err := jsoniter.Marshal(content) + if err != nil { + return nil, fmt.Errorf("failed to marshal content: %w", err) + } + text = string(data) + } + + // Use goutext.ExtractJSON for fault-tolerant parsing + parsed := goutext.ExtractJSON(text) + if parsed == nil { + // Try to use the text as the message directly + return &SimulatorOutput{ + Message: text, + GoalAchieved: false, + }, nil + } + + // Parse as SimulatorOutput + output := &SimulatorOutput{} + if m, ok := parsed.(map[string]interface{}); ok { + if msg, ok := m["message"].(string); ok { + output.Message = msg + } + if achieved, ok := m["goal_achieved"].(bool); ok { + output.GoalAchieved = achieved + } + if reasoning, ok := m["reasoning"].(string); ok { + output.Reasoning = reasoning + } + } + + if output.Message == "" { + return nil, fmt.Errorf("simulator returned empty message") + } + + return output, nil +} + +// checkCheckpoints validates checkpoints against current output +func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string { + reachedIDs := make([]string, 0) + + for _, cp := range checkpoints { + cpResult := result.Checkpoints[cp.ID] + if cpResult.Reached { + continue // Already reached + } + + // Check "after" constraint + if len(cp.After) > 0 { + allAfterReached := true + for _, afterID := range cp.After { + if afterResult, ok := result.Checkpoints[afterID]; ok { + if !afterResult.Reached { + allAfterReached = false + break + } + } + } + if !allAfterReached { + continue // Dependencies not met + } + } + + // Validate using asserter + tempCase := &Case{Assert: cp.Assert} + passed, msg := r.asserter.Validate(tempCase, output) + + if passed { + cpResult.Reached = true + cpResult.Passed = true + cpResult.ReachedAtTurn = len(result.Turns) + 1 + cpResult.Message = msg + reachedIDs = append(reachedIDs, cp.ID) + } + } + + return reachedIDs +} + +// allRequiredCheckpointsReached checks if all required checkpoints are reached +func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool { + for _, cp := range result.Checkpoints { + if cp.Required && !cp.Reached { + return false + } + } + return true +} diff --git a/agent/test/dynamic_runner_test.go b/agent/test/dynamic_runner_test.go new file mode 100644 index 00000000..a262e564 --- /dev/null +++ b/agent/test/dynamic_runner_test.go @@ -0,0 +1,319 @@ +package test_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/agent" + "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + testutils "github.com/yaoapp/yao/test" +) + +func TestCase_IsDynamicMode(t *testing.T) { + tests := []struct { + name string + tc *test.Case + expected bool + }{ + { + name: "standard mode - no simulator", + tc: &test.Case{ + ID: "T001", + Input: "Hello", + }, + expected: false, + }, + { + name: "standard mode - simulator but no checkpoints", + tc: &test.Case{ + ID: "T002", + Input: "Hello", + Simulator: &test.Simulator{Use: "tests.simulator-agent"}, + }, + expected: false, + }, + { + name: "standard mode - checkpoints but no simulator", + tc: &test.Case{ + ID: "T003", + Input: "Hello", + Checkpoints: []*test.Checkpoint{ + {ID: "cp1", Assert: map[string]interface{}{"type": "contains", "value": "hi"}}, + }, + }, + expected: false, + }, + { + name: "dynamic mode - has both simulator and checkpoints", + tc: &test.Case{ + ID: "T004", + Simulator: &test.Simulator{Use: "tests.simulator-agent"}, + Checkpoints: []*test.Checkpoint{ + {ID: "cp1", Assert: map[string]interface{}{"type": "contains", "value": "hi"}}, + }, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tc.IsDynamicMode() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCase_GetMaxTurns(t *testing.T) { + tests := []struct { + name string + tc *test.Case + expected int + }{ + { + name: "default max turns", + tc: &test.Case{ID: "T001"}, + expected: 20, + }, + { + name: "custom max turns", + tc: &test.Case{ID: "T002", MaxTurns: 10}, + expected: 10, + }, + { + name: "zero max turns uses default", + tc: &test.Case{ID: "T003", MaxTurns: 0}, + expected: 20, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.tc.GetMaxTurns() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCheckpoint_IsRequired(t *testing.T) { + boolTrue := true + boolFalse := false + + tests := []struct { + name string + cp *test.Checkpoint + expected bool + }{ + { + name: "default is required", + cp: &test.Checkpoint{ID: "cp1"}, + expected: true, + }, + { + name: "explicitly required", + cp: &test.Checkpoint{ID: "cp2", Required: &boolTrue}, + expected: true, + }, + { + name: "explicitly not required", + cp: &test.Checkpoint{ID: "cp3", Required: &boolFalse}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.cp.IsRequired() + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestDynamicResult_ToResult(t *testing.T) { + dr := &test.DynamicResult{ + ID: "T001", + Status: test.StatusPassed, + TotalTurns: 3, + DurationMs: 5000, + Turns: []*test.TurnResult{ + {Turn: 1, Input: "Hello", Output: "Hi there!"}, + {Turn: 2, Input: "How are you?", Output: "I'm doing well!"}, + {Turn: 3, Input: "Goodbye", Output: "Bye!"}, + }, + Checkpoints: map[string]*test.CheckpointResult{ + "greet": {ID: "greet", Reached: true, ReachedAtTurn: 1, Required: true}, + "bye": {ID: "bye", Reached: true, ReachedAtTurn: 3, Required: true}, + }, + } + + result := dr.ToResult() + + assert.Equal(t, "T001", result.ID) + assert.Equal(t, test.StatusPassed, result.Status) + assert.Equal(t, int64(5000), result.DurationMs) + assert.Equal(t, "Hello", result.Input) + assert.Equal(t, "Bye!", result.Output) + + // Check metadata + assert.NotNil(t, result.Metadata) + assert.Equal(t, "dynamic", result.Metadata["mode"]) + assert.Equal(t, 3, result.Metadata["total_turns"]) +} + +func TestDynamicRunner_Integration(t *testing.T) { + // Skip if running in short mode + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + // Prepare test environment + testutils.Prepare(t, config.Conf) + defer testutils.Clean() + + // Load agents + err := agent.Load(config.Conf) + if err != nil { + t.Skipf("Failed to load agents: %v", err) + } + + // Create a dynamic test case + tc := &test.Case{ + ID: "dynamic-greeting", + Simulator: &test.Simulator{ + Use: "tests.simulator-agent", + Options: &test.SimulatorOptions{ + Metadata: map[string]interface{}{ + "persona": "Friendly user", + "goal": "Have a brief greeting exchange", + }, + }, + }, + Input: "Hello!", + Checkpoints: []*test.Checkpoint{ + { + ID: "greeting", + Description: "Agent should greet back", + Assert: map[string]interface{}{ + "type": "regex", + "value": "(?i)(hello|hi|hey|greetings)", + }, + }, + }, + MaxTurns: 3, + } + + // Verify it's dynamic mode + assert.True(t, tc.IsDynamicMode()) + + // Create runner options + opts := &test.Options{ + Verbose: true, + Timeout: 30 * time.Second, + } + + // Create dynamic runner + runner := test.NewDynamicRunner(opts) + assert.NotNil(t, runner) + + // Note: Full integration test would require the simulator agent to be loaded + // and would make actual LLM calls. For CI, we test the structure and logic. +} + +func TestDynamicRunner_CheckpointOrdering(t *testing.T) { + // Test that checkpoints with "after" constraints are properly ordered + testutils.Prepare(t, config.Conf) + defer testutils.Clean() + + // Load agents + err := agent.Load(config.Conf) + if err != nil { + t.Skipf("Failed to load agents: %v", err) + } + + // Create a test case with ordered checkpoints + tc := &test.Case{ + ID: "ordered-checkpoints", + Simulator: &test.Simulator{ + Use: "tests.simulator-agent", + Options: &test.SimulatorOptions{ + Metadata: map[string]interface{}{ + "persona": "Customer", + "goal": "Complete a purchase", + }, + }, + }, + Checkpoints: []*test.Checkpoint{ + { + ID: "ask_product", + Description: "Agent asks about product", + Assert: map[string]interface{}{ + "type": "contains", + "value": "product", + }, + }, + { + ID: "confirm_order", + Description: "Agent confirms order", + After: []string{"ask_product"}, + Assert: map[string]interface{}{ + "type": "contains", + "value": "confirm", + }, + }, + { + ID: "complete", + Description: "Order completed", + After: []string{"confirm_order"}, + Assert: map[string]interface{}{ + "type": "contains", + "value": "complete", + }, + }, + }, + MaxTurns: 10, + } + + // Verify checkpoint structure + assert.Len(t, tc.Checkpoints, 3) + assert.Empty(t, tc.Checkpoints[0].After) + assert.Equal(t, []string{"ask_product"}, tc.Checkpoints[1].After) + assert.Equal(t, []string{"confirm_order"}, tc.Checkpoints[2].After) +} + +func TestSimulatorInput_Structure(t *testing.T) { + // Test SimulatorInput structure + input := &test.SimulatorInput{ + Persona: "Test user", + Goal: "Complete task", + TurnNumber: 3, + MaxTurns: 10, + CheckpointsReached: []string{"cp1", "cp2"}, + CheckpointsPending: []string{"cp3"}, + Extra: map[string]interface{}{ + "style": "formal", + }, + } + + assert.Equal(t, "Test user", input.Persona) + assert.Equal(t, "Complete task", input.Goal) + assert.Equal(t, 3, input.TurnNumber) + assert.Equal(t, 10, input.MaxTurns) + assert.Len(t, input.CheckpointsReached, 2) + assert.Len(t, input.CheckpointsPending, 1) + assert.Equal(t, "formal", input.Extra["style"]) +} + +func TestSimulatorOutput_Structure(t *testing.T) { + // Test SimulatorOutput structure + output := &test.SimulatorOutput{ + Message: "I'd like to buy a product", + GoalAchieved: false, + Reasoning: "Continuing toward purchase goal", + } + + assert.Equal(t, "I'd like to buy a product", output.Message) + assert.False(t, output.GoalAchieved) + assert.Equal(t, "Continuing toward purchase goal", output.Reasoning) +} diff --git a/agent/test/dynamic_types.go b/agent/test/dynamic_types.go new file mode 100644 index 00000000..d75ec2c4 --- /dev/null +++ b/agent/test/dynamic_types.go @@ -0,0 +1,159 @@ +package test + +import "github.com/yaoapp/yao/agent/context" + +// DynamicResult represents the result of a dynamic (simulator-driven) test +type DynamicResult struct { + // ID is the test case identifier + ID string `json:"id"` + + // Status is the overall test status + Status Status `json:"status"` + + // Turns contains results for each conversation turn + Turns []*TurnResult `json:"turns"` + + // Checkpoints maps checkpoint ID to its result + Checkpoints map[string]*CheckpointResult `json:"checkpoints"` + + // TotalTurns is the number of turns executed + TotalTurns int `json:"total_turns"` + + // DurationMs is the total execution time in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Error contains error message if status is failed/error/timeout + Error string `json:"error,omitempty"` +} + +// TurnResult represents the result of a single conversation turn +type TurnResult struct { + // Turn is the turn number (1-based) + Turn int `json:"turn"` + + // Input is the user message (from simulator or initial input) + Input interface{} `json:"input"` + + // Output is the agent's response + Output interface{} `json:"output,omitempty"` + + // CheckpointsReached lists checkpoint IDs reached in this turn + CheckpointsReached []string `json:"checkpoints_reached,omitempty"` + + // DurationMs is the turn execution time in milliseconds + DurationMs int64 `json:"duration_ms"` + + // Error contains error message if this turn failed + Error string `json:"error,omitempty"` +} + +// CheckpointResult represents the result of a checkpoint validation +type CheckpointResult struct { + // ID is the checkpoint identifier + ID string `json:"id"` + + // Reached indicates if the checkpoint was reached + Reached bool `json:"reached"` + + // ReachedAtTurn is the turn number when checkpoint was reached (0 if not reached) + ReachedAtTurn int `json:"reached_at_turn,omitempty"` + + // Required indicates if this checkpoint is required + Required bool `json:"required"` + + // Passed indicates if the checkpoint assertion passed + Passed bool `json:"passed"` + + // Message contains assertion result message + Message string `json:"message,omitempty"` +} + +// SimulatorInput is the input sent to the simulator agent +type SimulatorInput struct { + // Persona describes the user being simulated + Persona string `json:"persona,omitempty"` + + // Goal is what the user is trying to achieve + Goal string `json:"goal,omitempty"` + + // Conversation is the message history + Conversation []context.Message `json:"conversation"` + + // TurnNumber is the current turn (1-based) + TurnNumber int `json:"turn_number"` + + // MaxTurns is the maximum allowed turns + MaxTurns int `json:"max_turns"` + + // CheckpointsReached lists checkpoint IDs already reached + CheckpointsReached []string `json:"checkpoints_reached,omitempty"` + + // CheckpointsPending lists checkpoint IDs still pending + CheckpointsPending []string `json:"checkpoints_pending,omitempty"` + + // Extra metadata from simulator options + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// SimulatorOutput is the expected output from the simulator agent +type SimulatorOutput struct { + // Message is the simulated user message + Message string `json:"message"` + + // GoalAchieved indicates if the user's goal has been accomplished + GoalAchieved bool `json:"goal_achieved"` + + // Reasoning explains the simulator's response strategy + Reasoning string `json:"reasoning,omitempty"` +} + +// ToResult converts DynamicResult to standard Result for reporting +func (dr *DynamicResult) ToResult() *Result { + result := &Result{ + ID: dr.ID, + Status: dr.Status, + DurationMs: dr.DurationMs, + Error: dr.Error, + } + + // Store dynamic-specific data in metadata + result.Metadata = map[string]interface{}{ + "mode": "dynamic", + "total_turns": dr.TotalTurns, + "turns": dr.Turns, + "checkpoints": dr.Checkpoints, + } + + // Set input from first turn + if len(dr.Turns) > 0 { + result.Input = dr.Turns[0].Input + } + + // Set output from last turn + if len(dr.Turns) > 0 { + result.Output = dr.Turns[len(dr.Turns)-1].Output + } + + return result +} + +// IsDynamicMode checks if a test case should run in dynamic mode +func (tc *Case) IsDynamicMode() bool { + return tc.Simulator != nil && len(tc.Checkpoints) > 0 +} + +// GetMaxTurns returns the max turns for dynamic mode +func (tc *Case) GetMaxTurns() int { + if tc.MaxTurns > 0 { + return tc.MaxTurns + } + return 20 // Default max turns +} + +// IsRequired returns true if the checkpoint is required +func (cp *Checkpoint) IsRequired() bool { + if cp.Required == nil { + return true // Default to required + } + return *cp.Required +} diff --git a/agent/test/output.go b/agent/test/output.go index a49f1f6e..aaedcf04 100644 --- a/agent/test/output.go +++ b/agent/test/output.go @@ -303,6 +303,45 @@ func (w *OutputWriter) ScriptTestSummary(summary *ScriptTestSummary, duration ti fmt.Printf("%s\n", formatDuration(duration)) } +// DynamicTestStart outputs the start of a dynamic test +func (w *OutputWriter) DynamicTestStart(id string, checkpointCount int) { + color.New(color.FgWhite).Printf("► [%s] ", id) + color.New(color.FgCyan).Printf("(dynamic, %d checkpoints)\n", checkpointCount) +} + +// DynamicTurn outputs a single turn in dynamic testing +func (w *OutputWriter) DynamicTurn(turn int, inputSummary string, checkpointsReached, total int) { + if w.verbose { + color.New(color.FgHiBlack).Printf("│ ├─ Turn %d: %s ", turn, inputSummary) + color.New(color.FgCyan).Printf("[%d/%d checkpoints]\n", checkpointsReached, total) + } +} + +// DynamicCheckpoint outputs a checkpoint being reached +func (w *OutputWriter) DynamicCheckpoint(checkpointID string) { + if w.verbose { + color.New(color.FgGreen).Printf("│ │ └─ ✓ checkpoint: %s\n", checkpointID) + } +} + +// DynamicTestResult outputs the result of a dynamic test +func (w *OutputWriter) DynamicTestResult(status Status, turns int, checkpoints int, duration time.Duration) { + color.New(color.FgHiBlack).Printf(" └─ ") + + switch status { + case StatusPassed: + color.New(color.FgGreen).Printf("PASSED") + case StatusFailed: + color.New(color.FgRed).Printf("FAILED") + case StatusError: + color.New(color.FgRed).Printf("ERROR") + case StatusTimeout: + color.New(color.FgRed).Printf("TIMEOUT") + } + + color.New(color.FgHiBlack).Printf(" (%d turns, %d checkpoints, %s)\n", turns, checkpoints, formatDuration(duration)) +} + // StabilityResult prints stability analysis result for a test case func (w *OutputWriter) StabilityResult(sr *StabilityResult) { color.New(color.FgWhite).Printf(" [%s] ", sr.ID) diff --git a/agent/test/runner.go b/agent/test/runner.go index 7afbee70..3f18e9a8 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -377,6 +377,11 @@ func (r *Executor) runParallel(ast *assistant.Assistant, testCases []*Case, agen // runSingleTest runs a single test case func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID string, runNum int) *Result { + // Check if this is a dynamic mode test + if tc.IsDynamicMode() { + return r.runDynamicTest(ast, tc, agentID) + } + // Get input summary for display inputSummary := SummarizeInput(tc.Input, 50) r.output.TestStart(tc.ID, inputSummary, runNum) @@ -489,6 +494,58 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str return result } +// runDynamicTest runs a dynamic (simulator-driven) test case +func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID string) *Result { + // Output test start for dynamic mode + r.output.DynamicTestStart(tc.ID, len(tc.Checkpoints)) + + startTime := time.Now() + + // Execute before script if specified + var beforeData interface{} + if tc.Before != "" { + var err error + beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath) + if err != nil { + result := &Result{ + ID: tc.ID, + Status: StatusError, + Error: fmt.Sprintf("before script failed: %s", err.Error()), + DurationMs: time.Since(startTime).Milliseconds(), + } + r.output.TestResult(result.Status, time.Since(startTime)) + r.output.TestError(result.Error) + return result + } + } + + // Create dynamic runner and execute + dynamicRunner := NewDynamicRunner(r.opts) + dynamicResult := dynamicRunner.RunDynamic(ast, tc, agentID) + + // Convert to standard result + result := dynamicResult.ToResult() + + // Execute after script if specified + defer func() { + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) + } + } + }() + + // Output result + duration := time.Duration(result.DurationMs) * time.Millisecond + r.output.DynamicTestResult(result.Status, dynamicResult.TotalTurns, len(tc.Checkpoints), duration) + + if result.Error != "" { + r.output.TestError(result.Error) + } + + return result +} + // isBeforeError checks if the error message indicates a before script failure func isBeforeError(errMsg string) bool { return len(errMsg) > 0 && errMsg[:min(len(errMsg), 20)] == "before script failed" diff --git a/agent/test/types.go b/agent/test/types.go index 4d78561d..cddbeb8a 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -156,6 +156,10 @@ type Options struct { // DryRun generates test cases without running them // Useful for previewing agent-generated test cases DryRun bool `json:"dry_run,omitempty"` + + // Simulator is the default simulator agent ID for dynamic mode + // Can be overridden per test case in JSONL + Simulator string `json:"simulator,omitempty"` } // ContextConfig represents custom context configuration from JSON file @@ -395,6 +399,60 @@ type Case struct { // After script function (e.g., "scripts:tests.env.After") // Called after the test case completes (pass or fail) After string `json:"after,omitempty"` + + // Dynamic Mode Fields + // =============================== + + // Simulator configures the user simulator for dynamic testing + // When set, the test runs in dynamic mode with multi-turn conversation + Simulator *Simulator `json:"simulator,omitempty"` + + // Checkpoints define validation points for dynamic testing + // Each checkpoint is checked after every agent response + Checkpoints []*Checkpoint `json:"checkpoints,omitempty"` + + // MaxTurns is the maximum number of conversation turns (default: 20) + MaxTurns int `json:"max_turns,omitempty"` +} + +// Simulator configures the user simulator for dynamic testing +type Simulator struct { + // Use is the simulator agent ID (no prefix needed) + Use string `json:"use"` + + // Options for the simulator agent + Options *SimulatorOptions `json:"options,omitempty"` +} + +// SimulatorOptions configures simulator behavior +type SimulatorOptions struct { + // Metadata passed to the simulator agent + // Common fields: persona, goal, style + Metadata map[string]interface{} `json:"metadata,omitempty"` + + // Connector overrides the simulator's default connector + Connector string `json:"connector,omitempty"` +} + +// Checkpoint defines a validation point in dynamic testing +type Checkpoint struct { + // ID is the unique identifier for this checkpoint + ID string `json:"id"` + + // Description is a human-readable description + Description string `json:"description,omitempty"` + + // Assert defines the assertion to validate + // Same format as Case.Assert + Assert interface{} `json:"assert"` + + // After specifies checkpoint IDs that must be reached before this one + // Used to enforce ordering (e.g., "ask_type" must come before "confirm") + After []string `json:"after,omitempty"` + + // Required indicates if this checkpoint must be reached (default: true) + // Optional checkpoints don't cause test failure if not reached + Required *bool `json:"required,omitempty"` } // CaseOptions represents per-test-case context options diff --git a/cmd/agent/test.go b/cmd/agent/test.go index 7447c463..ebe543d1 100644 --- a/cmd/agent/test.go +++ b/cmd/agent/test.go @@ -36,6 +36,7 @@ var ( testBefore string // --before flag for global BeforeAll hook testAfter string // --after flag for global AfterAll hook testDryRun bool // --dry-run flag for generating tests without running + testSimulator string // --simulator flag for default simulator agent in dynamic mode ) // TestCmd is the agent test command @@ -163,6 +164,7 @@ var TestCmd = &cobra.Command{ BeforeAll: testBefore, AfterAll: testAfter, DryRun: testDryRun, + Simulator: testSimulator, } // Merge with defaults @@ -253,6 +255,7 @@ func init() { TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)")) TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)")) TestCmd.Flags().BoolVar(&testDryRun, "dry-run", false, L("Generate test cases without running them")) + TestCmd.Flags().StringVar(&testSimulator, "simulator", "", L("Default simulator agent for dynamic mode (e.g., tests.simulator-agent)")) // Mark input as required TestCmd.MarkFlagRequired("input") From e9b3d46a6fcbb46196382cef22af765b513fdf7a Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 11:28:19 +0800 Subject: [PATCH 15/17] Enhance README.md for Agent Test Framework - Updated the introduction to clarify the framework's capabilities, emphasizing support for standard testing, dynamic testing, agent-driven assertions, and CI integration. - Revised section headers for better organization, changing "Agent Tests" to "Standard Tests" and adding new sections for "Agent-Driven Input" and "Dynamic Mode." - Included detailed examples for generating test cases using agents and running dynamic tests with simulators. - Improved clarity on input modes and script test modes, ensuring users understand the requirements and options available for testing. - Added notes on the importance of the `-n` flag for agent-driven input mode and clarified the distinction between script and agent-driven test case generation. --- agent/test/README.md | 886 ++++++++++++++++++++----------------------- 1 file changed, 413 insertions(+), 473 deletions(-) diff --git a/agent/test/README.md b/agent/test/README.md index bdbd4e14..d8687362 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -1,10 +1,10 @@ # Agent Test Framework -A testing framework for Yao AI agents with support for assertions, stability analysis, and CI integration. +A comprehensive testing framework for Yao AI agents with support for standard testing, dynamic (simulator-driven) testing, agent-driven assertions, and CI integration. ## Quick Start -### Agent Tests +### Standard Tests ```bash # Test with direct message (auto-detect agent from current directory) @@ -24,6 +24,26 @@ yao agent test -i tests/inputs.jsonl -o report.html yao agent test -i tests/inputs.jsonl --runs 5 ``` +### Agent-Driven Input + +```bash +# Generate test cases using an agent +yao agent test -i "agents:tests.generator-agent?count=10" -n assistants.expense + +# Preview generated tests without running (dry-run) +yao agent test -i "agents:tests.generator-agent?count=5" -n assistants.expense --dry-run +``` + +### Dynamic Mode (Simulator) + +```bash +# Run dynamic tests with simulator +yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent + +# See detailed turn-by-turn output +yao agent test -i tests/dynamic.jsonl -v +``` + ### Script Tests ```bash @@ -39,7 +59,7 @@ yao agent test -i scripts.expense.setup --ctx tests/context.json -v ## Input Modes -The `-i` flag supports three input modes: +The `-i` flag supports multiple input modes: ### 1. JSONL File Mode @@ -64,123 +84,124 @@ yao agent test -i "Extract keywords from this text" yao agent test -i "Hello" -n workers.system.keyword ``` -Output is printed to stdout (or saved to `-o` if specified). +### 3. Agent-Driven Input Mode -### 3. Script Test Mode - -Test agent handler scripts (hooks, tools, setup functions): +Generate test cases using a generator agent: + +```bash +# Basic usage (-n specifies the target agent to test) +yao agent test -i "agents:tests.generator-agent" -n assistants.expense + +# With parameters +yao agent test -i "agents:tests.generator-agent?count=10&focus=edge-cases" -n assistants.expense + +# Dry-run to preview generated tests +yao agent test -i "agents:tests.generator-agent?count=5" -n assistants.expense --dry-run +``` + +**Note**: The `-n` flag is **required** for agent-driven input mode to specify which agent to test. The generator agent creates test cases for the target agent. + +### 4. Script Test Mode + +Test agent handler scripts: ```bash -# Run all tests in a script module yao agent test -i scripts.expense.setup -v - -# Run specific tests with filtering -yao agent test -i scripts.expense.setup --run "TestSystemReady" - -# Run with custom context -yao agent test -i scripts.expense.setup --ctx tests/context.json -v ``` Script test input format: `scripts..` (e.g., `scripts.expense.setup` → `assistants/expense/src/setup_test.ts`). -**Writing Test Scripts:** +### 5. Script-Generated Input Mode -Test scripts should be placed alongside the source files with `_test.ts` or `_test.js` suffix: +Generate test cases using a script: -``` -assistants/expense/src/ -├── setup.ts # Source file -├── setup_test.ts # Test file -├── tools.ts -└── tools_test.ts +```bash +yao agent test -i "scripts:tests.gen.Generate" -n assistants.expense ``` -Test functions must follow the naming convention `Test*` and accept `(t: testing.T, ctx: agent.Context)`: +**Note**: `scripts.xxx` (with dot) runs script tests, while `scripts:xxx` (with colon) generates test cases from a script. -```typescript -// assistants/expense/src/setup_test.ts -import { SystemReady } from "./setup"; +## Test Modes -// Test function signature: function Test*(t: testing.T, ctx: agent.Context) -export function TestSystemReady(t: testing.T, ctx: agent.Context) { - const result = SystemReady(ctx); +### Standard Mode - // Use t.assert for assertions - t.assert.True(result.success, "SystemReady should succeed"); - t.assert.Equal(result.status, "ready", "Status should be ready"); - t.assert.NotNil(result.data, "Data should not be nil"); -} +Single call to agent with optional message history. Each test is independent and stateless. -export function TestSystemReadyError(t: testing.T, ctx: agent.Context) { - // Access context properties - console.log("Testing with user:", ctx.authorized.user_id); - - // Test error handling - const result = SystemReady(ctx); - t.assert.False(result.error, "Should not have error"); +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": { + "type": "contains", + "value": "Hi" + } } ``` -**Available Assertions:** +### Dynamic Mode -| Method | Description | -| -------------------------------- | ------------------------------ | -| `t.assert.True(value, msg)` | Assert value is true | -| `t.assert.False(value, msg)` | Assert value is false | -| `t.assert.Equal(a, b, msg)` | Assert a equals b | -| `t.assert.NotEqual(a, b, msg)` | Assert a not equals b | -| `t.assert.Nil(value, msg)` | Assert value is null/undefined | -| `t.assert.NotNil(value, msg)` | Assert value is not nil | -| `t.assert.Contains(s, sub, msg)` | Assert string contains substr | -| `t.assert.Len(arr, n, msg)` | Assert array/string length | +Simulator-driven testing with checkpoint validation. A simulator agent generates user messages while checkpoints verify agent behavior. -**Test Control:** - -| Method | Description | -| -------------- | ---------------------------- | -| `t.Log(msg)` | Log a message | -| `t.Error(msg)` | Mark test as failed with msg | -| `t.Fatal(msg)` | Mark failed and stop test | -| `t.Skip(msg)` | Skip this test | +```jsonl +{ + "id": "T001", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Customer", + "goal": "Order a latte" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)hello" + } + }, + { + "id": "ask_size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + } + ], + "max_turns": 10 +} +``` ## Command Line Options -| Flag | Description | Default | -| ------------- | -------------------------------------------------- | -------------------------- | -| `-i` | Input: JSONL file path, message, or script ID | (required) | -| `-o` | Output file path | `output-{timestamp}.jsonl` | -| `-n` | Agent ID (optional, auto-detected) | auto-detect | -| `-c` | Override connector | agent default | -| `-u` | Test user ID | `test-user` | -| `-t` | Test team ID | `test-team` | -| `--ctx` | Path to context JSON file for custom authorization | - | -| `-r` | Reporter agent ID | built-in | -| `--runs` | Runs per test (stability analysis) | 1 | -| `--run` | Regex pattern to filter which tests to run | - | -| `--timeout` | Timeout per test | 5m | -| `--parallel` | Parallel test cases | 1 | -| `-v` | Verbose output | false | -| `--fail-fast` | Stop on first failure | false | - -## Agent Resolution - -The agent is resolved in the following priority order: - -1. **Explicit `-n` flag**: `yao agent test -i "msg" -n my.agent` -2. **Path-based detection**: Traverse up from input file to find `package.yao` -3. **Current directory**: For direct message mode, look for `package.yao` in cwd - -Example directory structure: - -``` -assistants/workers/system/keyword/ -├── package.yao <- Agent definition (auto-detected) -├── prompts.yml -├── src/ -│ └── index.ts -└── tests/ - └── inputs.jsonl <- Input file -``` +| Flag | Description | Default | +| ------------- | -------------------------------------------------------- | -------------------------- | +| `-i` | Input: JSONL file, message, `agents:xxx`, or `scripts:x` | (required) | +| `-o` | Output file path | `output-{timestamp}.jsonl` | +| `-n` | Agent ID (optional, auto-detected) | auto-detect | +| `-a` | Application directory | auto-detect | +| `-e` | Environment file | - | +| `-c` | Override connector | agent default | +| `-u` | Test user ID | `test-user` | +| `-t` | Test team ID | `test-team` | +| `-r` | Reporter agent ID for custom report | built-in | +| `-v` | Verbose output | false | +| `--ctx` | Path to context JSON file for custom authorization | - | +| `--simulator` | Default simulator agent ID for dynamic mode | - | +| `--before` | Global BeforeAll hook (e.g., `env_test.BeforeAll`) | - | +| `--after` | Global AfterAll hook (e.g., `env_test.AfterAll`) | - | +| `--runs` | Runs per test (stability analysis) | 1 | +| `--run` | Regex pattern to filter which tests to run | - | +| `--timeout` | Timeout per test | 5m | +| `--parallel` | Parallel test cases | 1 | +| `--fail-fast` | Stop on first failure | false | +| `--dry-run` | Generate test cases without running them | false | ## Input Format (JSONL) @@ -194,35 +215,57 @@ Each line is a JSON object: {"id": "T005", "input": "Skip this", "skip": true} ``` -### Fields +### Standard Mode Fields -| Field | Type | Required | Description | -| ---------- | ------------------------------ | -------- | ------------------------------ | -| `id` | string | Yes | Test case ID | -| `input` | string \| Message \| []Message | Yes | Test input | -| `assert` | Assertion \| []Assertion | No | Assertion rules | -| `expected` | any | No | Expected output (exact match) | -| `user` | string | No | Override user ID | -| `team` | string | No | Override team ID | -| `options` | Options | No | Context options (see below) | -| `timeout` | string | No | Override timeout (e.g., "30s") | -| `skip` | bool | No | Skip this test | -| `metadata` | map | No | Additional metadata | +| Field | Type | Required | Description | +| ---------- | ------------------------------ | -------- | ------------------------------------- | +| `id` | string | Yes | Test case ID | +| `input` | string \| Message \| []Message | Yes | Test input | +| `assert` | Assertion \| []Assertion | No | Assertion rules | +| `expected` | any | No | Expected output (exact match) | +| `user` | string | No | Override user ID for this test | +| `team` | string | No | Override team ID for this test | +| `metadata` | map | No | Additional metadata for hooks | +| `options` | Options | No | Context options | +| `timeout` | string | No | Override timeout (e.g., "30s") | +| `skip` | bool | No | Skip this test | +| `before` | string | No | Before hook (e.g., `env_test.Before`) | +| `after` | string | No | After hook (e.g., `env_test.After`) | + +### Dynamic Mode Fields + +| Field | Type | Required | Description | +| ----------------------------- | ------ | -------- | -------------------------------------- | +| `id` | string | Yes | Test case ID | +| `input` | string | Yes | Initial user message | +| `simulator` | object | Yes | Simulator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | +| `simulator.options` | object | No | Simulator options | +| `simulator.options.metadata` | map | No | Metadata (persona, goal, etc.) | +| `simulator.options.connector` | string | No | Override simulator connector | +| `checkpoints` | array | Yes | Checkpoints to verify | +| `checkpoints[].id` | string | Yes | Checkpoint identifier | +| `checkpoints[].description` | string | No | Human-readable description | +| `checkpoints[].assert` | object | Yes | Assertion to validate | +| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | +| `checkpoints[].required` | bool | No | Is checkpoint required (default: true) | +| `max_turns` | int | No | Maximum turns (default: 20) | +| `timeout` | string | No | Override timeout (e.g., "2m") | ### Options -The `options` field allows per-test-case configuration that maps to `context.Options`: +The `options` field allows per-test-case configuration: -| Field | Type | Description | -| ------------------------ | ------ | -------------------------------------------- | -| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | -| `mode` | string | Agent mode (default: `"chat"`) | -| `search` | bool | Enable/disable search mode (default: `true`) | -| `disable_global_prompts` | bool | Temporarily disable global prompts | -| `metadata` | map | Custom data passed to hooks (e.g., scenario) | -| `skip` | object | Skip configuration (see below) | +| Field | Type | Description | +| ------------------------ | ------ | ------------------------------------------ | +| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | +| `mode` | string | Agent mode (default: `"chat"`) | +| `search` | bool | Enable/disable search mode | +| `disable_global_prompts` | bool | Temporarily disable global prompts | +| `metadata` | map | Custom data passed to hooks | +| `skip` | object | Skip configuration (see below) | -#### Options.skip +### Options.skip | Field | Type | Description | | --------- | ---- | ----------------------- | @@ -232,38 +275,6 @@ The `options` field allows per-test-case configuration that maps to `context.Opt | `keyword` | bool | Skip keyword extraction | | `search` | bool | Skip auto search | -**Example with options:** - -```jsonl -{ - "id": "T001", - "input": "Query users with status active", - "options": { - "connector": "deepseek.v3", - "metadata": { - "scenario": "filter" - }, - "skip": { - "trace": true - } - }, - "assert": { - "type": "json_path", - "path": "from", - "value": "users" - } -} -``` - -**Using metadata for hook scenarios:** - -The `options.metadata` field is passed to agent hooks. For example, a Create Hook can read `options.metadata.scenario` to select different prompt presets: - -```jsonl -{"id": "T001", "input": "...", "options": {"metadata": {"scenario": "aggregation"}}} -{"id": "T002", "input": "...", "options": {"metadata": {"scenario": "join"}}} -``` - ### Input Types | Type | Description | Example | @@ -272,116 +283,11 @@ The `options.metadata` field is passed to agent hooks. For example, a Create Hoo | `Message` | Single message | `{"role": "user", "content": "..."}` | | `[]Message` | Conversation history | `[{"role": "user", ...}, {"role": "assistant", ...}]` | -### File Attachments - -Test inputs support file attachments (images, audio, documents) using the `file://` protocol. Files are loaded and converted to appropriate formats for the LLM. - -**Supported file types:** - -| Type | Extensions | Format | -| ------ | ---------------------------------------------------------------------- | ------------------------------ | -| Image | `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp` | Base64 data URL in `image_url` | -| Audio | `.wav`, `.mp3`, `.flac`, `.ogg`, `.m4a` | Base64 in `input_audio` | -| Doc | `.pdf`, `.doc`, `.docx`, `.xls`, `.xlsx`, `.txt`, `.csv`, `.json` | Base64 data URL in `file` | -| Source | `.yao`, `.ts`, `.js`, `.go`, `.py`, `.rs`, `.java`, `.sql`, `.yaml`... | Base64 data URL in `file` | - -**File path resolution:** - -- **Relative paths**: Resolved relative to the JSONL input file's directory (for file mode) or current working directory (for message mode) -- **Absolute paths**: Used as-is - -**Example with image attachment:** - -```jsonl -{ - "id": "T001", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Please analyze this invoice" - }, - { - "type": "image", - "source": "file://fixtures/invoice.jpg" - } - ] - }, - "assert": { - "type": "contains", - "value": "amount" - } -} -``` - -**Example with multiple attachments:** - -```jsonl -{ - "id": "T002", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Process these receipts" - }, - { - "type": "image", - "source": "file://fixtures/receipt1.png" - }, - { - "type": "image", - "source": "file://fixtures/receipt2.png" - }, - { - "type": "file", - "source": "file://fixtures/policy.pdf", - "name": "expense_policy.pdf" - } - ] - } -} -``` - -**Example with audio:** - -```jsonl -{ - "id": "T003", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Transcribe this audio" - }, - { - "type": "audio", - "source": "file://fixtures/recording.wav" - } - ] - } -} -``` - -**Content part types:** - -| Type | Fields | Description | -| ----------- | --------------------------------------- | -------------------------------- | -| `text` | `text` | Text content | -| `image` | `source` (file://) or `url` | Image attachment | -| `image_url` | `image_url: {url, detail?}` | Direct image URL (OpenAI format) | -| `audio` | `source` (file://) or `data`, `format` | Audio attachment | -| `file` | `source` (file://) or `url`, `filename` | Document attachment | -| `data` | `data: {sources: [...]}` | Data source references | - ## Assertions Use `assert` for flexible validation. If `assert` is defined, it takes precedence over `expected`. -### Assertion Types +### Static Assertions | Type | Description | Example | | -------------- | ----------------------------- | --------------------------------------------------------- | @@ -391,60 +297,45 @@ Use `assert` for flexible validation. If `assert` is defined, it takes precedenc | `json_path` | Extract JSON path and compare | `{"type": "json_path", "path": "$.field", "value": true}` | | `regex` | Match regex pattern | `{"type": "regex", "value": "\\d+"}` | | `type` | Check output type | `{"type": "type", "value": "object"}` | -| `script` | Run custom assertion script | `{"type": "script", "script": "scripts.test.Check"}` | -### Assertion Options +### Assertion Fields -| Field | Type | Description | -| --------- | ------ | --------------------------- | -| `type` | string | Assertion type (required) | -| `value` | any | Expected value or pattern | -| `path` | string | JSON path (for `json_path`) | -| `script` | string | Script name (for `script`) | -| `message` | string | Custom failure message | -| `negate` | bool | Invert the result | +| Field | Type | Description | +| --------- | ------ | -------------------------------------------------------- | +| `type` | string | Assertion type (required) | +| `value` | any | Expected value or pattern | +| `path` | string | JSON path for `json_path` type | +| `script` | string | Script name for `script` type | +| `use` | string | Agent/script ID for `agent` type (with `agents:` prefix) | +| `options` | object | Options for agent assertions | +| `message` | string | Custom failure message | +| `negate` | bool | Invert the assertion result | -### Examples +### Agent-Driven Assertions -**JSON path validation:** +For semantic or fuzzy validation using an LLM: ```jsonl { "id": "T001", - "input": "What's the weather?", + "input": "Hello", "assert": { - "type": "json_path", - "path": "need_search", - "value": true + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be friendly and helpful" } } ``` -**Multiple assertions (all must pass):** +The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`. + +### Script Assertions + +For custom validation logic: ```jsonl { - "id": "T002", - "input": "Hello", - "assert": [ - { - "type": "json_path", - "path": "need_search", - "value": false - }, - { - "type": "not_contains", - "value": "error" - } - ] -} -``` - -**Custom script assertion:** - -```jsonl -{ - "id": "T003", + "id": "T001", "input": "Test", "assert": { "type": "script", @@ -453,64 +344,225 @@ Use `assert` for flexible validation. If `assert` is defined, it takes precedenc } ``` -Script receives `(output, input, expected)` and returns: +### Multiple Assertions -```javascript -// Boolean -return true; - -// Or detailed result -return { pass: true, message: "Validation passed" }; -``` - -**Negated assertion:** +All assertions must pass: ```jsonl { - "id": "T004", + "id": "T001", "input": "Hello", - "assert": { - "type": "contains", - "value": "error", - "negate": true - } + "assert": [ + { + "type": "contains", + "value": "Hi" + }, + { + "type": "not_contains", + "value": "error" + }, + { + "type": "json_path", + "path": "status", + "value": "ok" + } + ] } ``` -### JSON Path Notes +## File Attachments -- Supports dot-notation: `$.field.subfield` or `field.subfield` -- Supports array indexing: `field[0]`, `field[0].subfield`, `field[0].nested[1]` -- Supports multiple expected values (OR logic): `"value": ["a", "b"]` - passes if actual matches any -- Auto-extracts JSON from markdown code blocks (` ```json ... ``` `) -- Works with both string output and structured objects - -**Array index examples:** - -```jsonl -{"id": "T001", "assert": {"type": "json_path", "path": "wheres[0].like", "value": "%test%"}} -{"id": "T002", "assert": {"type": "json_path", "path": "wheres[0].in[0]", "value": "pending"}} -{"id": "T003", "assert": {"type": "json_path", "path": "joins[0].from", "value": "users"}} -{"id": "T004", "assert": {"type": "json_path", "path": "groups[0]", "value": "category"}} -``` - -**Multiple expected values (OR logic):** +Test inputs support file attachments using the `file://` protocol: ```jsonl { - "id": "T005", - "assert": { - "type": "json_path", - "path": "error", - "value": [ - "missing_schema", - "missing_query" + "id": "T001", + "input": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Analyze this image" + }, + { + "type": "image", + "source": "file://fixtures/receipt.jpg" + } ] } } ``` -This passes if `error` equals either `"missing_schema"` or `"missing_query"`. +Supported types: images (jpg, png, gif, webp), audio (wav, mp3), documents (pdf, doc, txt). + +## Before/After Hooks + +### Per-Test Hooks + +Defined in JSONL, scripts located in agent's `src/` directory: + +```jsonl +{ + "id": "T001", + "input": "Test", + "before": "env_test.Before", + "after": "env_test.After" +} +``` + +### Global Hooks + +Via CLI flags: + +```bash +yao agent test -i tests/inputs.jsonl --before env_test.BeforeAll --after env_test.AfterAll +``` + +### Hook Script Example + +```typescript +// assistants/expense/src/env_test.ts + +export function Before(ctx: Context, testCase: TestCase): any { + // Setup: create test data + const userId = Process("models.user.Create", { name: "Test User" }); + return { userId }; // Passed to After +} + +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + beforeData: any +) { + // Cleanup: delete test data + if (beforeData?.userId) { + Process("models.user.Delete", beforeData.userId); + } +} + +export function BeforeAll(ctx: Context, testCases: TestCase[]): any { + Process("models.migrate"); + return { initialized: true }; +} + +export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { + Process("models.cleanup"); +} +``` + +## Script Testing + +Test agent handler scripts with the `t.assert` API: + +```typescript +// assistants/expense/src/setup_test.ts +import { SystemReady } from "./setup"; + +export function TestSystemReady(t: TestingT, ctx: Context) { + const result = SystemReady(ctx); + + t.assert.True(result.success, "Should succeed"); + t.assert.Equal(result.status, "ready", "Status should be ready"); + t.assert.NotNil(result.data, "Data should not be nil"); +} + +export function TestWithAgentAssertion(t: TestingT, ctx: Context) { + const response = Process("agents.expense.Stream", ctx, messages); + + // Static assertion + t.assert.Contains(response.content, "confirm"); + + // Agent-driven assertion + t.assert.Agent(response.content, "tests.validator-agent", { + criteria: "Response should ask for confirmation", + }); +} +``` + +### Available Assertions + +| Method | Description | +| -------------------------------- | ------------------------------ | +| `t.assert.True(value, msg)` | Assert value is true | +| `t.assert.False(value, msg)` | Assert value is false | +| `t.assert.Equal(a, b, msg)` | Assert a equals b | +| `t.assert.NotEqual(a, b, msg)` | Assert a not equals b | +| `t.assert.Nil(value, msg)` | Assert value is null/undefined | +| `t.assert.NotNil(value, msg)` | Assert value is not nil | +| `t.assert.Contains(s, sub, msg)` | Assert string contains substr | +| `t.assert.Len(arr, n, msg)` | Assert array/string length | +| `t.assert.Agent(resp, id, opts)` | Agent-driven assertion | + +## Dynamic Mode + +For testing complex conversation flows where the path is unpredictable: + +```jsonl +{ + "id": "coffee-order", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Customer ordering a latte", + "goal": "Complete the coffee order" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "description": "Agent greets customer", + "assert": { + "type": "regex", + "value": "(?i)(hello|hi|help)" + } + }, + { + "id": "ask_size", + "description": "Agent asks for size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + }, + { + "id": "confirm", + "description": "Agent confirms order", + "after": [ + "ask_size" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } + } + ], + "max_turns": 10 +} +``` + +### Console Output (Dynamic Mode) + +``` +► [coffee-order] (dynamic, 3 checkpoints) +ℹ Dynamic test: coffee-order (max 10 turns) +ℹ Turn 1: User: I want to order coffee +ℹ Turn 1: Agent: Hello! What can I get for you? +ℹ ✓ checkpoint: greeting +ℹ Turn 2: User: A medium latte please +ℹ Turn 2: Agent: What size would you like? +ℹ ✓ checkpoint: ask_size +ℹ Turn 3: User: Medium +ℹ Turn 3: Agent: Let me confirm: Medium latte. Correct? +ℹ ✓ checkpoint: confirm + └─ PASSED (3 turns, 3 checkpoints, 8.5s) +``` ## Output Formats @@ -523,18 +575,6 @@ Determined by `-o` file extension: | `.md` | Markdown | Human-readable | | `.html` | HTML | Interactive web report | -### Default Output Path - -When `-o` is not specified in file mode: - -``` -{input_directory}/output-{timestamp}.jsonl -``` - -Example: `tests/output-20241217100000.jsonl` - -In direct message mode without `-o`, output is printed to stdout. - ## Stability Analysis Run each test multiple times to measure consistency: @@ -543,15 +583,6 @@ Run each test multiple times to measure consistency: yao agent test -i tests/inputs.jsonl --runs 5 -o stability.json ``` -Output includes: - -- Pass rate per test -- Stability classification (stable, mostly_stable, unstable, highly_unstable) -- Average/min/max duration -- Standard deviation - -### Stability Classification - | Pass Rate | Classification | | --------- | --------------- | | 100% | Stable | @@ -559,49 +590,14 @@ Output includes: | 50-79% | Unstable | | < 50% | Highly Unstable | -## Test Environment - -The test framework creates a context with configurable environment: - -| Setting | Flag | Default | -| ---------- | ---- | ----------- | -| User ID | `-u` | `test-user` | -| Team ID | `-t` | `test-team` | -| Locale | - | `en-us` | -| ClientType | - | `test` | -| ClientIP | - | `127.0.0.1` | - -Priority: Command line flags > Test case fields > Defaults - -## Custom Reporter Agent - -Use `-r` to specify a custom agent for report generation: - -```bash -yao agent test -i tests/inputs.jsonl -r report.beautiful -o report.html -``` - -The reporter agent receives: - -```json -{ - "report": { "summary": {...}, "results": [...] }, - "format": "html", - "options": { "verbose": true } -} -``` - ## CI Integration ```bash # Exit code: 0 = all passed, 1 = failures -yao agent test -i tests/inputs.jsonl -o results.jsonl --fail-fast +yao agent test -i tests/inputs.jsonl --fail-fast -# Parse JSONL results -cat results.jsonl | jq 'select(.type == "summary")' - -# Run script tests -yao agent test -i scripts.expense.setup --fail-fast +# Run with parallel execution +yao agent test -i tests/inputs.jsonl --parallel 4 ``` ### GitHub Actions Example @@ -609,96 +605,40 @@ yao agent test -i scripts.expense.setup --fail-fast ```yaml - name: Run Agent Tests run: | - yao agent test -i assistants/keyword/tests/inputs.jsonl \ + yao agent test -i assistants/expense/tests/inputs.jsonl \ -u ci-user -t ci-team \ --runs 3 \ -o report.json +- name: Run Dynamic Tests + run: | + yao agent test -i assistants/expense/tests/dynamic.jsonl \ + --simulator tests.simulator-agent \ + -v + - name: Run Script Tests run: | yao agent test -i scripts.expense.setup -v - yao agent test -i scripts.expense.tools -v - -- name: Run Script Tests with Custom Context - run: | - yao agent test -i scripts.expense.setup \ - --ctx tests/context.json \ - --run "TestSystem.*" \ - -v - -- name: Check Stability - run: | - jq -e '.results | all(.pass_rate >= 80)' report.json ``` -## Examples +## Format Rules Reference -### Agent Tests +| Context | Format | Example | +| ---------------------- | ------------------------ | ----------------------------------------- | +| `-i agents:xxx` (CLI) | Colon prefix | `agents:tests.generator` | +| `-i scripts:xxx` (CLI) | Colon prefix | `scripts:tests.gen.Generate` | +| `-i scripts.xxx` (CLI) | Dot prefix (test mode) | `scripts.expense.setup` | +| JSONL assertion `use` | Prefix required | `"use": "agents:tests.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "tests.simulator-agent"` | +| `--simulator` flag | No prefix (agent only) | `--simulator tests.simulator-agent` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "tests.validator")` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | -```bash -# Quick development test (auto-detect agent) -cd assistants/keyword -yao agent test -i "Extract keywords: AI and ML" +**Script input modes**: -# Quick development test (specify agent) -yao agent test -i "Hello" -n workers.system.keyword - -# Full test suite with HTML report -yao agent test -i tests/inputs.jsonl -o report.html -v - -# Override connector -yao agent test -i tests/inputs.jsonl -c openai.gpt4 - -# Stability analysis -yao agent test -i tests/inputs.jsonl --runs 10 -o stability.json - -# Parallel execution with timeout -yao agent test -i tests/inputs.jsonl --parallel 4 --timeout 2m - -# Custom test environment -yao agent test -i tests/inputs.jsonl -u admin -t prod-team - -# Custom reporter agent -yao agent test -i tests/inputs.jsonl -r report.beautiful -o custom-report.md - -# Full example with all options -yao agent test -i tests/inputs.jsonl \ - -n keyword.agent \ - -c deepseek.v3 \ - -u test-user \ - -t test-team \ - --runs 3 \ - --timeout 10m \ - --parallel 4 \ - -r report.html \ - -o report.html -``` - -### Script Tests - -```bash -# Run all tests in a script module -yao agent test -i scripts.expense.setup -v - -# Run specific tests with regex filter -yao agent test -i scripts.expense.setup --run "TestSystemReady" - -# Run tests matching a pattern -yao agent test -i scripts.expense.setup --run "TestSystem.*" -v - -# Run with custom context (authorization, metadata, etc.) -yao agent test -i scripts.expense.setup --ctx tests/context.json -v - -# Run with specific user/team -yao agent test -i scripts.expense.setup -u admin -t ops-team -v - -# Combine options -yao agent test -i scripts.expense.setup \ - --ctx tests/context.json \ - --run "TestSystem.*" \ - --timeout 30s \ - -v -``` +- `scripts.xxx` (dot) - Run script tests (`*_test.ts` functions) +- `scripts:xxx` (colon) - Generate test cases from a script ## Exit Codes From ea10a7105013b661690db010c76282f596af2020 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 11:42:45 +0800 Subject: [PATCH 16/17] Enhance README.md with Detailed Input Format Examples for Agent Testing - Expanded the input format section to include comprehensive examples organized by scenario, covering various testing cases such as simple text input, assertions, conversation history, and dynamic mode. - Added scenarios demonstrating the use of file attachments, agent-driven assertions, and options for test configuration. - Improved clarity on the structure and requirements for JSONL input, ensuring users have clear guidance on how to format their test cases effectively. - Removed outdated TODO_V2.md file to streamline documentation and focus on the updated README content. --- agent/test/README.md | 450 +++++++++++++++++++++++++++++++++++++++++- agent/test/TODO_V2.md | 122 ------------ 2 files changed, 444 insertions(+), 128 deletions(-) delete mode 100644 agent/test/TODO_V2.md diff --git a/agent/test/README.md b/agent/test/README.md index d8687362..8a530134 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -205,14 +205,265 @@ Simulator-driven testing with checkpoint validation. A simulator agent generates ## Input Format (JSONL) -Each line is a JSON object: +Each line is a JSON object. Below are examples organized by scenario. + +### Scenario 1: Simple Text Input + +Basic test with string input: ```jsonl -{"id": "T001", "input": "Simple text"} -{"id": "T002", "input": {"role": "user", "content": "Message with role"}} -{"id": "T003", "input": [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello"}, {"role": "user", "content": "Follow-up"}]} -{"id": "T004", "input": "Test", "assert": {"type": "json_path", "path": "field", "value": true}} -{"id": "T005", "input": "Skip this", "skip": true} +{"id": "greeting-basic", "input": "Hello, how are you?"} +{"id": "greeting-chinese", "input": "你好，请问有什么可以帮助你的？"} +``` + +### Scenario 2: With Assertions + +Validate response content: + +```jsonl +{"id": "keyword-extract", "input": "Extract keywords from: AI and machine learning", "assert": {"type": "contains", "value": "AI"}} +{"id": "json-response", "input": "What's the weather?", "assert": {"type": "json_path", "path": "need_search", "value": true}} +{"id": "no-error", "input": "Help me", "assert": {"type": "not_contains", "value": "error"}} +``` + +### Scenario 3: Multiple Assertions + +All assertions must pass: + +```jsonl +{ + "id": "expense-submit", + "input": "Submit $500 travel expense", + "assert": [ + { + "type": "contains", + "value": "expense" + }, + { + "type": "not_contains", + "value": "error" + }, + { + "type": "regex", + "value": "(?i)(submitted|created|confirmed)" + } + ] +} +``` + +### Scenario 4: Conversation History + +Test with multi-turn context: + +```jsonl +{ + "id": "expense-confirm", + "input": [ + { + "role": "user", + "content": "Submit an expense" + }, + { + "role": "assistant", + "content": "What type of expense?" + }, + { + "role": "user", + "content": "Travel, $500" + }, + { + "role": "assistant", + "content": "Please confirm: $500 travel expense" + }, + { + "role": "user", + "content": "Yes, confirm" + } + ], + "assert": { + "type": "regex", + "value": "(?i)(submitted|created)" + } +} +``` + +### Scenario 5: With File Attachments + +Test with images or documents: + +```jsonl +{ + "id": "receipt-analyze", + "input": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Analyze this receipt" + }, + { + "type": "image", + "source": "file://fixtures/receipt.jpg" + } + ] + }, + "assert": { + "type": "contains", + "value": "amount" + } +} +``` + +### Scenario 6: Agent-Driven Assertion + +Use LLM to validate response semantics: + +```jsonl +{ + "id": "helpful-response", + "input": "How do I reset my password?", + "assert": { + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should provide clear step-by-step instructions" + } +} +``` + +### Scenario 7: With Options + +Override connector or skip features: + +```jsonl +{"id": "fast-model", "input": "Quick question", "options": {"connector": "deepseek.v3", "skip": {"history": true, "trace": true}}} +{"id": "scenario-test", "input": "Query users", "options": {"metadata": {"scenario": "filter"}}, "assert": {"type": "json_path", "path": "from", "value": "users"}} +``` + +### Scenario 8: With Before/After Hooks + +Setup and teardown for each test: + +```jsonl +{ + "id": "with-user-data", + "input": "Show my expenses", + "before": "env_test.Before", + "after": "env_test.After", + "assert": { + "type": "contains", + "value": "expense" + } +} +``` + +### Scenario 9: Skip Test + +Temporarily disable a test: + +```jsonl +{ + "id": "wip-feature", + "input": "New feature test", + "skip": true +} +``` + +### Scenario 10: Dynamic Mode (Simulator) + +Multi-turn testing with user simulator: + +```jsonl +{ + "id": "coffee-order", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Regular customer", + "goal": "Order a medium latte" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)(hello|hi|help)" + } + }, + { + "id": "ask-size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + }, + { + "id": "confirm", + "after": [ + "ask-size" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } + } + ], + "max_turns": 10 +} +``` + +### Scenario 11: Dynamic Mode with Optional Checkpoint + +Some checkpoints are optional: + +```jsonl +{ + "id": "expense-flow", + "input": "Submit expense", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "New employee", + "goal": "Submit $500 travel expense" + } + } + }, + "checkpoints": [ + { + "id": "ask-type", + "assert": { + "type": "regex", + "value": "(?i)type" + } + }, + { + "id": "suggest-category", + "required": false, + "assert": { + "type": "contains", + "value": "category" + } + }, + { + "id": "confirm", + "after": [ + "ask-type" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } + } + ], + "max_turns": 15 +} ``` ### Standard Mode Fields @@ -640,6 +891,193 @@ yao agent test -i tests/inputs.jsonl --parallel 4 - `scripts.xxx` (dot) - Run script tests (`*_test.ts` functions) - `scripts:xxx` (colon) - Generate test cases from a script +## Built-in Test Agents + +The framework provides three specialized agents for testing: + +### Generator Agent (`tests.generator-agent`) + +Generates test cases based on target agent description. + +**package.yao**: + +```json +{ + "name": "Test Case Generator", + "connector": "gpt-4o", + "description": "Generates test cases for agent testing", + "options": { "temperature": 0.7 }, + "automated": true +} +``` + +**prompts.yml**: + +```yaml +- role: system + content: | + You are a test case generator. Generate test cases based on the target agent. + + ## Input Format + - `target_agent`: Agent info (id, description, tools) + - `count`: Number of test cases (default: 5) + - `focus`: Focus area (e.g., "edge-cases", "happy-path") + + ## Output Format + JSON array of test cases: + [ + { + "id": "test-id", + "input": "User message", + "assert": [{"type": "contains", "value": "expected"}] + } + ] +``` + +**Usage**: + +```bash +yao agent test -i "agents:tests.generator-agent?count=10" -n assistants.expense +``` + +### Validator Agent (`tests.validator-agent`) + +Validates agent responses for agent-driven assertions. + +**package.yao**: + +```json +{ + "name": "Response Validator", + "connector": "gpt-4o", + "description": "Validates responses against criteria", + "options": { "temperature": 0 }, + "automated": true +} +``` + +**prompts.yml**: + +```yaml +- role: system + content: | + You are a response validator. Evaluate whether the response meets the criteria. + + ## Input Format + - `output`: The response to validate + - `criteria`: The validation rules + - `input`: Original input (optional) + + ## Output Format + JSON object (no markdown): + {"passed": true/false, "reason": "explanation"} + + ## Examples + Input: {"output": "Paris is the capital", "criteria": "factually accurate"} + Output: {"passed": true, "reason": "Statement is correct"} +``` + +**Usage in JSONL**: + +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": { + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be friendly" + } +} +``` + +**Usage in script tests**: + +```typescript +t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be helpful", +}); +``` + +### Simulator Agent (`tests.simulator-agent`) + +Simulates user behavior for dynamic mode testing. + +**package.yao**: + +```json +{ + "name": "User Simulator", + "connector": "gpt-4o", + "description": "Simulates user behavior for dynamic testing", + "options": { "temperature": 0.7 }, + "automated": true +} +``` + +**prompts.yml**: + +```yaml +- role: system + content: | + You are a user simulator. Generate realistic user messages based on persona and goal. + + ## Input Format + - `persona`: User description (e.g., "New employee") + - `goal`: What user wants to achieve + - `conversation`: Previous messages + - `turn_number`: Current turn + - `max_turns`: Maximum turns + + ## Output Format + JSON object: + { + "message": "User response", + "goal_achieved": false, + "reasoning": "Strategy explanation" + } + + ## Guidelines + 1. Stay in character + 2. Work toward the goal + 3. Be realistic (include natural variations) + 4. Set goal_achieved: true when done +``` + +**Usage in JSONL**: + +```jsonl +{ + "id": "dynamic-test", + "input": "I need help", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "New employee", + "goal": "Submit expense report" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)hello" + } + } + ], + "max_turns": 10 +} +``` + +**Usage via CLI**: + +```bash +yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent +``` + ## Exit Codes | Code | Description | diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md deleted file mode 100644 index 51cd45bd..00000000 --- a/agent/test/TODO_V2.md +++ /dev/null @@ -1,122 +0,0 @@ -# Agent Test Framework V2 - TODO - -> 详细实施计划见 [UPGRADE_PLAN.md](./UPGRADE_PLAN.md) - -## Format Rules - -| Context | Format | Example | -| --------------------- | ------------------------ | ---------------------------------------------- | -| `-i` flag (CLI) | Prefix required | `agents:workers.test.gen`, `scripts:tests.gen` | -| JSONL assertion `use` | Prefix required | `"use": "agents:workers.test.validator"` | -| JSONL `simulator.use` | No prefix (agent only) | `"use": "workers.test.user-simulator"` | -| `--simulator` flag | No prefix (agent only) | `--simulator workers.test.user-simulator` | -| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "workers.test.val")` | -| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | -| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | - -## Phase 1: Before/After Scripts ✅ - -**新增文件**: `script_hooks.go` - -- [x] `types.go`: 添加 `Before`, `After` 字段到 `Case` -- [x] `types.go`: 添加 `BeforeAll`, `AfterAll` 字段到 `Options` -- [x] `script_hooks.go`: 实现 `HookExecutor` -- [x] `script_hooks.go`: 通过 V8 直接执行 `*_test.ts` 脚本 -- [x] `runner.go`: 集成 before/after 到 `runSingleTest` -- [x] `runner.go`: 集成 beforeAll/afterAll 到 `RunTests` -- [x] `cmd/agent/test.go`: 添加 `--before`, `--after` flags -- [x] `test/utils.go`: 添加 `LoadAgentTestScripts()` 通用函数 -- [x] 创建示例脚本 `assistants/tests/hooks-test/src/env_test.ts` -- [x] 创建单元测试 `script_hooks_test.go` (黑盒测试) - -## Phase 2: Agent-Driven Assertions ✅ - -**修改文件**: `assert.go`, `script_assert.go` - -- [x] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion` -- [x] `assert.go`: 实现 `assertAgent` 方法 -- [x] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型 -- [x] `assert.go`: 使用 `goutext.ExtractJSON` 容错解析 LLM 响应 -- [x] `script_assert.go`: 添加 `assertAgentMethod` 到 `newAssertObject` -- [x] 创建示例 validator agent (`assistants/tests/validator-agent`) -- [x] 创建单元测试 `assert_agent_test.go` (JSONL 断言 + JSAPI 断言) - -## Phase 3: Agent-Driven Input ✅ - -**新增文件**: `input_source.go` - -> 用 Agent 生成测试用例，生成后使用标准模式执行。相对简单。 - -**准备工作**: - -- [x] 创建 generator agent (`yao-dev-app/assistants/tests/generator-agent`) -- [x] 编写 generator agent 的 prompts.yml - -**实现**: - -- [x] `input_source.go`: 实现 `ParseInputSource` -- [x] `input_source.go`: 实现 `GenerateTestCases` -- [x] `loader.go`: 添加 `LoadFromAgent` 方法 -- [x] `loader.go`: 添加 `LoadFromScript` 方法 -- [x] `runner.go`: 在 `RunTests` 支持不同输入源 -- [x] `cmd/agent/test.go`: 添加 `--dry-run` flag - -**测试**: - -- [x] 创建单元测试 `input_source_test.go` - -## Phase 4: Dynamic Mode (Simulator + Checkpoints) ✅ - -**新增文件**: `dynamic_runner.go`, `dynamic_types.go` - -> 运行时使用 Simulator Agent 动态生成对话，需要多轮循环和 checkpoint 匹配。依赖 Phase 3 的 Agent 调用经验。 - -**准备工作**: - -- [x] 创建 simulator agent (`yao-dev-app/assistants/tests/simulator-agent`) -- [x] 编写 simulator agent 的 prompts.yml (模拟用户行为) - -**实现**: - -- [x] `types.go`: 添加 `Simulator`, `Checkpoints` 字段到 `Case` -- [x] `dynamic_types.go`: 定义 `Checkpoint`, `DynamicResult` 等类型 -- [x] `dynamic_runner.go`: 实现 `DynamicRunner` -- [x] `dynamic_runner.go`: 实现 checkpoint 匹配逻辑 -- [x] `dynamic_runner.go`: 实现终止条件判断 -- [x] `runner.go`: 在 `runSingleTest` 判断并调用动态模式 - -**测试**: - -- [x] 创建单元测试 `dynamic_runner_test.go` - -## Phase 5: Console Output Optimization ✅ - -**修改文件**: `output.go` - -- [x] `output.go`: 添加 `DynamicTestStart` 方法 -- [x] `output.go`: 添加 `DynamicTurn` 方法 -- [x] `output.go`: 添加 `DynamicCheckpoint` 方法 -- [x] `output.go`: 添加 `DynamicTestResult` 方法 -- [x] 动态模式输出效果已验证 - -## Already Implemented ✅ - -- [x] Message history support (`input` as array) -- [x] File attachments (`file://` protocol) -- [x] `--parallel` flag -- [x] `--fail-fast` flag -- [x] `-v` verbose mode -- [x] Script testing (`*_test.ts`) -- [x] Before/After hooks (Phase 1) -- [x] Agent-driven assertions (Phase 2) -- [x] Agent-driven input (Phase 3) -- [x] `--dry-run` flag -- [x] Dynamic mode (Phase 4) -- [x] `--simulator` flag -- [x] Console output optimization (Phase 5) - -## Open Questions - -1. **Message Generation**: 是否提供 helper 从脚本生成 message history? -2. **Snapshot Testing**: 是否支持 "golden file" 对比? -3. **Retry Logic**: 测试失败是否支持自动重试? From 006dc8b3daca34116d3b9094d643c4878568c59f Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 11:53:24 +0800 Subject: [PATCH 17/17] Enhance README.md with Comprehensive Hook Documentation for Agent Testing - Added detailed sections on Before/After hooks, including types, execution order, and function signatures. - Provided examples for common use cases such as database setup/teardown and conditional setup based on metadata. - Clarified parameters for hook functions, improving guidance for users on implementing hooks in their tests. - Enhanced overall documentation to support better understanding of the testing framework's capabilities. --- agent/test/README.md | 175 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 5 deletions(-) diff --git a/agent/test/README.md b/agent/test/README.md index 8a530134..d49bdbd2 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -648,6 +648,31 @@ Supported types: images (jpg, png, gif, webp), audio (wav, mp3), documents (pdf, ## Before/After Hooks +Hooks allow you to run setup and teardown code before and after tests. Hook scripts must be placed in the agent's `src/` directory with `_test.ts` suffix. + +### Hook Types + +| Hook | Scope | When Called | Use Case | +| ----------- | -------- | --------------------- | ------------------------------- | +| `Before` | Per-test | Before each test case | Create test data, setup context | +| `After` | Per-test | After each test case | Cleanup test data, log results | +| `BeforeAll` | Global | Once before all tests | Database migration, init | +| `AfterAll` | Global | Once after all tests | Global cleanup, report | + +### Execution Order + +``` +BeforeAll (global) + ├─ Before (test 1) + │ └─ Test 1 execution + │ └─ After (test 1) + ├─ Before (test 2) + │ └─ Test 2 execution + │ └─ After (test 2) + └─ ... +AfterAll (global) +``` + ### Per-Test Hooks Defined in JSONL, scripts located in agent's `src/` directory: @@ -669,39 +694,179 @@ Via CLI flags: yao agent test -i tests/inputs.jsonl --before env_test.BeforeAll --after env_test.AfterAll ``` -### Hook Script Example +### Hook Function Signatures ```typescript // assistants/expense/src/env_test.ts +/** + * Before - Called before each test case + * @param ctx - Agent context with user/team info + * @param testCase - The test case about to run + * @returns any - Data passed to After hook (optional) + */ export function Before(ctx: Context, testCase: TestCase): any { - // Setup: create test data const userId = Process("models.user.Create", { name: "Test User" }); - return { userId }; // Passed to After + return { userId }; // This data is passed to After } +/** + * After - Called after each test case (pass or fail) + * @param ctx - Agent context + * @param testCase - The test case that ran + * @param result - Test result with status, output, duration + * @param beforeData - Data returned from Before hook + */ export function After( ctx: Context, testCase: TestCase, result: TestResult, beforeData: any ) { - // Cleanup: delete test data if (beforeData?.userId) { Process("models.user.Delete", beforeData.userId); } + if (result.status === "failed") { + console.log(`Test ${testCase.id} failed: ${result.error}`); + } } +/** + * BeforeAll - Called once before all tests + * @param ctx - Agent context + * @param testCases - Array of all test cases + * @returns any - Data passed to AfterAll hook (optional) + */ export function BeforeAll(ctx: Context, testCases: TestCase[]): any { Process("models.migrate"); - return { initialized: true }; + return { initialized: true, count: testCases.length }; } +/** + * AfterAll - Called once after all tests complete + * @param ctx - Agent context + * @param results - Array of all test results + * @param beforeData - Data returned from BeforeAll hook + */ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { + const passed = results.filter((r) => r.status === "passed").length; + console.log(`Tests completed: ${passed}/${results.length} passed`); Process("models.cleanup"); } ``` +### Hook Parameters + +**Context** - Agent execution context: + +```typescript +interface Context { + user_id: string; // Test user ID + team_id: string; // Test team ID + locale: string; // Locale (e.g., "en-us") + metadata: object; // Custom metadata from test case +} +``` + +**TestCase** - Test case definition: + +```typescript +interface TestCase { + id: string; // Test case ID + input: any; // Test input (string, Message, or Message[]) + assert?: object; // Assertion rules + expected?: any; // Expected output + user?: string; // Override user ID + team?: string; // Override team ID + metadata?: object; // Custom metadata + options?: object; // Context options + timeout?: string; // Timeout (e.g., "30s") + skip?: boolean; // Skip flag + before?: string; // Before hook reference + after?: string; // After hook reference +} +``` + +**TestResult** - Test execution result: + +```typescript +interface TestResult { + id: string; // Test case ID + status: string; // "passed" | "failed" | "error" | "skipped" | "timeout" + input: any; // Actual input sent + output: any; // Agent response + expected?: any; // Expected output (if defined) + error?: string; // Error message (if failed) + duration_ms: number; // Execution time in milliseconds + assertions?: object[]; // Assertion results +} +``` + +### Common Use Cases + +**Database Setup/Teardown**: + +```typescript +export function Before(ctx: Context, testCase: TestCase): any { + // Create test records + const user = Process("models.user.Create", { + name: "Test", + email: "test@example.com", + }); + const expense = Process("models.expense.Create", { + user_id: user.id, + amount: 100, + }); + return { user, expense }; +} + +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + data: any +) { + // Clean up in reverse order + if (data?.expense) Process("models.expense.Delete", data.expense.id); + if (data?.user) Process("models.user.Delete", data.user.id); +} +``` + +**Conditional Setup Based on Metadata**: + +```typescript +export function Before(ctx: Context, testCase: TestCase): any { + const scenario = testCase.metadata?.scenario || "default"; + + if (scenario === "empty_db") { + Process("models.expense.DeleteAll"); + } else if (scenario === "with_data") { + Process("scripts.tests.seed.LoadTestData"); + } + + return { scenario }; +} +``` + +**Logging and Debugging**: + +```typescript +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + data: any +) { + if (result.status === "failed") { + console.log("=== Test Failed ==="); + console.log("Test ID:", testCase.id); + console.log("Input:", JSON.stringify(testCase.input)); + console.log("Output:", JSON.stringify(result.output)); + console.log("Error:", result.error); + } +} +``` + ## Script Testing Test agent handler scripts with the `t.assert` API: