From e9b3d46a6fcbb46196382cef22af765b513fdf7a Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 26 Dec 2025 11:28:19 +0800 Subject: [PATCH] Enhance README.md for Agent Test Framework - Updated the introduction to clarify the framework's capabilities, emphasizing support for standard testing, dynamic testing, agent-driven assertions, and CI integration. - Revised section headers for better organization, changing "Agent Tests" to "Standard Tests" and adding new sections for "Agent-Driven Input" and "Dynamic Mode." - Included detailed examples for generating test cases using agents and running dynamic tests with simulators. - Improved clarity on input modes and script test modes, ensuring users understand the requirements and options available for testing. - Added notes on the importance of the `-n` flag for agent-driven input mode and clarified the distinction between script and agent-driven test case generation. --- agent/test/README.md | 886 ++++++++++++++++++++----------------------- 1 file changed, 413 insertions(+), 473 deletions(-) diff --git a/agent/test/README.md b/agent/test/README.md index bdbd4e14..d8687362 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -1,10 +1,10 @@ # Agent Test Framework -A testing framework for Yao AI agents with support for assertions, stability analysis, and CI integration. +A comprehensive testing framework for Yao AI agents with support for standard testing, dynamic (simulator-driven) testing, agent-driven assertions, and CI integration. ## Quick Start -### Agent Tests +### Standard Tests ```bash # Test with direct message (auto-detect agent from current directory) @@ -24,6 +24,26 @@ yao agent test -i tests/inputs.jsonl -o report.html yao agent test -i tests/inputs.jsonl --runs 5 ``` +### Agent-Driven Input + +```bash +# Generate test cases using an agent +yao agent test -i "agents:tests.generator-agent?count=10" -n assistants.expense + +# Preview generated tests without running (dry-run) +yao agent test -i "agents:tests.generator-agent?count=5" -n assistants.expense --dry-run +``` + +### Dynamic Mode (Simulator) + +```bash +# Run dynamic tests with simulator +yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent + +# See detailed turn-by-turn output +yao agent test -i tests/dynamic.jsonl -v +``` + ### Script Tests ```bash @@ -39,7 +59,7 @@ yao agent test -i scripts.expense.setup --ctx tests/context.json -v ## Input Modes -The `-i` flag supports three input modes: +The `-i` flag supports multiple input modes: ### 1. JSONL File Mode @@ -64,123 +84,124 @@ yao agent test -i "Extract keywords from this text" yao agent test -i "Hello" -n workers.system.keyword ``` -Output is printed to stdout (or saved to `-o` if specified). +### 3. Agent-Driven Input Mode -### 3. Script Test Mode - -Test agent handler scripts (hooks, tools, setup functions): +Generate test cases using a generator agent: + +```bash +# Basic usage (-n specifies the target agent to test) +yao agent test -i "agents:tests.generator-agent" -n assistants.expense + +# With parameters +yao agent test -i "agents:tests.generator-agent?count=10&focus=edge-cases" -n assistants.expense + +# Dry-run to preview generated tests +yao agent test -i "agents:tests.generator-agent?count=5" -n assistants.expense --dry-run +``` + +**Note**: The `-n` flag is **required** for agent-driven input mode to specify which agent to test. The generator agent creates test cases for the target agent. + +### 4. Script Test Mode + +Test agent handler scripts: ```bash -# Run all tests in a script module yao agent test -i scripts.expense.setup -v - -# Run specific tests with filtering -yao agent test -i scripts.expense.setup --run "TestSystemReady" - -# Run with custom context -yao agent test -i scripts.expense.setup --ctx tests/context.json -v ``` Script test input format: `scripts..` (e.g., `scripts.expense.setup` → `assistants/expense/src/setup_test.ts`). -**Writing Test Scripts:** +### 5. Script-Generated Input Mode -Test scripts should be placed alongside the source files with `_test.ts` or `_test.js` suffix: +Generate test cases using a script: -``` -assistants/expense/src/ -├── setup.ts # Source file -├── setup_test.ts # Test file -├── tools.ts -└── tools_test.ts +```bash +yao agent test -i "scripts:tests.gen.Generate" -n assistants.expense ``` -Test functions must follow the naming convention `Test*` and accept `(t: testing.T, ctx: agent.Context)`: +**Note**: `scripts.xxx` (with dot) runs script tests, while `scripts:xxx` (with colon) generates test cases from a script. -```typescript -// assistants/expense/src/setup_test.ts -import { SystemReady } from "./setup"; +## Test Modes -// Test function signature: function Test*(t: testing.T, ctx: agent.Context) -export function TestSystemReady(t: testing.T, ctx: agent.Context) { - const result = SystemReady(ctx); +### Standard Mode - // Use t.assert for assertions - t.assert.True(result.success, "SystemReady should succeed"); - t.assert.Equal(result.status, "ready", "Status should be ready"); - t.assert.NotNil(result.data, "Data should not be nil"); -} +Single call to agent with optional message history. Each test is independent and stateless. -export function TestSystemReadyError(t: testing.T, ctx: agent.Context) { - // Access context properties - console.log("Testing with user:", ctx.authorized.user_id); - - // Test error handling - const result = SystemReady(ctx); - t.assert.False(result.error, "Should not have error"); +```jsonl +{ + "id": "T001", + "input": "Hello", + "assert": { + "type": "contains", + "value": "Hi" + } } ``` -**Available Assertions:** +### Dynamic Mode -| Method | Description | -| -------------------------------- | ------------------------------ | -| `t.assert.True(value, msg)` | Assert value is true | -| `t.assert.False(value, msg)` | Assert value is false | -| `t.assert.Equal(a, b, msg)` | Assert a equals b | -| `t.assert.NotEqual(a, b, msg)` | Assert a not equals b | -| `t.assert.Nil(value, msg)` | Assert value is null/undefined | -| `t.assert.NotNil(value, msg)` | Assert value is not nil | -| `t.assert.Contains(s, sub, msg)` | Assert string contains substr | -| `t.assert.Len(arr, n, msg)` | Assert array/string length | +Simulator-driven testing with checkpoint validation. A simulator agent generates user messages while checkpoints verify agent behavior. -**Test Control:** - -| Method | Description | -| -------------- | ---------------------------- | -| `t.Log(msg)` | Log a message | -| `t.Error(msg)` | Mark test as failed with msg | -| `t.Fatal(msg)` | Mark failed and stop test | -| `t.Skip(msg)` | Skip this test | +```jsonl +{ + "id": "T001", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Customer", + "goal": "Order a latte" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "assert": { + "type": "regex", + "value": "(?i)hello" + } + }, + { + "id": "ask_size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + } + ], + "max_turns": 10 +} +``` ## Command Line Options -| Flag | Description | Default | -| ------------- | -------------------------------------------------- | -------------------------- | -| `-i` | Input: JSONL file path, message, or script ID | (required) | -| `-o` | Output file path | `output-{timestamp}.jsonl` | -| `-n` | Agent ID (optional, auto-detected) | auto-detect | -| `-c` | Override connector | agent default | -| `-u` | Test user ID | `test-user` | -| `-t` | Test team ID | `test-team` | -| `--ctx` | Path to context JSON file for custom authorization | - | -| `-r` | Reporter agent ID | built-in | -| `--runs` | Runs per test (stability analysis) | 1 | -| `--run` | Regex pattern to filter which tests to run | - | -| `--timeout` | Timeout per test | 5m | -| `--parallel` | Parallel test cases | 1 | -| `-v` | Verbose output | false | -| `--fail-fast` | Stop on first failure | false | - -## Agent Resolution - -The agent is resolved in the following priority order: - -1. **Explicit `-n` flag**: `yao agent test -i "msg" -n my.agent` -2. **Path-based detection**: Traverse up from input file to find `package.yao` -3. **Current directory**: For direct message mode, look for `package.yao` in cwd - -Example directory structure: - -``` -assistants/workers/system/keyword/ -├── package.yao <- Agent definition (auto-detected) -├── prompts.yml -├── src/ -│ └── index.ts -└── tests/ - └── inputs.jsonl <- Input file -``` +| Flag | Description | Default | +| ------------- | -------------------------------------------------------- | -------------------------- | +| `-i` | Input: JSONL file, message, `agents:xxx`, or `scripts:x` | (required) | +| `-o` | Output file path | `output-{timestamp}.jsonl` | +| `-n` | Agent ID (optional, auto-detected) | auto-detect | +| `-a` | Application directory | auto-detect | +| `-e` | Environment file | - | +| `-c` | Override connector | agent default | +| `-u` | Test user ID | `test-user` | +| `-t` | Test team ID | `test-team` | +| `-r` | Reporter agent ID for custom report | built-in | +| `-v` | Verbose output | false | +| `--ctx` | Path to context JSON file for custom authorization | - | +| `--simulator` | Default simulator agent ID for dynamic mode | - | +| `--before` | Global BeforeAll hook (e.g., `env_test.BeforeAll`) | - | +| `--after` | Global AfterAll hook (e.g., `env_test.AfterAll`) | - | +| `--runs` | Runs per test (stability analysis) | 1 | +| `--run` | Regex pattern to filter which tests to run | - | +| `--timeout` | Timeout per test | 5m | +| `--parallel` | Parallel test cases | 1 | +| `--fail-fast` | Stop on first failure | false | +| `--dry-run` | Generate test cases without running them | false | ## Input Format (JSONL) @@ -194,35 +215,57 @@ Each line is a JSON object: {"id": "T005", "input": "Skip this", "skip": true} ``` -### Fields +### Standard Mode Fields -| Field | Type | Required | Description | -| ---------- | ------------------------------ | -------- | ------------------------------ | -| `id` | string | Yes | Test case ID | -| `input` | string \| Message \| []Message | Yes | Test input | -| `assert` | Assertion \| []Assertion | No | Assertion rules | -| `expected` | any | No | Expected output (exact match) | -| `user` | string | No | Override user ID | -| `team` | string | No | Override team ID | -| `options` | Options | No | Context options (see below) | -| `timeout` | string | No | Override timeout (e.g., "30s") | -| `skip` | bool | No | Skip this test | -| `metadata` | map | No | Additional metadata | +| Field | Type | Required | Description | +| ---------- | ------------------------------ | -------- | ------------------------------------- | +| `id` | string | Yes | Test case ID | +| `input` | string \| Message \| []Message | Yes | Test input | +| `assert` | Assertion \| []Assertion | No | Assertion rules | +| `expected` | any | No | Expected output (exact match) | +| `user` | string | No | Override user ID for this test | +| `team` | string | No | Override team ID for this test | +| `metadata` | map | No | Additional metadata for hooks | +| `options` | Options | No | Context options | +| `timeout` | string | No | Override timeout (e.g., "30s") | +| `skip` | bool | No | Skip this test | +| `before` | string | No | Before hook (e.g., `env_test.Before`) | +| `after` | string | No | After hook (e.g., `env_test.After`) | + +### Dynamic Mode Fields + +| Field | Type | Required | Description | +| ----------------------------- | ------ | -------- | -------------------------------------- | +| `id` | string | Yes | Test case ID | +| `input` | string | Yes | Initial user message | +| `simulator` | object | Yes | Simulator configuration | +| `simulator.use` | string | Yes | Simulator agent ID (no prefix) | +| `simulator.options` | object | No | Simulator options | +| `simulator.options.metadata` | map | No | Metadata (persona, goal, etc.) | +| `simulator.options.connector` | string | No | Override simulator connector | +| `checkpoints` | array | Yes | Checkpoints to verify | +| `checkpoints[].id` | string | Yes | Checkpoint identifier | +| `checkpoints[].description` | string | No | Human-readable description | +| `checkpoints[].assert` | object | Yes | Assertion to validate | +| `checkpoints[].after` | array | No | Checkpoint IDs that must occur first | +| `checkpoints[].required` | bool | No | Is checkpoint required (default: true) | +| `max_turns` | int | No | Maximum turns (default: 20) | +| `timeout` | string | No | Override timeout (e.g., "2m") | ### Options -The `options` field allows per-test-case configuration that maps to `context.Options`: +The `options` field allows per-test-case configuration: -| Field | Type | Description | -| ------------------------ | ------ | -------------------------------------------- | -| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | -| `mode` | string | Agent mode (default: `"chat"`) | -| `search` | bool | Enable/disable search mode (default: `true`) | -| `disable_global_prompts` | bool | Temporarily disable global prompts | -| `metadata` | map | Custom data passed to hooks (e.g., scenario) | -| `skip` | object | Skip configuration (see below) | +| Field | Type | Description | +| ------------------------ | ------ | ------------------------------------------ | +| `connector` | string | Override connector (e.g., `"deepseek.v3"`) | +| `mode` | string | Agent mode (default: `"chat"`) | +| `search` | bool | Enable/disable search mode | +| `disable_global_prompts` | bool | Temporarily disable global prompts | +| `metadata` | map | Custom data passed to hooks | +| `skip` | object | Skip configuration (see below) | -#### Options.skip +### Options.skip | Field | Type | Description | | --------- | ---- | ----------------------- | @@ -232,38 +275,6 @@ The `options` field allows per-test-case configuration that maps to `context.Opt | `keyword` | bool | Skip keyword extraction | | `search` | bool | Skip auto search | -**Example with options:** - -```jsonl -{ - "id": "T001", - "input": "Query users with status active", - "options": { - "connector": "deepseek.v3", - "metadata": { - "scenario": "filter" - }, - "skip": { - "trace": true - } - }, - "assert": { - "type": "json_path", - "path": "from", - "value": "users" - } -} -``` - -**Using metadata for hook scenarios:** - -The `options.metadata` field is passed to agent hooks. For example, a Create Hook can read `options.metadata.scenario` to select different prompt presets: - -```jsonl -{"id": "T001", "input": "...", "options": {"metadata": {"scenario": "aggregation"}}} -{"id": "T002", "input": "...", "options": {"metadata": {"scenario": "join"}}} -``` - ### Input Types | Type | Description | Example | @@ -272,116 +283,11 @@ The `options.metadata` field is passed to agent hooks. For example, a Create Hoo | `Message` | Single message | `{"role": "user", "content": "..."}` | | `[]Message` | Conversation history | `[{"role": "user", ...}, {"role": "assistant", ...}]` | -### File Attachments - -Test inputs support file attachments (images, audio, documents) using the `file://` protocol. Files are loaded and converted to appropriate formats for the LLM. - -**Supported file types:** - -| Type | Extensions | Format | -| ------ | ---------------------------------------------------------------------- | ------------------------------ | -| Image | `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp` | Base64 data URL in `image_url` | -| Audio | `.wav`, `.mp3`, `.flac`, `.ogg`, `.m4a` | Base64 in `input_audio` | -| Doc | `.pdf`, `.doc`, `.docx`, `.xls`, `.xlsx`, `.txt`, `.csv`, `.json` | Base64 data URL in `file` | -| Source | `.yao`, `.ts`, `.js`, `.go`, `.py`, `.rs`, `.java`, `.sql`, `.yaml`... | Base64 data URL in `file` | - -**File path resolution:** - -- **Relative paths**: Resolved relative to the JSONL input file's directory (for file mode) or current working directory (for message mode) -- **Absolute paths**: Used as-is - -**Example with image attachment:** - -```jsonl -{ - "id": "T001", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Please analyze this invoice" - }, - { - "type": "image", - "source": "file://fixtures/invoice.jpg" - } - ] - }, - "assert": { - "type": "contains", - "value": "amount" - } -} -``` - -**Example with multiple attachments:** - -```jsonl -{ - "id": "T002", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Process these receipts" - }, - { - "type": "image", - "source": "file://fixtures/receipt1.png" - }, - { - "type": "image", - "source": "file://fixtures/receipt2.png" - }, - { - "type": "file", - "source": "file://fixtures/policy.pdf", - "name": "expense_policy.pdf" - } - ] - } -} -``` - -**Example with audio:** - -```jsonl -{ - "id": "T003", - "input": { - "role": "user", - "content": [ - { - "type": "text", - "text": "Transcribe this audio" - }, - { - "type": "audio", - "source": "file://fixtures/recording.wav" - } - ] - } -} -``` - -**Content part types:** - -| Type | Fields | Description | -| ----------- | --------------------------------------- | -------------------------------- | -| `text` | `text` | Text content | -| `image` | `source` (file://) or `url` | Image attachment | -| `image_url` | `image_url: {url, detail?}` | Direct image URL (OpenAI format) | -| `audio` | `source` (file://) or `data`, `format` | Audio attachment | -| `file` | `source` (file://) or `url`, `filename` | Document attachment | -| `data` | `data: {sources: [...]}` | Data source references | - ## Assertions Use `assert` for flexible validation. If `assert` is defined, it takes precedence over `expected`. -### Assertion Types +### Static Assertions | Type | Description | Example | | -------------- | ----------------------------- | --------------------------------------------------------- | @@ -391,60 +297,45 @@ Use `assert` for flexible validation. If `assert` is defined, it takes precedenc | `json_path` | Extract JSON path and compare | `{"type": "json_path", "path": "$.field", "value": true}` | | `regex` | Match regex pattern | `{"type": "regex", "value": "\\d+"}` | | `type` | Check output type | `{"type": "type", "value": "object"}` | -| `script` | Run custom assertion script | `{"type": "script", "script": "scripts.test.Check"}` | -### Assertion Options +### Assertion Fields -| Field | Type | Description | -| --------- | ------ | --------------------------- | -| `type` | string | Assertion type (required) | -| `value` | any | Expected value or pattern | -| `path` | string | JSON path (for `json_path`) | -| `script` | string | Script name (for `script`) | -| `message` | string | Custom failure message | -| `negate` | bool | Invert the result | +| Field | Type | Description | +| --------- | ------ | -------------------------------------------------------- | +| `type` | string | Assertion type (required) | +| `value` | any | Expected value or pattern | +| `path` | string | JSON path for `json_path` type | +| `script` | string | Script name for `script` type | +| `use` | string | Agent/script ID for `agent` type (with `agents:` prefix) | +| `options` | object | Options for agent assertions | +| `message` | string | Custom failure message | +| `negate` | bool | Invert the assertion result | -### Examples +### Agent-Driven Assertions -**JSON path validation:** +For semantic or fuzzy validation using an LLM: ```jsonl { "id": "T001", - "input": "What's the weather?", + "input": "Hello", "assert": { - "type": "json_path", - "path": "need_search", - "value": true + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be friendly and helpful" } } ``` -**Multiple assertions (all must pass):** +The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`. + +### Script Assertions + +For custom validation logic: ```jsonl { - "id": "T002", - "input": "Hello", - "assert": [ - { - "type": "json_path", - "path": "need_search", - "value": false - }, - { - "type": "not_contains", - "value": "error" - } - ] -} -``` - -**Custom script assertion:** - -```jsonl -{ - "id": "T003", + "id": "T001", "input": "Test", "assert": { "type": "script", @@ -453,64 +344,225 @@ Use `assert` for flexible validation. If `assert` is defined, it takes precedenc } ``` -Script receives `(output, input, expected)` and returns: +### Multiple Assertions -```javascript -// Boolean -return true; - -// Or detailed result -return { pass: true, message: "Validation passed" }; -``` - -**Negated assertion:** +All assertions must pass: ```jsonl { - "id": "T004", + "id": "T001", "input": "Hello", - "assert": { - "type": "contains", - "value": "error", - "negate": true - } + "assert": [ + { + "type": "contains", + "value": "Hi" + }, + { + "type": "not_contains", + "value": "error" + }, + { + "type": "json_path", + "path": "status", + "value": "ok" + } + ] } ``` -### JSON Path Notes +## File Attachments -- Supports dot-notation: `$.field.subfield` or `field.subfield` -- Supports array indexing: `field[0]`, `field[0].subfield`, `field[0].nested[1]` -- Supports multiple expected values (OR logic): `"value": ["a", "b"]` - passes if actual matches any -- Auto-extracts JSON from markdown code blocks (` ```json ... ``` `) -- Works with both string output and structured objects - -**Array index examples:** - -```jsonl -{"id": "T001", "assert": {"type": "json_path", "path": "wheres[0].like", "value": "%test%"}} -{"id": "T002", "assert": {"type": "json_path", "path": "wheres[0].in[0]", "value": "pending"}} -{"id": "T003", "assert": {"type": "json_path", "path": "joins[0].from", "value": "users"}} -{"id": "T004", "assert": {"type": "json_path", "path": "groups[0]", "value": "category"}} -``` - -**Multiple expected values (OR logic):** +Test inputs support file attachments using the `file://` protocol: ```jsonl { - "id": "T005", - "assert": { - "type": "json_path", - "path": "error", - "value": [ - "missing_schema", - "missing_query" + "id": "T001", + "input": { + "role": "user", + "content": [ + { + "type": "text", + "text": "Analyze this image" + }, + { + "type": "image", + "source": "file://fixtures/receipt.jpg" + } ] } } ``` -This passes if `error` equals either `"missing_schema"` or `"missing_query"`. +Supported types: images (jpg, png, gif, webp), audio (wav, mp3), documents (pdf, doc, txt). + +## Before/After Hooks + +### Per-Test Hooks + +Defined in JSONL, scripts located in agent's `src/` directory: + +```jsonl +{ + "id": "T001", + "input": "Test", + "before": "env_test.Before", + "after": "env_test.After" +} +``` + +### Global Hooks + +Via CLI flags: + +```bash +yao agent test -i tests/inputs.jsonl --before env_test.BeforeAll --after env_test.AfterAll +``` + +### Hook Script Example + +```typescript +// assistants/expense/src/env_test.ts + +export function Before(ctx: Context, testCase: TestCase): any { + // Setup: create test data + const userId = Process("models.user.Create", { name: "Test User" }); + return { userId }; // Passed to After +} + +export function After( + ctx: Context, + testCase: TestCase, + result: TestResult, + beforeData: any +) { + // Cleanup: delete test data + if (beforeData?.userId) { + Process("models.user.Delete", beforeData.userId); + } +} + +export function BeforeAll(ctx: Context, testCases: TestCase[]): any { + Process("models.migrate"); + return { initialized: true }; +} + +export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { + Process("models.cleanup"); +} +``` + +## Script Testing + +Test agent handler scripts with the `t.assert` API: + +```typescript +// assistants/expense/src/setup_test.ts +import { SystemReady } from "./setup"; + +export function TestSystemReady(t: TestingT, ctx: Context) { + const result = SystemReady(ctx); + + t.assert.True(result.success, "Should succeed"); + t.assert.Equal(result.status, "ready", "Status should be ready"); + t.assert.NotNil(result.data, "Data should not be nil"); +} + +export function TestWithAgentAssertion(t: TestingT, ctx: Context) { + const response = Process("agents.expense.Stream", ctx, messages); + + // Static assertion + t.assert.Contains(response.content, "confirm"); + + // Agent-driven assertion + t.assert.Agent(response.content, "tests.validator-agent", { + criteria: "Response should ask for confirmation", + }); +} +``` + +### Available Assertions + +| Method | Description | +| -------------------------------- | ------------------------------ | +| `t.assert.True(value, msg)` | Assert value is true | +| `t.assert.False(value, msg)` | Assert value is false | +| `t.assert.Equal(a, b, msg)` | Assert a equals b | +| `t.assert.NotEqual(a, b, msg)` | Assert a not equals b | +| `t.assert.Nil(value, msg)` | Assert value is null/undefined | +| `t.assert.NotNil(value, msg)` | Assert value is not nil | +| `t.assert.Contains(s, sub, msg)` | Assert string contains substr | +| `t.assert.Len(arr, n, msg)` | Assert array/string length | +| `t.assert.Agent(resp, id, opts)` | Agent-driven assertion | + +## Dynamic Mode + +For testing complex conversation flows where the path is unpredictable: + +```jsonl +{ + "id": "coffee-order", + "input": "I want to order coffee", + "simulator": { + "use": "tests.simulator-agent", + "options": { + "metadata": { + "persona": "Customer ordering a latte", + "goal": "Complete the coffee order" + } + } + }, + "checkpoints": [ + { + "id": "greeting", + "description": "Agent greets customer", + "assert": { + "type": "regex", + "value": "(?i)(hello|hi|help)" + } + }, + { + "id": "ask_size", + "description": "Agent asks for size", + "after": [ + "greeting" + ], + "assert": { + "type": "regex", + "value": "(?i)size" + } + }, + { + "id": "confirm", + "description": "Agent confirms order", + "after": [ + "ask_size" + ], + "assert": { + "type": "regex", + "value": "(?i)confirm" + } + } + ], + "max_turns": 10 +} +``` + +### Console Output (Dynamic Mode) + +``` +► [coffee-order] (dynamic, 3 checkpoints) +ℹ Dynamic test: coffee-order (max 10 turns) +ℹ Turn 1: User: I want to order coffee +ℹ Turn 1: Agent: Hello! What can I get for you? +ℹ ✓ checkpoint: greeting +ℹ Turn 2: User: A medium latte please +ℹ Turn 2: Agent: What size would you like? +ℹ ✓ checkpoint: ask_size +ℹ Turn 3: User: Medium +ℹ Turn 3: Agent: Let me confirm: Medium latte. Correct? +ℹ ✓ checkpoint: confirm + └─ PASSED (3 turns, 3 checkpoints, 8.5s) +``` ## Output Formats @@ -523,18 +575,6 @@ Determined by `-o` file extension: | `.md` | Markdown | Human-readable | | `.html` | HTML | Interactive web report | -### Default Output Path - -When `-o` is not specified in file mode: - -``` -{input_directory}/output-{timestamp}.jsonl -``` - -Example: `tests/output-20241217100000.jsonl` - -In direct message mode without `-o`, output is printed to stdout. - ## Stability Analysis Run each test multiple times to measure consistency: @@ -543,15 +583,6 @@ Run each test multiple times to measure consistency: yao agent test -i tests/inputs.jsonl --runs 5 -o stability.json ``` -Output includes: - -- Pass rate per test -- Stability classification (stable, mostly_stable, unstable, highly_unstable) -- Average/min/max duration -- Standard deviation - -### Stability Classification - | Pass Rate | Classification | | --------- | --------------- | | 100% | Stable | @@ -559,49 +590,14 @@ Output includes: | 50-79% | Unstable | | < 50% | Highly Unstable | -## Test Environment - -The test framework creates a context with configurable environment: - -| Setting | Flag | Default | -| ---------- | ---- | ----------- | -| User ID | `-u` | `test-user` | -| Team ID | `-t` | `test-team` | -| Locale | - | `en-us` | -| ClientType | - | `test` | -| ClientIP | - | `127.0.0.1` | - -Priority: Command line flags > Test case fields > Defaults - -## Custom Reporter Agent - -Use `-r` to specify a custom agent for report generation: - -```bash -yao agent test -i tests/inputs.jsonl -r report.beautiful -o report.html -``` - -The reporter agent receives: - -```json -{ - "report": { "summary": {...}, "results": [...] }, - "format": "html", - "options": { "verbose": true } -} -``` - ## CI Integration ```bash # Exit code: 0 = all passed, 1 = failures -yao agent test -i tests/inputs.jsonl -o results.jsonl --fail-fast +yao agent test -i tests/inputs.jsonl --fail-fast -# Parse JSONL results -cat results.jsonl | jq 'select(.type == "summary")' - -# Run script tests -yao agent test -i scripts.expense.setup --fail-fast +# Run with parallel execution +yao agent test -i tests/inputs.jsonl --parallel 4 ``` ### GitHub Actions Example @@ -609,96 +605,40 @@ yao agent test -i scripts.expense.setup --fail-fast ```yaml - name: Run Agent Tests run: | - yao agent test -i assistants/keyword/tests/inputs.jsonl \ + yao agent test -i assistants/expense/tests/inputs.jsonl \ -u ci-user -t ci-team \ --runs 3 \ -o report.json +- name: Run Dynamic Tests + run: | + yao agent test -i assistants/expense/tests/dynamic.jsonl \ + --simulator tests.simulator-agent \ + -v + - name: Run Script Tests run: | yao agent test -i scripts.expense.setup -v - yao agent test -i scripts.expense.tools -v - -- name: Run Script Tests with Custom Context - run: | - yao agent test -i scripts.expense.setup \ - --ctx tests/context.json \ - --run "TestSystem.*" \ - -v - -- name: Check Stability - run: | - jq -e '.results | all(.pass_rate >= 80)' report.json ``` -## Examples +## Format Rules Reference -### Agent Tests +| Context | Format | Example | +| ---------------------- | ------------------------ | ----------------------------------------- | +| `-i agents:xxx` (CLI) | Colon prefix | `agents:tests.generator` | +| `-i scripts:xxx` (CLI) | Colon prefix | `scripts:tests.gen.Generate` | +| `-i scripts.xxx` (CLI) | Dot prefix (test mode) | `scripts.expense.setup` | +| JSONL assertion `use` | Prefix required | `"use": "agents:tests.validator"` | +| JSONL `simulator.use` | No prefix (agent only) | `"use": "tests.simulator-agent"` | +| `--simulator` flag | No prefix (agent only) | `--simulator tests.simulator-agent` | +| `t.assert.Agent()` | No prefix (method-bound) | `t.assert.Agent(resp, "tests.validator")` | +| JSONL `before/after` | No prefix (in src/) | `"before": "env_test.Before"` | +| `--before/--after` | No prefix (in src/) | `--before env_test.BeforeAll` | -```bash -# Quick development test (auto-detect agent) -cd assistants/keyword -yao agent test -i "Extract keywords: AI and ML" +**Script input modes**: -# Quick development test (specify agent) -yao agent test -i "Hello" -n workers.system.keyword - -# Full test suite with HTML report -yao agent test -i tests/inputs.jsonl -o report.html -v - -# Override connector -yao agent test -i tests/inputs.jsonl -c openai.gpt4 - -# Stability analysis -yao agent test -i tests/inputs.jsonl --runs 10 -o stability.json - -# Parallel execution with timeout -yao agent test -i tests/inputs.jsonl --parallel 4 --timeout 2m - -# Custom test environment -yao agent test -i tests/inputs.jsonl -u admin -t prod-team - -# Custom reporter agent -yao agent test -i tests/inputs.jsonl -r report.beautiful -o custom-report.md - -# Full example with all options -yao agent test -i tests/inputs.jsonl \ - -n keyword.agent \ - -c deepseek.v3 \ - -u test-user \ - -t test-team \ - --runs 3 \ - --timeout 10m \ - --parallel 4 \ - -r report.html \ - -o report.html -``` - -### Script Tests - -```bash -# Run all tests in a script module -yao agent test -i scripts.expense.setup -v - -# Run specific tests with regex filter -yao agent test -i scripts.expense.setup --run "TestSystemReady" - -# Run tests matching a pattern -yao agent test -i scripts.expense.setup --run "TestSystem.*" -v - -# Run with custom context (authorization, metadata, etc.) -yao agent test -i scripts.expense.setup --ctx tests/context.json -v - -# Run with specific user/team -yao agent test -i scripts.expense.setup -u admin -t ops-team -v - -# Combine options -yao agent test -i scripts.expense.setup \ - --ctx tests/context.json \ - --run "TestSystem.*" \ - --timeout 30s \ - -v -``` +- `scripts.xxx` (dot) - Run script tests (`*_test.ts` functions) +- `scripts:xxx` (colon) - Generate test cases from a script ## Exit Codes