From d7b84bdf364ebb1e946d775722a5537fce8675ad Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 25 Dec 2025 17:35:30 +0800 Subject: [PATCH] Enhance Documentation and Add JSONL Support for Agent Assertions - Updated the DESIGN_V2.md file to clarify the usage of agent-driven assertions in JSONL test cases, including detailed examples and API specifications. - Introduced a new section on script testing with agent assertions, outlining the implementation and usage of the `t.assert.Agent()` method. - Modified the TODO_V2.md file to reflect the addition of JSONL support for agent assertions and outlined tasks for further development in this area. --- agent/test/DESIGN_V2.md | 112 ++++++++++++++++++++++++++++++++++++++-- agent/test/TODO_V2.md | 11 ++++ 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index da13502f..4ad6fbc3 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -385,7 +385,7 @@ For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `c ```jsonl { "type": "agent", - "use": "agents:workers.test.validator", + "use": "workers.test.validator", "options": { "connector": "openai-gpt4", "metadata": { @@ -397,9 +397,9 @@ For fuzzy, semantic, or context-aware validation. Uses `options` aligned with `c } ``` -### Script Assertions +### Script Assertions (in JSONL) -For custom validation logic: +For custom validation logic in JSONL test cases: ```jsonl { @@ -427,7 +427,7 @@ Mix static and agent-driven assertions: }, { "type": "agent", - "use": "agents:workers.test.validator", + "use": "workers.test.validator", "options": { "metadata": { "criteria": "Confirmation message should include expense amount and be polite" @@ -438,6 +438,110 @@ Mix static and agent-driven assertions: } ``` +## Script Testing with Agent Assertions + +Script tests can also use Agent-driven assertions via the `t.assert.Agent()` method. + +### API + +```typescript +// t.assert.Agent(response, agentID, options?) -> ValidatorResult +// agentID: Direct agent ID without prefix, e.g., "workers.test.validator" +interface ValidatorResult { + passed: boolean; + score?: number; + reason: string; + suggestions?: string[]; +} +``` + +### Usage in Script Tests + +```typescript +// tests/expense_test.ts +export function TestExpenseResponse(t: TestingT, ctx: Context) { + // Call the agent being tested + const response = Process("agents.expense.Stream", ctx, [ + { role: "user", content: "How do I submit an expense?" }, + ]); + + // Static assertions + t.assert.NotNil(response); + t.assert.Contains(response.content, "expense"); + + // Agent-driven assertion - automatically fails test if validation fails + t.assert.Agent(response.content, "workers.test.validator", { + metadata: { + criteria: + "Response should explain the expense submission process clearly", + expected_topics: ["receipt", "approval", "deadline"], + tone: "helpful", + }, + }); +} +``` + +### With Conversation Context + +```typescript +export function TestMultiTurnExpense(t: TestingT, ctx: Context) { + const messages = [ + { role: "user", content: "I need to submit a travel expense" }, + { role: "assistant", content: "I'd be happy to help..." }, + { role: "user", content: "It's for a flight to Beijing, $2000" }, + ]; + + const response = Process("agents.expense.Stream", ctx, messages); + + // Agent assertion with conversation context + // Automatically fails test and logs suggestions if validation fails + t.assert.Agent(response.content, "workers.test.validator", { + metadata: { + criteria: + "Response should confirm the expense details and ask for receipt", + conversation: messages, + }, + }); +} +``` + +### Implementation + +The `t.assert.Agent()` method internally: + +1. Prepares `context.Options` with `test_mode: "validator"` +2. Calls the validator agent via `Assistant.Stream()` +3. Parses structured output (JSON) +4. Returns `ValidatorResult` + +```go +// In script_assert.go +func assertAgentMethod(iso *v8go.Isolate, t *TestingT, agentCtx *context.Context) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + // Parse arguments: response, agentID, options + response := args[0].String() + agentID := args[1].String() // Direct agent ID, e.g., "workers.test.validator" + options := parseOptions(args[2]) + + // Prepare validator options + validatorOpts := &context.Options{ + Skip: &context.Skip{History: true}, + Metadata: map[string]any{ + "test_mode": "validator", + ...options.Metadata, + }, + } + + // Call validator agent + assistant, _ := agent.Get(agentID) + result, _ := assistant.Stream(agentCtx, messages, validatorOpts) + + // Parse and return result + return toJsValue(parseValidatorResult(result)) + }) +} +``` + ## Test Case Format ### Single-Turn (Existing) diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index 2119ccd5..137c1e8f 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -55,6 +55,7 @@ ## Phase 6: Agent-Driven Assertions +### In JSONL Test Cases - [ ] Add `agent` assertion type to assertion parser - [ ] Support `options` field in assertion (aligned with `context.Options`) - [ ] Implement validator agent invocation via `Assistant.Stream()` @@ -62,7 +63,17 @@ - [ ] Pass conversation context and criteria in `options.metadata` - [ ] Support score-based pass/fail threshold (configurable in `options.metadata`) - [ ] Add `suggestions` to assertion error output + +### In Script Tests +- [ ] Add `t.assert.Agent(response, agentID, options?)` method +- [ ] `agentID` is direct ID (e.g., `workers.test.validator`), no prefix needed +- [ ] Invoke validator agent with context +- [ ] Return `ValidatorResult` object to JavaScript +- [ ] Support passing conversation history in options + +### Shared - [ ] Create example validator agent with prompt template +- [ ] Document `ValidatorResult` interface ## Phase 7: Error Handling & Reporting