diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index be1a2b9a..9b956550 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -990,15 +990,16 @@ Existing single-turn tests work unchanged: ## Current Implementation Status -| Feature | Status | Notes | -| ----------------------- | ---------- | ---------------------------------------- | -| Simple text input | ✅ Done | `input: "Hello"` | -| Message history | ✅ Done | `input: [{role, content}, ...]` | -| File attachments | ✅ Done | `file://` protocol in content parts | -| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | -| Agent-driven assertions | 🔲 Planned | `type: "agent"` with validator agent | -| Dynamic mode | 🔲 Planned | Simulator + Checkpoints | -| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | +| Feature | Status | Notes | +| ----------------------- | ---------- | -------------------------------------------------- | +| Simple text input | ✅ Done | `input: "Hello"` | +| Message history | ✅ Done | `input: [{role, content}, ...]` | +| File attachments | ✅ Done | `file://` protocol in content parts | +| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | +| Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | +| Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | +| Dynamic mode | 🔲 Planned | Simulator + Checkpoints | +| Agent-driven input | 🔲 Planned | `-i agents:xxx` for test generation | ## Open Questions diff --git a/agent/test/TODO_V2.md b/agent/test/TODO_V2.md index aa1fb513..8179ffc4 100644 --- a/agent/test/TODO_V2.md +++ b/agent/test/TODO_V2.md @@ -29,15 +29,17 @@ - [x] 创建示例脚本 `assistants/tests/hooks-test/src/env_test.ts` - [x] 创建单元测试 `script_hooks_test.go` (黑盒测试) -## Phase 2: Agent-Driven Assertions +## Phase 2: Agent-Driven Assertions ✅ **修改文件**: `assert.go`, `script_assert.go` -- [ ] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion` -- [ ] `assert.go`: 实现 `assertAgent` 方法 -- [ ] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型 -- [ ] `script_assert.go`: 添加 `AssertAgent` 方法到 `TestingT` -- [ ] 创建示例 validator agent +- [x] `types.go`: 添加 `Use`, `Options` 字段到 `Assertion` +- [x] `assert.go`: 实现 `assertAgent` 方法 +- [x] `assert.go`: 在 `evaluateAssertion` 添加 `agent` 类型 +- [x] `assert.go`: 使用 `goutext.ExtractJSON` 容错解析 LLM 响应 +- [x] `script_assert.go`: 添加 `assertAgentMethod` 到 `newAssertObject` +- [x] 创建示例 validator agent (`assistants/tests/validator-agent`) +- [x] 创建单元测试 `assert_agent_test.go` (JSONL 断言 + JSAPI 断言) ## Phase 3: Dynamic Mode (Simulator + Checkpoints) @@ -82,6 +84,7 @@ - [x] `-v` verbose mode - [x] Script testing (`*_test.ts`) - [x] Before/After hooks (Phase 1) +- [x] Agent-driven assertions (Phase 2) ## Open Questions diff --git a/agent/test/assert.go b/agent/test/assert.go index fc3c97e7..505f0e62 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -9,6 +9,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/process" + goutext "github.com/yaoapp/gou/text" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" ) // Asserter handles test assertions @@ -115,6 +118,9 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion { if s, ok := m["script"].(string); ok { assertion.Script = s } + if u, ok := m["use"].(string); ok { + assertion.Use = u + } if msg, ok := m["message"].(string); ok { assertion.Message = msg } @@ -122,6 +128,17 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion { assertion.Negate = n } + // Parse options for agent assertions + if opts, ok := m["options"].(map[string]interface{}); ok { + assertion.Options = &AssertionOptions{} + if c, ok := opts["connector"].(string); ok { + assertion.Options.Connector = c + } + if meta, ok := opts["metadata"].(map[string]interface{}); ok { + assertion.Options.Metadata = meta + } + } + return assertion } @@ -147,6 +164,8 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa result = a.assertType(assertion, output) case "script": result = a.assertScript(assertion, output, input) + case "agent": + result = a.assertAgent(assertion, output, input) default: result.Passed = false result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type) @@ -229,17 +248,14 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass var jsonData interface{} switch v := output.(type) { case string: - // Try to parse as JSON - if err := jsoniter.Unmarshal([]byte(v), &jsonData); err != nil { - // Try to extract JSON from markdown code blocks - extracted := extractJSONFromText(v) - if extracted != nil { - jsonData = extracted - } else { - result.Passed = false - result.Message = fmt.Sprintf("output is not valid JSON: %s", err.Error()) - return result - } + // Use gou/text to extract JSON (handles markdown, auto-repair, etc.) + extracted := goutext.ExtractJSON(v) + if extracted != nil { + jsonData = extracted + } else { + result.Passed = false + result.Message = fmt.Sprintf("output is not valid JSON: %s", v) + return result } case map[string]interface{}, []interface{}: jsonData = v @@ -478,6 +494,158 @@ func (a *Asserter) getType(v interface{}) string { } } +// assertAgent uses an agent to validate the output +func (a *Asserter) assertAgent(assertion *Assertion, output, input interface{}) *AssertionResult { + result := &AssertionResult{ + Assertion: assertion, + Actual: output, + } + + // Parse use field: "agents:tests.validator-agent" + if !strings.HasPrefix(assertion.Use, "agents:") { + result.Passed = false + result.Message = "agent assertion requires 'use' field with 'agents:' prefix" + return result + } + + agentID := strings.TrimPrefix(assertion.Use, "agents:") + + // Get assistant + ast, err := assistant.Get(agentID) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("failed to get validator agent: %s", err.Error()) + return result + } + + // Build validation request + validationInput := map[string]interface{}{ + "output": output, + "input": input, + } + + // Add criteria from Value field + if assertion.Value != nil { + validationInput["criteria"] = assertion.Value + } + + // Add metadata from options + if assertion.Options != nil && assertion.Options.Metadata != nil { + for k, v := range assertion.Options.Metadata { + validationInput[k] = v + } + } + + // Build context options - skip history and trace for validator + opts := &context.Options{ + Skip: &context.Skip{ + History: true, + Trace: true, + Output: true, + }, + Metadata: map[string]interface{}{ + "test_mode": "validator", + }, + } + if assertion.Options != nil && assertion.Options.Connector != "" { + opts.Connector = assertion.Options.Connector + } + + // Create context and call agent + env := NewEnvironment("", "") + ctx := NewTestContext("validator", agentID, env) + defer ctx.Release() + + // Convert validation input to JSON string for the message + inputJSON, err := json.Marshal(validationInput) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error()) + return result + } + + messages := []context.Message{{ + Role: context.RoleUser, + Content: string(inputJSON), + }} + + response, err := ast.Stream(ctx, messages, opts) + if err != nil { + result.Passed = false + result.Message = fmt.Sprintf("validator agent error: %s", err.Error()) + return result + } + + // Parse response + return a.parseValidatorResponse(response, result) +} + +// parseValidatorResponse parses the validator agent's response +func (a *Asserter) parseValidatorResponse(response *context.Response, result *AssertionResult) *AssertionResult { + output := extractValidatorOutput(response) + + // Expected format: { "passed": bool, "reason": string, "score": float, "suggestions": [] } + if outputMap, ok := output.(map[string]interface{}); ok { + if passed, ok := outputMap["passed"].(bool); ok { + result.Passed = passed + } else { + result.Passed = false + result.Message = "validator response missing 'passed' field" + return result + } + if reason, ok := outputMap["reason"].(string); ok { + result.Message = reason + } + // Store score and suggestions in expected field for reference + result.Expected = outputMap + } else { + result.Passed = false + result.Message = "validator agent returned invalid response format" + } + + return result +} + +// extractValidatorOutput extracts the output from a validator response +func extractValidatorOutput(response *context.Response) interface{} { + if response == nil || response.Completion == nil { + return nil + } + + // Get content from completion + content := response.Completion.Content + if content == nil { + return nil + } + + // Try to get text content + var text string + switch v := content.(type) { + case string: + text = v + default: + // Try to marshal and use as-is + data, err := json.Marshal(content) + if err != nil { + return nil + } + text = string(data) + } + + if text == "" { + return nil + } + + // Use gou/text to extract JSON (handles markdown code blocks, auto-repair, etc.) + result := goutext.ExtractJSON(text) + if result != nil { + return result + } + + // Return raw text if extraction fails + return text +} + // assertScript runs a custom assertion script func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) *AssertionResult { result := &AssertionResult{ @@ -559,30 +727,3 @@ func (a *Asserter) toString(v interface{}) string { return string(b) } } - -// extractJSONFromText tries to extract JSON from text (e.g., markdown code blocks) -func extractJSONFromText(text string) interface{} { - // Try to find JSON in code blocks - patterns := []string{ - "```json\n", - "```\n", - } - - for _, start := range patterns { - if idx := strings.Index(text, start); idx >= 0 { - text = text[idx+len(start):] - if endIdx := strings.Index(text, "```"); endIdx >= 0 { - text = text[:endIdx] - } - break - } - } - - // Try to parse - var result interface{} - if err := jsoniter.Unmarshal([]byte(strings.TrimSpace(text)), &result); err == nil { - return result - } - - return nil -} diff --git a/agent/test/assert_agent_test.go b/agent/test/assert_agent_test.go new file mode 100644 index 00000000..20dbc4c9 --- /dev/null +++ b/agent/test/assert_agent_test.go @@ -0,0 +1,275 @@ +package test_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/agent" + agenttest "github.com/yaoapp/yao/agent/test" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" + "rogchap.com/v8go" +) + +func TestAsserter_AgentAssertion(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + tests := []struct { + name string + tc *agenttest.Case + output interface{} + expected bool + skipMsg string + }{ + { + name: "agent assertion - pass", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should be a greeting", + }, + }, + output: "Hello! How can I help you today?", + expected: true, + }, + { + name: "agent assertion - fail", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response should provide a detailed technical answer", + }, + }, + output: "I don't know.", + expected: false, + }, + { + name: "agent assertion - missing prefix", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "tests.validator-agent", // Missing agents: prefix + "value": "Should pass", + }, + }, + output: "Hello", + expected: false, // Should fail due to missing prefix + }, + { + name: "agent assertion - with metadata", + tc: &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "Response is helpful", + "options": map[string]interface{}{ + "metadata": map[string]interface{}{ + "context": "customer support", + }, + }, + }, + }, + output: "I'd be happy to help you with your order. Let me look that up for you.", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.skipMsg != "" { + t.Skip(tt.skipMsg) + } + + passed, errMsg := asserter.Validate(tt.tc, tt.output) + if passed != tt.expected { + t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg) + } + }) + } +} + +func TestAsserter_AgentAssertion_InvalidAgent(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + tc := &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:nonexistent.agent", + "value": "Should fail", + }, + } + + passed, errMsg := asserter.Validate(tc, "Hello") + assert.False(t, passed, "Should fail for nonexistent agent") + assert.Contains(t, errMsg, "failed to get validator agent", "Error should mention agent loading failure") +} + +func TestAsserter_MapToAssertion_WithUseAndOptions(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + asserter := agenttest.NewAsserter() + + // Test that mapToAssertion correctly parses use and options fields + tc := &agenttest.Case{ + Assert: map[string]interface{}{ + "type": "agent", + "use": "agents:tests.validator-agent", + "value": "criteria here", + "options": map[string]interface{}{ + "connector": "gpt-4o", + "metadata": map[string]interface{}{ + "key": "value", + }, + }, + }, + } + + // Validate triggers parseAssertions internally + // We just verify it doesn't panic and processes correctly + _, _ = asserter.Validate(tc, "test output") + // If we get here without panic, the parsing worked +} + +// TestTestingT_AssertAgent tests the JSAPI t.assert.Agent() method +func TestTestingT_AssertAgent(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Load agent (includes assistants) + err := agent.Load(config.Conf) + if err != nil { + t.Fatalf("Failed to load agent: %v", err) + } + + tests := []struct { + name string + script string + shouldFail bool + }{ + { + name: "JSAPI agent assertion - pass", + script: ` + function test(t) { + var response = "Hello! How can I help you today?"; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be a friendly greeting" + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - JSON response", + script: ` + function test(t) { + var response = { + status: "success", + data: { user: "john", email: "john@example.com" }, + message: "User created successfully" + }; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should be a successful API response with user data" + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - with metadata", + script: ` + function test(t) { + var response = "I'd be happy to help you with your order."; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response is helpful and professional", + metadata: { context: "customer support" } + }); + } + test(__test_t); + `, + shouldFail: false, + }, + { + name: "JSAPI agent assertion - fail case", + script: ` + function test(t) { + var response = "I don't know."; + t.assert.Agent(response, "tests.validator-agent", { + criteria: "Response should provide a detailed technical explanation" + }); + } + test(__test_t); + `, + shouldFail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create TestingT + testingT := agenttest.NewTestingT(tt.name) + + // Create V8 isolate and context + iso := v8go.NewIsolate() + defer iso.Dispose() + + v8ctx := v8go.NewContext(iso) + defer v8ctx.Close() + + // Create testing object + testObj, err := agenttest.NewTestingTObject(v8ctx, testingT) + if err != nil { + t.Fatalf("Failed to create testing object: %v", err) + } + + // Set testing object as global + global := v8ctx.Global() + global.Set("__test_t", testObj) + + // Run the test script + _, err = v8ctx.RunScript(tt.script, "test.js") + + // Check results + if tt.shouldFail { + assert.True(t, testingT.Failed(), "Test should have failed") + } else { + if err != nil { + t.Errorf("Script execution error: %v", err) + } + assert.False(t, testingT.Failed(), "Test should have passed, errors: %v", testingT.Errors()) + } + }) + } +} + +// Ensure v8 is used (for script loading) +var _ = v8.Scripts diff --git a/agent/test/script_assert.go b/agent/test/script_assert.go index c0cc0245..c2c22ffd 100644 --- a/agent/test/script_assert.go +++ b/agent/test/script_assert.go @@ -261,6 +261,9 @@ func newAssertObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) { // JSON path assertion assertObj.Set("JSONPath", assertJSONPathMethod(iso, t)) + // Agent-driven assertion + assertObj.Set("Agent", assertAgentMethod(iso, t)) + // Create instance instance, err := assertObj.NewInstance(v8ctx) if err != nil { @@ -924,6 +927,70 @@ func assertJSONPathMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate }) } +// assertAgentMethod implements assert.Agent(response, agentID, options?) +// Uses a validator agent to check the response +// agentID is the direct agent ID (no "agents:" prefix needed) +func assertAgentMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate { + return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + v8ctx := info.Context() + args := info.Args() + if len(args) < 2 { + t.fail("Agent requires response and agentID arguments", &ScriptAssertionInfo{Type: "Agent"}) + return v8go.Undefined(iso) + } + + response, _ := bridge.GoValue(args[0], v8ctx) + agentID := args[1].String() + + // Get options if provided + var options map[string]interface{} + if len(args) > 2 && args[2].IsObject() { + optVal, _ := bridge.GoValue(args[2], v8ctx) + options, _ = optVal.(map[string]interface{}) + } + + // Build assertion with agents: prefix + assertion := &Assertion{ + Type: "agent", + Use: "agents:" + agentID, + } + + // Extract criteria and metadata from options + if options != nil { + if criteria, ok := options["criteria"]; ok { + assertion.Value = criteria + } + if metadata, ok := options["metadata"].(map[string]interface{}); ok { + assertion.Options = &AssertionOptions{Metadata: metadata} + } + if connector, ok := options["connector"].(string); ok { + if assertion.Options == nil { + assertion.Options = &AssertionOptions{} + } + assertion.Options.Connector = connector + } + } + + // Use the asserter to validate + asserter := &Asserter{} + result := asserter.assertAgent(assertion, response, nil) + + if !result.Passed { + msg := result.Message + if msg == "" { + msg = "agent assertion failed" + } + t.fail(msg, &ScriptAssertionInfo{ + Type: "Agent", + Actual: response, + Message: msg, + }) + } + + return v8go.Undefined(iso) + }) +} + // Helper functions // deepEqual performs deep equality comparison diff --git a/agent/test/types.go b/agent/test/types.go index f74a0307..75387879 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -436,6 +436,7 @@ type Assertion struct { // - "script": run a custom assertion script // - "type": check output type (string, object, array, number, boolean) // - "schema": validate against JSON schema + // - "agent": use an agent to validate the response Type string `json:"type"` // Value is the expected value or pattern (depends on type) @@ -448,6 +449,14 @@ type Assertion struct { // The script receives (output, input, expected) and returns {pass: bool, message: string} Script string `json:"script,omitempty"` + // Use specifies the agent/script for validation + // For agent assertions: "agents:tests.validator-agent" (with prefix) + // For script assertions: "scripts:tests.validate" (with prefix) + Use string `json:"use,omitempty"` + + // Options for agent-driven assertions (aligned with context.Options) + Options *AssertionOptions `json:"options,omitempty"` + // Message is a custom failure message Message string `json:"message,omitempty"` @@ -455,6 +464,15 @@ type Assertion struct { Negate bool `json:"negate,omitempty"` } +// AssertionOptions for agent-driven assertions +type AssertionOptions struct { + // Connector overrides the agent's default connector + Connector string `json:"connector,omitempty"` + + // Metadata contains custom data passed to the validator agent + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + // AssertionResult represents the result of an assertion type AssertionResult struct { // Passed indicates whether the assertion passed