diff --git a/agent/test/README.md b/agent/test/README.md index 2135f018..f7749f3b 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -607,6 +607,33 @@ For semantic or fuzzy validation using an LLM: The validator agent receives the output and criteria, then returns `{"passed": true/false, "reason": "..."}`. +**How it works:** + +1. The framework builds a validation request with the agent's response (including tool result messages) +2. The validator agent evaluates the response against the criteria +3. The validator returns a JSON response with `passed` and `reason` + +**Output in test report (for checkpoints):** + +```json +{ + "agent_validation": { + "passed": true, + "reason": "Response explicitly confirms setup completion", + "criteria": "Response should be friendly and helpful", + "input": "Hello! How can I help you today?", + "response": { + "passed": true, + "reason": "Response explicitly confirms setup completion" + } + } +} +``` + +- `input`: The content sent to the validator (agent response + tool result messages) +- `response`: The raw JSON response from the validator agent +- `criteria`: The validation criteria from the test case + ### Script Assertions For custom validation logic: @@ -1037,6 +1064,35 @@ interface TurnResult { } ``` +### Checkpoint Result Structure + +Each checkpoint in the output includes: + +```typescript +interface CheckpointResult { + id: string; // Checkpoint identifier + reached: boolean; // Whether checkpoint was reached + reached_at_turn?: number; // Turn number when reached (if reached) + required: boolean; // Whether checkpoint is required + passed: boolean; // Whether assertion passed + message?: string; // Assertion result message + agent_validation?: { + // Agent assertion details (for type: "agent") + passed: boolean; // Validator's determination + reason: string; // Explanation from validator + criteria: string; // Validation criteria checked + input: any; // Content sent to validator + response: { + // Raw validator response + passed: boolean; + reason: string; + }; + }; +} +``` + +**Note**: For agent-based assertions (`type: "agent"`), the `agent_validation` field provides full transparency into the validation process. The `input` field contains the combined output (agent text response + tool result messages) that was validated. + ## Output Formats Determined by `-o` file extension: diff --git a/agent/test/assert.go b/agent/test/assert.go index 505f0e62..ad0c4864 100644 --- a/agent/test/assert.go +++ b/agent/test/assert.go @@ -42,6 +42,45 @@ func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) { return true, "" } +// ValidateWithDetails validates the output and returns detailed results +// This is useful for agent assertions where we want to capture the validator's response +func (a *Asserter) ValidateWithDetails(tc *Case, output interface{}) *AssertionResult { + if tc.Assert == nil { + return &AssertionResult{Passed: true} + } + + assertions := a.parseAssertions(tc.Assert) + if len(assertions) == 0 { + return &AssertionResult{Passed: true} + } + + // For single assertion, return its full result + if len(assertions) == 1 { + return a.evaluateAssertion(assertions[0], output, tc.Input) + } + + // For multiple assertions, combine results + var failures []string + for _, assertion := range assertions { + result := a.evaluateAssertion(assertion, output, tc.Input) + if !result.Passed { + msg := result.Message + if assertion.Message != "" { + msg = assertion.Message + } + failures = append(failures, msg) + } + } + + if len(failures) > 0 { + return &AssertionResult{ + Passed: false, + Message: strings.Join(failures, "; "), + } + } + return &AssertionResult{Passed: true} +} + // validateAssertions validates output against assertion rules func (a *Asserter) validateAssertions(tc *Case, output interface{}) (bool, string) { assertions := a.parseAssertions(tc.Assert) diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go index 6530c7ad..6dc66e33 100644 --- a/agent/test/dynamic_runner.go +++ b/agent/test/dynamic_runner.go @@ -194,8 +194,8 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s // Add assistant response to messages, including tool calls if any messages = appendAssistantMessages(messages, response) - // Check checkpoints against this response - reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result) + // Check checkpoints against this response (including tool results) + reachedIDs := r.checkCheckpoints(tc.Checkpoints, response, result) turnResult.CheckpointsReached = reachedIDs if r.opts.Verbose && len(reachedIDs) > 0 { @@ -368,10 +368,15 @@ func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*Sim return output, nil } -// checkCheckpoints validates checkpoints against current output -func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string { +// checkCheckpoints validates checkpoints against current response +// It checks both the completion content and tool results for comprehensive validation +func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, response *context.Response, result *DynamicResult) []string { reachedIDs := make([]string, 0) + // Build combined output for checkpoint validation + // This includes both content and tool result messages + combinedOutput := buildCombinedOutput(response) + for _, cp := range checkpoints { cpResult := result.Checkpoints[cp.ID] if cpResult.Reached { @@ -394,22 +399,161 @@ func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output inter } } - // Validate using asserter + // Validate using asserter against combined output with full details tempCase := &Case{Assert: cp.Assert} - passed, msg := r.asserter.Validate(tempCase, output) + assertResult := r.asserter.ValidateWithDetails(tempCase, combinedOutput) - if passed { + if assertResult.Passed { cpResult.Reached = true cpResult.Passed = true cpResult.ReachedAtTurn = len(result.Turns) + 1 - cpResult.Message = msg + cpResult.Message = assertResult.Message reachedIDs = append(reachedIDs, cp.ID) } + + // Store agent validation details if this is an agent assertion + if isAgentAssertion(cp.Assert) { + // Extract criteria from assertion value + var criteria string + if assertMap, ok := cp.Assert.(map[string]interface{}); ok { + if c, ok := assertMap["value"].(string); ok { + criteria = c + } + } + + cpResult.AgentValidation = &AgentValidationResult{ + Passed: assertResult.Passed, + Criteria: criteria, + Input: combinedOutput, // Content sent to validator for checking + } + + // Extract reason and store full response from validator + if assertResult.Expected != nil { + if validatorResponse, ok := assertResult.Expected.(map[string]interface{}); ok { + if reason, ok := validatorResponse["reason"].(string); ok { + cpResult.AgentValidation.Reason = reason + } + // Store the full validator response + cpResult.AgentValidation.Response = validatorResponse + } + } + } } return reachedIDs } +// isAgentAssertion checks if the assertion is an agent-based assertion +func isAgentAssertion(assert interface{}) bool { + if assertMap, ok := assert.(map[string]interface{}); ok { + if assertType, ok := assertMap["type"].(string); ok { + return assertType == "agent" + } + } + return false +} + +// truncateForReport truncates content for report output +func truncateForReport(content interface{}, maxLen int) interface{} { + if content == nil { + return nil + } + + str, ok := content.(string) + if !ok { + return content + } + + if len(str) <= maxLen { + return str + } + + return str[:maxLen] + "... (truncated)" +} + +// buildCombinedOutput builds a combined output string from response +// that includes both completion content and tool result messages +func buildCombinedOutput(response *context.Response) string { + if response == nil { + return "" + } + + var parts []string + + // Add completion content + if response.Completion != nil && response.Completion.Content != nil { + if content := extractContentString(response.Completion.Content); content != "" { + parts = append(parts, content) + } + } + + // Add tool result messages + if len(response.Tools) > 0 { + for _, tool := range response.Tools { + if tool.Result != nil { + // Try to extract message from result + if resultMap, ok := tool.Result.(map[string]interface{}); ok { + if msg, exists := resultMap["message"]; exists && msg != nil { + if msgStr, ok := msg.(string); ok && msgStr != "" { + parts = append(parts, msgStr) + } + } + } + } + } + } + + // Join all parts with newline for comprehensive matching + return joinNonEmpty(parts, "\n") +} + +// extractContentString extracts string content from various types +func extractContentString(content interface{}) string { + if content == nil { + return "" + } + + switch v := content.(type) { + case string: + return v + case []interface{}: + // Handle array content (e.g., multimodal content) + var texts []string + for _, item := range v { + if m, ok := item.(map[string]interface{}); ok { + if text, ok := m["text"].(string); ok { + texts = append(texts, text) + } + } + } + return joinNonEmpty(texts, "\n") + default: + // Try to marshal to string + if data, err := jsoniter.MarshalToString(content); err == nil { + return data + } + return "" + } +} + +// joinNonEmpty joins non-empty strings with separator +func joinNonEmpty(parts []string, sep string) string { + var nonEmpty []string + for _, p := range parts { + if p != "" { + nonEmpty = append(nonEmpty, p) + } + } + if len(nonEmpty) == 0 { + return "" + } + result := nonEmpty[0] + for i := 1; i < len(nonEmpty); i++ { + result += sep + nonEmpty[i] + } + return result +} + // allRequiredCheckpointsReached checks if all required checkpoints are reached func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool { for _, cp := range result.Checkpoints { diff --git a/agent/test/dynamic_types.go b/agent/test/dynamic_types.go index 27cb0634..ea215779 100644 --- a/agent/test/dynamic_types.go +++ b/agent/test/dynamic_types.go @@ -93,6 +93,27 @@ type CheckpointResult struct { // Message contains assertion result message Message string `json:"message,omitempty"` + + // AgentValidation contains the agent validator's response (for agent assertions) + AgentValidation *AgentValidationResult `json:"agent_validation,omitempty"` +} + +// AgentValidationResult contains the result from an agent-based assertion +type AgentValidationResult struct { + // Passed indicates if the agent validator determined the assertion passed + Passed bool `json:"passed"` + + // Reason is the explanation from the agent validator + Reason string `json:"reason,omitempty"` + + // Criteria is the validation criteria that was checked + Criteria string `json:"criteria,omitempty"` + + // Input is the content that was sent to validator for checking + Input interface{} `json:"input,omitempty"` + + // Response is the raw response from the validator agent + Response interface{} `json:"response,omitempty"` } // SimulatorInput is the input sent to the simulator agent