- Added support for `tool_called` and `tool_result` assertions in the Asserter, allowing for validation of tool execution and arguments. - Introduced methods to check if specific tools were called and to validate their results against expected patterns. - Updated the `README.md` to include detailed documentation on new assertion types, including usage examples and value formats. - Enhanced the dynamic runner to set the response for tool-related assertions, improving the overall testing framework's capabilities.
1043 lines
26 KiB
Go
1043 lines
26 KiB
Go
package test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
|
|
jsoniter "github.com/json-iterator/go"
|
|
"github.com/yaoapp/gou/process"
|
|
goutext "github.com/yaoapp/gou/text"
|
|
"github.com/yaoapp/yao/agent/assistant"
|
|
"github.com/yaoapp/yao/agent/context"
|
|
)
|
|
|
|
// Asserter handles test assertions
|
|
type Asserter struct {
|
|
// response holds the current response for tool-related assertions
|
|
response *context.Response
|
|
}
|
|
|
|
// NewAsserter creates a new asserter
|
|
func NewAsserter() *Asserter {
|
|
return &Asserter{}
|
|
}
|
|
|
|
// WithResponse sets the response for tool-related assertions
|
|
func (a *Asserter) WithResponse(response *context.Response) *Asserter {
|
|
a.response = response
|
|
return a
|
|
}
|
|
|
|
// Validate validates the output against the test case's assertions
|
|
// Returns (passed, error message)
|
|
func (a *Asserter) Validate(tc *Case, output interface{}) (bool, string) {
|
|
// If assert is defined, use assertion rules
|
|
if tc.Assert != nil {
|
|
return a.validateAssertions(tc, output)
|
|
}
|
|
|
|
// If expected is defined, use simple comparison
|
|
if tc.Expected != nil {
|
|
if validateOutput(output, tc.Expected) {
|
|
return true, ""
|
|
}
|
|
return false, "output does not match expected"
|
|
}
|
|
|
|
// No assertions defined - pass if we got output without error
|
|
return true, ""
|
|
}
|
|
|
|
// ValidateWithDetails validates the output and returns detailed results
|
|
// This is useful for agent assertions where we want to capture the validator's response
|
|
func (a *Asserter) ValidateWithDetails(tc *Case, output interface{}) *AssertionResult {
|
|
if tc.Assert == nil {
|
|
return &AssertionResult{Passed: true}
|
|
}
|
|
|
|
assertions := a.parseAssertions(tc.Assert)
|
|
if len(assertions) == 0 {
|
|
return &AssertionResult{Passed: true}
|
|
}
|
|
|
|
// For single assertion, return its full result
|
|
if len(assertions) == 1 {
|
|
return a.evaluateAssertion(assertions[0], output, tc.Input)
|
|
}
|
|
|
|
// For multiple assertions, combine results
|
|
var failures []string
|
|
for _, assertion := range assertions {
|
|
result := a.evaluateAssertion(assertion, output, tc.Input)
|
|
if !result.Passed {
|
|
msg := result.Message
|
|
if assertion.Message != "" {
|
|
msg = assertion.Message
|
|
}
|
|
failures = append(failures, msg)
|
|
}
|
|
}
|
|
|
|
if len(failures) > 0 {
|
|
return &AssertionResult{
|
|
Passed: false,
|
|
Message: strings.Join(failures, "; "),
|
|
}
|
|
}
|
|
return &AssertionResult{Passed: true}
|
|
}
|
|
|
|
// validateAssertions validates output against assertion rules
|
|
func (a *Asserter) validateAssertions(tc *Case, output interface{}) (bool, string) {
|
|
assertions := a.parseAssertions(tc.Assert)
|
|
if len(assertions) == 0 {
|
|
return true, ""
|
|
}
|
|
|
|
var failures []string
|
|
for _, assertion := range assertions {
|
|
result := a.evaluateAssertion(assertion, output, tc.Input)
|
|
if !result.Passed {
|
|
msg := result.Message
|
|
if assertion.Message != "" {
|
|
msg = assertion.Message
|
|
}
|
|
failures = append(failures, msg)
|
|
}
|
|
}
|
|
|
|
if len(failures) > 0 {
|
|
return false, strings.Join(failures, "; ")
|
|
}
|
|
return true, ""
|
|
}
|
|
|
|
// parseAssertions parses the assert field into a list of assertions
|
|
func (a *Asserter) parseAssertions(assert interface{}) []*Assertion {
|
|
if assert == nil {
|
|
return nil
|
|
}
|
|
|
|
var assertions []*Assertion
|
|
|
|
switch v := assert.(type) {
|
|
case map[string]interface{}:
|
|
// Single assertion object
|
|
assertion := a.mapToAssertion(v)
|
|
if assertion != nil {
|
|
assertions = append(assertions, assertion)
|
|
}
|
|
|
|
case []interface{}:
|
|
// Array of assertions
|
|
for _, item := range v {
|
|
if m, ok := item.(map[string]interface{}); ok {
|
|
assertion := a.mapToAssertion(m)
|
|
if assertion != nil {
|
|
assertions = append(assertions, assertion)
|
|
}
|
|
}
|
|
}
|
|
|
|
case string:
|
|
// Shorthand: just a type name (e.g., "contains")
|
|
assertions = append(assertions, &Assertion{Type: v})
|
|
}
|
|
|
|
return assertions
|
|
}
|
|
|
|
// mapToAssertion converts a map to an Assertion
|
|
func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion {
|
|
assertion := &Assertion{}
|
|
|
|
if t, ok := m["type"].(string); ok {
|
|
assertion.Type = t
|
|
}
|
|
if v, ok := m["value"]; ok {
|
|
assertion.Value = v
|
|
}
|
|
if p, ok := m["path"].(string); ok {
|
|
assertion.Path = p
|
|
}
|
|
if s, ok := m["script"].(string); ok {
|
|
assertion.Script = s
|
|
}
|
|
if u, ok := m["use"].(string); ok {
|
|
assertion.Use = u
|
|
}
|
|
if msg, ok := m["message"].(string); ok {
|
|
assertion.Message = msg
|
|
}
|
|
if n, ok := m["negate"].(bool); ok {
|
|
assertion.Negate = n
|
|
}
|
|
|
|
// Parse options for agent assertions
|
|
if opts, ok := m["options"].(map[string]interface{}); ok {
|
|
assertion.Options = &AssertionOptions{}
|
|
if c, ok := opts["connector"].(string); ok {
|
|
assertion.Options.Connector = c
|
|
}
|
|
if meta, ok := opts["metadata"].(map[string]interface{}); ok {
|
|
assertion.Options.Metadata = meta
|
|
}
|
|
}
|
|
|
|
return assertion
|
|
}
|
|
|
|
// evaluateAssertion evaluates a single assertion
|
|
func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
switch assertion.Type {
|
|
case "equals", "":
|
|
result = a.assertEquals(assertion, output)
|
|
case "contains":
|
|
result = a.assertContains(assertion, output)
|
|
case "not_contains":
|
|
result = a.assertNotContains(assertion, output)
|
|
case "json_path":
|
|
result = a.assertJSONPath(assertion, output)
|
|
case "regex":
|
|
result = a.assertRegex(assertion, output)
|
|
case "type":
|
|
result = a.assertType(assertion, output)
|
|
case "script":
|
|
result = a.assertScript(assertion, output, input)
|
|
case "agent":
|
|
result = a.assertAgent(assertion, output, input)
|
|
case "tool_called":
|
|
result = a.assertToolCalled(assertion)
|
|
case "tool_result":
|
|
result = a.assertToolResult(assertion)
|
|
default:
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type)
|
|
}
|
|
|
|
// Apply negate
|
|
if assertion.Negate {
|
|
result.Passed = !result.Passed
|
|
if result.Passed {
|
|
result.Message = "negated assertion passed"
|
|
} else {
|
|
result.Message = "negated: " + result.Message
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// assertEquals checks for exact equality
|
|
func (a *Asserter) assertEquals(assertion *Assertion, output interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Actual: output,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
if validateOutput(output, assertion.Value) {
|
|
result.Passed = true
|
|
result.Message = "values are equal"
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("expected %v, got %v", assertion.Value, output)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// assertContains checks if output contains the expected value
|
|
func (a *Asserter) assertContains(assertion *Assertion, output interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Actual: output,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
outputStr := a.toString(output)
|
|
expectedStr := a.toString(assertion.Value)
|
|
|
|
if strings.Contains(outputStr, expectedStr) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("output contains '%s'", expectedStr)
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("output does not contain '%s'", expectedStr)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// assertNotContains checks if output does not contain the expected value
|
|
func (a *Asserter) assertNotContains(assertion *Assertion, output interface{}) *AssertionResult {
|
|
result := a.assertContains(assertion, output)
|
|
result.Passed = !result.Passed
|
|
if result.Passed {
|
|
result.Message = fmt.Sprintf("output does not contain '%s'", a.toString(assertion.Value))
|
|
} else {
|
|
result.Message = fmt.Sprintf("output should not contain '%s'", a.toString(assertion.Value))
|
|
}
|
|
return result
|
|
}
|
|
|
|
// assertJSONPath extracts a value using JSON path and compares
|
|
func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
// Convert output to JSON if needed
|
|
var jsonData interface{}
|
|
switch v := output.(type) {
|
|
case string:
|
|
// Use gou/text to extract JSON (handles markdown, auto-repair, etc.)
|
|
extracted := goutext.ExtractJSON(v)
|
|
if extracted != nil {
|
|
jsonData = extracted
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("output is not valid JSON: %s", v)
|
|
return result
|
|
}
|
|
case map[string]interface{}, []interface{}:
|
|
jsonData = v
|
|
default:
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("output is not a JSON object or array, got: %T = %v", output, truncateOutput(output, 200))
|
|
return result
|
|
}
|
|
|
|
// Extract value using simple path (e.g., "$.need_search" or "need_search")
|
|
path := strings.TrimPrefix(assertion.Path, "$.")
|
|
actual := a.extractPath(jsonData, path)
|
|
result.Actual = actual
|
|
|
|
// Compare expected value with actual value
|
|
// First try direct comparison (handles array-to-array comparison)
|
|
if validateOutput(actual, assertion.Value) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("path '%s' equals expected value", assertion.Path)
|
|
return result
|
|
}
|
|
|
|
// If expected is an array and direct comparison failed, check if actual matches ANY element (IN semantics)
|
|
// This is for cases like: expected: ["a", "b"], actual: "a" (actual is one of expected)
|
|
if expectedArr, ok := assertion.Value.([]interface{}); ok {
|
|
// Only apply IN semantics if actual is NOT an array (otherwise it was already compared above)
|
|
if _, actualIsArr := actual.([]interface{}); !actualIsArr {
|
|
for _, expectedItem := range expectedArr {
|
|
if validateOutput(actual, expectedItem) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("path '%s' equals one of expected values", assertion.Path)
|
|
return result
|
|
}
|
|
}
|
|
}
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("path '%s': expected %v, got %v", assertion.Path, assertion.Value, actual)
|
|
} else {
|
|
// Direct comparison already failed above
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("path '%s': expected %v, got %v", assertion.Path, assertion.Value, actual)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// truncateOutput truncates output for error messages
|
|
func truncateOutput(output interface{}, maxLen int) string {
|
|
var s string
|
|
switch v := output.(type) {
|
|
case string:
|
|
s = v
|
|
case nil:
|
|
return "<nil>"
|
|
default:
|
|
bytes, err := jsoniter.Marshal(v)
|
|
if err != nil {
|
|
s = fmt.Sprintf("%v", v)
|
|
} else {
|
|
s = string(bytes)
|
|
}
|
|
}
|
|
|
|
if len(s) > maxLen {
|
|
return s[:maxLen] + "..."
|
|
}
|
|
return s
|
|
}
|
|
|
|
// extractPath extracts a value from JSON using dot-notation path with array index support
|
|
// Supports: "field", "field.nested", "field[0]", "field[0].nested", "field.nested[0].value"
|
|
func (a *Asserter) extractPath(data interface{}, path string) interface{} {
|
|
current := data
|
|
|
|
// Parse path into segments, handling both dots and array indices
|
|
// e.g., "wheres[0].like" -> ["wheres", "[0]", "like"]
|
|
segments := parsePathSegments(path)
|
|
|
|
for _, segment := range segments {
|
|
if segment == "" {
|
|
continue
|
|
}
|
|
|
|
// Check if this is an array index like "[0]"
|
|
if strings.HasPrefix(segment, "[") && strings.HasSuffix(segment, "]") {
|
|
indexStr := segment[1 : len(segment)-1]
|
|
index, err := strconv.Atoi(indexStr)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
arr, ok := current.([]interface{})
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
if index < 0 || index >= len(arr) {
|
|
return nil
|
|
}
|
|
current = arr[index]
|
|
} else {
|
|
// Regular field access
|
|
switch v := current.(type) {
|
|
case map[string]interface{}:
|
|
current = v[segment]
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
return current
|
|
}
|
|
|
|
// parsePathSegments splits a path like "wheres[0].like" into ["wheres", "[0]", "like"]
|
|
func parsePathSegments(path string) []string {
|
|
var segments []string
|
|
var current strings.Builder
|
|
|
|
for i := 0; i < len(path); i++ {
|
|
ch := path[i]
|
|
switch ch {
|
|
case '.':
|
|
if current.Len() > 0 {
|
|
segments = append(segments, current.String())
|
|
current.Reset()
|
|
}
|
|
case '[':
|
|
if current.Len() > 0 {
|
|
segments = append(segments, current.String())
|
|
current.Reset()
|
|
}
|
|
// Find the closing bracket
|
|
j := i + 1
|
|
for j < len(path) && path[j] != ']' {
|
|
j++
|
|
}
|
|
if j < len(path) {
|
|
segments = append(segments, path[i:j+1]) // Include "[" and "]"
|
|
i = j
|
|
}
|
|
default:
|
|
current.WriteByte(ch)
|
|
}
|
|
}
|
|
|
|
if current.Len() > 0 {
|
|
segments = append(segments, current.String())
|
|
}
|
|
|
|
return segments
|
|
}
|
|
|
|
// assertRegex checks if output matches a regex pattern
|
|
func (a *Asserter) assertRegex(assertion *Assertion, output interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Actual: output,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
pattern, ok := assertion.Value.(string)
|
|
if !ok {
|
|
result.Passed = false
|
|
result.Message = "regex pattern must be a string"
|
|
return result
|
|
}
|
|
|
|
re, err := regexp.Compile(pattern)
|
|
if err != nil {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("invalid regex pattern: %s", err.Error())
|
|
return result
|
|
}
|
|
|
|
outputStr := a.toString(output)
|
|
if re.MatchString(outputStr) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("output matches pattern '%s'", pattern)
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("output does not match pattern '%s'", pattern)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// assertType checks the type of the output
|
|
func (a *Asserter) assertType(assertion *Assertion, output interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Actual: output,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
expectedType, ok := assertion.Value.(string)
|
|
if !ok {
|
|
result.Passed = false
|
|
result.Message = "type assertion value must be a string"
|
|
return result
|
|
}
|
|
|
|
actualType := a.getType(output)
|
|
result.Actual = actualType
|
|
|
|
if actualType == expectedType {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("output is of type '%s'", expectedType)
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("expected type '%s', got '%s'", expectedType, actualType)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// getType returns the type name of a value
|
|
func (a *Asserter) getType(v interface{}) string {
|
|
if v == nil {
|
|
return "null"
|
|
}
|
|
|
|
switch v.(type) {
|
|
case string:
|
|
return "string"
|
|
case float64, float32, int, int64, int32:
|
|
return "number"
|
|
case bool:
|
|
return "boolean"
|
|
case []interface{}:
|
|
return "array"
|
|
case map[string]interface{}:
|
|
return "object"
|
|
default:
|
|
return fmt.Sprintf("%T", v)
|
|
}
|
|
}
|
|
|
|
// assertAgent uses an agent to validate the output
|
|
func (a *Asserter) assertAgent(assertion *Assertion, output, input interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Actual: output,
|
|
}
|
|
|
|
// Parse use field: "agents:tests.validator-agent"
|
|
if !strings.HasPrefix(assertion.Use, "agents:") {
|
|
result.Passed = false
|
|
result.Message = "agent assertion requires 'use' field with 'agents:' prefix"
|
|
return result
|
|
}
|
|
|
|
agentID := strings.TrimPrefix(assertion.Use, "agents:")
|
|
|
|
// Get assistant
|
|
ast, err := assistant.Get(agentID)
|
|
if err != nil {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("failed to get validator agent: %s", err.Error())
|
|
return result
|
|
}
|
|
|
|
// Build validation request
|
|
validationInput := map[string]interface{}{
|
|
"output": output,
|
|
"input": input,
|
|
}
|
|
|
|
// Add criteria from Value field
|
|
if assertion.Value != nil {
|
|
validationInput["criteria"] = assertion.Value
|
|
}
|
|
|
|
// Add metadata from options
|
|
if assertion.Options != nil && assertion.Options.Metadata != nil {
|
|
for k, v := range assertion.Options.Metadata {
|
|
validationInput[k] = v
|
|
}
|
|
}
|
|
|
|
// Build context options - skip history and trace for validator
|
|
opts := &context.Options{
|
|
Skip: &context.Skip{
|
|
History: true,
|
|
Trace: true,
|
|
Output: true,
|
|
},
|
|
Metadata: map[string]interface{}{
|
|
"test_mode": "validator",
|
|
},
|
|
}
|
|
if assertion.Options != nil && assertion.Options.Connector != "" {
|
|
opts.Connector = assertion.Options.Connector
|
|
}
|
|
|
|
// Create context and call agent
|
|
env := NewEnvironment("", "")
|
|
ctx := NewTestContext("validator", agentID, env)
|
|
defer ctx.Release()
|
|
|
|
// Convert validation input to JSON string for the message
|
|
inputJSON, err := json.Marshal(validationInput)
|
|
if err != nil {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error())
|
|
return result
|
|
}
|
|
|
|
messages := []context.Message{{
|
|
Role: context.RoleUser,
|
|
Content: string(inputJSON),
|
|
}}
|
|
|
|
response, err := ast.Stream(ctx, messages, opts)
|
|
if err != nil {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("validator agent error: %s", err.Error())
|
|
return result
|
|
}
|
|
|
|
// Parse response
|
|
return a.parseValidatorResponse(response, result)
|
|
}
|
|
|
|
// parseValidatorResponse parses the validator agent's response
|
|
func (a *Asserter) parseValidatorResponse(response *context.Response, result *AssertionResult) *AssertionResult {
|
|
output := extractValidatorOutput(response)
|
|
|
|
// Expected format: { "passed": bool, "reason": string, "score": float, "suggestions": [] }
|
|
if outputMap, ok := output.(map[string]interface{}); ok {
|
|
if passed, ok := outputMap["passed"].(bool); ok {
|
|
result.Passed = passed
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = "validator response missing 'passed' field"
|
|
return result
|
|
}
|
|
if reason, ok := outputMap["reason"].(string); ok {
|
|
result.Message = reason
|
|
}
|
|
// Store score and suggestions in expected field for reference
|
|
result.Expected = outputMap
|
|
} else {
|
|
result.Passed = false
|
|
result.Message = "validator agent returned invalid response format"
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// extractValidatorOutput extracts the output from a validator response
|
|
func extractValidatorOutput(response *context.Response) interface{} {
|
|
if response == nil || response.Completion == nil {
|
|
return nil
|
|
}
|
|
|
|
// Get content from completion
|
|
content := response.Completion.Content
|
|
if content == nil {
|
|
return nil
|
|
}
|
|
|
|
// Try to get text content
|
|
var text string
|
|
switch v := content.(type) {
|
|
case string:
|
|
text = v
|
|
default:
|
|
// Try to marshal and use as-is
|
|
data, err := json.Marshal(content)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
text = string(data)
|
|
}
|
|
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
|
|
// Use gou/text to extract JSON (handles markdown code blocks, auto-repair, etc.)
|
|
result := goutext.ExtractJSON(text)
|
|
if result != nil {
|
|
return result
|
|
}
|
|
|
|
// Return raw text if extraction fails
|
|
return text
|
|
}
|
|
|
|
// assertScript runs a custom assertion script
|
|
func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Actual: output,
|
|
}
|
|
|
|
if assertion.Script == "" {
|
|
result.Passed = false
|
|
result.Message = "script assertion requires a script name"
|
|
return result
|
|
}
|
|
|
|
// Build script arguments
|
|
args := []interface{}{
|
|
output,
|
|
input,
|
|
assertion.Value,
|
|
}
|
|
|
|
// Run the script as a process
|
|
p, err := process.Of(assertion.Script, args...)
|
|
if err != nil {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("failed to create process: %s", err.Error())
|
|
return result
|
|
}
|
|
|
|
res, err := p.Exec()
|
|
if err != nil {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("script execution failed: %s", err.Error())
|
|
return result
|
|
}
|
|
|
|
// Parse script result
|
|
// Expected format: { "pass": bool, "message": string }
|
|
switch v := res.(type) {
|
|
case bool:
|
|
result.Passed = v
|
|
if v {
|
|
result.Message = "script assertion passed"
|
|
} else {
|
|
result.Message = "script assertion failed"
|
|
}
|
|
|
|
case map[string]interface{}:
|
|
if pass, ok := v["pass"].(bool); ok {
|
|
result.Passed = pass
|
|
}
|
|
if msg, ok := v["message"].(string); ok {
|
|
result.Message = msg
|
|
}
|
|
|
|
default:
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("script returned unexpected type: %T", res)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// assertToolCalled checks if a specific tool was called
|
|
// value can be:
|
|
// - string: exact tool name to match
|
|
// - []string: any of the tool names
|
|
// - map with "name" and optional "arguments" for more specific matching
|
|
func (a *Asserter) assertToolCalled(assertion *Assertion) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
if a.response == nil {
|
|
result.Passed = false
|
|
result.Message = "no response available for tool_called assertion"
|
|
return result
|
|
}
|
|
|
|
if len(a.response.Tools) == 0 {
|
|
result.Passed = false
|
|
result.Message = "no tools were called"
|
|
return result
|
|
}
|
|
|
|
// Get tool names that were called
|
|
calledTools := make([]string, 0, len(a.response.Tools))
|
|
for _, tool := range a.response.Tools {
|
|
calledTools = append(calledTools, tool.Tool)
|
|
}
|
|
result.Actual = calledTools
|
|
|
|
switch v := assertion.Value.(type) {
|
|
case string:
|
|
// Simple case: check if tool name matches (supports prefix matching)
|
|
for _, tool := range a.response.Tools {
|
|
if matchToolName(tool.Tool, v) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("tool '%s' was called", v)
|
|
return result
|
|
}
|
|
}
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("tool '%s' was not called, called: %v", v, calledTools)
|
|
|
|
case []interface{}:
|
|
// Check if any of the specified tools were called
|
|
for _, expected := range v {
|
|
if expectedStr, ok := expected.(string); ok {
|
|
for _, tool := range a.response.Tools {
|
|
if matchToolName(tool.Tool, expectedStr) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("tool '%s' was called", expectedStr)
|
|
return result
|
|
}
|
|
}
|
|
}
|
|
}
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("none of the expected tools were called, called: %v", calledTools)
|
|
|
|
case map[string]interface{}:
|
|
// Advanced case: match name and optionally arguments
|
|
expectedName, _ := v["name"].(string)
|
|
expectedArgs := v["arguments"]
|
|
|
|
for _, tool := range a.response.Tools {
|
|
if matchToolName(tool.Tool, expectedName) {
|
|
// If arguments specified, check them too
|
|
if expectedArgs != nil {
|
|
if matchArguments(tool.Arguments, expectedArgs) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("tool '%s' was called with matching arguments", expectedName)
|
|
return result
|
|
}
|
|
} else {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("tool '%s' was called", expectedName)
|
|
return result
|
|
}
|
|
}
|
|
}
|
|
result.Passed = false
|
|
if expectedArgs != nil {
|
|
result.Message = fmt.Sprintf("tool '%s' was not called with expected arguments", expectedName)
|
|
} else {
|
|
result.Message = fmt.Sprintf("tool '%s' was not called, called: %v", expectedName, calledTools)
|
|
}
|
|
|
|
default:
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("invalid tool_called value type: %T", assertion.Value)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// assertToolResult checks the result of a tool call
|
|
// value should be a map with "tool" (name) and "result" (expected result pattern)
|
|
func (a *Asserter) assertToolResult(assertion *Assertion) *AssertionResult {
|
|
result := &AssertionResult{
|
|
Assertion: assertion,
|
|
Expected: assertion.Value,
|
|
}
|
|
|
|
if a.response == nil {
|
|
result.Passed = false
|
|
result.Message = "no response available for tool_result assertion"
|
|
return result
|
|
}
|
|
|
|
if len(a.response.Tools) == 0 {
|
|
result.Passed = false
|
|
result.Message = "no tools were called"
|
|
return result
|
|
}
|
|
|
|
spec, ok := assertion.Value.(map[string]interface{})
|
|
if !ok {
|
|
result.Passed = false
|
|
result.Message = "tool_result assertion requires a map with 'tool' and 'result' fields"
|
|
return result
|
|
}
|
|
|
|
toolName, _ := spec["tool"].(string)
|
|
expectedResult := spec["result"]
|
|
|
|
if toolName == "" {
|
|
result.Passed = false
|
|
result.Message = "tool_result assertion requires 'tool' field"
|
|
return result
|
|
}
|
|
|
|
// Find the tool call
|
|
for _, tool := range a.response.Tools {
|
|
if matchToolName(tool.Tool, toolName) {
|
|
result.Actual = tool.Result
|
|
|
|
// Check if there was an error
|
|
if tool.Error != "" {
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("tool '%s' returned error: %s", toolName, tool.Error)
|
|
return result
|
|
}
|
|
|
|
// If no expected result specified, just check success (no error)
|
|
if expectedResult == nil {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("tool '%s' executed successfully", toolName)
|
|
return result
|
|
}
|
|
|
|
// Match result
|
|
if matchResult(tool.Result, expectedResult) {
|
|
result.Passed = true
|
|
result.Message = fmt.Sprintf("tool '%s' result matches expected", toolName)
|
|
return result
|
|
}
|
|
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("tool '%s' result does not match expected", toolName)
|
|
return result
|
|
}
|
|
}
|
|
|
|
result.Passed = false
|
|
result.Message = fmt.Sprintf("tool '%s' was not called", toolName)
|
|
return result
|
|
}
|
|
|
|
// matchToolName checks if a tool name matches the expected pattern
|
|
// Supports exact match and suffix match (e.g., "setup" matches "agents_expense_tools__setup")
|
|
func matchToolName(actual, expected string) bool {
|
|
if actual == expected {
|
|
return true
|
|
}
|
|
// Support suffix matching (tool name without namespace prefix)
|
|
if strings.HasSuffix(actual, "__"+expected) || strings.HasSuffix(actual, "."+expected) {
|
|
return true
|
|
}
|
|
// Support contains matching for partial names
|
|
if strings.Contains(actual, expected) {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// matchArguments checks if tool arguments match expected pattern
|
|
func matchArguments(actual, expected interface{}) bool {
|
|
expectedMap, ok := expected.(map[string]interface{})
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
actualMap, ok := actual.(map[string]interface{})
|
|
if !ok {
|
|
// Try parsing as JSON string
|
|
if actualStr, ok := actual.(string); ok {
|
|
var parsed map[string]interface{}
|
|
if err := jsoniter.UnmarshalFromString(actualStr, &parsed); err == nil {
|
|
actualMap = parsed
|
|
} else {
|
|
return false
|
|
}
|
|
} else {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Check that all expected keys exist and match
|
|
for key, expectedVal := range expectedMap {
|
|
actualVal, exists := actualMap[key]
|
|
if !exists {
|
|
return false
|
|
}
|
|
if !validateOutput(actualVal, expectedVal) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// matchResult checks if tool result matches expected pattern
|
|
func matchResult(actual, expected interface{}) bool {
|
|
switch exp := expected.(type) {
|
|
case map[string]interface{}:
|
|
actualMap, ok := actual.(map[string]interface{})
|
|
if !ok {
|
|
return false
|
|
}
|
|
// Check that all expected keys exist and match
|
|
for key, expectedVal := range exp {
|
|
actualVal, exists := actualMap[key]
|
|
if !exists {
|
|
return false
|
|
}
|
|
if !matchResult(actualVal, expectedVal) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
|
|
case string:
|
|
// Support regex pattern matching for strings
|
|
if strings.HasPrefix(exp, "regex:") {
|
|
pattern := strings.TrimPrefix(exp, "regex:")
|
|
re, err := regexp.Compile(pattern)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
actualStr := fmt.Sprintf("%v", actual)
|
|
return re.MatchString(actualStr)
|
|
}
|
|
return fmt.Sprintf("%v", actual) == exp
|
|
|
|
case bool:
|
|
actualBool, ok := actual.(bool)
|
|
return ok && actualBool == exp
|
|
|
|
default:
|
|
return validateOutput(actual, expected)
|
|
}
|
|
}
|
|
|
|
// toString converts a value to string for comparison
|
|
func (a *Asserter) toString(v interface{}) string {
|
|
if v == nil {
|
|
return ""
|
|
}
|
|
|
|
switch val := v.(type) {
|
|
case string:
|
|
return val
|
|
case []byte:
|
|
return string(val)
|
|
default:
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
return fmt.Sprintf("%v", v)
|
|
}
|
|
return string(b)
|
|
}
|
|
}
|