5a63d9280b
- Added support for `tool_called` and `tool_result` assertions in the Asserter, allowing for validation of tool execution and arguments. - Introduced methods to check if specific tools were called and to validate their results against expected patterns. - Updated the `README.md` to include detailed documentation on new assertion types, including usage examples and value formats. - Enhanced the dynamic runner to set the response for tool-related assertions, improving the overall testing framework's capabilities.
856 lines
20 KiB
Go
856 lines
20 KiB
Go
package test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/yaoapp/yao/agent/context"
|
|
)
|
|
|
|
func TestAsserter_JSONPath_ArrayEquality(t *testing.T) {
|
|
asserter := NewAsserter()
|
|
|
|
tests := []struct {
|
|
name string
|
|
tc *Case
|
|
output interface{}
|
|
expected bool
|
|
errMsg string
|
|
}{
|
|
{
|
|
name: "array equals array - same content",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "search_types",
|
|
"value": []interface{}{"db"},
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"search_types": []interface{}{"db"},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "array equals array - different content",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "search_types",
|
|
"value": []interface{}{"db"},
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"search_types": []interface{}{"web"},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "array equals array - multiple elements",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "search_types",
|
|
"value": []interface{}{"web", "db"},
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"search_types": []interface{}{"web", "db"},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "array equals array - different order",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "search_types",
|
|
"value": []interface{}{"db", "web"},
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"search_types": []interface{}{"web", "db"},
|
|
},
|
|
expected: false, // Order matters for array equality
|
|
},
|
|
{
|
|
name: "scalar in array - match",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "status",
|
|
"value": []interface{}{"active", "pending"},
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"status": "active",
|
|
},
|
|
expected: true, // "active" is one of ["active", "pending"]
|
|
},
|
|
{
|
|
name: "scalar in array - no match",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "status",
|
|
"value": []interface{}{"active", "pending"},
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"status": "inactive",
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "simple value comparison",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "need_search",
|
|
"value": true,
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"need_search": true,
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "nested path",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "result.count",
|
|
"value": float64(5),
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"result": map[string]interface{}{
|
|
"count": float64(5),
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "array index access",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "items[0].name",
|
|
"value": "first",
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"items": []interface{}{
|
|
map[string]interface{}{"name": "first"},
|
|
map[string]interface{}{"name": "second"},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
passed, errMsg := asserter.Validate(tt.tc, tt.output)
|
|
if passed != tt.expected {
|
|
t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsserter_Contains(t *testing.T) {
|
|
asserter := NewAsserter()
|
|
|
|
tests := []struct {
|
|
name string
|
|
tc *Case
|
|
output interface{}
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "string contains substring",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "contains",
|
|
"value": "hello",
|
|
},
|
|
},
|
|
output: "hello world",
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "string does not contain",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "contains",
|
|
"value": "goodbye",
|
|
},
|
|
},
|
|
output: "hello world",
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "JSON contains field",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "contains",
|
|
"value": "success",
|
|
},
|
|
},
|
|
output: map[string]interface{}{
|
|
"status": "success",
|
|
},
|
|
expected: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
passed, _ := asserter.Validate(tt.tc, tt.output)
|
|
if passed != tt.expected {
|
|
t.Errorf("Expected passed=%v, got passed=%v", tt.expected, passed)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsserter_MultipleAssertions(t *testing.T) {
|
|
asserter := NewAsserter()
|
|
|
|
tc := &Case{
|
|
Assert: []interface{}{
|
|
map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "need_search",
|
|
"value": true,
|
|
},
|
|
map[string]interface{}{
|
|
"type": "json_path",
|
|
"path": "search_types",
|
|
"value": []interface{}{"web"},
|
|
},
|
|
},
|
|
}
|
|
|
|
output := map[string]interface{}{
|
|
"need_search": true,
|
|
"search_types": []interface{}{"web"},
|
|
}
|
|
|
|
passed, errMsg := asserter.Validate(tc, output)
|
|
if !passed {
|
|
t.Errorf("Expected all assertions to pass, got error: %s", errMsg)
|
|
}
|
|
}
|
|
|
|
func TestAsserter_Negate(t *testing.T) {
|
|
asserter := NewAsserter()
|
|
|
|
tc := &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "contains",
|
|
"value": "error",
|
|
"negate": true,
|
|
},
|
|
}
|
|
|
|
output := "success message"
|
|
|
|
passed, _ := asserter.Validate(tc, output)
|
|
if !passed {
|
|
t.Error("Expected negated assertion to pass")
|
|
}
|
|
}
|
|
|
|
func TestAsserter_Regex(t *testing.T) {
|
|
asserter := NewAsserter()
|
|
|
|
tests := []struct {
|
|
name string
|
|
tc *Case
|
|
output interface{}
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "regex matches",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "regex",
|
|
"value": `\d{3}-\d{4}`,
|
|
},
|
|
},
|
|
output: "Phone: 123-4567",
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "regex does not match",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "regex",
|
|
"value": `\d{3}-\d{4}`,
|
|
},
|
|
},
|
|
output: "No phone number here",
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
passed, _ := asserter.Validate(tt.tc, tt.output)
|
|
if passed != tt.expected {
|
|
t.Errorf("Expected passed=%v, got passed=%v", tt.expected, passed)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsserter_ToolCalled(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
tc *Case
|
|
response *context.Response
|
|
expected bool
|
|
errMsg string
|
|
}{
|
|
{
|
|
name: "tool called - exact match",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": "agents_expense_tools__setup",
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{Tool: "agents_expense_tools__setup", Result: map[string]interface{}{"success": true}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool called - suffix match",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": "setup",
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{Tool: "agents_expense_tools__setup", Result: map[string]interface{}{"success": true}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool not called",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": "setup",
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "tool called - wrong tool",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": "setup",
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{Tool: "agents_expense_tools__submit", Result: map[string]interface{}{"success": true}},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "tool called - any of multiple",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": []interface{}{"setup", "init"},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{Tool: "agents_expense_tools__init", Result: map[string]interface{}{"success": true}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool called - with arguments (map)",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": map[string]interface{}{
|
|
"name": "setup",
|
|
"arguments": map[string]interface{}{
|
|
"action": "init",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Arguments: map[string]interface{}{"action": "init", "config": map[string]interface{}{}},
|
|
Result: map[string]interface{}{"success": true},
|
|
},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool called - with arguments (JSON string)",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": map[string]interface{}{
|
|
"name": "setup",
|
|
"arguments": map[string]interface{}{
|
|
"action": "init",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Arguments: `{"action":"init","config":{"default_currency":"USD"}}`,
|
|
Result: map[string]interface{}{"success": true},
|
|
},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool called - wrong arguments",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": map[string]interface{}{
|
|
"name": "setup",
|
|
"arguments": map[string]interface{}{
|
|
"action": "update",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Arguments: map[string]interface{}{"action": "init"},
|
|
Result: map[string]interface{}{"success": true},
|
|
},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "no response",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": "setup",
|
|
},
|
|
},
|
|
response: nil,
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
asserter := NewAsserter().WithResponse(tt.response)
|
|
passed, errMsg := asserter.Validate(tt.tc, nil)
|
|
if passed != tt.expected {
|
|
t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsserter_ToolResult(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
tc *Case
|
|
response *context.Response
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "tool result - success check",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
"result": map[string]interface{}{
|
|
"success": true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Result: map[string]interface{}{"success": true, "message": "Setup complete"},
|
|
},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool result - message check with regex",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
"result": map[string]interface{}{
|
|
"message": "regex:(?i)setup.*complete",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Result: map[string]interface{}{"success": true, "message": "Setup complete!"},
|
|
},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool result - no expected result (just check no error)",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Result: map[string]interface{}{"success": true},
|
|
},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "tool result - tool has error",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Error: "permission denied",
|
|
},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "tool result - result mismatch",
|
|
tc: &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
"result": map[string]interface{}{
|
|
"success": true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
response: &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Result: map[string]interface{}{"success": false},
|
|
},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
asserter := NewAsserter().WithResponse(tt.response)
|
|
passed, errMsg := asserter.Validate(tt.tc, nil)
|
|
if passed != tt.expected {
|
|
t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAsserter_MultipleToolAssertions(t *testing.T) {
|
|
// This tests the exact scenario from setup-006: tool_called + tool_result
|
|
response := &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Arguments: `{"action":"init","config":{"default_currency":"USD","categories":[{"id":"meals","name":"Meals","daily_limit":100}]}}`,
|
|
Result: map[string]interface{}{
|
|
"success": true,
|
|
"action": "init",
|
|
"config": map[string]interface{}{
|
|
"default_currency": "USD",
|
|
},
|
|
"message": "Setup complete!",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
asserter := NewAsserter().WithResponse(response)
|
|
|
|
tc := &Case{
|
|
Assert: []interface{}{
|
|
map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": map[string]interface{}{
|
|
"name": "setup",
|
|
"arguments": map[string]interface{}{
|
|
"action": "init",
|
|
},
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
"result": map[string]interface{}{
|
|
"success": true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
result := asserter.ValidateWithDetails(tc, nil)
|
|
if !result.Passed {
|
|
t.Errorf("Expected multiple tool assertions to pass, got: %s", result.Message)
|
|
}
|
|
}
|
|
|
|
func TestAsserter_SharedAsserterWithResponse(t *testing.T) {
|
|
// Test that a shared asserter correctly uses WithResponse
|
|
asserter := NewAsserter()
|
|
|
|
// First call without response - should fail
|
|
tc := &Case{
|
|
Assert: map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": "setup",
|
|
},
|
|
}
|
|
result := asserter.ValidateWithDetails(tc, nil)
|
|
if result.Passed {
|
|
t.Error("Expected tool_called to fail without response")
|
|
}
|
|
if result.Message != "no response available for tool_called assertion" {
|
|
t.Errorf("Unexpected message: %s", result.Message)
|
|
}
|
|
|
|
// Now set response
|
|
response := &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Result: map[string]interface{}{"success": true},
|
|
},
|
|
},
|
|
}
|
|
asserter.WithResponse(response)
|
|
|
|
// Should pass now
|
|
result = asserter.ValidateWithDetails(tc, nil)
|
|
if !result.Passed {
|
|
t.Errorf("Expected tool_called to pass with response, got: %s", result.Message)
|
|
}
|
|
}
|
|
|
|
func TestAsserter_Setup006Scenario(t *testing.T) {
|
|
// Exact reproduction of setup-006 scenario
|
|
// Turn 2: tool was called with action: init, result has success: true
|
|
response := &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Arguments: `{"action":"init","config":{"default_currency":"USD","categories":[{"id":"meals","name":"Meals","daily_limit":100},{"id":"travel","name":"Travel","daily_limit":500}]}}`,
|
|
Result: map[string]interface{}{
|
|
"config": map[string]interface{}{
|
|
"categories": []interface{}{
|
|
map[string]interface{}{"daily_limit": float64(100), "id": "meals", "name": "Meals"},
|
|
map[string]interface{}{"daily_limit": float64(500), "id": "travel", "name": "Travel"},
|
|
},
|
|
"default_currency": "USD",
|
|
},
|
|
"message": "Setup complete! The expense system has been initialized successfully with the configured settings. You can now start submitting expenses.",
|
|
"success": true,
|
|
"action": "init",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// This is the exact assert from setup-006's quick_complete checkpoint
|
|
assertDef := []interface{}{
|
|
map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": map[string]interface{}{
|
|
"name": "setup",
|
|
"arguments": map[string]interface{}{
|
|
"action": "init",
|
|
},
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
"result": map[string]interface{}{
|
|
"success": true,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
asserter := NewAsserter().WithResponse(response)
|
|
tc := &Case{Assert: assertDef}
|
|
|
|
result := asserter.ValidateWithDetails(tc, nil)
|
|
if !result.Passed {
|
|
t.Errorf("Expected setup-006 scenario to pass, got: %s", result.Message)
|
|
}
|
|
|
|
// Also test individual assertions
|
|
t.Run("tool_called only", func(t *testing.T) {
|
|
tc2 := &Case{Assert: assertDef[0]}
|
|
result2 := asserter.ValidateWithDetails(tc2, nil)
|
|
if !result2.Passed {
|
|
t.Errorf("Expected tool_called to pass, got: %s", result2.Message)
|
|
}
|
|
})
|
|
|
|
t.Run("tool_result only", func(t *testing.T) {
|
|
tc3 := &Case{Assert: assertDef[1]}
|
|
result3 := asserter.ValidateWithDetails(tc3, nil)
|
|
if !result3.Passed {
|
|
t.Errorf("Expected tool_result to pass, got: %s", result3.Message)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAsserter_Setup003Scenario(t *testing.T) {
|
|
// Exact reproduction of setup-003 scenario
|
|
// Turn 3: tool was called with action: update
|
|
response := &context.Response{
|
|
Tools: []context.ToolCallResponse{
|
|
{
|
|
Tool: "agents_expense_tools__setup",
|
|
Arguments: `{"action":"update","config":{"categories":[{"daily_limit":500,"id":"meals","name":"Business Meals"}]}}`,
|
|
Result: map[string]interface{}{
|
|
"action": "update",
|
|
"config": map[string]interface{}{
|
|
"categories": []interface{}{
|
|
map[string]interface{}{"daily_limit": float64(500), "id": "meals", "name": "Business Meals"},
|
|
},
|
|
},
|
|
"message": "Configuration updated successfully! Your changes have been saved.",
|
|
"success": true,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// This is the exact assert from setup-003's update_complete checkpoint
|
|
assertDef := []interface{}{
|
|
map[string]interface{}{
|
|
"type": "tool_called",
|
|
"value": map[string]interface{}{
|
|
"name": "setup",
|
|
"arguments": map[string]interface{}{
|
|
"action": "update",
|
|
},
|
|
},
|
|
},
|
|
map[string]interface{}{
|
|
"type": "tool_result",
|
|
"value": map[string]interface{}{
|
|
"tool": "setup",
|
|
"result": map[string]interface{}{
|
|
"success": true,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
asserter := NewAsserter().WithResponse(response)
|
|
tc := &Case{Assert: assertDef}
|
|
|
|
result := asserter.ValidateWithDetails(tc, nil)
|
|
if !result.Passed {
|
|
t.Errorf("Expected setup-003 scenario to pass, got: %s", result.Message)
|
|
}
|
|
|
|
// Test individual assertions
|
|
t.Run("tool_called with action:update", func(t *testing.T) {
|
|
tc2 := &Case{Assert: assertDef[0]}
|
|
result2 := asserter.ValidateWithDetails(tc2, nil)
|
|
if !result2.Passed {
|
|
t.Errorf("Expected tool_called to pass, got: %s", result2.Message)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestMatchToolName(t *testing.T) {
|
|
tests := []struct {
|
|
actual string
|
|
expected string
|
|
match bool
|
|
}{
|
|
{"agents_expense_tools__setup", "agents_expense_tools__setup", true},
|
|
{"agents_expense_tools__setup", "setup", true},
|
|
{"agents.expense.tools.setup", "setup", true},
|
|
{"agents_expense_tools__setup", "init", false},
|
|
{"setup", "setup", true},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.actual+"_"+tt.expected, func(t *testing.T) {
|
|
if matchToolName(tt.actual, tt.expected) != tt.match {
|
|
t.Errorf("matchToolName(%q, %q) = %v, want %v", tt.actual, tt.expected, !tt.match, tt.match)
|
|
}
|
|
})
|
|
}
|
|
}
|