opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1397 from trheyi/main

Browse source 
Agent Test Framework V2 - Dynamic Mode, Agent-Driven Features & Hooks
This commit is contained in:
Max 2025-12-26 11:57:28 +08:00 • committed by GitHub
commit b7e46d4fff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 5920 additions and 583 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -53,3 +53,4 @@ agent/assistant/hook/*.test.md
agent/search/TODO.md
agent/search/job-logs.txt
agent/test/MULTI_TURN_DESIGN.md
agent/test/UPGRADE_PLAN.md

1017
agent/test/DESIGN_V2.md Normal file
View file

File diff suppressed because it is too large Load diff

1575
agent/test/README.md
View file

File diff suppressed because it is too large Load diff

217
agent/test/assert.go
View file

@ -9,6 +9,9 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/process"
goutext "github.com/yaoapp/gou/text"
"github.com/yaoapp/yao/agent/assistant"
"github.com/yaoapp/yao/agent/context"
)
// Asserter handles test assertions
@ -115,6 +118,9 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion {
if s, ok := m["script"].(string); ok {
assertion.Script = s
}
if u, ok := m["use"].(string); ok {
assertion.Use = u
}
if msg, ok := m["message"].(string); ok {
assertion.Message = msg
}
@ -122,6 +128,17 @@ func (a *Asserter) mapToAssertion(m map[string]interface{}) *Assertion {
assertion.Negate = n
}
// Parse options for agent assertions
if opts, ok := m["options"].(map[string]interface{}); ok {
assertion.Options = &AssertionOptions{}
if c, ok := opts["connector"].(string); ok {
assertion.Options.Connector = c
}
if meta, ok := opts["metadata"].(map[string]interface{}); ok {
assertion.Options.Metadata = meta
}
}
return assertion
}
@ -147,6 +164,8 @@ func (a *Asserter) evaluateAssertion(assertion *Assertion, output, input interfa
result = a.assertType(assertion, output)
case "script":
result = a.assertScript(assertion, output, input)
case "agent":
result = a.assertAgent(assertion, output, input)
default:
result.Passed = false
result.Message = fmt.Sprintf("unknown assertion type: %s", assertion.Type)
@ -229,17 +248,14 @@ func (a *Asserter) assertJSONPath(assertion *Assertion, output interface{}) *Ass
var jsonData interface{}
switch v := output.(type) {
case string:
// Try to parse as JSON
if err := jsoniter.Unmarshal([]byte(v), &jsonData); err != nil {
// Try to extract JSON from markdown code blocks
extracted := extractJSONFromText(v)
if extracted != nil {
jsonData = extracted
} else {
result.Passed = false
result.Message = fmt.Sprintf("output is not valid JSON: %s", err.Error())
return result
}
// Use gou/text to extract JSON (handles markdown, auto-repair, etc.)
extracted := goutext.ExtractJSON(v)
if extracted != nil {
jsonData = extracted
} else {
result.Passed = false
result.Message = fmt.Sprintf("output is not valid JSON: %s", v)
return result
}
case map[string]interface{}, []interface{}:
jsonData = v
@ -478,6 +494,158 @@ func (a *Asserter) getType(v interface{}) string {
}
}
// assertAgent uses an agent to validate the output
func (a *Asserter) assertAgent(assertion *Assertion, output, input interface{}) *AssertionResult {
result := &AssertionResult{
Assertion: assertion,
Actual: output,
}
// Parse use field: "agents:tests.validator-agent"
if !strings.HasPrefix(assertion.Use, "agents:") {
result.Passed = false
result.Message = "agent assertion requires 'use' field with 'agents:' prefix"
return result
}
agentID := strings.TrimPrefix(assertion.Use, "agents:")
// Get assistant
ast, err := assistant.Get(agentID)
if err != nil {
result.Passed = false
result.Message = fmt.Sprintf("failed to get validator agent: %s", err.Error())
return result
}
// Build validation request
validationInput := map[string]interface{}{
"output": output,
"input": input,
}
// Add criteria from Value field
if assertion.Value != nil {
validationInput["criteria"] = assertion.Value
}
// Add metadata from options
if assertion.Options != nil && assertion.Options.Metadata != nil {
for k, v := range assertion.Options.Metadata {
validationInput[k] = v
}
}
// Build context options - skip history and trace for validator
opts := &context.Options{
Skip: &context.Skip{
History: true,
Trace: true,
Output: true,
},
Metadata: map[string]interface{}{
"test_mode": "validator",
},
}
if assertion.Options != nil && assertion.Options.Connector != "" {
opts.Connector = assertion.Options.Connector
}
// Create context and call agent
env := NewEnvironment("", "")
ctx := NewTestContext("validator", agentID, env)
defer ctx.Release()
// Convert validation input to JSON string for the message
inputJSON, err := json.Marshal(validationInput)
if err != nil {
result.Passed = false
result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error())
return result
}
messages := []context.Message{{
Role: context.RoleUser,
Content: string(inputJSON),
}}
response, err := ast.Stream(ctx, messages, opts)
if err != nil {
result.Passed = false
result.Message = fmt.Sprintf("validator agent error: %s", err.Error())
return result
}
// Parse response
return a.parseValidatorResponse(response, result)
}
// parseValidatorResponse parses the validator agent's response
func (a *Asserter) parseValidatorResponse(response *context.Response, result *AssertionResult) *AssertionResult {
output := extractValidatorOutput(response)
// Expected format: { "passed": bool, "reason": string, "score": float, "suggestions": [] }
if outputMap, ok := output.(map[string]interface{}); ok {
if passed, ok := outputMap["passed"].(bool); ok {
result.Passed = passed
} else {
result.Passed = false
result.Message = "validator response missing 'passed' field"
return result
}
if reason, ok := outputMap["reason"].(string); ok {
result.Message = reason
}
// Store score and suggestions in expected field for reference
result.Expected = outputMap
} else {
result.Passed = false
result.Message = "validator agent returned invalid response format"
}
return result
}
// extractValidatorOutput extracts the output from a validator response
func extractValidatorOutput(response *context.Response) interface{} {
if response == nil || response.Completion == nil {
return nil
}
// Get content from completion
content := response.Completion.Content
if content == nil {
return nil
}
// Try to get text content
var text string
switch v := content.(type) {
case string:
text = v
default:
// Try to marshal and use as-is
data, err := json.Marshal(content)
if err != nil {
return nil
}
text = string(data)
}
if text == "" {
return nil
}
// Use gou/text to extract JSON (handles markdown code blocks, auto-repair, etc.)
result := goutext.ExtractJSON(text)
if result != nil {
return result
}
// Return raw text if extraction fails
return text
}
// assertScript runs a custom assertion script
func (a *Asserter) assertScript(assertion *Assertion, output, input interface{}) *AssertionResult {
result := &AssertionResult{
@ -559,30 +727,3 @@ func (a *Asserter) toString(v interface{}) string {
return string(b)
}
}
// extractJSONFromText tries to extract JSON from text (e.g., markdown code blocks)
func extractJSONFromText(text string) interface{} {
// Try to find JSON in code blocks
patterns := []string{
"```json\n",
"```\n",
}
for _, start := range patterns {
if idx := strings.Index(text, start); idx >= 0 {
text = text[idx+len(start):]
if endIdx := strings.Index(text, "```"); endIdx >= 0 {
text = text[:endIdx]
}
break
}
}
// Try to parse
var result interface{}
if err := jsoniter.Unmarshal([]byte(strings.TrimSpace(text)), &result); err == nil {
return result
}
return nil
}

275
agent/test/assert_agent_test.go Normal file
View file

@ -0,0 +1,275 @@
package test_test
import (
"testing"
"github.com/stretchr/testify/assert"
v8 "github.com/yaoapp/gou/runtime/v8"
"github.com/yaoapp/yao/agent"
agenttest "github.com/yaoapp/yao/agent/test"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
"rogchap.com/v8go"
)
func TestAsserter_AgentAssertion(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent (includes assistants)
err := agent.Load(config.Conf)
if err != nil {
t.Fatalf("Failed to load agent: %v", err)
}
asserter := agenttest.NewAsserter()
tests := []struct {
name string
tc *agenttest.Case
output interface{}
expected bool
skipMsg string
}{
{
name: "agent assertion - pass",
tc: &agenttest.Case{
Assert: map[string]interface{}{
"type": "agent",
"use": "agents:tests.validator-agent",
"value": "Response should be a greeting",
},
},
output: "Hello! How can I help you today?",
expected: true,
},
{
name: "agent assertion - fail",
tc: &agenttest.Case{
Assert: map[string]interface{}{
"type": "agent",
"use": "agents:tests.validator-agent",
"value": "Response should provide a detailed technical answer",
},
},
output: "I don't know.",
expected: false,
},
{
name: "agent assertion - missing prefix",
tc: &agenttest.Case{
Assert: map[string]interface{}{
"type": "agent",
"use": "tests.validator-agent", // Missing agents: prefix
"value": "Should pass",
},
},
output: "Hello",
expected: false, // Should fail due to missing prefix
},
{
name: "agent assertion - with metadata",
tc: &agenttest.Case{
Assert: map[string]interface{}{
"type": "agent",
"use": "agents:tests.validator-agent",
"value": "Response is helpful",
"options": map[string]interface{}{
"metadata": map[string]interface{}{
"context": "customer support",
},
},
},
},
output: "I'd be happy to help you with your order. Let me look that up for you.",
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.skipMsg != "" {
t.Skip(tt.skipMsg)
}
passed, errMsg := asserter.Validate(tt.tc, tt.output)
if passed != tt.expected {
t.Errorf("Expected passed=%v, got passed=%v, error: %s", tt.expected, passed, errMsg)
}
})
}
}
func TestAsserter_AgentAssertion_InvalidAgent(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent (includes assistants)
err := agent.Load(config.Conf)
if err != nil {
t.Fatalf("Failed to load agent: %v", err)
}
asserter := agenttest.NewAsserter()
tc := &agenttest.Case{
Assert: map[string]interface{}{
"type": "agent",
"use": "agents:nonexistent.agent",
"value": "Should fail",
},
}
passed, errMsg := asserter.Validate(tc, "Hello")
assert.False(t, passed, "Should fail for nonexistent agent")
assert.Contains(t, errMsg, "failed to get validator agent", "Error should mention agent loading failure")
}
func TestAsserter_MapToAssertion_WithUseAndOptions(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent (includes assistants)
err := agent.Load(config.Conf)
if err != nil {
t.Fatalf("Failed to load agent: %v", err)
}
asserter := agenttest.NewAsserter()
// Test that mapToAssertion correctly parses use and options fields
tc := &agenttest.Case{
Assert: map[string]interface{}{
"type": "agent",
"use": "agents:tests.validator-agent",
"value": "criteria here",
"options": map[string]interface{}{
"connector": "gpt-4o",
"metadata": map[string]interface{}{
"key": "value",
},
},
},
}
// Validate triggers parseAssertions internally
// We just verify it doesn't panic and processes correctly
_, _ = asserter.Validate(tc, "test output")
// If we get here without panic, the parsing worked
}
// TestTestingT_AssertAgent tests the JSAPI t.assert.Agent() method
func TestTestingT_AssertAgent(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent (includes assistants)
err := agent.Load(config.Conf)
if err != nil {
t.Fatalf("Failed to load agent: %v", err)
}
tests := []struct {
name string
script string
shouldFail bool
}{
{
name: "JSAPI agent assertion - pass",
script: `
function test(t) {
var response = "Hello! How can I help you today?";
t.assert.Agent(response, "tests.validator-agent", {
criteria: "Response should be a friendly greeting"
});
}
test(__test_t);
`,
shouldFail: false,
},
{
name: "JSAPI agent assertion - JSON response",
script: `
function test(t) {
var response = {
status: "success",
data: { user: "john", email: "john@example.com" },
message: "User created successfully"
};
t.assert.Agent(response, "tests.validator-agent", {
criteria: "Response should be a successful API response with user data"
});
}
test(__test_t);
`,
shouldFail: false,
},
{
name: "JSAPI agent assertion - with metadata",
script: `
function test(t) {
var response = "I'd be happy to help you with your order.";
t.assert.Agent(response, "tests.validator-agent", {
criteria: "Response is helpful and professional",
metadata: { context: "customer support" }
});
}
test(__test_t);
`,
shouldFail: false,
},
{
name: "JSAPI agent assertion - fail case",
script: `
function test(t) {
var response = "I don't know.";
t.assert.Agent(response, "tests.validator-agent", {
criteria: "Response should provide a detailed technical explanation"
});
}
test(__test_t);
`,
shouldFail: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create TestingT
testingT := agenttest.NewTestingT(tt.name)
// Create V8 isolate and context
iso := v8go.NewIsolate()
defer iso.Dispose()
v8ctx := v8go.NewContext(iso)
defer v8ctx.Close()
// Create testing object
testObj, err := agenttest.NewTestingTObject(v8ctx, testingT)
if err != nil {
t.Fatalf("Failed to create testing object: %v", err)
}
// Set testing object as global
global := v8ctx.Global()
global.Set("__test_t", testObj)
// Run the test script
_, err = v8ctx.RunScript(tt.script, "test.js")
// Check results
if tt.shouldFail {
assert.True(t, testingT.Failed(), "Test should have failed")
} else {
if err != nil {
t.Errorf("Script execution error: %v", err)
}
assert.False(t, testingT.Failed(), "Test should have passed, errors: %v", testingT.Errors())
}
})
}
}
// Ensure v8 is used (for script loading)
var _ = v8.Scripts

250
agent/test/dynamic_integration_test.go Normal file
View file

@ -0,0 +1,250 @@
package test_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/yao/agent"
agenttest "github.com/yaoapp/yao/agent/test"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
// TestDynamicRunner_CoffeeOrder tests a complete dynamic mode flow:
// Simulator acts as a customer ordering coffee, agent handles the order
func TestDynamicRunner_CoffeeOrder(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Create a temporary JSONL file with a dynamic test case
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl")
// Dynamic test case: customer ordering coffee (JSONL must be single line)
testCase := `{"id": "coffee-order-flow", "name": "Complete Coffee Order", "input": "Hi, I would like to order a coffee please", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "A customer who wants to order a medium latte with oat milk", "goal": "Successfully complete a coffee order"}}}, "checkpoints": [{"id": "greeting", "description": "Agent greets and asks for order", "assert": {"type": "regex", "value": "(?i)(order|like|help)"}}, {"id": "ask_size", "description": "Agent asks for size", "after": ["greeting"], "assert": {"type": "regex", "value": "(?i)size"}}, {"id": "confirm_order", "description": "Agent confirms the order", "after": ["ask_size"], "assert": {"type": "regex", "value": "(?i)confirm"}}], "max_turns": 8}`
err = os.WriteFile(inputFile, []byte(testCase), 0644)
require.NoError(t, err, "Failed to write test file")
// Run dynamic test
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.dynamic-test-agent",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
require.NotNil(t, report.Summary, "Summary should not be nil")
// Log results
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
// Check results
if len(report.Results) > 0 {
result := report.Results[0]
t.Logf("Test [%s] Status: %s", result.ID, result.Status)
// Check metadata for dynamic mode info
if result.Metadata != nil {
if mode, ok := result.Metadata["mode"].(string); ok {
assert.Equal(t, "dynamic", mode, "Should be dynamic mode")
}
if turns, ok := result.Metadata["total_turns"].(int); ok {
t.Logf("Total turns: %d", turns)
}
}
if result.Error != "" {
t.Logf("Error: %s", result.Error)
}
}
}
// TestDynamicRunner_WithInitialInput tests dynamic mode with initial user input
func TestDynamicRunner_WithInitialInput(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Create a test case with initial input
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl")
// Start with user's first message (JSONL must be single line)
testCase := `{"id": "coffee-with-initial", "name": "Coffee Order with Initial Message", "input": "Hi, I want to order a coffee", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Customer ordering a large cappuccino", "goal": "Complete the coffee order"}}}, "checkpoints": [{"id": "acknowledge", "description": "Agent acknowledges the order request", "assert": {"type": "regex", "value": "(?i)(coffee|order|help)"}}, {"id": "ask_details", "description": "Agent asks for more details", "after": ["acknowledge"], "assert": {"type": "regex", "value": "(?i)(size|type|what)"}}], "max_turns": 5}`
err = os.WriteFile(inputFile, []byte(testCase), 0644)
require.NoError(t, err, "Failed to write test file")
// Run test
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.dynamic-test-agent",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
}
// TestDynamicRunner_OptionalCheckpoint tests optional checkpoint behavior
func TestDynamicRunner_OptionalCheckpoint(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl")
// Test with one required and one optional checkpoint (JSONL must be single line)
testCase := `{"id": "optional-checkpoint-test", "name": "Test with Optional Checkpoint", "input": "Hello", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Simple customer", "goal": "Get a greeting response"}}}, "checkpoints": [{"id": "greeting_response", "description": "Agent responds with greeting", "assert": {"type": "regex", "value": "(?i)(hello|hi|help)"}}, {"id": "special_offer", "description": "Agent mentions special offer (optional)", "required": false, "assert": {"type": "contains", "value": "special offer"}}], "max_turns": 3}`
err = os.WriteFile(inputFile, []byte(testCase), 0644)
require.NoError(t, err, "Failed to write test file")
// Run test
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.dynamic-test-agent",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
// Test should pass even if optional checkpoint is not reached
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
// If the required checkpoint is reached, the test should pass
if len(report.Results) > 0 && report.Results[0].Metadata != nil {
if checkpoints, ok := report.Results[0].Metadata["checkpoints"].(map[string]*agenttest.CheckpointResult); ok {
for id, cp := range checkpoints {
t.Logf("Checkpoint [%s]: reached=%v, required=%v", id, cp.Reached, cp.Required)
}
}
}
}
// TestDynamicRunner_MaxTurnsExceeded tests behavior when max turns is exceeded
func TestDynamicRunner_MaxTurnsExceeded(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl")
// Test case with impossible checkpoint and low max_turns (JSONL must be single line)
testCase := `{"id": "max-turns-test", "name": "Test Max Turns Exceeded", "input": "Hello", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Persistent customer", "goal": "Keep talking"}}}, "checkpoints": [{"id": "impossible", "description": "This checkpoint will never be reached", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_NEVER_APPEARS_12345"}}], "max_turns": 2}`
err = os.WriteFile(inputFile, []byte(testCase), 0644)
require.NoError(t, err, "Failed to write test file")
// Run test
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.dynamic-test-agent",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
// Test should fail due to max turns exceeded
assert.Equal(t, 1, report.Summary.Failed, "Test should fail")
if len(report.Results) > 0 {
result := report.Results[0]
assert.Equal(t, agenttest.StatusFailed, result.Status, "Status should be failed")
assert.Contains(t, result.Error, "max turns", "Error should mention max turns")
t.Logf("Error (expected): %s", result.Error)
}
}
// TestDynamicRunner_CheckpointOrdering tests that checkpoint ordering is enforced
func TestDynamicRunner_CheckpointOrderingEnforced(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "dynamic-inputs.jsonl")
// Test case with ordered checkpoints (JSONL must be single line)
testCase := `{"id": "ordered-checkpoints", "name": "Test Checkpoint Ordering", "input": "I want to order coffee", "simulator": {"use": "tests.simulator-agent", "options": {"metadata": {"persona": "Customer ordering step by step", "goal": "Complete coffee order following the flow"}}}, "checkpoints": [{"id": "step1_greeting", "description": "Agent greets", "assert": {"type": "regex", "value": "(?i)(hello|hi|help|order)"}}, {"id": "step2_size", "description": "Agent asks about size", "after": ["step1_greeting"], "assert": {"type": "regex", "value": "(?i)size"}}, {"id": "step3_confirm", "description": "Agent confirms", "after": ["step2_size"], "assert": {"type": "regex", "value": "(?i)confirm"}}], "max_turns": 10}`
err = os.WriteFile(inputFile, []byte(testCase), 0644)
require.NoError(t, err, "Failed to write test file")
// Run test
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.dynamic-test-agent",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
// Log checkpoint order
if len(report.Results) > 0 && report.Results[0].Metadata != nil {
if checkpoints, ok := report.Results[0].Metadata["checkpoints"].(map[string]*agenttest.CheckpointResult); ok {
for id, cp := range checkpoints {
t.Logf("Checkpoint [%s]: reached=%v, at_turn=%d", id, cp.Reached, cp.ReachedAtTurn)
}
}
}
}

413
agent/test/dynamic_runner.go Normal file
View file

@ -0,0 +1,413 @@
package test
import (
"fmt"
"time"
jsoniter "github.com/json-iterator/go"
goutext "github.com/yaoapp/gou/text"
"github.com/yaoapp/yao/agent/assistant"
"github.com/yaoapp/yao/agent/context"
)
// DynamicRunner handles dynamic (simulator-driven) test execution
type DynamicRunner struct {
opts *Options
output *OutputWriter
asserter *Asserter
}
// NewDynamicRunner creates a new dynamic runner
func NewDynamicRunner(opts *Options) *DynamicRunner {
return &DynamicRunner{
opts: opts,
output: NewOutputWriter(opts.Verbose),
asserter: NewAsserter(),
}
}
// RunDynamic executes a dynamic test case
func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID string) *DynamicResult {
startTime := time.Now()
result := &DynamicResult{
ID: tc.ID,
Turns: make([]*TurnResult, 0),
Checkpoints: make(map[string]*CheckpointResult),
}
// Initialize checkpoints
for _, cp := range tc.Checkpoints {
result.Checkpoints[cp.ID] = &CheckpointResult{
ID: cp.ID,
Reached: false,
Required: cp.IsRequired(),
}
}
// Get simulator agent
simAST, err := assistant.Get(tc.Simulator.Use)
if err != nil {
result.Status = StatusError
result.Error = fmt.Sprintf("failed to get simulator agent: %s", err.Error())
result.DurationMs = time.Since(startTime).Milliseconds()
return result
}
// Get configuration
maxTurns := tc.GetMaxTurns()
timeout := tc.GetTimeout(r.opts.Timeout)
// Build simulator metadata
simMetadata := make(map[string]interface{})
if tc.Simulator.Options != nil && tc.Simulator.Options.Metadata != nil {
for k, v := range tc.Simulator.Options.Metadata {
simMetadata[k] = v
}
}
// Conversation history
messages := make([]context.Message, 0)
// Get initial input if provided
initialMessages, err := tc.GetMessages()
if err == nil && len(initialMessages) > 0 {
messages = append(messages, initialMessages...)
}
// Output dynamic test start
if r.opts.Verbose {
r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns)
}
// Conversation loop
for turn := 1; turn <= maxTurns; turn++ {
turnStart := time.Now()
turnResult := &TurnResult{Turn: turn}
// Check timeout
if time.Since(startTime) > timeout {
result.Status = StatusTimeout
result.Error = fmt.Sprintf("timeout after %s", timeout)
result.DurationMs = time.Since(startTime).Milliseconds()
result.TotalTurns = turn - 1
return result
}
// For turns after the first, get input from simulator
if turn > 1 || len(messages) == 0 {
simInput := r.buildSimulatorInput(tc, messages, result, turn, maxTurns, simMetadata)
simOutput, err := r.callSimulator(simAST, tc, simInput)
if err != nil {
turnResult.Error = fmt.Sprintf("simulator error: %s", err.Error())
result.Turns = append(result.Turns, turnResult)
result.Status = StatusError
result.Error = turnResult.Error
result.DurationMs = time.Since(startTime).Milliseconds()
result.TotalTurns = turn
return result
}
// Check if goal achieved
if simOutput.GoalAchieved {
if r.opts.Verbose {
r.output.Info(" Turn %d: Simulator signaled goal achieved", turn)
}
// Check if all required checkpoints reached
if r.allRequiredCheckpointsReached(result) {
result.Status = StatusPassed
} else {
result.Status = StatusFailed
result.Error = "simulator signaled goal achieved but not all required checkpoints reached"
}
result.DurationMs = time.Since(startTime).Milliseconds()
result.TotalTurns = turn - 1
return result
}
// Add user message
userMessage := context.Message{
Role: context.RoleUser,
Content: simOutput.Message,
}
messages = append(messages, userMessage)
turnResult.Input = simOutput.Message
if r.opts.Verbose {
r.output.Info(" Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50))
}
} else {
// Use initial input for first turn
if len(messages) > 0 {
lastMsg := messages[len(messages)-1]
turnResult.Input = lastMsg.Content
if r.opts.Verbose {
r.output.Info(" Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50))
}
}
}
// Call target agent
ctx := NewTestContextFromOptions(
fmt.Sprintf("dynamic-%s-%d", tc.ID, turn),
agentID,
r.opts,
tc,
)
opts := buildContextOptions(tc, r.opts)
response, err := ast.Stream(ctx, messages, opts)
ctx.Release()
if err != nil {
turnResult.Error = err.Error()
turnResult.DurationMs = time.Since(turnStart).Milliseconds()
result.Turns = append(result.Turns, turnResult)
result.Status = StatusError
result.Error = fmt.Sprintf("agent error at turn %d: %s", turn, err.Error())
result.DurationMs = time.Since(startTime).Milliseconds()
result.TotalTurns = turn
return result
}
// Extract output
output := extractOutput(response)
turnResult.Output = output
turnResult.DurationMs = time.Since(turnStart).Milliseconds()
if r.opts.Verbose {
r.output.Info(" Turn %d: Agent: %s", turn, truncateOutput(output, 50))
}
// Add assistant response to messages
messages = append(messages, context.Message{
Role: context.RoleAssistant,
Content: output,
})
// Check checkpoints against this response
reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result)
turnResult.CheckpointsReached = reachedIDs
if r.opts.Verbose && len(reachedIDs) > 0 {
for _, id := range reachedIDs {
r.output.Info(" ✓ checkpoint: %s", id)
}
}
result.Turns = append(result.Turns, turnResult)
// Check if all required checkpoints reached
if r.allRequiredCheckpointsReached(result) {
result.Status = StatusPassed
result.DurationMs = time.Since(startTime).Milliseconds()
result.TotalTurns = turn
return result
}
}
// Max turns exceeded
result.Status = StatusFailed
result.Error = fmt.Sprintf("max turns (%d) exceeded without reaching all checkpoints", maxTurns)
result.DurationMs = time.Since(startTime).Milliseconds()
result.TotalTurns = maxTurns
return result
}
// buildSimulatorInput builds the input for the simulator agent
func (r *DynamicRunner) buildSimulatorInput(
tc *Case,
messages []context.Message,
result *DynamicResult,
turn, maxTurns int,
metadata map[string]interface{},
) *SimulatorInput {
input := &SimulatorInput{
Conversation: messages,
TurnNumber: turn,
MaxTurns: maxTurns,
}
// Extract persona and goal from metadata
if persona, ok := metadata["persona"].(string); ok {
input.Persona = persona
}
if goal, ok := metadata["goal"].(string); ok {
input.Goal = goal
}
// Build checkpoint lists
input.CheckpointsReached = make([]string, 0)
input.CheckpointsPending = make([]string, 0)
for id, cp := range result.Checkpoints {
if cp.Reached {
input.CheckpointsReached = append(input.CheckpointsReached, id)
} else {
input.CheckpointsPending = append(input.CheckpointsPending, id)
}
}
// Store extra metadata
input.Extra = make(map[string]interface{})
for k, v := range metadata {
if k != "persona" && k != "goal" {
input.Extra[k] = v
}
}
return input
}
// callSimulator calls the simulator agent and parses the response
func (r *DynamicRunner) callSimulator(simAST *assistant.Assistant, tc *Case, input *SimulatorInput) (*SimulatorOutput, error) {
// Create context
env := NewEnvironment("", "")
ctx := NewTestContext("simulator", tc.Simulator.Use, env)
defer ctx.Release()
// Build options - skip history and trace
opts := &context.Options{
Skip: &context.Skip{
History: true,
Trace: true,
Output: true,
},
Metadata: map[string]interface{}{
"test_mode": "simulator",
},
}
// Override connector if specified
if tc.Simulator.Options != nil && tc.Simulator.Options.Connector != "" {
opts.Connector = tc.Simulator.Options.Connector
}
// Build message
inputJSON, err := jsoniter.Marshal(input)
if err != nil {
return nil, fmt.Errorf("failed to marshal simulator input: %w", err)
}
messages := []context.Message{{
Role: context.RoleUser,
Content: string(inputJSON),
}}
// Call simulator
response, err := simAST.Stream(ctx, messages, opts)
if err != nil {
return nil, fmt.Errorf("simulator agent error: %w", err)
}
// Parse response
return r.parseSimulatorResponse(response)
}
// parseSimulatorResponse parses the simulator agent's response
func (r *DynamicRunner) parseSimulatorResponse(response *context.Response) (*SimulatorOutput, error) {
if response == nil || response.Completion == nil {
return nil, fmt.Errorf("empty response from simulator")
}
// Extract content
content := response.Completion.Content
if content == nil {
return nil, fmt.Errorf("no content in simulator response")
}
// Convert to string
var text string
switch v := content.(type) {
case string:
text = v
default:
data, err := jsoniter.Marshal(content)
if err != nil {
return nil, fmt.Errorf("failed to marshal content: %w", err)
}
text = string(data)
}
// Use goutext.ExtractJSON for fault-tolerant parsing
parsed := goutext.ExtractJSON(text)
if parsed == nil {
// Try to use the text as the message directly
return &SimulatorOutput{
Message: text,
GoalAchieved: false,
}, nil
}
// Parse as SimulatorOutput
output := &SimulatorOutput{}
if m, ok := parsed.(map[string]interface{}); ok {
if msg, ok := m["message"].(string); ok {
output.Message = msg
}
if achieved, ok := m["goal_achieved"].(bool); ok {
output.GoalAchieved = achieved
}
if reasoning, ok := m["reasoning"].(string); ok {
output.Reasoning = reasoning
}
}
if output.Message == "" {
return nil, fmt.Errorf("simulator returned empty message")
}
return output, nil
}
// checkCheckpoints validates checkpoints against current output
func (r *DynamicRunner) checkCheckpoints(checkpoints []*Checkpoint, output interface{}, result *DynamicResult) []string {
reachedIDs := make([]string, 0)
for _, cp := range checkpoints {
cpResult := result.Checkpoints[cp.ID]
if cpResult.Reached {
continue // Already reached
}
// Check "after" constraint
if len(cp.After) > 0 {
allAfterReached := true
for _, afterID := range cp.After {
if afterResult, ok := result.Checkpoints[afterID]; ok {
if !afterResult.Reached {
allAfterReached = false
break
}
}
}
if !allAfterReached {
continue // Dependencies not met
}
}
// Validate using asserter
tempCase := &Case{Assert: cp.Assert}
passed, msg := r.asserter.Validate(tempCase, output)
if passed {
cpResult.Reached = true
cpResult.Passed = true
cpResult.ReachedAtTurn = len(result.Turns) + 1
cpResult.Message = msg
reachedIDs = append(reachedIDs, cp.ID)
}
}
return reachedIDs
}
// allRequiredCheckpointsReached checks if all required checkpoints are reached
func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) bool {
for _, cp := range result.Checkpoints {
if cp.Required && !cp.Reached {
return false
}
}
return true
}

319
agent/test/dynamic_runner_test.go Normal file
View file

@ -0,0 +1,319 @@
package test_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/agent"
"github.com/yaoapp/yao/agent/test"
"github.com/yaoapp/yao/config"
testutils "github.com/yaoapp/yao/test"
)
func TestCase_IsDynamicMode(t *testing.T) {
tests := []struct {
name string
tc *test.Case
expected bool
}{
{
name: "standard mode - no simulator",
tc: &test.Case{
ID: "T001",
Input: "Hello",
},
expected: false,
},
{
name: "standard mode - simulator but no checkpoints",
tc: &test.Case{
ID: "T002",
Input: "Hello",
Simulator: &test.Simulator{Use: "tests.simulator-agent"},
},
expected: false,
},
{
name: "standard mode - checkpoints but no simulator",
tc: &test.Case{
ID: "T003",
Input: "Hello",
Checkpoints: []*test.Checkpoint{
{ID: "cp1", Assert: map[string]interface{}{"type": "contains", "value": "hi"}},
},
},
expected: false,
},
{
name: "dynamic mode - has both simulator and checkpoints",
tc: &test.Case{
ID: "T004",
Simulator: &test.Simulator{Use: "tests.simulator-agent"},
Checkpoints: []*test.Checkpoint{
{ID: "cp1", Assert: map[string]interface{}{"type": "contains", "value": "hi"}},
},
},
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tc.IsDynamicMode()
assert.Equal(t, tt.expected, result)
})
}
}
func TestCase_GetMaxTurns(t *testing.T) {
tests := []struct {
name string
tc *test.Case
expected int
}{
{
name: "default max turns",
tc: &test.Case{ID: "T001"},
expected: 20,
},
{
name: "custom max turns",
tc: &test.Case{ID: "T002", MaxTurns: 10},
expected: 10,
},
{
name: "zero max turns uses default",
tc: &test.Case{ID: "T003", MaxTurns: 0},
expected: 20,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.tc.GetMaxTurns()
assert.Equal(t, tt.expected, result)
})
}
}
func TestCheckpoint_IsRequired(t *testing.T) {
boolTrue := true
boolFalse := false
tests := []struct {
name string
cp *test.Checkpoint
expected bool
}{
{
name: "default is required",
cp: &test.Checkpoint{ID: "cp1"},
expected: true,
},
{
name: "explicitly required",
cp: &test.Checkpoint{ID: "cp2", Required: &boolTrue},
expected: true,
},
{
name: "explicitly not required",
cp: &test.Checkpoint{ID: "cp3", Required: &boolFalse},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := tt.cp.IsRequired()
assert.Equal(t, tt.expected, result)
})
}
}
func TestDynamicResult_ToResult(t *testing.T) {
dr := &test.DynamicResult{
ID: "T001",
Status: test.StatusPassed,
TotalTurns: 3,
DurationMs: 5000,
Turns: []*test.TurnResult{
{Turn: 1, Input: "Hello", Output: "Hi there!"},
{Turn: 2, Input: "How are you?", Output: "I'm doing well!"},
{Turn: 3, Input: "Goodbye", Output: "Bye!"},
},
Checkpoints: map[string]*test.CheckpointResult{
"greet": {ID: "greet", Reached: true, ReachedAtTurn: 1, Required: true},
"bye": {ID: "bye", Reached: true, ReachedAtTurn: 3, Required: true},
},
}
result := dr.ToResult()
assert.Equal(t, "T001", result.ID)
assert.Equal(t, test.StatusPassed, result.Status)
assert.Equal(t, int64(5000), result.DurationMs)
assert.Equal(t, "Hello", result.Input)
assert.Equal(t, "Bye!", result.Output)
// Check metadata
assert.NotNil(t, result.Metadata)
assert.Equal(t, "dynamic", result.Metadata["mode"])
assert.Equal(t, 3, result.Metadata["total_turns"])
}
func TestDynamicRunner_Integration(t *testing.T) {
// Skip if running in short mode
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
// Prepare test environment
testutils.Prepare(t, config.Conf)
defer testutils.Clean()
// Load agents
err := agent.Load(config.Conf)
if err != nil {
t.Skipf("Failed to load agents: %v", err)
}
// Create a dynamic test case
tc := &test.Case{
ID: "dynamic-greeting",
Simulator: &test.Simulator{
Use: "tests.simulator-agent",
Options: &test.SimulatorOptions{
Metadata: map[string]interface{}{
"persona": "Friendly user",
"goal": "Have a brief greeting exchange",
},
},
},
Input: "Hello!",
Checkpoints: []*test.Checkpoint{
{
ID: "greeting",
Description: "Agent should greet back",
Assert: map[string]interface{}{
"type": "regex",
"value": "(?i)(hello|hi|hey|greetings)",
},
},
},
MaxTurns: 3,
}
// Verify it's dynamic mode
assert.True(t, tc.IsDynamicMode())
// Create runner options
opts := &test.Options{
Verbose: true,
Timeout: 30 * time.Second,
}
// Create dynamic runner
runner := test.NewDynamicRunner(opts)
assert.NotNil(t, runner)
// Note: Full integration test would require the simulator agent to be loaded
// and would make actual LLM calls. For CI, we test the structure and logic.
}
func TestDynamicRunner_CheckpointOrdering(t *testing.T) {
// Test that checkpoints with "after" constraints are properly ordered
testutils.Prepare(t, config.Conf)
defer testutils.Clean()
// Load agents
err := agent.Load(config.Conf)
if err != nil {
t.Skipf("Failed to load agents: %v", err)
}
// Create a test case with ordered checkpoints
tc := &test.Case{
ID: "ordered-checkpoints",
Simulator: &test.Simulator{
Use: "tests.simulator-agent",
Options: &test.SimulatorOptions{
Metadata: map[string]interface{}{
"persona": "Customer",
"goal": "Complete a purchase",
},
},
},
Checkpoints: []*test.Checkpoint{
{
ID: "ask_product",
Description: "Agent asks about product",
Assert: map[string]interface{}{
"type": "contains",
"value": "product",
},
},
{
ID: "confirm_order",
Description: "Agent confirms order",
After: []string{"ask_product"},
Assert: map[string]interface{}{
"type": "contains",
"value": "confirm",
},
},
{
ID: "complete",
Description: "Order completed",
After: []string{"confirm_order"},
Assert: map[string]interface{}{
"type": "contains",
"value": "complete",
},
},
},
MaxTurns: 10,
}
// Verify checkpoint structure
assert.Len(t, tc.Checkpoints, 3)
assert.Empty(t, tc.Checkpoints[0].After)
assert.Equal(t, []string{"ask_product"}, tc.Checkpoints[1].After)
assert.Equal(t, []string{"confirm_order"}, tc.Checkpoints[2].After)
}
func TestSimulatorInput_Structure(t *testing.T) {
// Test SimulatorInput structure
input := &test.SimulatorInput{
Persona: "Test user",
Goal: "Complete task",
TurnNumber: 3,
MaxTurns: 10,
CheckpointsReached: []string{"cp1", "cp2"},
CheckpointsPending: []string{"cp3"},
Extra: map[string]interface{}{
"style": "formal",
},
}
assert.Equal(t, "Test user", input.Persona)
assert.Equal(t, "Complete task", input.Goal)
assert.Equal(t, 3, input.TurnNumber)
assert.Equal(t, 10, input.MaxTurns)
assert.Len(t, input.CheckpointsReached, 2)
assert.Len(t, input.CheckpointsPending, 1)
assert.Equal(t, "formal", input.Extra["style"])
}
func TestSimulatorOutput_Structure(t *testing.T) {
// Test SimulatorOutput structure
output := &test.SimulatorOutput{
Message: "I'd like to buy a product",
GoalAchieved: false,
Reasoning: "Continuing toward purchase goal",
}
assert.Equal(t, "I'd like to buy a product", output.Message)
assert.False(t, output.GoalAchieved)
assert.Equal(t, "Continuing toward purchase goal", output.Reasoning)
}

159
agent/test/dynamic_types.go Normal file
View file

@ -0,0 +1,159 @@
package test
import "github.com/yaoapp/yao/agent/context"
// DynamicResult represents the result of a dynamic (simulator-driven) test
type DynamicResult struct {
// ID is the test case identifier
ID string `json:"id"`
// Status is the overall test status
Status Status `json:"status"`
// Turns contains results for each conversation turn
Turns []*TurnResult `json:"turns"`
// Checkpoints maps checkpoint ID to its result
Checkpoints map[string]*CheckpointResult `json:"checkpoints"`
// TotalTurns is the number of turns executed
TotalTurns int `json:"total_turns"`
// DurationMs is the total execution time in milliseconds
DurationMs int64 `json:"duration_ms"`
// Error contains error message if status is failed/error/timeout
Error string `json:"error,omitempty"`
}
// TurnResult represents the result of a single conversation turn
type TurnResult struct {
// Turn is the turn number (1-based)
Turn int `json:"turn"`
// Input is the user message (from simulator or initial input)
Input interface{} `json:"input"`
// Output is the agent's response
Output interface{} `json:"output,omitempty"`
// CheckpointsReached lists checkpoint IDs reached in this turn
CheckpointsReached []string `json:"checkpoints_reached,omitempty"`
// DurationMs is the turn execution time in milliseconds
DurationMs int64 `json:"duration_ms"`
// Error contains error message if this turn failed
Error string `json:"error,omitempty"`
}
// CheckpointResult represents the result of a checkpoint validation
type CheckpointResult struct {
// ID is the checkpoint identifier
ID string `json:"id"`
// Reached indicates if the checkpoint was reached
Reached bool `json:"reached"`
// ReachedAtTurn is the turn number when checkpoint was reached (0 if not reached)
ReachedAtTurn int `json:"reached_at_turn,omitempty"`
// Required indicates if this checkpoint is required
Required bool `json:"required"`
// Passed indicates if the checkpoint assertion passed
Passed bool `json:"passed"`
// Message contains assertion result message
Message string `json:"message,omitempty"`
}
// SimulatorInput is the input sent to the simulator agent
type SimulatorInput struct {
// Persona describes the user being simulated
Persona string `json:"persona,omitempty"`
// Goal is what the user is trying to achieve
Goal string `json:"goal,omitempty"`
// Conversation is the message history
Conversation []context.Message `json:"conversation"`
// TurnNumber is the current turn (1-based)
TurnNumber int `json:"turn_number"`
// MaxTurns is the maximum allowed turns
MaxTurns int `json:"max_turns"`
// CheckpointsReached lists checkpoint IDs already reached
CheckpointsReached []string `json:"checkpoints_reached,omitempty"`
// CheckpointsPending lists checkpoint IDs still pending
CheckpointsPending []string `json:"checkpoints_pending,omitempty"`
// Extra metadata from simulator options
Extra map[string]interface{} `json:"extra,omitempty"`
}
// SimulatorOutput is the expected output from the simulator agent
type SimulatorOutput struct {
// Message is the simulated user message
Message string `json:"message"`
// GoalAchieved indicates if the user's goal has been accomplished
GoalAchieved bool `json:"goal_achieved"`
// Reasoning explains the simulator's response strategy
Reasoning string `json:"reasoning,omitempty"`
}
// ToResult converts DynamicResult to standard Result for reporting
func (dr *DynamicResult) ToResult() *Result {
result := &Result{
ID: dr.ID,
Status: dr.Status,
DurationMs: dr.DurationMs,
Error: dr.Error,
}
// Store dynamic-specific data in metadata
result.Metadata = map[string]interface{}{
"mode": "dynamic",
"total_turns": dr.TotalTurns,
"turns": dr.Turns,
"checkpoints": dr.Checkpoints,
}
// Set input from first turn
if len(dr.Turns) > 0 {
result.Input = dr.Turns[0].Input
}
// Set output from last turn
if len(dr.Turns) > 0 {
result.Output = dr.Turns[len(dr.Turns)-1].Output
}
return result
}
// IsDynamicMode checks if a test case should run in dynamic mode
func (tc *Case) IsDynamicMode() bool {
return tc.Simulator != nil && len(tc.Checkpoints) > 0
}
// GetMaxTurns returns the max turns for dynamic mode
func (tc *Case) GetMaxTurns() int {
if tc.MaxTurns > 0 {
return tc.MaxTurns
}
return 20 // Default max turns
}
// IsRequired returns true if the checkpoint is required
func (cp *Checkpoint) IsRequired() bool {
if cp.Required == nil {
return true // Default to required
}
return *cp.Required
}

392
agent/test/input_source.go Normal file
View file

@ -0,0 +1,392 @@
package test
import (
"fmt"
"net/url"
"strconv"
"strings"
jsoniter "github.com/json-iterator/go"
goutext "github.com/yaoapp/gou/text"
"github.com/yaoapp/yao/agent/assistant"
"github.com/yaoapp/yao/agent/context"
)
// InputSourceType represents the type of input source
type InputSourceType string
const (
// InputSourceFile indicates input from a JSONL file
InputSourceFile InputSourceType = "file"
// InputSourceMessage indicates input from a direct message string
InputSourceMessage InputSourceType = "message"
// InputSourceScript indicates script test mode
InputSourceScript InputSourceType = "script"
// InputSourceAgent indicates input generated by an agent
InputSourceAgent InputSourceType = "agent"
)
// InputSource represents a parsed input source
type InputSource struct {
Type InputSourceType // file, message, script, agent
Value string // path, message, script ref, or agent ID
Params map[string]interface{} // query parameters (for agent source)
}
// ParseInputSource parses the -i flag value into an InputSource
// Supported formats:
// - "agents:workers.test.generator" - Agent-generated test cases
// - "agents:workers.test.generator?count=10&focus=edge-cases" - With parameters
// - "scripts.tests.gen" - Script-generated test cases
// - "./tests/inputs.jsonl" - JSONL file
// - "Hello, how are you?" - Direct message
func ParseInputSource(input string) *InputSource {
// Check for agents: prefix
if strings.HasPrefix(input, "agents:") {
return parseAgentSource(strings.TrimPrefix(input, "agents:"))
}
// Check for scripts: prefix (for generator scripts)
if strings.HasPrefix(input, "scripts:") {
return &InputSource{
Type: InputSourceScript,
Value: strings.TrimPrefix(input, "scripts:"),
}
}
// Check for script test mode (scripts.xxx format without prefix)
if strings.HasPrefix(input, "scripts.") {
return &InputSource{
Type: InputSourceScript,
Value: input,
}
}
// Check for file extension
if strings.HasSuffix(input, ".jsonl") || strings.HasSuffix(input, ".json") {
return &InputSource{
Type: InputSourceFile,
Value: input,
}
}
// Check if it looks like a file path
if strings.Contains(input, "/") || strings.Contains(input, "\\") {
return &InputSource{
Type: InputSourceFile,
Value: input,
}
}
// Default to message
return &InputSource{
Type: InputSourceMessage,
Value: input,
}
}
// parseAgentSource parses an agent source string with optional query parameters
// Format: "agent.id" or "agent.id?count=10&focus=edge-cases"
func parseAgentSource(input string) *InputSource {
source := &InputSource{
Type: InputSourceAgent,
Params: make(map[string]interface{}),
}
// Check for query parameters
if idx := strings.Index(input, "?"); idx >= 0 {
source.Value = input[:idx]
queryStr := input[idx+1:]
// Parse query parameters
values, err := url.ParseQuery(queryStr)
if err == nil {
for key, vals := range values {
if len(vals) > 0 {
// Try to parse as number
if num, err := strconv.Atoi(vals[0]); err == nil {
source.Params[key] = num
} else if num, err := strconv.ParseFloat(vals[0], 64); err == nil {
source.Params[key] = num
} else if vals[0] == "true" {
source.Params[key] = true
} else if vals[0] == "false" {
source.Params[key] = false
} else {
source.Params[key] = vals[0]
}
}
}
}
} else {
source.Value = input
}
return source
}
// GeneratorInput represents the input sent to a generator agent
type GeneratorInput struct {
TargetAgent *TargetAgentInfo `json:"target_agent"`
Count int `json:"count,omitempty"`
Focus string `json:"focus,omitempty"`
Extra map[string]interface{} `json:"extra,omitempty"`
}
// TargetAgentInfo contains information about the agent being tested
type TargetAgentInfo struct {
ID string `json:"id"`
Description string `json:"description,omitempty"`
Tools []map[string]interface{} `json:"tools,omitempty"`
}
// GenerateTestCases generates test cases using a generator agent
func GenerateTestCases(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) {
// Get generator assistant
ast, err := assistant.Get(agentID)
if err != nil {
return nil, fmt.Errorf("failed to get generator agent %s: %w", agentID, err)
}
// Build generation request
genInput := &GeneratorInput{
TargetAgent: targetInfo,
Count: 5, // Default count
}
// Apply parameters
if params != nil {
if count, ok := params["count"].(int); ok {
genInput.Count = count
}
if focus, ok := params["focus"].(string); ok {
genInput.Focus = focus
}
// Store extra parameters
genInput.Extra = make(map[string]interface{})
for k, v := range params {
if k != "count" && k != "focus" {
genInput.Extra[k] = v
}
}
}
// Create context
env := NewEnvironment("", "")
ctx := NewTestContext("generator", agentID, env)
defer ctx.Release()
// Build options - skip history and trace for efficiency
opts := &context.Options{
Skip: &context.Skip{
History: true,
Trace: true,
Output: true,
},
Metadata: map[string]interface{}{
"test_mode": "generator",
},
}
// Build message
inputJSON, err := jsoniter.Marshal(genInput)
if err != nil {
return nil, fmt.Errorf("failed to marshal generator input: %w", err)
}
messages := []context.Message{{
Role: context.RoleUser,
Content: string(inputJSON),
}}
// Call generator agent
response, err := ast.Stream(ctx, messages, opts)
if err != nil {
return nil, fmt.Errorf("generator agent error: %w", err)
}
// Extract and parse response
return parseGeneratedCases(response)
}
// parseGeneratedCases parses the generator agent's response into test cases
func parseGeneratedCases(response *context.Response) ([]*Case, error) {
if response == nil || response.Completion == nil {
return nil, fmt.Errorf("empty response from generator agent")
}
// Extract content
content := response.Completion.Content
if content == nil {
return nil, fmt.Errorf("no content in generator response")
}
// Convert content to string
var text string
switch v := content.(type) {
case string:
text = v
default:
data, err := jsoniter.Marshal(content)
if err != nil {
return nil, fmt.Errorf("failed to marshal content: %w", err)
}
text = string(data)
}
// Use goutext.ExtractJSON for fault-tolerant parsing
parsed := goutext.ExtractJSON(text)
if parsed == nil {
return nil, fmt.Errorf("failed to parse generator response as JSON: %s", truncateOutput(text, 200))
}
// Convert to []*Case
return convertToCases(parsed)
}
// convertToCases converts parsed JSON to test cases
func convertToCases(parsed interface{}) ([]*Case, error) {
// Handle array of cases
arr, ok := parsed.([]interface{})
if !ok {
// Maybe it's a single case wrapped in an object
if obj, ok := parsed.(map[string]interface{}); ok {
if cases, ok := obj["cases"].([]interface{}); ok {
arr = cases
} else if testCases, ok := obj["test_cases"].([]interface{}); ok {
arr = testCases
} else {
// Single case
arr = []interface{}{obj}
}
} else {
return nil, fmt.Errorf("expected array of test cases, got %T", parsed)
}
}
cases := make([]*Case, 0, len(arr))
for i, item := range arr {
caseMap, ok := item.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("test case %d is not an object", i)
}
tc, err := mapToCase(caseMap)
if err != nil {
return nil, fmt.Errorf("failed to parse test case %d: %w", i, err)
}
cases = append(cases, tc)
}
return cases, nil
}
// mapToCase converts a map to a Case struct
func mapToCase(m map[string]interface{}) (*Case, error) {
tc := &Case{}
// Required: id
if id, ok := m["id"].(string); ok {
tc.ID = id
} else {
return nil, fmt.Errorf("missing required field 'id'")
}
// Required: input
if input, ok := m["input"]; ok {
tc.Input = input
} else {
return nil, fmt.Errorf("missing required field 'input'")
}
// Optional: assertions/assert
if assertions, ok := m["assertions"]; ok {
tc.Assert = assertions
} else if assert, ok := m["assert"]; ok {
tc.Assert = assert
}
// Optional: options - convert map to CaseOptions
if options, ok := m["options"].(map[string]interface{}); ok {
tc.Options = mapToCaseOptions(options)
}
// Optional: before/after
if before, ok := m["before"].(string); ok {
tc.Before = before
}
if after, ok := m["after"].(string); ok {
tc.After = after
}
// Optional: timeout
if timeout, ok := m["timeout"].(string); ok {
tc.Timeout = timeout
}
return tc, nil
}
// ToInputMode converts InputSourceType to InputMode for backward compatibility
func (s *InputSource) ToInputMode() InputMode {
switch s.Type {
case InputSourceFile:
return InputModeFile
case InputSourceMessage:
return InputModeMessage
case InputSourceScript:
return InputModeScript
case InputSourceAgent:
// Agent source generates cases, then runs in file mode
return InputModeFile
default:
return InputModeMessage
}
}
// mapToCaseOptions converts a map to CaseOptions
func mapToCaseOptions(m map[string]interface{}) *CaseOptions {
opts := &CaseOptions{}
if connector, ok := m["connector"].(string); ok {
opts.Connector = connector
}
if mode, ok := m["mode"].(string); ok {
opts.Mode = mode
}
if disableGlobalPrompts, ok := m["disable_global_prompts"].(bool); ok {
opts.DisableGlobalPrompts = disableGlobalPrompts
}
if search, ok := m["search"].(bool); ok {
opts.Search = &search
}
if metadata, ok := m["metadata"].(map[string]interface{}); ok {
opts.Metadata = metadata
}
if skip, ok := m["skip"].(map[string]interface{}); ok {
opts.Skip = &CaseSkipOptions{}
if history, ok := skip["history"].(bool); ok {
opts.Skip.History = history
}
if trace, ok := skip["trace"].(bool); ok {
opts.Skip.Trace = trace
}
if output, ok := skip["output"].(bool); ok {
opts.Skip.Output = output
}
if keyword, ok := skip["keyword"].(bool); ok {
opts.Skip.Keyword = keyword
}
if searchSkip, ok := skip["search"].(bool); ok {
opts.Skip.Search = searchSkip
}
}
return opts
}

181
agent/test/input_source_test.go Normal file
View file

@ -0,0 +1,181 @@
package test_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/yao/agent"
agenttest "github.com/yaoapp/yao/agent/test"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestParseInputSource(t *testing.T) {
tests := []struct {
name string
input string
wantType agenttest.InputSourceType
wantValue string
wantParams map[string]interface{}
}{
{
name: "JSONL file",
input: "./tests/inputs.jsonl",
wantType: agenttest.InputSourceFile,
wantValue: "./tests/inputs.jsonl",
},
{
name: "JSON file",
input: "./tests/inputs.json",
wantType: agenttest.InputSourceFile,
wantValue: "./tests/inputs.json",
},
{
name: "direct message",
input: "Hello, how are you?",
wantType: agenttest.InputSourceMessage,
wantValue: "Hello, how are you?",
},
{
name: "agent source simple",
input: "agents:tests.generator-agent",
wantType: agenttest.InputSourceAgent,
wantValue: "tests.generator-agent",
},
{
name: "agent source with params",
input: "agents:tests.generator-agent?count=10&focus=edge-cases",
wantType: agenttest.InputSourceAgent,
wantValue: "tests.generator-agent",
wantParams: map[string]interface{}{
"count": 10,
"focus": "edge-cases",
},
},
{
name: "agent source with boolean param",
input: "agents:tests.generator-agent?verbose=true",
wantType: agenttest.InputSourceAgent,
wantValue: "tests.generator-agent",
wantParams: map[string]interface{}{
"verbose": true,
},
},
{
name: "script source with prefix",
input: "scripts:tests.gen.Generate",
wantType: agenttest.InputSourceScript,
wantValue: "tests.gen.Generate",
},
{
name: "script test mode",
input: "scripts.tests.gen",
wantType: agenttest.InputSourceScript,
wantValue: "scripts.tests.gen",
},
{
name: "path with separator",
input: "/path/to/inputs.jsonl",
wantType: agenttest.InputSourceFile,
wantValue: "/path/to/inputs.jsonl",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
source := agenttest.ParseInputSource(tt.input)
assert.Equal(t, tt.wantType, source.Type, "Type mismatch")
assert.Equal(t, tt.wantValue, source.Value, "Value mismatch")
if tt.wantParams != nil {
for k, v := range tt.wantParams {
assert.Equal(t, v, source.Params[k], "Param %s mismatch", k)
}
}
})
}
}
func TestInputSource_ToInputMode(t *testing.T) {
tests := []struct {
name string
source *agenttest.InputSource
wantMode agenttest.InputMode
}{
{
name: "file source",
source: &agenttest.InputSource{Type: agenttest.InputSourceFile},
wantMode: agenttest.InputModeFile,
},
{
name: "message source",
source: &agenttest.InputSource{Type: agenttest.InputSourceMessage},
wantMode: agenttest.InputModeMessage,
},
{
name: "script source",
source: &agenttest.InputSource{Type: agenttest.InputSourceScript},
wantMode: agenttest.InputModeScript,
},
{
name: "agent source",
source: &agenttest.InputSource{Type: agenttest.InputSourceAgent},
wantMode: agenttest.InputModeFile, // Agent generates cases, then runs in file mode
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mode := tt.source.ToInputMode()
assert.Equal(t, tt.wantMode, mode)
})
}
}
func TestGenerateTestCases(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent (includes assistants)
err := agent.Load(config.Conf)
if err != nil {
t.Fatalf("Failed to load agent: %v", err)
}
// Test generating test cases from the generator agent
targetInfo := &agenttest.TargetAgentInfo{
ID: "tests.next",
Description: "A simple test agent for greeting",
}
params := map[string]interface{}{
"count": 3,
"focus": "happy-path",
}
cases, err := agenttest.GenerateTestCases("tests.generator-agent", targetInfo, params)
if err != nil {
t.Fatalf("Failed to generate test cases: %v", err)
}
// Verify we got some test cases
assert.NotEmpty(t, cases, "Should generate at least one test case")
// Verify each case has required fields
for _, tc := range cases {
assert.NotEmpty(t, tc.ID, "Test case should have ID")
assert.NotNil(t, tc.Input, "Test case should have Input")
}
t.Logf("Generated %d test cases", len(cases))
for _, tc := range cases {
t.Logf(" - %s", tc.ID)
}
}
func TestMapToCaseOptions(t *testing.T) {
// Test that options map is correctly converted
source := agenttest.ParseInputSource("agents:test?count=5")
assert.Equal(t, 5, source.Params["count"])
}

6
agent/test/interfaces.go
View file

@ -43,6 +43,12 @@ type Loader interface {
// LoadFile loads test cases from a JSONL file
LoadFile(path string) ([]*Case, error)
// LoadFromAgent generates test cases using a generator agent
LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error)
// LoadFromScript generates test cases using a script
LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error)
}
// Resolver is the interface for resolving agent information

33
agent/test/loader.go
View file

@ -8,6 +8,7 @@ import (
"time"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/process"
)
// JSONLLoader loads test cases from JSONL files
@ -143,3 +144,35 @@ func FilterByIDs(cases []*Case, ids []string) []*Case {
return idSet[tc.ID]
})
}
// LoadFromAgent generates test cases using a generator agent
func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) {
return GenerateTestCases(agentID, targetInfo, params)
}
// LoadFromScript generates test cases using a script
// scriptRef format: "module.FunctionName" (e.g., "tests.gen.Generate")
func (l *JSONLLoader) LoadFromScript(scriptRef string, targetInfo *TargetAgentInfo) ([]*Case, error) {
// Parse script reference
parts := strings.Split(scriptRef, ".")
if len(parts) < 2 {
return nil, fmt.Errorf("invalid script reference format: %s (expected 'module.Function')", scriptRef)
}
// Build process name: scripts.module.Function
processName := "scripts." + scriptRef
// Execute via process
p, err := process.Of(processName, targetInfo)
if err != nil {
return nil, fmt.Errorf("failed to create process %s: %w", processName, err)
}
result, err := p.Exec()
if err != nil {
return nil, fmt.Errorf("script execution failed: %w", err)
}
// Parse result as test cases
return convertToCases(result)
}

39
agent/test/output.go
View file

@ -303,6 +303,45 @@ func (w *OutputWriter) ScriptTestSummary(summary *ScriptTestSummary, duration ti
fmt.Printf("%s\n", formatDuration(duration))
}
// DynamicTestStart outputs the start of a dynamic test
func (w *OutputWriter) DynamicTestStart(id string, checkpointCount int) {
color.New(color.FgWhite).Printf("► [%s] ", id)
color.New(color.FgCyan).Printf("(dynamic, %d checkpoints)\n", checkpointCount)
}
// DynamicTurn outputs a single turn in dynamic testing
func (w *OutputWriter) DynamicTurn(turn int, inputSummary string, checkpointsReached, total int) {
if w.verbose {
color.New(color.FgHiBlack).Printf("│ ├─ Turn %d: %s ", turn, inputSummary)
color.New(color.FgCyan).Printf("[%d/%d checkpoints]\n", checkpointsReached, total)
}
}
// DynamicCheckpoint outputs a checkpoint being reached
func (w *OutputWriter) DynamicCheckpoint(checkpointID string) {
if w.verbose {
color.New(color.FgGreen).Printf("│ │ └─ ✓ checkpoint: %s\n", checkpointID)
}
}
// DynamicTestResult outputs the result of a dynamic test
func (w *OutputWriter) DynamicTestResult(status Status, turns int, checkpoints int, duration time.Duration) {
color.New(color.FgHiBlack).Printf(" └─ ")
switch status {
case StatusPassed:
color.New(color.FgGreen).Printf("PASSED")
case StatusFailed:
color.New(color.FgRed).Printf("FAILED")
case StatusError:
color.New(color.FgRed).Printf("ERROR")
case StatusTimeout:
color.New(color.FgRed).Printf("TIMEOUT")
}
color.New(color.FgHiBlack).Printf(" (%d turns, %d checkpoints, %s)\n", turns, checkpoints, formatDuration(duration))
}
// StabilityResult prints stability analysis result for a test case
func (w *OutputWriter) StabilityResult(sr *StabilityResult) {
color.New(color.FgWhite).Printf(" [%s] ", sr.ID)

268
agent/test/runner.go
View file

@ -1,11 +1,12 @@
package test
import (
"bufio"
stdContext "context"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"time"
@ -16,19 +17,22 @@ import (
// Executor executes test cases against an agent
type Executor struct {
opts *Options
output *OutputWriter
resolver Resolver
loader Loader
opts *Options
output *OutputWriter
resolver Resolver
loader Loader
hookExecutor *HookExecutor
agentPath string // Path to the agent being tested
}
// NewRunner creates a new test runner
func NewRunner(opts *Options) *Executor {
return &Executor{
opts: opts,
output: NewOutputWriter(opts.Verbose),
resolver: NewResolver(),
loader: NewLoader(),
opts: opts,
output: NewOutputWriter(opts.Verbose),
resolver: NewResolver(),
loader: NewLoader(),
hookExecutor: NewHookExecutor(opts.Verbose),
}
}
@ -161,21 +165,65 @@ func (r *Executor) RunTests() (*Report, error) {
}
r.output.Info("Agent: %s", agentInfo.ID)
r.agentPath = agentInfo.Path // Store agent path for hook execution
if r.opts.Connector != "" {
r.output.Info("Connector: %s (override)", r.opts.Connector)
} else if agentInfo.Connector != "" {
r.output.Info("Connector: %s", agentInfo.Connector)
}
// Load test cases
// Load test cases based on input source
var testCases []*Case
inputSource := ParseInputSource(r.opts.Input)
// File mode - load from JSONL
testCases, err = r.loader.LoadFile(r.opts.Input)
if err != nil {
return nil, fmt.Errorf("failed to load test cases: %w", err)
switch inputSource.Type {
case InputSourceAgent:
// Generate test cases using agent
r.output.Info("Generating test cases from agent: %s", inputSource.Value)
targetInfo := &TargetAgentInfo{
ID: agentInfo.ID,
Description: agentInfo.Description,
}
testCases, err = r.loader.LoadFromAgent(inputSource.Value, targetInfo, inputSource.Params)
if err != nil {
return nil, fmt.Errorf("failed to generate test cases: %w", err)
}
r.output.Info("Generated: %d test cases", len(testCases))
case InputSourceScript:
// Generate test cases using script (if it's a generator script, not test script)
// Note: scripts. prefix without "scripts:" is handled by RunScriptTests
if strings.HasPrefix(r.opts.Input, "scripts:") {
scriptRef := strings.TrimPrefix(r.opts.Input, "scripts:")
r.output.Info("Generating test cases from script: %s", scriptRef)
targetInfo := &TargetAgentInfo{
ID: agentInfo.ID,
Description: agentInfo.Description,
}
testCases, err = r.loader.LoadFromScript(scriptRef, targetInfo)
if err != nil {
return nil, fmt.Errorf("failed to generate test cases from script: %w", err)
}
r.output.Info("Generated: %d test cases", len(testCases))
} else {
// This is a test script (scripts.xxx format), handled by RunScriptTests
return nil, fmt.Errorf("script test mode should be handled by RunScriptTests")
}
default:
// File mode - load from JSONL
testCases, err = r.loader.LoadFile(r.opts.Input)
if err != nil {
return nil, fmt.Errorf("failed to load test cases: %w", err)
}
r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases))
}
// Handle dry-run mode - just output the generated test cases
if r.opts.DryRun {
r.output.Info("Dry-run mode: outputting generated test cases")
return r.outputDryRun(testCases, agentInfo)
}
r.output.Info("Input: %s (%d test cases)", r.opts.Input, len(testCases))
// Filter skipped tests
activeTests := FilterSkipped(testCases)
@ -222,6 +270,27 @@ func (r *Executor) RunTests() (*Report, error) {
},
}
// Execute global BeforeAll if specified
var globalBeforeData interface{}
if r.opts.BeforeAll != "" {
r.output.Info("BeforeAll: %s", r.opts.BeforeAll)
var err error
globalBeforeData, err = r.hookExecutor.ExecuteBeforeAll(r.opts.BeforeAll, activeTests, agentInfo.Path)
if err != nil {
return nil, fmt.Errorf("beforeAll script failed: %w", err)
}
}
// Ensure AfterAll runs even if tests fail
defer func() {
if r.opts.AfterAll != "" {
r.output.Info("AfterAll: %s", r.opts.AfterAll)
if err := r.hookExecutor.ExecuteAfterAll(r.opts.AfterAll, report.Results, globalBeforeData, agentInfo.Path); err != nil {
r.output.Warning("afterAll script failed: %s", err.Error())
}
}
}()
// Run tests
r.output.SubHeader("Running Tests")
@ -308,6 +377,11 @@ func (r *Executor) runParallel(ast *assistant.Assistant, testCases []*Case, agen
// runSingleTest runs a single test case
func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID string, runNum int) *Result {
// Check if this is a dynamic mode test
if tc.IsDynamicMode() {
return r.runDynamicTest(ast, tc, agentID)
}
// Get input summary for display
inputSummary := SummarizeInput(tc.Input, 50)
r.output.TestStart(tc.ID, inputSummary, runNum)
@ -322,6 +396,31 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str
Options: tc.Options,
}
// Execute before script if specified
var beforeData interface{}
if tc.Before != "" {
var err error
beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath)
if err != nil {
result.Status = StatusError
result.Error = fmt.Sprintf("before script failed: %s", err.Error())
result.DurationMs = time.Since(startTime).Milliseconds()
r.output.TestResult(result.Status, time.Since(startTime))
r.output.TestError(result.Error)
// Note: after script is NOT called when before fails
return result
}
}
// Ensure after script runs even if test fails (but only if before succeeded)
defer func() {
if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) {
if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil {
r.output.Warning("after script failed: %s", err.Error())
}
}
}()
// Parse input to messages with file loading support
// BaseDir is derived from the input file directory
inputOpts := r.getInputOptions()
@ -395,6 +494,71 @@ func (r *Executor) runSingleTest(ast *assistant.Assistant, tc *Case, agentID str
return result
}
// runDynamicTest runs a dynamic (simulator-driven) test case
func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID string) *Result {
// Output test start for dynamic mode
r.output.DynamicTestStart(tc.ID, len(tc.Checkpoints))
startTime := time.Now()
// Execute before script if specified
var beforeData interface{}
if tc.Before != "" {
var err error
beforeData, err = r.hookExecutor.ExecuteBefore(tc.Before, tc, r.agentPath)
if err != nil {
result := &Result{
ID: tc.ID,
Status: StatusError,
Error: fmt.Sprintf("before script failed: %s", err.Error()),
DurationMs: time.Since(startTime).Milliseconds(),
}
r.output.TestResult(result.Status, time.Since(startTime))
r.output.TestError(result.Error)
return result
}
}
// Create dynamic runner and execute
dynamicRunner := NewDynamicRunner(r.opts)
dynamicResult := dynamicRunner.RunDynamic(ast, tc, agentID)
// Convert to standard result
result := dynamicResult.ToResult()
// Execute after script if specified
defer func() {
if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) {
if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil {
r.output.Warning("after script failed: %s", err.Error())
}
}
}()
// Output result
duration := time.Duration(result.DurationMs) * time.Millisecond
r.output.DynamicTestResult(result.Status, dynamicResult.TotalTurns, len(tc.Checkpoints), duration)
if result.Error != "" {
r.output.TestError(result.Error)
}
return result
}
// isBeforeError checks if the error message indicates a before script failure
func isBeforeError(errMsg string) bool {
return len(errMsg) > 0 && errMsg[:min(len(errMsg), 20)] == "before script failed"
}
// min returns the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
// runStabilityTests runs each test case multiple times for stability analysis
func (r *Executor) runStabilityTests(ast *assistant.Assistant, testCases []*Case, agentID string) []*StabilityResult {
results := make([]*StabilityResult, 0, len(testCases))
@ -499,20 +663,6 @@ func (r *Executor) writeOutput(report *Report) error {
return reporter.Write(report, file)
}
// writeJSONLine writes a JSON line to the writer
func writeJSONLine(writer *bufio.Writer, data interface{}) error {
line, err := jsoniter.Marshal(data)
if err != nil {
return err
}
_, err = writer.Write(line)
if err != nil {
return err
}
_, err = writer.WriteString("\n")
return err
}
// buildContextOptions builds context.Options from test case and runner options
// Priority: test case options > runner options > defaults
func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options {
@ -593,6 +743,12 @@ func isEmptyValue(v interface{}) bool {
return true
}
// Use reflection to check for typed nil (e.g., *NextHookResponse(nil))
rv := reflect.ValueOf(v)
if rv.Kind() == reflect.Ptr && rv.IsNil() {
return true
}
switch val := v.(type) {
case string:
return val == ""
@ -600,6 +756,12 @@ func isEmptyValue(v interface{}) bool {
return len(val) == 0
case []interface{}:
return len(val) == 0
case *context.NextHookResponse:
// Check if NextHookResponse is effectively empty
if val == nil {
return true
}
return val.Data == nil && val.Delegate == nil
}
return false
@ -633,3 +795,51 @@ func (r *Executor) getInputOptions() *InputOptions {
return opts
}
// outputDryRun outputs generated test cases without running them
func (r *Executor) outputDryRun(testCases []*Case, agentInfo *AgentInfo) (*Report, error) {
r.output.Info("Generated Test Cases:")
// Output each test case as JSONL
for _, tc := range testCases {
data, err := jsoniter.Marshal(tc)
if err != nil {
r.output.Warning("Failed to marshal test case %s: %s", tc.ID, err.Error())
continue
}
fmt.Println(string(data))
}
// Write to output file if specified
if r.opts.OutputFile != "" {
file, err := os.Create(r.opts.OutputFile)
if err != nil {
return nil, fmt.Errorf("failed to create output file: %w", err)
}
defer file.Close()
for _, tc := range testCases {
data, err := jsoniter.Marshal(tc)
if err != nil {
continue
}
file.WriteString(string(data) + "\n")
}
r.output.Info("Output written to: %s", r.opts.OutputFile)
}
// Return a minimal report
connector := r.opts.Connector
if connector == "" {
connector = agentInfo.Connector
}
return &Report{
Summary: &Summary{
Total: len(testCases),
AgentID: agentInfo.ID,
Connector: connector,
},
}, nil
}

280
agent/test/runner_integration_test.go Normal file
View file

@ -0,0 +1,280 @@
package test_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/yao/agent"
agenttest "github.com/yaoapp/yao/agent/test"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
// TestRunner_AgentDrivenInput tests the complete flow:
// 1. Use generator-agent to generate test cases
// 2. Run the generated tests against simple-greeting agent
func TestRunner_AgentDrivenInput(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Test with agent-driven input
opts := &agenttest.Options{
Input: "agents:tests.generator-agent?count=3",
AgentID: "tests.simple-greeting",
Verbose: true,
InputMode: agenttest.InputModeFile, // Will be overridden by ParseInputSource
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
require.NotNil(t, report.Summary, "Summary should not be nil")
// Verify report
assert.Greater(t, report.Summary.Total, 0, "Should have at least one test case")
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
}
// TestRunner_AgentDrivenInput_DryRun tests dry-run mode
func TestRunner_AgentDrivenInput_DryRun(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Test with dry-run mode
opts := &agenttest.Options{
Input: "agents:tests.generator-agent?count=2",
AgentID: "tests.simple-greeting",
DryRun: true,
Verbose: true,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Dry-run should not return error")
require.NotNil(t, report, "Report should not be nil")
// In dry-run mode, tests are generated but not executed
// So Passed and Failed should both be 0, but Total should have the count
assert.Greater(t, report.Summary.Total, 0, "Should have generated test cases")
t.Logf("Generated %d test cases in dry-run mode", report.Summary.Total)
}
// TestRunner_FileInput tests loading test cases from JSONL file
func TestRunner_FileInput(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Create a temporary JSONL file with test cases
// Use case-insensitive contains for robustness
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
testCases := `{"id": "greeting-hello", "input": "Hello", "assert": {"type": "regex", "value": "(?i)hello"}}
{"id": "greeting-hi", "input": "Hi there", "assert": {"type": "regex", "value": "(?i)(hi|hello)"}}
{"id": "greeting-morning", "input": "Good morning", "assert": {"type": "regex", "value": "(?i)(hello|morning|good)"}}`
err = os.WriteFile(inputFile, []byte(testCases), 0644)
require.NoError(t, err, "Failed to write test file")
// Run tests from file
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.simple-greeting",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
// Verify report
assert.Equal(t, 3, report.Summary.Total, "Should have 3 test cases")
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
// Check results for debugging
if report.Results != nil {
for _, r := range report.Results {
t.Logf(" [%s] Status: %s, Output: %v", r.ID, r.Status, r.Output)
}
}
}
// TestRunner_DirectMessage tests direct message mode
func TestRunner_DirectMessage(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Test with direct message
opts := &agenttest.Options{
Input: "Hello, how are you?",
AgentID: "tests.simple-greeting",
Verbose: true,
InputMode: agenttest.InputModeMessage,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
// Direct message mode returns a minimal report
assert.Equal(t, 1, report.Summary.Total, "Should have 1 test case")
assert.Equal(t, 1, report.Summary.Passed, "Direct message should pass")
}
// TestRunner_WithBeforeAfter tests before/after hooks
func TestRunner_WithBeforeAfter(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Create a temporary JSONL file with test cases that use hooks
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
// Note: hooks-test agent has env_test.ts with Before/After functions
testCases := `{"id": "hook-test-1", "input": "Hello", "assert": {"type": "contains", "value": "hello"}, "before": "env_test.Before", "after": "env_test.After"}`
err = os.WriteFile(inputFile, []byte(testCases), 0644)
require.NoError(t, err, "Failed to write test file")
// Run tests with hooks (using hooks-test agent which has the hook scripts)
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.hooks-test",
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
t.Logf("Total: %d, Passed: %d, Failed: %d",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
}
// TestRunner_Parallel tests parallel execution
func TestRunner_Parallel(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Create a temporary JSONL file with multiple test cases
// Use regex for case-insensitive matching
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
testCases := `{"id": "parallel-1", "input": "Hello", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}}
{"id": "parallel-2", "input": "Hi", "assert": {"type": "regex", "value": "(?i)(hello|hi)"}}
{"id": "parallel-3", "input": "Hey", "assert": {"type": "regex", "value": "(?i)(hello|hi|hey)"}}
{"id": "parallel-4", "input": "Good day", "assert": {"type": "regex", "value": "(?i)(hello|good|day)"}}`
err = os.WriteFile(inputFile, []byte(testCases), 0644)
require.NoError(t, err, "Failed to write test file")
// Run tests in parallel
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.simple-greeting",
Parallel: 2, // Run 2 tests in parallel
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error")
require.NotNil(t, report, "Report should not be nil")
assert.Equal(t, 4, report.Summary.Total, "Should have 4 test cases")
t.Logf("Total: %d, Passed: %d, Failed: %d (parallel: 2)",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
}
// TestRunner_FailFast tests fail-fast behavior
func TestRunner_FailFast(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agents
err := agent.Load(config.Conf)
require.NoError(t, err, "Failed to load agents")
// Create a temporary JSONL file with a failing test first
tmpDir := t.TempDir()
inputFile := filepath.Join(tmpDir, "inputs.jsonl")
// First test will fail (expects "impossible" which won't be in response)
testCases := `{"id": "fail-first", "input": "Hello", "assert": {"type": "contains", "value": "IMPOSSIBLE_STRING_12345"}}
{"id": "should-skip", "input": "Hi", "assert": {"type": "contains", "value": "hi"}}`
err = os.WriteFile(inputFile, []byte(testCases), 0644)
require.NoError(t, err, "Failed to write test file")
// Run tests with fail-fast
opts := &agenttest.Options{
Input: inputFile,
AgentID: "tests.simple-greeting",
FailFast: true,
Verbose: true,
InputMode: agenttest.InputModeFile,
}
opts = agenttest.MergeOptions(opts, agenttest.DefaultOptions())
runner := agenttest.NewRunner(opts)
report, err := runner.Run()
require.NoError(t, err, "Runner should not return error (fail-fast is not an error)")
require.NotNil(t, report, "Report should not be nil")
// With fail-fast, only the first test should run
assert.Equal(t, 1, report.Summary.Failed, "First test should fail")
// The second test might not run due to fail-fast
t.Logf("Total: %d, Passed: %d, Failed: %d (fail-fast enabled)",
report.Summary.Total, report.Summary.Passed, report.Summary.Failed)
}

67
agent/test/script_assert.go
View file

@ -261,6 +261,9 @@ func newAssertObject(v8ctx *v8go.Context, t *TestingT) (*v8go.Value, error) {
// JSON path assertion
assertObj.Set("JSONPath", assertJSONPathMethod(iso, t))
// Agent-driven assertion
assertObj.Set("Agent", assertAgentMethod(iso, t))
// Create instance
instance, err := assertObj.NewInstance(v8ctx)
if err != nil {
@ -924,6 +927,70 @@ func assertJSONPathMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate
})
}
// assertAgentMethod implements assert.Agent(response, agentID, options?)
// Uses a validator agent to check the response
// agentID is the direct agent ID (no "agents:" prefix needed)
func assertAgentMethod(iso *v8go.Isolate, t *TestingT) *v8go.FunctionTemplate {
return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value {
v8ctx := info.Context()
args := info.Args()
if len(args) < 2 {
t.fail("Agent requires response and agentID arguments", &ScriptAssertionInfo{Type: "Agent"})
return v8go.Undefined(iso)
}
response, _ := bridge.GoValue(args[0], v8ctx)
agentID := args[1].String()
// Get options if provided
var options map[string]interface{}
if len(args) > 2 && args[2].IsObject() {
optVal, _ := bridge.GoValue(args[2], v8ctx)
options, _ = optVal.(map[string]interface{})
}
// Build assertion with agents: prefix
assertion := &Assertion{
Type: "agent",
Use: "agents:" + agentID,
}
// Extract criteria and metadata from options
if options != nil {
if criteria, ok := options["criteria"]; ok {
assertion.Value = criteria
}
if metadata, ok := options["metadata"].(map[string]interface{}); ok {
assertion.Options = &AssertionOptions{Metadata: metadata}
}
if connector, ok := options["connector"].(string); ok {
if assertion.Options == nil {
assertion.Options = &AssertionOptions{}
}
assertion.Options.Connector = connector
}
}
// Use the asserter to validate
asserter := &Asserter{}
result := asserter.assertAgent(assertion, response, nil)
if !result.Passed {
msg := result.Message
if msg == "" {
msg = "agent assertion failed"
}
t.fail(msg, &ScriptAssertionInfo{
Type: "Agent",
Actual: response,
Message: msg,
})
}
return v8go.Undefined(iso)
})
}
// Helper functions
// deepEqual performs deep equality comparison

592
agent/test/script_hooks.go Normal file
View file

@ -0,0 +1,592 @@
package test
import (
"fmt"
"path/filepath"
"strings"
"github.com/yaoapp/gou/application"
v8 "github.com/yaoapp/gou/runtime/v8"
"github.com/yaoapp/gou/runtime/v8/bridge"
"github.com/yaoapp/yao/agent/context"
"rogchap.com/v8go"
)
// HookExecutor executes before/after scripts from *_test.ts files
// Scripts are loaded via V8 and executed directly, not via Process()
type HookExecutor struct {
verbose bool
output *OutputWriter
loadedDirs map[string]bool // Track which directories have been loaded
agentContext *context.Context
}
// NewHookExecutor creates a new hook executor
func NewHookExecutor(verbose bool) *HookExecutor {
return &HookExecutor{
verbose: verbose,
output: NewOutputWriter(verbose),
loadedDirs: make(map[string]bool),
}
}
// SetAgentContext sets the agent context for script execution
func (h *HookExecutor) SetAgentContext(ctx *context.Context) {
h.agentContext = ctx
}
// HookRef represents a parsed hook reference
// Format: "src/env_test.ts:Before" or just "Before" (uses default test file)
type HookRef struct {
ScriptFile string // e.g., "env_test.ts"
Function string // e.g., "Before"
}
// ParseHookRef parses a hook reference string
// Formats:
// - "Before" -> uses first *_test.ts file found
// - "env_test.Before" -> uses src/env_test.ts
// - "src/env_test.Before" -> uses src/env_test.ts
func ParseHookRef(ref string) (*HookRef, error) {
if ref == "" {
return nil, fmt.Errorf("empty hook reference")
}
// Split by last dot to get function name
lastDot := strings.LastIndex(ref, ".")
if lastDot == -1 {
// Just function name, will use default test file
return &HookRef{
ScriptFile: "", // Will be resolved later
Function: ref,
}, nil
}
scriptPart := ref[:lastDot]
funcName := ref[lastDot+1:]
// Normalize script file name
scriptFile := scriptPart
if !strings.HasSuffix(scriptFile, "_test") {
scriptFile += "_test"
}
scriptFile += ".ts"
// Remove "src/" prefix if present
scriptFile = strings.TrimPrefix(scriptFile, "src/")
return &HookRef{
ScriptFile: scriptFile,
Function: funcName,
}, nil
}
// LoadTestScripts loads all *_test.ts scripts from the agent's src directory
// Returns the script IDs that were loaded
func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
srcDir := filepath.Join(agentPath, "src")
// Check if already loaded
if h.loadedDirs[srcDir] {
return nil, nil
}
// Check if src directory exists
exists, err := application.App.Exists(srcDir)
if err != nil {
return nil, err
}
if !exists {
return nil, nil // No src directory, not an error
}
var loadedScripts []string
exts := []string{"*_test.ts", "*_test.js"}
err = application.App.Walk(srcDir, func(root, file string, isdir bool) error {
if isdir {
return nil
}
// Only load *_test.ts/js files
base := filepath.Base(file)
if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") {
return nil
}
// Generate script ID
scriptID := generateHookScriptID(file, srcDir)
// Load the script
_, err := v8.Load(file, scriptID)
if err != nil {
if h.verbose {
h.output.Warning("Failed to load hook script %s: %v", base, err)
}
return nil // Continue loading other scripts
}
loadedScripts = append(loadedScripts, scriptID)
if h.verbose {
h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID)
}
return nil
}, exts...)
if err != nil {
return nil, fmt.Errorf("failed to walk src directory: %w", err)
}
h.loadedDirs[srcDir] = true
return loadedScripts, nil
}
// generateHookScriptID generates a script ID for hook scripts
// Example: assistants/test/src/env_test.ts -> hook.env_test
func generateHookScriptID(filePath string, srcDir string) string {
filePath = filepath.ToSlash(filePath)
srcDir = filepath.ToSlash(srcDir)
relPath := strings.TrimPrefix(filePath, srcDir+"/")
relPath = strings.TrimPrefix(relPath, "/")
relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath))
return "hook." + strings.ReplaceAll(relPath, "/", ".")
}
// FindTestScript finds a loaded test script by pattern
// If scriptFile is empty, returns the first *_test script found
func (h *HookExecutor) FindTestScript(scriptFile string) (*v8.Script, string, error) {
if scriptFile != "" {
// Look for specific script
scriptID := "hook." + strings.TrimSuffix(scriptFile, ".ts")
scriptID = strings.TrimSuffix(scriptID, ".js")
if script, ok := v8.Scripts[scriptID]; ok {
return script, scriptID, nil
}
return nil, "", fmt.Errorf("hook script not found: %s (id: %s)", scriptFile, scriptID)
}
// Find first *_test script
for id, script := range v8.Scripts {
if strings.HasPrefix(id, "hook.") && strings.Contains(id, "_test") {
return script, id, nil
}
}
return nil, "", fmt.Errorf("no hook test script found")
}
// ExecuteBefore executes a Before function from a test script
func (h *HookExecutor) ExecuteBefore(ref string, testCase *Case, agentPath string) (interface{}, error) {
hookRef, err := ParseHookRef(ref)
if err != nil {
return nil, err
}
// Ensure scripts are loaded
if _, err := h.LoadTestScripts(agentPath); err != nil {
return nil, fmt.Errorf("failed to load test scripts: %w", err)
}
// Find the script
script, scriptID, err := h.FindTestScript(hookRef.ScriptFile)
if err != nil {
return nil, err
}
if h.verbose {
h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID)
}
// Execute the function
return h.executeHookFunction(script, hookRef.Function, testCase, nil, nil)
}
// ExecuteAfter executes an After function from a test script
func (h *HookExecutor) ExecuteAfter(ref string, testCase *Case, result *Result, beforeData interface{}, agentPath string) error {
hookRef, err := ParseHookRef(ref)
if err != nil {
return err
}
// Ensure scripts are loaded
if _, err := h.LoadTestScripts(agentPath); err != nil {
return fmt.Errorf("failed to load test scripts: %w", err)
}
// Find the script
script, scriptID, err := h.FindTestScript(hookRef.ScriptFile)
if err != nil {
return err
}
if h.verbose {
h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID)
}
// Execute the function
_, err = h.executeHookFunction(script, hookRef.Function, testCase, result, beforeData)
return err
}
// ExecuteBeforeAll executes a BeforeAll function
func (h *HookExecutor) ExecuteBeforeAll(ref string, testCases []*Case, agentPath string) (interface{}, error) {
hookRef, err := ParseHookRef(ref)
if err != nil {
return nil, err
}
// Ensure scripts are loaded
if _, err := h.LoadTestScripts(agentPath); err != nil {
return nil, fmt.Errorf("failed to load test scripts: %w", err)
}
// Find the script
script, scriptID, err := h.FindTestScript(hookRef.ScriptFile)
if err != nil {
return nil, err
}
if h.verbose {
h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID)
}
// Execute with test cases array
return h.executeHookFunctionWithCases(script, hookRef.Function, testCases)
}
// ExecuteAfterAll executes an AfterAll function
func (h *HookExecutor) ExecuteAfterAll(ref string, results []*Result, beforeData interface{}, agentPath string) error {
hookRef, err := ParseHookRef(ref)
if err != nil {
return err
}
// Ensure scripts are loaded
if _, err := h.LoadTestScripts(agentPath); err != nil {
return fmt.Errorf("failed to load test scripts: %w", err)
}
// Find the script
script, scriptID, err := h.FindTestScript(hookRef.ScriptFile)
if err != nil {
return err
}
if h.verbose {
h.output.Verbose("Executing %s from %s", hookRef.Function, scriptID)
}
// Execute with results array
_, err = h.executeHookFunctionWithResults(script, hookRef.Function, results, beforeData)
return err
}
// executeHookFunction executes a hook function with test case context
func (h *HookExecutor) executeHookFunction(script *v8.Script, funcName string, testCase *Case, result *Result, beforeData interface{}) (interface{}, error) {
// Create script context
scriptCtx, err := script.NewContext("", nil)
if err != nil {
return nil, fmt.Errorf("failed to create script context: %w", err)
}
defer scriptCtx.Close()
v8ctx := scriptCtx.Context
// Set share data
if err := h.setShareData(v8ctx); err != nil {
return nil, err
}
// Get the function
global := v8ctx.Global()
fnValue, err := global.Get(funcName)
if err != nil {
return nil, fmt.Errorf("failed to get function %s: %w", funcName, err)
}
if fnValue.IsUndefined() || fnValue.IsNull() {
return nil, fmt.Errorf("function %s not defined", funcName)
}
if !fnValue.IsFunction() {
return nil, fmt.Errorf("%s is not a function", funcName)
}
fn, err := fnValue.AsFunction()
if err != nil {
return nil, fmt.Errorf("failed to convert to function: %w", err)
}
// Build arguments
args, err := h.buildHookArgs(v8ctx, testCase, result, beforeData)
if err != nil {
return nil, err
}
// Convert to v8go.Valuer slice for Call
valuerArgs := make([]v8go.Valuer, len(args))
for i, arg := range args {
valuerArgs[i] = arg
}
// Call the function
jsResult, err := fn.Call(global, valuerArgs...)
if err != nil {
return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
}
// Convert result to Go value
if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() {
return nil, nil
}
goResult, err := bridge.GoValue(jsResult, v8ctx)
if err != nil {
return nil, fmt.Errorf("failed to convert result: %w", err)
}
// Extract data field if present
if resultMap, ok := goResult.(map[string]interface{}); ok {
if data, exists := resultMap["data"]; exists {
return data, nil
}
}
return goResult, nil
}
// executeHookFunctionWithCases executes BeforeAll with test cases array
func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName string, testCases []*Case) (interface{}, error) {
scriptCtx, err := script.NewContext("", nil)
if err != nil {
return nil, fmt.Errorf("failed to create script context: %w", err)
}
defer scriptCtx.Close()
v8ctx := scriptCtx.Context
if err := h.setShareData(v8ctx); err != nil {
return nil, err
}
global := v8ctx.Global()
fnValue, err := global.Get(funcName)
if err != nil {
return nil, fmt.Errorf("failed to get function %s: %w", funcName, err)
}
if fnValue.IsUndefined() || fnValue.IsNull() {
return nil, fmt.Errorf("function %s not defined", funcName)
}
if !fnValue.IsFunction() {
return nil, fmt.Errorf("%s is not a function", funcName)
}
fn, err := fnValue.AsFunction()
if err != nil {
return nil, fmt.Errorf("failed to convert to function: %w", err)
}
// Convert test cases to JS array
casesJS, err := h.testCasesToJS(v8ctx, testCases)
if err != nil {
return nil, err
}
jsResult, err := fn.Call(global, casesJS)
if err != nil {
return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
}
if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() {
return nil, nil
}
goResult, err := bridge.GoValue(jsResult, v8ctx)
if err != nil {
return nil, fmt.Errorf("failed to convert result: %w", err)
}
if resultMap, ok := goResult.(map[string]interface{}); ok {
if data, exists := resultMap["data"]; exists {
return data, nil
}
}
return goResult, nil
}
// executeHookFunctionWithResults executes AfterAll with results array
func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcName string, results []*Result, beforeData interface{}) (interface{}, error) {
scriptCtx, err := script.NewContext("", nil)
if err != nil {
return nil, fmt.Errorf("failed to create script context: %w", err)
}
defer scriptCtx.Close()
v8ctx := scriptCtx.Context
if err := h.setShareData(v8ctx); err != nil {
return nil, err
}
global := v8ctx.Global()
fnValue, err := global.Get(funcName)
if err != nil {
return nil, fmt.Errorf("failed to get function %s: %w", funcName, err)
}
if fnValue.IsUndefined() || fnValue.IsNull() {
return nil, fmt.Errorf("function %s not defined", funcName)
}
if !fnValue.IsFunction() {
return nil, fmt.Errorf("%s is not a function", funcName)
}
fn, err := fnValue.AsFunction()
if err != nil {
return nil, fmt.Errorf("failed to convert to function: %w", err)
}
// Convert results to JS array
resultsJS, err := h.resultsToJS(v8ctx, results)
if err != nil {
return nil, err
}
// Convert beforeData to JS
beforeDataJS, err := bridge.JsValue(v8ctx, beforeData)
if err != nil {
return nil, fmt.Errorf("failed to convert beforeData: %w", err)
}
jsResult, err := fn.Call(global, resultsJS, beforeDataJS)
if err != nil {
return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
}
if jsResult == nil || jsResult.IsUndefined() || jsResult.IsNull() {
return nil, nil
}
goResult, err := bridge.GoValue(jsResult, v8ctx)
if err != nil {
return nil, fmt.Errorf("failed to convert result: %w", err)
}
return goResult, nil
}
// setShareData sets the share data for script execution
func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error {
var authorized map[string]interface{}
if h.agentContext != nil && h.agentContext.Authorized != nil {
authorized = h.agentContext.Authorized.AuthorizedToMap()
}
return bridge.SetShareData(v8ctx, v8ctx.Global(), &bridge.Share{
Sid: "",
Root: false,
Global: nil,
Authorized: authorized,
})
}
// buildHookArgs builds the arguments for a hook function call
func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) {
var args []*v8go.Value
// Arg 1: testCase
if testCase != nil {
tcMap := map[string]interface{}{
"id": testCase.ID,
"input": testCase.Input,
}
if testCase.Metadata != nil {
tcMap["metadata"] = testCase.Metadata
}
if testCase.Assert != nil {
tcMap["assert"] = testCase.Assert
}
tcJS, err := bridge.JsValue(v8ctx, tcMap)
if err != nil {
return nil, fmt.Errorf("failed to convert testCase: %w", err)
}
args = append(args, tcJS)
}
// Arg 2: result (for After)
if result != nil {
resultMap := map[string]interface{}{
"id": result.ID,
"status": string(result.Status),
"duration_ms": result.DurationMs,
}
if result.Output != nil {
resultMap["output"] = result.Output
}
if result.Error != "" {
resultMap["error"] = result.Error
}
resultJS, err := bridge.JsValue(v8ctx, resultMap)
if err != nil {
return nil, fmt.Errorf("failed to convert result: %w", err)
}
args = append(args, resultJS)
}
// Arg 3: beforeData (for After)
if beforeData != nil {
beforeDataJS, err := bridge.JsValue(v8ctx, beforeData)
if err != nil {
return nil, fmt.Errorf("failed to convert beforeData: %w", err)
}
args = append(args, beforeDataJS)
}
return args, nil
}
// testCasesToJS converts test cases to a JS array
func (h *HookExecutor) testCasesToJS(v8ctx *v8go.Context, testCases []*Case) (*v8go.Value, error) {
cases := make([]map[string]interface{}, len(testCases))
for i, tc := range testCases {
cases[i] = map[string]interface{}{
"id": tc.ID,
"input": tc.Input,
}
if tc.Metadata != nil {
cases[i]["metadata"] = tc.Metadata
}
}
return bridge.JsValue(v8ctx, cases)
}
// resultsToJS converts results to a JS array
func (h *HookExecutor) resultsToJS(v8ctx *v8go.Context, results []*Result) (*v8go.Value, error) {
resultMaps := make([]map[string]interface{}, len(results))
for i, r := range results {
resultMaps[i] = map[string]interface{}{
"id": r.ID,
"status": string(r.Status),
"duration_ms": r.DurationMs,
}
if r.Output != nil {
resultMaps[i]["output"] = r.Output
}
if r.Error != "" {
resultMaps[i]["error"] = r.Error
}
}
return bridge.JsValue(v8ctx, resultMaps)
}

244
agent/test/script_hooks_test.go Normal file
View file

@ -0,0 +1,244 @@
package test_test
import (
"testing"
"github.com/stretchr/testify/assert"
v8 "github.com/yaoapp/gou/runtime/v8"
agenttest "github.com/yaoapp/yao/agent/test"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
const hooksTestAgent = "assistants/tests/hooks-test"
func TestParseHookRef(t *testing.T) {
tests := []struct {
name string
input string
wantFile string
wantFunc string
expectErr bool
}{
{
name: "function only",
input: "Before",
wantFile: "",
wantFunc: "Before",
},
{
name: "with script file",
input: "env_test.Before",
wantFile: "env_test.ts",
wantFunc: "Before",
},
{
name: "with src prefix",
input: "src/env_test.Before",
wantFile: "env_test.ts",
wantFunc: "Before",
},
{
name: "nested path",
input: "setup/db_test.Before",
wantFile: "setup/db_test.ts",
wantFunc: "Before",
},
{
name: "empty string",
input: "",
expectErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ref, err := agenttest.ParseHookRef(tt.input)
if tt.expectErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tt.wantFile, ref.ScriptFile)
assert.Equal(t, tt.wantFunc, ref.Function)
})
}
}
func TestHookExecutorLoadTestScripts(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts using the utility function
scripts := test.LoadAgentTestScripts(t, hooksTestAgent)
assert.NotEmpty(t, scripts, "Should load at least one test script")
// Verify the script was loaded into V8
found := false
for _, scriptID := range scripts {
if _, ok := v8.Scripts[scriptID]; ok {
found = true
t.Logf("Loaded script: %s", scriptID)
break
}
}
assert.True(t, found, "At least one script should be loaded into V8")
}
func TestHookExecutorExecuteBefore(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts
test.LoadAgentTestScripts(t, hooksTestAgent)
executor := agenttest.NewHookExecutor(true)
testCase := &agenttest.Case{
ID: "TEST001",
Input: "Hello World",
}
// Execute Before hook
beforeData, err := executor.ExecuteBefore("env_test.Before", testCase, hooksTestAgent)
assert.NoError(t, err)
assert.NotNil(t, beforeData)
// Verify returned data
dataMap, ok := beforeData.(map[string]interface{})
assert.True(t, ok, "beforeData should be a map")
assert.Equal(t, "TEST001", dataMap["test_id"])
assert.NotEmpty(t, dataMap["mock_user_id"])
assert.NotEmpty(t, dataMap["mock_session_id"])
}
func TestHookExecutorExecuteAfter(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts
test.LoadAgentTestScripts(t, hooksTestAgent)
executor := agenttest.NewHookExecutor(true)
testCase := &agenttest.Case{
ID: "TEST002",
Input: "Test input",
}
result := &agenttest.Result{
ID: "TEST002",
Status: agenttest.StatusPassed,
DurationMs: 100,
}
beforeData := map[string]interface{}{
"test_id": "TEST002",
"mock_user_id": "user_TEST002_12345",
"mock_session_id": "session_12345",
}
// Execute After hook
err := executor.ExecuteAfter("env_test.After", testCase, result, beforeData, hooksTestAgent)
assert.NoError(t, err)
}
func TestHookExecutorExecuteBeforeAll(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts
test.LoadAgentTestScripts(t, hooksTestAgent)
executor := agenttest.NewHookExecutor(true)
testCases := []*agenttest.Case{
{ID: "T001", Input: "Test 1"},
{ID: "T002", Input: "Test 2"},
{ID: "T003", Input: "Test 3"},
}
// Execute BeforeAll hook
globalData, err := executor.ExecuteBeforeAll("env_test.BeforeAll", testCases, hooksTestAgent)
assert.NoError(t, err)
assert.NotNil(t, globalData)
// Verify returned data
dataMap, ok := globalData.(map[string]interface{})
assert.True(t, ok, "globalData should be a map")
assert.NotEmpty(t, dataMap["suite_id"])
assert.Equal(t, float64(3), dataMap["test_count"]) // JSON numbers are float64
}
func TestHookExecutorExecuteAfterAll(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts
test.LoadAgentTestScripts(t, hooksTestAgent)
executor := agenttest.NewHookExecutor(true)
results := []*agenttest.Result{
{ID: "T001", Status: agenttest.StatusPassed, DurationMs: 100},
{ID: "T002", Status: agenttest.StatusFailed, DurationMs: 200, Error: "assertion failed"},
{ID: "T003", Status: agenttest.StatusPassed, DurationMs: 150},
}
globalData := map[string]interface{}{
"suite_id": "suite_12345",
"test_count": 3,
}
// Execute AfterAll hook
err := executor.ExecuteAfterAll("env_test.AfterAll", results, globalData, hooksTestAgent)
assert.NoError(t, err)
}
func TestHookExecutorFunctionNotFound(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts
test.LoadAgentTestScripts(t, hooksTestAgent)
executor := agenttest.NewHookExecutor(true)
testCase := &agenttest.Case{
ID: "TEST001",
Input: "Hello",
}
// Try to execute non-existent function
_, err := executor.ExecuteBefore("env_test.NonExistent", testCase, hooksTestAgent)
assert.Error(t, err)
assert.Contains(t, err.Error(), "not defined")
}
func TestHookExecutorScriptNotFound(t *testing.T) {
// Prepare test environment
test.Prepare(t, config.Conf)
defer test.Clean()
// Load agent test scripts
test.LoadAgentTestScripts(t, hooksTestAgent)
executor := agenttest.NewHookExecutor(true)
testCase := &agenttest.Case{
ID: "TEST001",
Input: "Hello",
}
// Try to execute from non-existent script
_, err := executor.ExecuteBefore("nonexistent_test.Before", testCase, hooksTestAgent)
assert.Error(t, err)
assert.Contains(t, err.Error(), "not found")
}

96
agent/test/types.go
View file

@ -144,6 +144,22 @@ type Options struct {
// Only tests matching the pattern will be executed
// Example: "TestSystem" matches TestSystemReady, TestSystemError, etc.
Run string `json:"run,omitempty"`
// BeforeAll is the global before script (e.g., "scripts:tests.env.BeforeAll")
// Called once before all test cases
BeforeAll string `json:"before_all,omitempty"`
// AfterAll is the global after script (e.g., "scripts:tests.env.AfterAll")
// Called once after all test cases
AfterAll string `json:"after_all,omitempty"`
// DryRun generates test cases without running them
// Useful for previewing agent-generated test cases
DryRun bool `json:"dry_run,omitempty"`
// Simulator is the default simulator agent ID for dynamic mode
// Can be overridden per test case in JSONL
Simulator string `json:"simulator,omitempty"`
}
// ContextConfig represents custom context configuration from JSON file
@ -375,6 +391,68 @@ type Case struct {
// Timeout overrides the default timeout for this test case
// Format: "30s", "1m", "2m30s"
Timeout string `json:"timeout,omitempty"`
// Before script function (e.g., "scripts:tests.env.Before")
// Called before the test case runs, returns data passed to After
Before string `json:"before,omitempty"`
// After script function (e.g., "scripts:tests.env.After")
// Called after the test case completes (pass or fail)
After string `json:"after,omitempty"`
// Dynamic Mode Fields
// ===============================
// Simulator configures the user simulator for dynamic testing
// When set, the test runs in dynamic mode with multi-turn conversation
Simulator *Simulator `json:"simulator,omitempty"`
// Checkpoints define validation points for dynamic testing
// Each checkpoint is checked after every agent response
Checkpoints []*Checkpoint `json:"checkpoints,omitempty"`
// MaxTurns is the maximum number of conversation turns (default: 20)
MaxTurns int `json:"max_turns,omitempty"`
}
// Simulator configures the user simulator for dynamic testing
type Simulator struct {
// Use is the simulator agent ID (no prefix needed)
Use string `json:"use"`
// Options for the simulator agent
Options *SimulatorOptions `json:"options,omitempty"`
}
// SimulatorOptions configures simulator behavior
type SimulatorOptions struct {
// Metadata passed to the simulator agent
// Common fields: persona, goal, style
Metadata map[string]interface{} `json:"metadata,omitempty"`
// Connector overrides the simulator's default connector
Connector string `json:"connector,omitempty"`
}
// Checkpoint defines a validation point in dynamic testing
type Checkpoint struct {
// ID is the unique identifier for this checkpoint
ID string `json:"id"`
// Description is a human-readable description
Description string `json:"description,omitempty"`
// Assert defines the assertion to validate
// Same format as Case.Assert
Assert interface{} `json:"assert"`
// After specifies checkpoint IDs that must be reached before this one
// Used to enforce ordering (e.g., "ask_type" must come before "confirm")
After []string `json:"after,omitempty"`
// Required indicates if this checkpoint must be reached (default: true)
// Optional checkpoints don't cause test failure if not reached
Required *bool `json:"required,omitempty"`
}
// CaseOptions represents per-test-case context options
@ -420,6 +498,7 @@ type Assertion struct {
// - "script": run a custom assertion script
// - "type": check output type (string, object, array, number, boolean)
// - "schema": validate against JSON schema
// - "agent": use an agent to validate the response
Type string `json:"type"`
// Value is the expected value or pattern (depends on type)
@ -432,6 +511,14 @@ type Assertion struct {
// The script receives (output, input, expected) and returns {pass: bool, message: string}
Script string `json:"script,omitempty"`
// Use specifies the agent/script for validation
// For agent assertions: "agents:tests.validator-agent" (with prefix)
// For script assertions: "scripts:tests.validate" (with prefix)
Use string `json:"use,omitempty"`
// Options for agent-driven assertions (aligned with context.Options)
Options *AssertionOptions `json:"options,omitempty"`
// Message is a custom failure message
Message string `json:"message,omitempty"`
@ -439,6 +526,15 @@ type Assertion struct {
Negate bool `json:"negate,omitempty"`
}
// AssertionOptions for agent-driven assertions
type AssertionOptions struct {
// Connector overrides the agent's default connector
Connector string `json:"connector,omitempty"`
// Metadata contains custom data passed to the validator agent
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
// AssertionResult represents the result of an assertion
type AssertionResult struct {
// Passed indicates whether the assertion passed

12
cmd/agent/test.go
View file

@ -33,6 +33,10 @@ var (
testParallel int
testVerbose bool
testFailFast bool
testBefore string // --before flag for global BeforeAll hook
testAfter string // --after flag for global AfterAll hook
testDryRun bool // --dry-run flag for generating tests without running
testSimulator string // --simulator flag for default simulator agent in dynamic mode
)
// TestCmd is the agent test command
@ -157,6 +161,10 @@ var TestCmd = &cobra.Command{
Parallel: testParallel,
Verbose: testVerbose,
FailFast: testFailFast,
BeforeAll: testBefore,
AfterAll: testAfter,
DryRun: testDryRun,
Simulator: testSimulator,
}
// Merge with defaults
@ -244,6 +252,10 @@ func init() {
TestCmd.Flags().IntVar(&testParallel, "parallel", 1, L("Number of parallel test cases"))
TestCmd.Flags().BoolVarP(&testVerbose, "verbose", "v", false, L("Verbose output"))
TestCmd.Flags().BoolVar(&testFailFast, "fail-fast", false, L("Stop on first failure"))
TestCmd.Flags().StringVar(&testBefore, "before", "", L("Global BeforeAll hook (e.g., env_test.BeforeAll)"))
TestCmd.Flags().StringVar(&testAfter, "after", "", L("Global AfterAll hook (e.g., env_test.AfterAll)"))
TestCmd.Flags().BoolVar(&testDryRun, "dry-run", false, L("Generate test cases without running them"))
TestCmd.Flags().StringVar(&testSimulator, "simulator", "", L("Default simulator agent for dynamic mode (e.g., tests.simulator-agent)"))
// Mark input as required
TestCmd.MarkFlagRequired("input")

67
test/utils.go
View file

@ -800,3 +800,70 @@ func GuardBearerJWT(c *gin.Context) {
claims := helper.JwtValidate(tokenString)
c.Set("__sid", claims.SID)
}
// LoadAgentTestScripts loads all *_test.ts/js scripts from an agent's src directory.
// This is useful for testing agent hooks (before/after scripts) and other agent-specific test scripts.
//
// Usage:
//
// test.Prepare(t, config.Conf)
// defer test.Clean()
// scripts := test.LoadAgentTestScripts(t, "assistants/tests/hooks-test")
//
// Parameters:
// - t: testing.T instance
// - agentRelPath: relative path to agent directory from app root (e.g., "assistants/tests/hooks-test")
//
// Returns:
// - []string: list of loaded script IDs (e.g., ["hook.env_test"])
func LoadAgentTestScripts(t *testing.T, agentRelPath string) []string {
srcDir := filepath.Join(agentRelPath, "src")
// Check if src directory exists
exists, err := application.App.Exists(srcDir)
if err != nil {
t.Fatalf("Failed to check src directory: %v", err)
}
if !exists {
t.Logf("No src directory found at %s, skipping", srcDir)
return nil
}
var loadedScripts []string
exts := []string{"*_test.ts", "*_test.js"}
err = application.App.Walk(srcDir, func(root, file string, isdir bool) error {
if isdir {
return nil
}
// Only load *_test.ts/js files
base := filepath.Base(file)
if !strings.HasSuffix(base, "_test.ts") && !strings.HasSuffix(base, "_test.js") {
return nil
}
// Generate script ID: hook.{relative_path_without_ext}
// e.g., assistants/tests/hooks-test/src/env_test.ts -> hook.env_test
relPath := strings.TrimPrefix(file, srcDir+"/")
relPath = strings.TrimPrefix(relPath, "/")
relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath))
scriptID := "hook." + strings.ReplaceAll(relPath, "/", ".")
// Load the script
_, err := v8.Load(file, scriptID)
if err != nil {
t.Logf("Warning: Failed to load hook script %s: %v", base, err)
return nil // Continue loading other scripts
}
loadedScripts = append(loadedScripts, scriptID)
return nil
}, exts...)
if err != nil {
t.Fatalf("Failed to walk src directory: %v", err)
}
return loadedScripts
}