Enhance Agent Response Structure and Dynamic Test Functionality
- Added support for tool call responses in the agent's response structure, allowing for better handling of tool execution results. - Updated the `TurnResult` and `TurnResponse` types to include full agent responses, including tool call details and next hook data. - Improved dynamic test execution by ensuring consistent chat session state across turns, enhancing the overall testing framework's capabilities. - Enhanced documentation in README.md to reflect changes in response structure and dynamic testing output format.
This commit is contained in:
Max
parent
4d1d17f1ab
commit
5fb5035d8c
9 changed files with 487 additions and 41 deletions
|
|
@ -30,6 +30,7 @@ func (ast *Assistant) processNextResponse(npc *NextProcessContext) (*agentContex
|
|||
Create: npc.CreateResponse,
|
||||
Next: npc.NextResponse.Data, // Put custom data in Next field
|
||||
Completion: npc.CompletionResponse,
|
||||
Tools: npc.ToolCallResponses,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -72,5 +73,6 @@ func (ast *Assistant) buildStandardResponse(npc *NextProcessContext) *agentConte
|
|||
Create: npc.CreateResponse,
|
||||
Next: npc.NextResponse,
|
||||
Completion: npc.CompletionResponse,
|
||||
Tools: npc.ToolCallResponses,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -349,6 +349,7 @@ type Response struct {
|
|||
Create *HookCreateResponse `json:"create,omitempty"` // Create response from the create hook
|
||||
Next interface{} `json:"next,omitempty"` // Next response from the next hook
|
||||
Completion *CompletionResponse `json:"completion,omitempty"` // Completion response from the completion hook
|
||||
Tools []ToolCallResponse `json:"tools,omitempty"` // Tool call results (if any tools were executed)
|
||||
}
|
||||
|
||||
// HookCreateResponse the response of the create hook
|
||||
|
|
|
|||
|
|
@ -203,6 +203,33 @@ Simulator-driven testing with checkpoint validation. A simulator agent generates
|
|||
| `--fail-fast` | Stop on first failure | false |
|
||||
| `--dry-run` | Generate test cases without running them | false |
|
||||
|
||||
## Custom Context File
|
||||
|
||||
Create a JSON file for custom authorization:
|
||||
|
||||
```json
|
||||
{
|
||||
"chat_id": "test-chat-001",
|
||||
"authorized": {
|
||||
"user_id": "test-user-123",
|
||||
"team_id": "test-team-456",
|
||||
"constraints": {
|
||||
"owner_only": true,
|
||||
"extra": { "department": "engineering" }
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"mode": "test"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use with `--ctx`:
|
||||
|
||||
```bash
|
||||
yao agent test -i scripts.expense.setup --ctx tests/context.json -v
|
||||
```
|
||||
|
||||
## Input Format (JSONL)
|
||||
|
||||
Each line is a JSON object. Below are examples organized by scenario.
|
||||
|
|
@ -761,9 +788,12 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) {
|
|||
|
||||
```typescript
|
||||
interface Context {
|
||||
user_id: string; // Test user ID
|
||||
team_id: string; // Test team ID
|
||||
locale: string; // Locale (e.g., "en-us")
|
||||
authorized: {
|
||||
user_id: string; // Test user ID
|
||||
team_id: string; // Test team ID
|
||||
constraints?: object; // Access constraints
|
||||
};
|
||||
metadata: object; // Custom metadata from test case
|
||||
}
|
||||
```
|
||||
|
|
@ -980,6 +1010,33 @@ For testing complex conversation flows where the path is unpredictable:
|
|||
└─ PASSED (3 turns, 3 checkpoints, 8.5s)
|
||||
```
|
||||
|
||||
### Dynamic Mode Output Structure
|
||||
|
||||
Each turn in the output includes:
|
||||
|
||||
```typescript
|
||||
interface TurnResult {
|
||||
turn: number; // Turn number (1-based)
|
||||
input: string; // User message
|
||||
output: any; // Agent response summary (for display)
|
||||
response: {
|
||||
// Full agent response (for detailed analysis)
|
||||
content: string; // LLM text content
|
||||
tool_calls: [
|
||||
{
|
||||
// Tool calls made
|
||||
tool: string; // Tool name
|
||||
arguments: any; // Call arguments
|
||||
result: any; // Execution result
|
||||
}
|
||||
];
|
||||
next: any; // Next hook data
|
||||
};
|
||||
checkpoints_reached: string[]; // Checkpoint IDs reached
|
||||
duration_ms: number; // Execution time
|
||||
}
|
||||
```
|
||||
|
||||
## Output Formats
|
||||
|
||||
Determined by `-o` file extension:
|
||||
|
|
|
|||
|
|
@ -77,7 +77,14 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
|
|||
|
||||
// Output dynamic test start
|
||||
if r.opts.Verbose {
|
||||
r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns)
|
||||
r.output.Verbose("Dynamic test: %s (max %d turns)", tc.ID, maxTurns)
|
||||
}
|
||||
|
||||
// Use consistent chatID across all turns to preserve session state (ctx.memory.chat)
|
||||
// Priority: context config > generated ID
|
||||
chatID := fmt.Sprintf("dynamic-%s", tc.ID)
|
||||
if r.opts.ContextData != nil && r.opts.ContextData.ChatID != "" {
|
||||
chatID = r.opts.ContextData.ChatID
|
||||
}
|
||||
|
||||
// Conversation loop
|
||||
|
|
@ -111,7 +118,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
|
|||
// Check if goal achieved
|
||||
if simOutput.GoalAchieved {
|
||||
if r.opts.Verbose {
|
||||
r.output.Info(" Turn %d: Simulator signaled goal achieved", turn)
|
||||
r.output.Verbose("Turn %d: Simulator signaled goal achieved", turn)
|
||||
}
|
||||
|
||||
// Check if all required checkpoints reached
|
||||
|
|
@ -135,7 +142,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
|
|||
turnResult.Input = simOutput.Message
|
||||
|
||||
if r.opts.Verbose {
|
||||
r.output.Info(" Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50))
|
||||
r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50))
|
||||
}
|
||||
} else {
|
||||
// Use initial input for first turn
|
||||
|
|
@ -143,14 +150,15 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
|
|||
lastMsg := messages[len(messages)-1]
|
||||
turnResult.Input = lastMsg.Content
|
||||
if r.opts.Verbose {
|
||||
r.output.Info(" Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50))
|
||||
r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Call target agent
|
||||
// Use consistent chatID across all turns to preserve session state (ctx.memory.chat)
|
||||
ctx := NewTestContextFromOptions(
|
||||
fmt.Sprintf("dynamic-%s-%d", tc.ID, turn),
|
||||
chatID,
|
||||
agentID,
|
||||
r.opts,
|
||||
tc,
|
||||
|
|
@ -171,20 +179,20 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
|
|||
return result
|
||||
}
|
||||
|
||||
// Extract output
|
||||
// Extract output (summary for display and conversation history)
|
||||
output := extractOutput(response)
|
||||
turnResult.Output = output
|
||||
turnResult.DurationMs = time.Since(turnStart).Milliseconds()
|
||||
|
||||
// Store full response for reporting
|
||||
turnResult.Response = buildTurnResponse(response)
|
||||
|
||||
if r.opts.Verbose {
|
||||
r.output.Info(" Turn %d: Agent: %s", turn, truncateOutput(output, 50))
|
||||
r.output.Verbose("Turn %d: Agent: %s", turn, truncateOutput(output, 50))
|
||||
}
|
||||
|
||||
// Add assistant response to messages
|
||||
messages = append(messages, context.Message{
|
||||
Role: context.RoleAssistant,
|
||||
Content: output,
|
||||
})
|
||||
// Add assistant response to messages, including tool calls if any
|
||||
messages = appendAssistantMessages(messages, response)
|
||||
|
||||
// Check checkpoints against this response
|
||||
reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result)
|
||||
|
|
@ -192,7 +200,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s
|
|||
|
||||
if r.opts.Verbose && len(reachedIDs) > 0 {
|
||||
for _, id := range reachedIDs {
|
||||
r.output.Info(" ✓ checkpoint: %s", id)
|
||||
r.output.Verbose(" ✓ checkpoint: %s", id)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -411,3 +419,114 @@ func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) boo
|
|||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// buildTurnResponse builds a TurnResponse from the agent response
|
||||
func buildTurnResponse(response *context.Response) *TurnResponse {
|
||||
if response == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
tr := &TurnResponse{}
|
||||
|
||||
// Extract completion content
|
||||
if response.Completion != nil {
|
||||
tr.Content = response.Completion.Content
|
||||
|
||||
// Extract tool calls from completion
|
||||
if len(response.Completion.ToolCalls) > 0 {
|
||||
for _, tc := range response.Completion.ToolCalls {
|
||||
tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{
|
||||
Tool: tc.Function.Name,
|
||||
Arguments: tc.Function.Arguments,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add tool results
|
||||
if len(response.Tools) > 0 {
|
||||
// If we already have tool calls from completion, match results
|
||||
if len(tr.ToolCalls) > 0 {
|
||||
for i, toolResult := range response.Tools {
|
||||
if i < len(tr.ToolCalls) {
|
||||
tr.ToolCalls[i].Result = toolResult.Result
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Create tool call entries from results
|
||||
for _, toolResult := range response.Tools {
|
||||
tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{
|
||||
Tool: toolResult.Tool,
|
||||
Arguments: toolResult.Arguments,
|
||||
Result: toolResult.Result,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract Next hook data
|
||||
if response.Next != nil && !isEmptyValue(response.Next) {
|
||||
tr.Next = response.Next
|
||||
}
|
||||
|
||||
return tr
|
||||
}
|
||||
|
||||
// appendAssistantMessages appends assistant messages to the conversation history
|
||||
// including tool calls and tool results if present
|
||||
func appendAssistantMessages(messages []context.Message, response *context.Response) []context.Message {
|
||||
if response == nil {
|
||||
return messages
|
||||
}
|
||||
|
||||
// Check if there are tool calls in the completion
|
||||
hasToolCalls := response.Completion != nil && len(response.Completion.ToolCalls) > 0
|
||||
|
||||
if hasToolCalls {
|
||||
// Add assistant message with tool calls
|
||||
assistantMsg := context.Message{
|
||||
Role: context.RoleAssistant,
|
||||
ToolCalls: response.Completion.ToolCalls,
|
||||
}
|
||||
// Include content if present
|
||||
if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) {
|
||||
assistantMsg.Content = response.Completion.Content
|
||||
}
|
||||
messages = append(messages, assistantMsg)
|
||||
|
||||
// Add tool result messages for each tool call
|
||||
for i, tc := range response.Completion.ToolCalls {
|
||||
toolCallID := tc.ID
|
||||
var resultContent string
|
||||
|
||||
// Get result from response.Tools if available
|
||||
if i < len(response.Tools) {
|
||||
resultJSON, err := jsoniter.MarshalToString(response.Tools[i].Result)
|
||||
if err == nil {
|
||||
resultContent = resultJSON
|
||||
} else {
|
||||
resultContent = fmt.Sprintf("%v", response.Tools[i].Result)
|
||||
}
|
||||
} else {
|
||||
resultContent = "{}"
|
||||
}
|
||||
|
||||
messages = append(messages, context.Message{
|
||||
Role: context.RoleTool,
|
||||
ToolCallID: &toolCallID,
|
||||
Content: resultContent,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
// No tool calls, just add content if present
|
||||
content := extractOutput(response)
|
||||
if content != nil && !isEmptyValue(content) {
|
||||
messages = append(messages, context.Message{
|
||||
Role: context.RoleAssistant,
|
||||
Content: content,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return messages
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,9 +34,12 @@ type TurnResult struct {
|
|||
// Input is the user message (from simulator or initial input)
|
||||
Input interface{} `json:"input"`
|
||||
|
||||
// Output is the agent's response
|
||||
// Output is the agent's response (summary for display and conversation history)
|
||||
Output interface{} `json:"output,omitempty"`
|
||||
|
||||
// Response is the full agent response including completion and tool results
|
||||
Response *TurnResponse `json:"response,omitempty"`
|
||||
|
||||
// CheckpointsReached lists checkpoint IDs reached in this turn
|
||||
CheckpointsReached []string `json:"checkpoints_reached,omitempty"`
|
||||
|
||||
|
|
@ -47,6 +50,30 @@ type TurnResult struct {
|
|||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// TurnResponse contains the full agent response for a turn
|
||||
type TurnResponse struct {
|
||||
// Content is the text content from LLM completion
|
||||
Content interface{} `json:"content,omitempty"`
|
||||
|
||||
// ToolCalls contains the tool calls made by the agent
|
||||
ToolCalls []ToolCallInfo `json:"tool_calls,omitempty"`
|
||||
|
||||
// Next is the data returned from Next hook
|
||||
Next interface{} `json:"next,omitempty"`
|
||||
}
|
||||
|
||||
// ToolCallInfo contains information about a tool call
|
||||
type ToolCallInfo struct {
|
||||
// Tool is the tool name
|
||||
Tool string `json:"tool"`
|
||||
|
||||
// Arguments are the tool call arguments
|
||||
Arguments interface{} `json:"arguments,omitempty"`
|
||||
|
||||
// Result is the tool execution result
|
||||
Result interface{} `json:"result,omitempty"`
|
||||
}
|
||||
|
||||
// CheckpointResult represents the result of a checkpoint validation
|
||||
type CheckpointResult struct {
|
||||
// ID is the checkpoint identifier
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -145,6 +146,13 @@ func FilterByIDs(cases []*Case, ids []string) []*Case {
|
|||
})
|
||||
}
|
||||
|
||||
// FilterByPattern returns test cases whose ID matches the given regex pattern
|
||||
func FilterByPattern(cases []*Case, pattern *regexp.Regexp) []*Case {
|
||||
return FilterTestCases(cases, func(tc *Case) bool {
|
||||
return pattern.MatchString(tc.ID)
|
||||
})
|
||||
}
|
||||
|
||||
// LoadFromAgent generates test cases using a generator agent
|
||||
func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) {
|
||||
return GenerateTestCases(agentID, targetInfo, params)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -232,6 +233,32 @@ func (r *Executor) RunTests() (*Report, error) {
|
|||
r.output.Warning("Skipped: %d test cases", skippedCount)
|
||||
}
|
||||
|
||||
// Filter by --run pattern if specified
|
||||
if r.opts.Run != "" {
|
||||
runPattern, err := regexp.Compile(r.opts.Run)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid --run pattern %q: %w", r.opts.Run, err)
|
||||
}
|
||||
activeTests = FilterByPattern(activeTests, runPattern)
|
||||
if len(activeTests) == 0 {
|
||||
return nil, fmt.Errorf("no test cases match pattern %q", r.opts.Run)
|
||||
}
|
||||
r.output.Info("Filter: %q (%d test cases match)", r.opts.Run, len(activeTests))
|
||||
}
|
||||
|
||||
// Load context config if specified
|
||||
if r.opts.ContextFile != "" {
|
||||
ctxConfig, err := LoadContextConfig(r.opts.ContextFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load context file: %w", err)
|
||||
}
|
||||
r.opts.ContextData = ctxConfig
|
||||
r.output.Info("Context: %s", r.opts.ContextFile)
|
||||
}
|
||||
|
||||
// Set options on hook executor (for context data access in hooks)
|
||||
r.hookExecutor.SetOptions(r.opts)
|
||||
|
||||
// Print test info
|
||||
if r.opts.Runs > 1 {
|
||||
r.output.Info("Runs: %d per test case (stability analysis)", r.opts.Runs)
|
||||
|
|
@ -526,14 +553,12 @@ func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID st
|
|||
// Convert to standard result
|
||||
result := dynamicResult.ToResult()
|
||||
|
||||
// Execute after script if specified
|
||||
defer func() {
|
||||
if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) {
|
||||
if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil {
|
||||
r.output.Warning("after script failed: %s", err.Error())
|
||||
}
|
||||
// Execute after script if specified (before outputting result)
|
||||
if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) {
|
||||
if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil {
|
||||
r.output.Warning("after script failed: %s", err.Error())
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Output result
|
||||
duration := time.Duration(result.DurationMs) * time.Millisecond
|
||||
|
|
@ -718,7 +743,7 @@ func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options {
|
|||
}
|
||||
|
||||
// extractOutput extracts the output from the agent response
|
||||
// Priority: Next hook data (if non-empty) > Completion content > nil
|
||||
// Priority: Next hook data > Completion content > Tool results message > nil
|
||||
func extractOutput(response *context.Response) interface{} {
|
||||
if response == nil {
|
||||
return nil
|
||||
|
|
@ -729,11 +754,76 @@ func extractOutput(response *context.Response) interface{} {
|
|||
if response.Next != nil && !isEmptyValue(response.Next) {
|
||||
return response.Next
|
||||
}
|
||||
// Fall back to raw completion content
|
||||
|
||||
// Fall back to completion response
|
||||
if response.Completion != nil {
|
||||
return response.Completion.Content
|
||||
// If content is non-empty, return it
|
||||
if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) {
|
||||
return response.Completion.Content
|
||||
}
|
||||
}
|
||||
|
||||
// If no content but tools were executed, extract message from tool results
|
||||
// This handles the case where LLM calls tools but doesn't generate text
|
||||
if len(response.Tools) > 0 {
|
||||
return extractToolResultMessage(response.Tools)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractToolResultMessage extracts the message field from tool results
|
||||
// Returns the first non-empty message found, or a summary of tool calls
|
||||
func extractToolResultMessage(tools []context.ToolCallResponse) interface{} {
|
||||
if len(tools) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to extract "message" field from tool results first
|
||||
for _, tool := range tools {
|
||||
if tool.Result != nil {
|
||||
// Try to get message from result map
|
||||
if resultMap, ok := tool.Result.(map[string]interface{}); ok {
|
||||
if msg, exists := resultMap["message"]; exists && msg != nil {
|
||||
if msgStr, ok := msg.(string); ok && msgStr != "" {
|
||||
return msgStr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No message found, generate a summary of tool calls
|
||||
var summaries []string
|
||||
for _, tool := range tools {
|
||||
toolName := tool.Tool
|
||||
if toolName == "" {
|
||||
toolName = "unknown"
|
||||
}
|
||||
// Extract key info from result if possible
|
||||
if tool.Result != nil {
|
||||
if resultMap, ok := tool.Result.(map[string]interface{}); ok {
|
||||
// Try common result fields
|
||||
if action, ok := resultMap["action"].(string); ok {
|
||||
summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, action))
|
||||
continue
|
||||
}
|
||||
if success, ok := resultMap["success"].(bool); ok {
|
||||
status := "failed"
|
||||
if success {
|
||||
status = "success"
|
||||
}
|
||||
summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, status))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
summaries = append(summaries, fmt.Sprintf("[%s]", toolName))
|
||||
}
|
||||
|
||||
if len(summaries) > 0 {
|
||||
return strings.Join(summaries, " ")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ type HookExecutor struct {
|
|||
output *OutputWriter
|
||||
loadedDirs map[string]bool // Track which directories have been loaded
|
||||
agentContext *context.Context
|
||||
opts *Options // Test options (includes ContextData from --ctx)
|
||||
}
|
||||
|
||||
// NewHookExecutor creates a new hook executor
|
||||
|
|
@ -35,6 +36,11 @@ func (h *HookExecutor) SetAgentContext(ctx *context.Context) {
|
|||
h.agentContext = ctx
|
||||
}
|
||||
|
||||
// SetOptions sets the test options for hook execution
|
||||
func (h *HookExecutor) SetOptions(opts *Options) {
|
||||
h.opts = opts
|
||||
}
|
||||
|
||||
// HookRef represents a parsed hook reference
|
||||
// Format: "src/env_test.ts:Before" or just "Before" (uses default test file)
|
||||
type HookRef struct {
|
||||
|
|
@ -86,13 +92,23 @@ func ParseHookRef(ref string) (*HookRef, error) {
|
|||
func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
|
||||
srcDir := filepath.Join(agentPath, "src")
|
||||
|
||||
// Check if already loaded
|
||||
// Convert to relative path for application.App
|
||||
// application.App expects paths relative to YAO_ROOT
|
||||
relSrcDir := srcDir
|
||||
if application.App != nil {
|
||||
if rel, err := filepath.Rel(application.App.Root(), srcDir); err == nil {
|
||||
relSrcDir = rel
|
||||
}
|
||||
}
|
||||
|
||||
// Check if already loaded (use absolute path as key)
|
||||
// No logging for already loaded - this is normal and happens frequently
|
||||
if h.loadedDirs[srcDir] {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Check if src directory exists
|
||||
exists, err := application.App.Exists(srcDir)
|
||||
exists, err := application.App.Exists(relSrcDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -103,7 +119,7 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
|
|||
var loadedScripts []string
|
||||
exts := []string{"*_test.ts", "*_test.js"}
|
||||
|
||||
err = application.App.Walk(srcDir, func(root, file string, isdir bool) error {
|
||||
err = application.App.Walk(relSrcDir, func(root, file string, isdir bool) error {
|
||||
if isdir {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -114,10 +130,10 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Generate script ID
|
||||
scriptID := generateHookScriptID(file, srcDir)
|
||||
// Generate script ID (use relative path for consistency)
|
||||
scriptID := generateHookScriptID(file, relSrcDir)
|
||||
|
||||
// Load the script
|
||||
// Load the script (file path from Walk is relative to App root)
|
||||
_, err := v8.Load(file, scriptID)
|
||||
if err != nil {
|
||||
if h.verbose {
|
||||
|
|
@ -127,10 +143,6 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
|
|||
}
|
||||
|
||||
loadedScripts = append(loadedScripts, scriptID)
|
||||
if h.verbose {
|
||||
h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}, exts...)
|
||||
|
||||
|
|
@ -139,6 +151,12 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) {
|
|||
}
|
||||
|
||||
h.loadedDirs[srcDir] = true
|
||||
|
||||
// Log summary only once when scripts are first loaded
|
||||
if h.verbose && len(loadedScripts) > 0 {
|
||||
h.output.Verbose("Loaded %d hook scripts from %s", len(loadedScripts), relSrcDir)
|
||||
}
|
||||
|
||||
return loadedScripts, nil
|
||||
}
|
||||
|
||||
|
|
@ -392,13 +410,19 @@ func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName
|
|||
return nil, fmt.Errorf("failed to convert to function: %w", err)
|
||||
}
|
||||
|
||||
// Build ctx argument
|
||||
ctxJS, err := h.buildCtxArg(v8ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert test cases to JS array
|
||||
casesJS, err := h.testCasesToJS(v8ctx, testCases)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jsResult, err := fn.Call(global, casesJS)
|
||||
jsResult, err := fn.Call(global, ctxJS, casesJS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
|
||||
}
|
||||
|
|
@ -454,6 +478,12 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam
|
|||
return nil, fmt.Errorf("failed to convert to function: %w", err)
|
||||
}
|
||||
|
||||
// Build ctx argument
|
||||
ctxJS, err := h.buildCtxArg(v8ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert results to JS array
|
||||
resultsJS, err := h.resultsToJS(v8ctx, results)
|
||||
if err != nil {
|
||||
|
|
@ -466,7 +496,7 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam
|
|||
return nil, fmt.Errorf("failed to convert beforeData: %w", err)
|
||||
}
|
||||
|
||||
jsResult, err := fn.Call(global, resultsJS, beforeDataJS)
|
||||
jsResult, err := fn.Call(global, ctxJS, resultsJS, beforeDataJS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hook function %s failed: %w", funcName, err)
|
||||
}
|
||||
|
|
@ -498,11 +528,105 @@ func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error {
|
|||
})
|
||||
}
|
||||
|
||||
// buildCtxArg builds the context argument for hook functions
|
||||
func (h *HookExecutor) buildCtxArg(v8ctx *v8go.Context) (*v8go.Value, error) {
|
||||
ctxMap := map[string]interface{}{
|
||||
"locale": "en",
|
||||
}
|
||||
|
||||
// Use ContextData from --ctx flag if available
|
||||
if h.opts != nil && h.opts.ContextData != nil {
|
||||
cfg := h.opts.ContextData
|
||||
if cfg.Locale != "" {
|
||||
ctxMap["locale"] = cfg.Locale
|
||||
}
|
||||
if cfg.Authorized != nil {
|
||||
authorized := map[string]interface{}{}
|
||||
if cfg.Authorized.UserID != "" {
|
||||
authorized["user_id"] = cfg.Authorized.UserID
|
||||
}
|
||||
if cfg.Authorized.TeamID != "" {
|
||||
authorized["team_id"] = cfg.Authorized.TeamID
|
||||
}
|
||||
if cfg.Authorized.TenantID != "" {
|
||||
authorized["tenant_id"] = cfg.Authorized.TenantID
|
||||
}
|
||||
if cfg.Authorized.Sub != "" {
|
||||
authorized["sub"] = cfg.Authorized.Sub
|
||||
}
|
||||
ctxMap["authorized"] = authorized
|
||||
}
|
||||
if cfg.Metadata != nil {
|
||||
ctxMap["metadata"] = cfg.Metadata
|
||||
}
|
||||
}
|
||||
|
||||
return bridge.JsValue(v8ctx, ctxMap)
|
||||
}
|
||||
|
||||
// buildHookArgs builds the arguments for a hook function call
|
||||
// Arguments order: ctx, testCase, result (for After), beforeData (for After)
|
||||
func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) {
|
||||
var args []*v8go.Value
|
||||
|
||||
// Arg 1: testCase
|
||||
// Arg 1: ctx (context) - build from opts.ContextData if available
|
||||
ctxMap := map[string]interface{}{
|
||||
"locale": "en",
|
||||
}
|
||||
|
||||
// Use ContextData from --ctx flag if available
|
||||
if h.opts != nil && h.opts.ContextData != nil {
|
||||
cfg := h.opts.ContextData
|
||||
if cfg.Locale != "" {
|
||||
ctxMap["locale"] = cfg.Locale
|
||||
}
|
||||
if cfg.Authorized != nil {
|
||||
authorized := map[string]interface{}{}
|
||||
if cfg.Authorized.UserID != "" {
|
||||
authorized["user_id"] = cfg.Authorized.UserID
|
||||
}
|
||||
if cfg.Authorized.TeamID != "" {
|
||||
authorized["team_id"] = cfg.Authorized.TeamID
|
||||
}
|
||||
if cfg.Authorized.TenantID != "" {
|
||||
authorized["tenant_id"] = cfg.Authorized.TenantID
|
||||
}
|
||||
if cfg.Authorized.Sub != "" {
|
||||
authorized["sub"] = cfg.Authorized.Sub
|
||||
}
|
||||
ctxMap["authorized"] = authorized
|
||||
}
|
||||
if cfg.Metadata != nil {
|
||||
ctxMap["metadata"] = cfg.Metadata
|
||||
}
|
||||
} else if testCase != nil {
|
||||
// Fallback to test case fields
|
||||
if testCase.UserID != "" {
|
||||
ctxMap["user_id"] = testCase.UserID
|
||||
}
|
||||
if testCase.TeamID != "" {
|
||||
ctxMap["team_id"] = testCase.TeamID
|
||||
}
|
||||
// Build authorized info
|
||||
authorized := map[string]interface{}{}
|
||||
if testCase.UserID != "" {
|
||||
authorized["user_id"] = testCase.UserID
|
||||
}
|
||||
if testCase.TeamID != "" {
|
||||
authorized["team_id"] = testCase.TeamID
|
||||
}
|
||||
if len(authorized) > 0 {
|
||||
ctxMap["authorized"] = authorized
|
||||
}
|
||||
}
|
||||
|
||||
ctxJS, err := bridge.JsValue(v8ctx, ctxMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert ctx: %w", err)
|
||||
}
|
||||
args = append(args, ctxJS)
|
||||
|
||||
// Arg 2: testCase
|
||||
if testCase != nil {
|
||||
tcMap := map[string]interface{}{
|
||||
"id": testCase.ID,
|
||||
|
|
@ -514,12 +638,20 @@ func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result
|
|||
if testCase.Assert != nil {
|
||||
tcMap["assert"] = testCase.Assert
|
||||
}
|
||||
// Include simulator options for dynamic tests
|
||||
if testCase.Simulator != nil {
|
||||
tcMap["simulator"] = testCase.Simulator
|
||||
}
|
||||
|
||||
tcJS, err := bridge.JsValue(v8ctx, tcMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert testCase: %w", err)
|
||||
}
|
||||
args = append(args, tcJS)
|
||||
} else {
|
||||
// Pass empty object if no testCase
|
||||
emptyJS, _ := bridge.JsValue(v8ctx, map[string]interface{}{})
|
||||
args = append(args, emptyJS)
|
||||
}
|
||||
|
||||
// Arg 2: result (for After)
|
||||
|
|
|
|||
|
|
@ -165,6 +165,10 @@ type Options struct {
|
|||
// ContextConfig represents custom context configuration from JSON file
|
||||
// This allows full customization of the test context including authorized info
|
||||
type ContextConfig struct {
|
||||
// ChatID is the chat session identifier
|
||||
// Used to maintain session state across turns in dynamic tests
|
||||
ChatID string `json:"chat_id,omitempty"`
|
||||
|
||||
// Authorized contains custom authorization data
|
||||
Authorized *AuthorizedConfig `json:"authorized,omitempty"`
|
||||
|
||||
|
|
@ -554,9 +558,15 @@ type AssertionResult struct {
|
|||
}
|
||||
|
||||
// GetEnvironment returns the effective test environment for this test case
|
||||
// Priority: command line flags > test case fields > defaults
|
||||
// Priority: command line flags > context config > test case fields > defaults
|
||||
func (tc *Case) GetEnvironment(opts *Options) *Environment {
|
||||
env := NewEnvironment("", "")
|
||||
// Start with context config if available, otherwise use defaults
|
||||
var env *Environment
|
||||
if opts != nil && opts.ContextData != nil {
|
||||
env = NewEnvironmentWithContext("", "", opts.ContextData)
|
||||
} else {
|
||||
env = NewEnvironment("", "")
|
||||
}
|
||||
|
||||
// Apply test case specific values
|
||||
if tc.UserID != "" {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue