diff --git a/agent/assistant/next.go b/agent/assistant/next.go index fc6c7c7a..accacaab 100644 --- a/agent/assistant/next.go +++ b/agent/assistant/next.go @@ -30,6 +30,7 @@ func (ast *Assistant) processNextResponse(npc *NextProcessContext) (*agentContex Create: npc.CreateResponse, Next: npc.NextResponse.Data, // Put custom data in Next field Completion: npc.CompletionResponse, + Tools: npc.ToolCallResponses, }, nil } @@ -72,5 +73,6 @@ func (ast *Assistant) buildStandardResponse(npc *NextProcessContext) *agentConte Create: npc.CreateResponse, Next: npc.NextResponse, Completion: npc.CompletionResponse, + Tools: npc.ToolCallResponses, } } diff --git a/agent/context/types.go b/agent/context/types.go index d567a72f..f5cbb372 100644 --- a/agent/context/types.go +++ b/agent/context/types.go @@ -349,6 +349,7 @@ type Response struct { Create *HookCreateResponse `json:"create,omitempty"` // Create response from the create hook Next interface{} `json:"next,omitempty"` // Next response from the next hook Completion *CompletionResponse `json:"completion,omitempty"` // Completion response from the completion hook + Tools []ToolCallResponse `json:"tools,omitempty"` // Tool call results (if any tools were executed) } // HookCreateResponse the response of the create hook diff --git a/agent/test/README.md b/agent/test/README.md index c476c4a4..2135f018 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -203,6 +203,33 @@ Simulator-driven testing with checkpoint validation. A simulator agent generates | `--fail-fast` | Stop on first failure | false | | `--dry-run` | Generate test cases without running them | false | +## Custom Context File + +Create a JSON file for custom authorization: + +```json +{ + "chat_id": "test-chat-001", + "authorized": { + "user_id": "test-user-123", + "team_id": "test-team-456", + "constraints": { + "owner_only": true, + "extra": { "department": "engineering" } + } + }, + "metadata": { + "mode": "test" + } +} +``` + +Use with `--ctx`: + +```bash +yao agent test -i scripts.expense.setup --ctx tests/context.json -v +``` + ## Input Format (JSONL) Each line is a JSON object. Below are examples organized by scenario. @@ -761,9 +788,12 @@ export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) { ```typescript interface Context { - user_id: string; // Test user ID - team_id: string; // Test team ID locale: string; // Locale (e.g., "en-us") + authorized: { + user_id: string; // Test user ID + team_id: string; // Test team ID + constraints?: object; // Access constraints + }; metadata: object; // Custom metadata from test case } ``` @@ -980,6 +1010,33 @@ For testing complex conversation flows where the path is unpredictable: └─ PASSED (3 turns, 3 checkpoints, 8.5s) ``` +### Dynamic Mode Output Structure + +Each turn in the output includes: + +```typescript +interface TurnResult { + turn: number; // Turn number (1-based) + input: string; // User message + output: any; // Agent response summary (for display) + response: { + // Full agent response (for detailed analysis) + content: string; // LLM text content + tool_calls: [ + { + // Tool calls made + tool: string; // Tool name + arguments: any; // Call arguments + result: any; // Execution result + } + ]; + next: any; // Next hook data + }; + checkpoints_reached: string[]; // Checkpoint IDs reached + duration_ms: number; // Execution time +} +``` + ## Output Formats Determined by `-o` file extension: diff --git a/agent/test/dynamic_runner.go b/agent/test/dynamic_runner.go index 2b6020df..6530c7ad 100644 --- a/agent/test/dynamic_runner.go +++ b/agent/test/dynamic_runner.go @@ -77,7 +77,14 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s // Output dynamic test start if r.opts.Verbose { - r.output.Info("Dynamic test: %s (max %d turns)", tc.ID, maxTurns) + r.output.Verbose("Dynamic test: %s (max %d turns)", tc.ID, maxTurns) + } + + // Use consistent chatID across all turns to preserve session state (ctx.memory.chat) + // Priority: context config > generated ID + chatID := fmt.Sprintf("dynamic-%s", tc.ID) + if r.opts.ContextData != nil && r.opts.ContextData.ChatID != "" { + chatID = r.opts.ContextData.ChatID } // Conversation loop @@ -111,7 +118,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s // Check if goal achieved if simOutput.GoalAchieved { if r.opts.Verbose { - r.output.Info(" Turn %d: Simulator signaled goal achieved", turn) + r.output.Verbose("Turn %d: Simulator signaled goal achieved", turn) } // Check if all required checkpoints reached @@ -135,7 +142,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s turnResult.Input = simOutput.Message if r.opts.Verbose { - r.output.Info(" Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50)) + r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(simOutput.Message, 50)) } } else { // Use initial input for first turn @@ -143,14 +150,15 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s lastMsg := messages[len(messages)-1] turnResult.Input = lastMsg.Content if r.opts.Verbose { - r.output.Info(" Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50)) + r.output.Verbose("Turn %d: User: %s", turn, truncateOutput(lastMsg.Content, 50)) } } } // Call target agent + // Use consistent chatID across all turns to preserve session state (ctx.memory.chat) ctx := NewTestContextFromOptions( - fmt.Sprintf("dynamic-%s-%d", tc.ID, turn), + chatID, agentID, r.opts, tc, @@ -171,20 +179,20 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s return result } - // Extract output + // Extract output (summary for display and conversation history) output := extractOutput(response) turnResult.Output = output turnResult.DurationMs = time.Since(turnStart).Milliseconds() + // Store full response for reporting + turnResult.Response = buildTurnResponse(response) + if r.opts.Verbose { - r.output.Info(" Turn %d: Agent: %s", turn, truncateOutput(output, 50)) + r.output.Verbose("Turn %d: Agent: %s", turn, truncateOutput(output, 50)) } - // Add assistant response to messages - messages = append(messages, context.Message{ - Role: context.RoleAssistant, - Content: output, - }) + // Add assistant response to messages, including tool calls if any + messages = appendAssistantMessages(messages, response) // Check checkpoints against this response reachedIDs := r.checkCheckpoints(tc.Checkpoints, output, result) @@ -192,7 +200,7 @@ func (r *DynamicRunner) RunDynamic(ast *assistant.Assistant, tc *Case, agentID s if r.opts.Verbose && len(reachedIDs) > 0 { for _, id := range reachedIDs { - r.output.Info(" ✓ checkpoint: %s", id) + r.output.Verbose(" ✓ checkpoint: %s", id) } } @@ -411,3 +419,114 @@ func (r *DynamicRunner) allRequiredCheckpointsReached(result *DynamicResult) boo } return true } + +// buildTurnResponse builds a TurnResponse from the agent response +func buildTurnResponse(response *context.Response) *TurnResponse { + if response == nil { + return nil + } + + tr := &TurnResponse{} + + // Extract completion content + if response.Completion != nil { + tr.Content = response.Completion.Content + + // Extract tool calls from completion + if len(response.Completion.ToolCalls) > 0 { + for _, tc := range response.Completion.ToolCalls { + tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{ + Tool: tc.Function.Name, + Arguments: tc.Function.Arguments, + }) + } + } + } + + // Add tool results + if len(response.Tools) > 0 { + // If we already have tool calls from completion, match results + if len(tr.ToolCalls) > 0 { + for i, toolResult := range response.Tools { + if i < len(tr.ToolCalls) { + tr.ToolCalls[i].Result = toolResult.Result + } + } + } else { + // Create tool call entries from results + for _, toolResult := range response.Tools { + tr.ToolCalls = append(tr.ToolCalls, ToolCallInfo{ + Tool: toolResult.Tool, + Arguments: toolResult.Arguments, + Result: toolResult.Result, + }) + } + } + } + + // Extract Next hook data + if response.Next != nil && !isEmptyValue(response.Next) { + tr.Next = response.Next + } + + return tr +} + +// appendAssistantMessages appends assistant messages to the conversation history +// including tool calls and tool results if present +func appendAssistantMessages(messages []context.Message, response *context.Response) []context.Message { + if response == nil { + return messages + } + + // Check if there are tool calls in the completion + hasToolCalls := response.Completion != nil && len(response.Completion.ToolCalls) > 0 + + if hasToolCalls { + // Add assistant message with tool calls + assistantMsg := context.Message{ + Role: context.RoleAssistant, + ToolCalls: response.Completion.ToolCalls, + } + // Include content if present + if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) { + assistantMsg.Content = response.Completion.Content + } + messages = append(messages, assistantMsg) + + // Add tool result messages for each tool call + for i, tc := range response.Completion.ToolCalls { + toolCallID := tc.ID + var resultContent string + + // Get result from response.Tools if available + if i < len(response.Tools) { + resultJSON, err := jsoniter.MarshalToString(response.Tools[i].Result) + if err == nil { + resultContent = resultJSON + } else { + resultContent = fmt.Sprintf("%v", response.Tools[i].Result) + } + } else { + resultContent = "{}" + } + + messages = append(messages, context.Message{ + Role: context.RoleTool, + ToolCallID: &toolCallID, + Content: resultContent, + }) + } + } else { + // No tool calls, just add content if present + content := extractOutput(response) + if content != nil && !isEmptyValue(content) { + messages = append(messages, context.Message{ + Role: context.RoleAssistant, + Content: content, + }) + } + } + + return messages +} diff --git a/agent/test/dynamic_types.go b/agent/test/dynamic_types.go index d75ec2c4..27cb0634 100644 --- a/agent/test/dynamic_types.go +++ b/agent/test/dynamic_types.go @@ -34,9 +34,12 @@ type TurnResult struct { // Input is the user message (from simulator or initial input) Input interface{} `json:"input"` - // Output is the agent's response + // Output is the agent's response (summary for display and conversation history) Output interface{} `json:"output,omitempty"` + // Response is the full agent response including completion and tool results + Response *TurnResponse `json:"response,omitempty"` + // CheckpointsReached lists checkpoint IDs reached in this turn CheckpointsReached []string `json:"checkpoints_reached,omitempty"` @@ -47,6 +50,30 @@ type TurnResult struct { Error string `json:"error,omitempty"` } +// TurnResponse contains the full agent response for a turn +type TurnResponse struct { + // Content is the text content from LLM completion + Content interface{} `json:"content,omitempty"` + + // ToolCalls contains the tool calls made by the agent + ToolCalls []ToolCallInfo `json:"tool_calls,omitempty"` + + // Next is the data returned from Next hook + Next interface{} `json:"next,omitempty"` +} + +// ToolCallInfo contains information about a tool call +type ToolCallInfo struct { + // Tool is the tool name + Tool string `json:"tool"` + + // Arguments are the tool call arguments + Arguments interface{} `json:"arguments,omitempty"` + + // Result is the tool execution result + Result interface{} `json:"result,omitempty"` +} + // CheckpointResult represents the result of a checkpoint validation type CheckpointResult struct { // ID is the checkpoint identifier diff --git a/agent/test/loader.go b/agent/test/loader.go index 52092b68..73fdad39 100644 --- a/agent/test/loader.go +++ b/agent/test/loader.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "regexp" "strings" "time" @@ -145,6 +146,13 @@ func FilterByIDs(cases []*Case, ids []string) []*Case { }) } +// FilterByPattern returns test cases whose ID matches the given regex pattern +func FilterByPattern(cases []*Case, pattern *regexp.Regexp) []*Case { + return FilterTestCases(cases, func(tc *Case) bool { + return pattern.MatchString(tc.ID) + }) +} + // LoadFromAgent generates test cases using a generator agent func (l *JSONLLoader) LoadFromAgent(agentID string, targetInfo *TargetAgentInfo, params map[string]interface{}) ([]*Case, error) { return GenerateTestCases(agentID, targetInfo, params) diff --git a/agent/test/runner.go b/agent/test/runner.go index 3f18e9a8..9159df4d 100644 --- a/agent/test/runner.go +++ b/agent/test/runner.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strings" "sync" "time" @@ -232,6 +233,32 @@ func (r *Executor) RunTests() (*Report, error) { r.output.Warning("Skipped: %d test cases", skippedCount) } + // Filter by --run pattern if specified + if r.opts.Run != "" { + runPattern, err := regexp.Compile(r.opts.Run) + if err != nil { + return nil, fmt.Errorf("invalid --run pattern %q: %w", r.opts.Run, err) + } + activeTests = FilterByPattern(activeTests, runPattern) + if len(activeTests) == 0 { + return nil, fmt.Errorf("no test cases match pattern %q", r.opts.Run) + } + r.output.Info("Filter: %q (%d test cases match)", r.opts.Run, len(activeTests)) + } + + // Load context config if specified + if r.opts.ContextFile != "" { + ctxConfig, err := LoadContextConfig(r.opts.ContextFile) + if err != nil { + return nil, fmt.Errorf("failed to load context file: %w", err) + } + r.opts.ContextData = ctxConfig + r.output.Info("Context: %s", r.opts.ContextFile) + } + + // Set options on hook executor (for context data access in hooks) + r.hookExecutor.SetOptions(r.opts) + // Print test info if r.opts.Runs > 1 { r.output.Info("Runs: %d per test case (stability analysis)", r.opts.Runs) @@ -526,14 +553,12 @@ func (r *Executor) runDynamicTest(ast *assistant.Assistant, tc *Case, agentID st // Convert to standard result result := dynamicResult.ToResult() - // Execute after script if specified - defer func() { - if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { - if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { - r.output.Warning("after script failed: %s", err.Error()) - } + // Execute after script if specified (before outputting result) + if tc.After != "" && (tc.Before == "" || beforeData != nil || result.Status != StatusError || !isBeforeError(result.Error)) { + if err := r.hookExecutor.ExecuteAfter(tc.After, tc, result, beforeData, r.agentPath); err != nil { + r.output.Warning("after script failed: %s", err.Error()) } - }() + } // Output result duration := time.Duration(result.DurationMs) * time.Millisecond @@ -718,7 +743,7 @@ func buildContextOptions(tc *Case, runnerOpts *Options) *context.Options { } // extractOutput extracts the output from the agent response -// Priority: Next hook data (if non-empty) > Completion content > nil +// Priority: Next hook data > Completion content > Tool results message > nil func extractOutput(response *context.Response) interface{} { if response == nil { return nil @@ -729,11 +754,76 @@ func extractOutput(response *context.Response) interface{} { if response.Next != nil && !isEmptyValue(response.Next) { return response.Next } - // Fall back to raw completion content + + // Fall back to completion response if response.Completion != nil { - return response.Completion.Content + // If content is non-empty, return it + if response.Completion.Content != nil && !isEmptyValue(response.Completion.Content) { + return response.Completion.Content + } } + // If no content but tools were executed, extract message from tool results + // This handles the case where LLM calls tools but doesn't generate text + if len(response.Tools) > 0 { + return extractToolResultMessage(response.Tools) + } + + return nil +} + +// extractToolResultMessage extracts the message field from tool results +// Returns the first non-empty message found, or a summary of tool calls +func extractToolResultMessage(tools []context.ToolCallResponse) interface{} { + if len(tools) == 0 { + return nil + } + + // Try to extract "message" field from tool results first + for _, tool := range tools { + if tool.Result != nil { + // Try to get message from result map + if resultMap, ok := tool.Result.(map[string]interface{}); ok { + if msg, exists := resultMap["message"]; exists && msg != nil { + if msgStr, ok := msg.(string); ok && msgStr != "" { + return msgStr + } + } + } + } + } + + // No message found, generate a summary of tool calls + var summaries []string + for _, tool := range tools { + toolName := tool.Tool + if toolName == "" { + toolName = "unknown" + } + // Extract key info from result if possible + if tool.Result != nil { + if resultMap, ok := tool.Result.(map[string]interface{}); ok { + // Try common result fields + if action, ok := resultMap["action"].(string); ok { + summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, action)) + continue + } + if success, ok := resultMap["success"].(bool); ok { + status := "failed" + if success { + status = "success" + } + summaries = append(summaries, fmt.Sprintf("[%s: %s]", toolName, status)) + continue + } + } + } + summaries = append(summaries, fmt.Sprintf("[%s]", toolName)) + } + + if len(summaries) > 0 { + return strings.Join(summaries, " ") + } return nil } diff --git a/agent/test/script_hooks.go b/agent/test/script_hooks.go index ea9349e1..a129b2ba 100644 --- a/agent/test/script_hooks.go +++ b/agent/test/script_hooks.go @@ -19,6 +19,7 @@ type HookExecutor struct { output *OutputWriter loadedDirs map[string]bool // Track which directories have been loaded agentContext *context.Context + opts *Options // Test options (includes ContextData from --ctx) } // NewHookExecutor creates a new hook executor @@ -35,6 +36,11 @@ func (h *HookExecutor) SetAgentContext(ctx *context.Context) { h.agentContext = ctx } +// SetOptions sets the test options for hook execution +func (h *HookExecutor) SetOptions(opts *Options) { + h.opts = opts +} + // HookRef represents a parsed hook reference // Format: "src/env_test.ts:Before" or just "Before" (uses default test file) type HookRef struct { @@ -86,13 +92,23 @@ func ParseHookRef(ref string) (*HookRef, error) { func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { srcDir := filepath.Join(agentPath, "src") - // Check if already loaded + // Convert to relative path for application.App + // application.App expects paths relative to YAO_ROOT + relSrcDir := srcDir + if application.App != nil { + if rel, err := filepath.Rel(application.App.Root(), srcDir); err == nil { + relSrcDir = rel + } + } + + // Check if already loaded (use absolute path as key) + // No logging for already loaded - this is normal and happens frequently if h.loadedDirs[srcDir] { return nil, nil } // Check if src directory exists - exists, err := application.App.Exists(srcDir) + exists, err := application.App.Exists(relSrcDir) if err != nil { return nil, err } @@ -103,7 +119,7 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { var loadedScripts []string exts := []string{"*_test.ts", "*_test.js"} - err = application.App.Walk(srcDir, func(root, file string, isdir bool) error { + err = application.App.Walk(relSrcDir, func(root, file string, isdir bool) error { if isdir { return nil } @@ -114,10 +130,10 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { return nil } - // Generate script ID - scriptID := generateHookScriptID(file, srcDir) + // Generate script ID (use relative path for consistency) + scriptID := generateHookScriptID(file, relSrcDir) - // Load the script + // Load the script (file path from Walk is relative to App root) _, err := v8.Load(file, scriptID) if err != nil { if h.verbose { @@ -127,10 +143,6 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { } loadedScripts = append(loadedScripts, scriptID) - if h.verbose { - h.output.Verbose("Loaded hook script: %s (id: %s)", base, scriptID) - } - return nil }, exts...) @@ -139,6 +151,12 @@ func (h *HookExecutor) LoadTestScripts(agentPath string) ([]string, error) { } h.loadedDirs[srcDir] = true + + // Log summary only once when scripts are first loaded + if h.verbose && len(loadedScripts) > 0 { + h.output.Verbose("Loaded %d hook scripts from %s", len(loadedScripts), relSrcDir) + } + return loadedScripts, nil } @@ -392,13 +410,19 @@ func (h *HookExecutor) executeHookFunctionWithCases(script *v8.Script, funcName return nil, fmt.Errorf("failed to convert to function: %w", err) } + // Build ctx argument + ctxJS, err := h.buildCtxArg(v8ctx) + if err != nil { + return nil, err + } + // Convert test cases to JS array casesJS, err := h.testCasesToJS(v8ctx, testCases) if err != nil { return nil, err } - jsResult, err := fn.Call(global, casesJS) + jsResult, err := fn.Call(global, ctxJS, casesJS) if err != nil { return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) } @@ -454,6 +478,12 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam return nil, fmt.Errorf("failed to convert to function: %w", err) } + // Build ctx argument + ctxJS, err := h.buildCtxArg(v8ctx) + if err != nil { + return nil, err + } + // Convert results to JS array resultsJS, err := h.resultsToJS(v8ctx, results) if err != nil { @@ -466,7 +496,7 @@ func (h *HookExecutor) executeHookFunctionWithResults(script *v8.Script, funcNam return nil, fmt.Errorf("failed to convert beforeData: %w", err) } - jsResult, err := fn.Call(global, resultsJS, beforeDataJS) + jsResult, err := fn.Call(global, ctxJS, resultsJS, beforeDataJS) if err != nil { return nil, fmt.Errorf("hook function %s failed: %w", funcName, err) } @@ -498,11 +528,105 @@ func (h *HookExecutor) setShareData(v8ctx *v8go.Context) error { }) } +// buildCtxArg builds the context argument for hook functions +func (h *HookExecutor) buildCtxArg(v8ctx *v8go.Context) (*v8go.Value, error) { + ctxMap := map[string]interface{}{ + "locale": "en", + } + + // Use ContextData from --ctx flag if available + if h.opts != nil && h.opts.ContextData != nil { + cfg := h.opts.ContextData + if cfg.Locale != "" { + ctxMap["locale"] = cfg.Locale + } + if cfg.Authorized != nil { + authorized := map[string]interface{}{} + if cfg.Authorized.UserID != "" { + authorized["user_id"] = cfg.Authorized.UserID + } + if cfg.Authorized.TeamID != "" { + authorized["team_id"] = cfg.Authorized.TeamID + } + if cfg.Authorized.TenantID != "" { + authorized["tenant_id"] = cfg.Authorized.TenantID + } + if cfg.Authorized.Sub != "" { + authorized["sub"] = cfg.Authorized.Sub + } + ctxMap["authorized"] = authorized + } + if cfg.Metadata != nil { + ctxMap["metadata"] = cfg.Metadata + } + } + + return bridge.JsValue(v8ctx, ctxMap) +} + // buildHookArgs builds the arguments for a hook function call +// Arguments order: ctx, testCase, result (for After), beforeData (for After) func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result *Result, beforeData interface{}) ([]*v8go.Value, error) { var args []*v8go.Value - // Arg 1: testCase + // Arg 1: ctx (context) - build from opts.ContextData if available + ctxMap := map[string]interface{}{ + "locale": "en", + } + + // Use ContextData from --ctx flag if available + if h.opts != nil && h.opts.ContextData != nil { + cfg := h.opts.ContextData + if cfg.Locale != "" { + ctxMap["locale"] = cfg.Locale + } + if cfg.Authorized != nil { + authorized := map[string]interface{}{} + if cfg.Authorized.UserID != "" { + authorized["user_id"] = cfg.Authorized.UserID + } + if cfg.Authorized.TeamID != "" { + authorized["team_id"] = cfg.Authorized.TeamID + } + if cfg.Authorized.TenantID != "" { + authorized["tenant_id"] = cfg.Authorized.TenantID + } + if cfg.Authorized.Sub != "" { + authorized["sub"] = cfg.Authorized.Sub + } + ctxMap["authorized"] = authorized + } + if cfg.Metadata != nil { + ctxMap["metadata"] = cfg.Metadata + } + } else if testCase != nil { + // Fallback to test case fields + if testCase.UserID != "" { + ctxMap["user_id"] = testCase.UserID + } + if testCase.TeamID != "" { + ctxMap["team_id"] = testCase.TeamID + } + // Build authorized info + authorized := map[string]interface{}{} + if testCase.UserID != "" { + authorized["user_id"] = testCase.UserID + } + if testCase.TeamID != "" { + authorized["team_id"] = testCase.TeamID + } + if len(authorized) > 0 { + ctxMap["authorized"] = authorized + } + } + + ctxJS, err := bridge.JsValue(v8ctx, ctxMap) + if err != nil { + return nil, fmt.Errorf("failed to convert ctx: %w", err) + } + args = append(args, ctxJS) + + // Arg 2: testCase if testCase != nil { tcMap := map[string]interface{}{ "id": testCase.ID, @@ -514,12 +638,20 @@ func (h *HookExecutor) buildHookArgs(v8ctx *v8go.Context, testCase *Case, result if testCase.Assert != nil { tcMap["assert"] = testCase.Assert } + // Include simulator options for dynamic tests + if testCase.Simulator != nil { + tcMap["simulator"] = testCase.Simulator + } tcJS, err := bridge.JsValue(v8ctx, tcMap) if err != nil { return nil, fmt.Errorf("failed to convert testCase: %w", err) } args = append(args, tcJS) + } else { + // Pass empty object if no testCase + emptyJS, _ := bridge.JsValue(v8ctx, map[string]interface{}{}) + args = append(args, emptyJS) } // Arg 2: result (for After) diff --git a/agent/test/types.go b/agent/test/types.go index cddbeb8a..6e727b47 100644 --- a/agent/test/types.go +++ b/agent/test/types.go @@ -165,6 +165,10 @@ type Options struct { // ContextConfig represents custom context configuration from JSON file // This allows full customization of the test context including authorized info type ContextConfig struct { + // ChatID is the chat session identifier + // Used to maintain session state across turns in dynamic tests + ChatID string `json:"chat_id,omitempty"` + // Authorized contains custom authorization data Authorized *AuthorizedConfig `json:"authorized,omitempty"` @@ -554,9 +558,15 @@ type AssertionResult struct { } // GetEnvironment returns the effective test environment for this test case -// Priority: command line flags > test case fields > defaults +// Priority: command line flags > context config > test case fields > defaults func (tc *Case) GetEnvironment(opts *Options) *Environment { - env := NewEnvironment("", "") + // Start with context config if available, otherwise use defaults + var env *Environment + if opts != nil && opts.ContextData != nil { + env = NewEnvironmentWithContext("", "", opts.ContextData) + } else { + env = NewEnvironment("", "") + } // Apply test case specific values if tc.UserID != "" {