package standard import ( "encoding/json" "fmt" "strings" "github.com/yaoapp/gou/process" robottypes "github.com/yaoapp/yao/agent/robot/types" "github.com/yaoapp/yao/assert" ) // ValidatorConfig configures validation behavior (decoupled from RunConfig) type ValidatorConfig struct { // ValidationThreshold is the minimum score to pass validation (default: 0.6) ValidationThreshold float64 } // DefaultValidatorConfig returns the default validator configuration func DefaultValidatorConfig() *ValidatorConfig { return &ValidatorConfig{ ValidationThreshold: 0.6, } } // Validator handles task result validation using a two-layer approach: // 1. Rule-based validation: Uses yao/assert for deterministic rules (type, contains, regex, json_path) // 2. Semantic validation: Calls Validation Agent for semantic understanding (ExpectedOutput) type Validator struct { ctx *robottypes.Context robot *robottypes.Robot config *ValidatorConfig asserter *assert.Asserter } // NewValidator creates a new task validator func NewValidator(ctx *robottypes.Context, robot *robottypes.Robot, config *ValidatorConfig) *Validator { if config == nil { config = DefaultValidatorConfig() } v := &Validator{ ctx: ctx, robot: robot, config: config, asserter: assert.New(), } // Configure asserter with robot-specific implementations v.asserter.WithAgentValidator(&robotAgentValidator{v: v}) v.asserter.WithScriptRunner(&robotScriptRunner{ctx: ctx}) return v } // Validate validates task output using two-layer validation (without multi-turn context) // Equivalent to ValidateWithContext(task, output, nil) // Use ValidateWithContext when you have a CallResult for better multi-turn support func (v *Validator) Validate(task *robottypes.Task, output interface{}) *robottypes.ValidationResult { return v.ValidateWithContext(task, output, nil) } // ValidateWithContext validates task output and determines execution state for multi-turn conversation. // It extends basic validation with: // - Complete: whether expected result is obtained // - NeedReply: whether to continue conversation // - ReplyContent: content for next turn // // Parameters: // - task: the task being executed // - output: the output from assistant/mcp/process // - callResult: the full call result (for detecting assistant's need for more info) func (v *Validator) ValidateWithContext(task *robottypes.Task, output interface{}, callResult *CallResult) *robottypes.ValidationResult { // If no validation rules and no expected output, return passed and complete if task.ExpectedOutput == "" && len(task.ValidationRules) == 0 { return &robottypes.ValidationResult{ Passed: true, Score: 1.0, Complete: v.hasValidOutput(output), } } result := &robottypes.ValidationResult{ Passed: true, Score: 1.0, } // Layer 1: Rule-based validation (using yao/assert) if len(task.ValidationRules) > 0 { ruleResult := v.validateRules(task.ValidationRules, output) if !ruleResult.Passed { // Rule validation failed - check if we should retry with feedback ruleResult.Complete = false ruleResult.NeedReply, ruleResult.ReplyContent = v.checkNeedReplyOnFailure(task, ruleResult) return ruleResult } // Merge rule validation results result.Issues = append(result.Issues, ruleResult.Issues...) result.Suggestions = append(result.Suggestions, ruleResult.Suggestions...) } // Layer 2: Semantic validation (using Validation Agent) // Only run if ExpectedOutput is set or there are agent-type rules if task.ExpectedOutput != "" || v.hasAgentRules(task.ValidationRules) { semanticResult := v.validateSemantic(task, output) result = v.mergeResults(result, semanticResult) } // Determine execution state result.Complete = v.isComplete(task, output, result) result.NeedReply, result.ReplyContent = v.checkNeedReply(task, output, callResult, result) return result } // hasValidOutput checks if output is non-empty and valid func (v *Validator) hasValidOutput(output interface{}) bool { if output == nil { return false } switch o := output.(type) { case string: return strings.TrimSpace(o) != "" case []interface{}: return len(o) > 0 case map[string]interface{}: return len(o) > 0 default: return true } } // isComplete determines if the expected result has been obtained func (v *Validator) isComplete(task *robottypes.Task, output interface{}, result *robottypes.ValidationResult) bool { // If validation failed, not complete if !result.Passed { return false } // Must have valid output if !v.hasValidOutput(output) { return false } // If score is below threshold, consider incomplete if result.Score < v.config.ValidationThreshold { return false } return true } // checkNeedReply determines if conversation should continue and generates reply content func (v *Validator) checkNeedReply(task *robottypes.Task, output interface{}, callResult *CallResult, result *robottypes.ValidationResult) (bool, string) { // If already complete, no need to reply if result.Complete { return false, "" } // Scenario 1: Assistant explicitly asks for more information if callResult != nil { text := callResult.GetText() if v.detectNeedMoreInfo(text) { return true, v.generateClarificationReply(task, text) } } // Scenario 2: Validation passed but output is incomplete/empty if result.Passed && !v.hasValidOutput(output) { return true, "Please continue and provide the complete result as specified in the task." } // Scenario 3: Validation failed with suggestions - can retry with feedback if !result.Passed && len(result.Suggestions) > 0 { return true, v.generateFeedbackReply(result) } // Scenario 4: Low confidence score - ask for improvement if result.Passed && result.Score < v.config.ValidationThreshold { return true, fmt.Sprintf("The result is partially correct (score: %.2f), but needs improvement. Please refine your response to better match the expected output: %s", result.Score, task.ExpectedOutput) } // No need to continue return false, "" } // checkNeedReplyOnFailure handles the case when rule validation fails func (v *Validator) checkNeedReplyOnFailure(task *robottypes.Task, result *robottypes.ValidationResult) (bool, string) { // If there are suggestions, we can try to fix if len(result.Suggestions) > 0 { return true, v.generateFeedbackReply(result) } // If there are issues, provide feedback if len(result.Issues) > 0 { var sb strings.Builder sb.WriteString("Your response did not pass validation. Please fix the following issues:\n\n") for _, issue := range result.Issues { sb.WriteString(fmt.Sprintf("- %s\n", issue)) } sb.WriteString(fmt.Sprintf("\nExpected output: %s", task.ExpectedOutput)) return true, sb.String() } return false, "" } // detectNeedMoreInfo checks if assistant's response indicates need for more information func (v *Validator) detectNeedMoreInfo(text string) bool { if text == "" { return false } textLower := strings.ToLower(text) keywords := []string{ "need more information", "please clarify", "could you provide", "can you specify", "what is the", "which one", "please provide", "i need to know", "could you tell me", "what do you mean", } for _, kw := range keywords { if strings.Contains(textLower, kw) { return true } } // Check for question marks at the end (likely asking for clarification) // Note: We require 2+ question marks to avoid false positives from rhetorical questions // or questions that are part of the output (e.g., "How can I help you?") // Single questions are often just conversational and don't need clarification trimmed := strings.TrimSpace(text) if strings.HasSuffix(trimmed, "?") { if strings.Count(text, "?") >= 2 { return true } } return false } // generateClarificationReply generates a reply when assistant asks for clarification func (v *Validator) generateClarificationReply(task *robottypes.Task, assistantText string) string { var sb strings.Builder sb.WriteString("Please proceed with the task based on the available information.\n\n") if task.ExpectedOutput != "" { sb.WriteString(fmt.Sprintf("**Expected Output**: %s\n\n", task.ExpectedOutput)) } sb.WriteString("If you need to make assumptions, please state them clearly and proceed with the most reasonable interpretation.") return sb.String() } // generateFeedbackReply generates a reply with validation feedback func (v *Validator) generateFeedbackReply(result *robottypes.ValidationResult) string { var sb strings.Builder sb.WriteString("## Validation Feedback\n\n") sb.WriteString("Your previous response needs improvement. Please address the following:\n\n") if len(result.Issues) > 0 { sb.WriteString("### Issues\n") for _, issue := range result.Issues { sb.WriteString(fmt.Sprintf("- %s\n", issue)) } sb.WriteString("\n") } if len(result.Suggestions) > 0 { sb.WriteString("### Suggestions\n") for _, suggestion := range result.Suggestions { sb.WriteString(fmt.Sprintf("- %s\n", suggestion)) } sb.WriteString("\n") } sb.WriteString("Please provide an improved response that addresses these points.") return sb.String() } // validateRules validates output against rule-based assertions func (v *Validator) validateRules(rules []string, output interface{}) *robottypes.ValidationResult { result := &robottypes.ValidationResult{ Passed: true, Score: 1.0, } // Parse rules into assertions assertions := v.parseRules(rules) if len(assertions) == 0 { return result } // Run assertions passed, message := v.asserter.Validate(assertions, output) if !passed { result.Passed = false result.Score = 0 result.Issues = append(result.Issues, message) } return result } // parseRules converts validation rules (strings or JSON) to assertions // Supports: // - Simple string rules: "output must be valid JSON" (converted to type check) // - JSON assertion objects: {"type": "contains", "value": "success"} func (v *Validator) parseRules(rules []string) []*assert.Assertion { var assertions []*assert.Assertion for _, rule := range rules { // Try to parse as JSON assertion if strings.HasPrefix(rule, "{") { var assertionMap map[string]interface{} if err := json.Unmarshal([]byte(rule), &assertionMap); err == nil { parsed := assert.ParseAssertions(assertionMap) assertions = append(assertions, parsed...) continue } } // Convert common string rules to assertions assertion := v.convertStringRule(rule) if assertion != nil { assertions = append(assertions, assertion) } } return assertions } // convertStringRule converts a human-readable rule string to an assertion // Examples: // - "output must be valid JSON" -> {"type": "type", "value": "object"} // - "must contain 'success'" -> {"type": "contains", "value": "success"} // - "count > 0" -> (passed to semantic validation) func (v *Validator) convertStringRule(rule string) *assert.Assertion { ruleLower := strings.ToLower(rule) // JSON type check if strings.Contains(ruleLower, "valid json") || strings.Contains(ruleLower, "json object") { return &assert.Assertion{ Type: "type", Value: "object", Message: rule, } } // Array type check if strings.Contains(ruleLower, "json array") || strings.Contains(ruleLower, "must be array") { return &assert.Assertion{ Type: "type", Value: "array", Message: rule, } } // Contains check if strings.Contains(ruleLower, "contain") { // Extract the value in quotes if start := strings.Index(rule, "'"); start != -1 { if end := strings.Index(rule[start+1:], "'"); end != -1 { value := rule[start+1 : start+1+end] return &assert.Assertion{ Type: "contains", Value: value, Message: rule, } } } if start := strings.Index(rule, "\""); start != -1 { if end := strings.Index(rule[start+1:], "\""); end != -1 { value := rule[start+1 : start+1+end] return &assert.Assertion{ Type: "contains", Value: value, Message: rule, } } } } // Not empty check - use regex to match at least one character if strings.Contains(ruleLower, "not empty") || strings.Contains(ruleLower, "non-empty") { return &assert.Assertion{ Type: "regex", Value: ".+", Message: rule, } } // For other rules, return nil (will be handled by semantic validation) return nil } // hasAgentRules checks if any rule requires agent-based validation func (v *Validator) hasAgentRules(rules []string) bool { for _, rule := range rules { if strings.HasPrefix(rule, "{") { var assertionMap map[string]interface{} if err := json.Unmarshal([]byte(rule), &assertionMap); err == nil { if assertionMap["type"] == "agent" { return true } } } } return false } // validateSemantic performs semantic validation using the Validation Agent func (v *Validator) validateSemantic(task *robottypes.Task, output interface{}) *robottypes.ValidationResult { // Get validation agent ID (per-robot config > global Uses > empty) validationAgentID := robottypes.ResolvePhaseAgent(v.robot.Config, "validation") if validationAgentID == "" { return &robottypes.ValidationResult{ Passed: false, Score: 0, Issues: []string{"no Validation Agent configured (set uses.validation in agent.yml or resources.phases in robot config)"}, } } // Build validation prompt validationPrompt := v.BuildSemanticPrompt(task, output) // Call validation agent caller := NewAgentCaller() caller.Connector = v.robot.LanguageModel caller.Workspace = v.robot.Workspace result, err := caller.CallWithMessages(v.ctx, validationAgentID, validationPrompt) if err != nil { return &robottypes.ValidationResult{ Passed: false, Score: 0, Issues: []string{fmt.Sprintf("Validation agent error: %s", err.Error())}, } } return v.ParseAgentResult(result) } // BuildSemanticPrompt builds the prompt for semantic validation // Format matches the Validation Agent's expected input structure: // 1. Task: task definition with expected_output and validation_rules // 2. Result: actual output from task execution // 3. Success Criteria: overall criteria (optional) func (v *Validator) BuildSemanticPrompt(task *robottypes.Task, output interface{}) string { var sb strings.Builder // Section 1: Task (matches Agent's expected "Task" input) sb.WriteString("## Task\n\n") sb.WriteString(fmt.Sprintf("**Task ID**: %s\n", task.ID)) sb.WriteString(fmt.Sprintf("**Executor**: %s (%s)\n\n", task.ExecutorID, task.ExecutorType)) // Task description (instructions) if len(task.Messages) > 0 { sb.WriteString("**Instructions**:\n") for _, msg := range task.Messages { if content, ok := msg.Content.(string); ok { sb.WriteString(content + "\n") } } sb.WriteString("\n") } // Expected output (primary criterion for semantic validation) if task.ExpectedOutput != "" { sb.WriteString(fmt.Sprintf("**expected_output**: %s\n\n", task.ExpectedOutput)) } // Validation rules semanticRules := v.getSemanticRules(task.ValidationRules) if len(semanticRules) > 0 { sb.WriteString("**validation_rules**:\n") for _, rule := range semanticRules { sb.WriteString(fmt.Sprintf("- %s\n", rule)) } sb.WriteString("\n") } // Section 2: Result (matches Agent's expected "Result" input) sb.WriteString("## Result\n\n") if output != nil { outputJSON, err := json.MarshalIndent(output, "", " ") if err == nil { sb.WriteString(fmt.Sprintf("```json\n%s\n```\n", string(outputJSON))) } else { sb.WriteString(fmt.Sprintf("%v\n", output)) } } else { sb.WriteString("(no output)\n") } // Section 3: Success Criteria (optional, from goals if available) // Note: This could be extended to include criteria from exec.Goals if needed sb.WriteString("\n## Success Criteria\n\n") if task.ExpectedOutput != "" { sb.WriteString(fmt.Sprintf("The task should produce: %s\n", task.ExpectedOutput)) } else { sb.WriteString("Complete the task successfully with valid output.\n") } return sb.String() } // getSemanticRules returns rules that need semantic validation (not convertible to assertions) func (v *Validator) getSemanticRules(rules []string) []string { var semanticRules []string for _, rule := range rules { // Skip JSON assertions (already handled) if strings.HasPrefix(rule, "{") { continue } // Skip rules that were converted to assertions if v.convertStringRule(rule) == nil { semanticRules = append(semanticRules, rule) } } return semanticRules } // ParseAgentResult parses the validation agent's response func (v *Validator) ParseAgentResult(result *CallResult) *robottypes.ValidationResult { validation := &robottypes.ValidationResult{ Passed: false, Score: 0, } // Try to parse as JSON data, err := result.GetJSON() if err != nil { // If not JSON, try to interpret the text response text := result.GetText() if text != "" { validation.Details = text // Simple heuristic: check for positive keywords textLower := strings.ToLower(text) positiveKeywords := []string{"passed", "valid", "correct", "success"} for _, keyword := range positiveKeywords { if strings.Contains(textLower, keyword) { validation.Passed = true validation.Score = 0.8 break } } } return validation } // Parse JSON fields if passed, ok := data["passed"].(bool); ok { validation.Passed = passed } if score, ok := data["score"].(float64); ok { validation.Score = score } if issues, ok := data["issues"].([]interface{}); ok { for _, issue := range issues { if s, ok := issue.(string); ok { validation.Issues = append(validation.Issues, s) } } } if suggestions, ok := data["suggestions"].([]interface{}); ok { for _, suggestion := range suggestions { if s, ok := suggestion.(string); ok { validation.Suggestions = append(validation.Suggestions, s) } } } if details, ok := data["details"].(string); ok { validation.Details = details } return validation } // mergeResults merges rule-based and semantic validation results func (v *Validator) mergeResults(ruleResult, semanticResult *robottypes.ValidationResult) *robottypes.ValidationResult { // If either failed, the overall result is failed if !ruleResult.Passed || !semanticResult.Passed { return &robottypes.ValidationResult{ Passed: false, Score: min(ruleResult.Score, semanticResult.Score), Issues: append(ruleResult.Issues, semanticResult.Issues...), Suggestions: append(ruleResult.Suggestions, semanticResult.Suggestions...), Details: semanticResult.Details, } } // Both passed return &robottypes.ValidationResult{ Passed: true, Score: (ruleResult.Score + semanticResult.Score) / 2, Issues: append(ruleResult.Issues, semanticResult.Issues...), Suggestions: append(ruleResult.Suggestions, semanticResult.Suggestions...), Details: semanticResult.Details, } } // ============================================================================ // Robot-specific implementations of assert interfaces // ============================================================================ // robotAgentValidator implements assert.AgentValidator for robot package type robotAgentValidator struct { v *Validator } // Validate validates output using an agent func (av *robotAgentValidator) Validate(agentID string, output, input, criteria interface{}, options *assert.AssertionOptions) *assert.Result { result := &assert.Result{} // Build validation request validationInput := map[string]interface{}{ "output": output, "input": input, } if criteria != nil { validationInput["criteria"] = criteria } inputJSON, err := json.Marshal(validationInput) if err != nil { result.Passed = false result.Message = fmt.Sprintf("failed to marshal validation input: %s", err.Error()) return result } // Call agent caller := NewAgentCaller() caller.Connector = av.v.robot.LanguageModel caller.Workspace = av.v.robot.Workspace callResult, err := caller.CallWithMessages(av.v.ctx, agentID, string(inputJSON)) if err != nil { result.Passed = false result.Message = fmt.Sprintf("agent validation error: %s", err.Error()) return result } // Parse response data, err := callResult.GetJSON() if err != nil { result.Passed = false result.Message = "agent returned invalid response format" return result } if passed, ok := data["passed"].(bool); ok { result.Passed = passed } if reason, ok := data["reason"].(string); ok { result.Message = reason } result.Expected = data return result } // robotScriptRunner implements assert.ScriptRunner for robot package type robotScriptRunner struct { ctx *robottypes.Context } // Run runs an assertion script using Yao process func (r *robotScriptRunner) Run(scriptName string, output, input, expected interface{}) (bool, string, error) { // Build script arguments args := []interface{}{output, input, expected} // Create and run the process proc, err := process.Of(scriptName, args...) if err != nil { return false, "", fmt.Errorf("failed to create process: %w", err) } // Set context for timeout and cancellation support if r.ctx != nil { proc.Context = r.ctx.Context } if err := proc.Execute(); err != nil { return false, "", fmt.Errorf("script execution failed: %w", err) } defer proc.Release() // Parse result - expected format: bool or { "pass": bool, "message": string } res := proc.Value() switch v := res.(type) { case bool: if v { return true, "script assertion passed", nil } return false, "script assertion failed", nil case map[string]interface{}: passed := false message := "" if pass, ok := v["pass"].(bool); ok { passed = pass } if msg, ok := v["message"].(string); ok { message = msg } return passed, message, nil default: return false, fmt.Sprintf("script returned unexpected type: %T", res), nil } }