package test import ( "bufio" "fmt" "html/template" "io" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/yaoapp/yao/agent/caller" "github.com/yaoapp/yao/agent/context" ) // JSONLReporter generates JSONL format reports (default) type JSONLReporter struct{} // NewJSONLReporter creates a new JSONL reporter func NewJSONLReporter() *JSONLReporter { return &JSONLReporter{} } // Generate generates a JSONL report (writes to stdout or file) func (r *JSONLReporter) Generate(report *Report) error { return nil // JSONL is written during test execution } // Write writes the report in JSONL format func (r *JSONLReporter) Write(report *Report, w io.Writer) error { writer := bufio.NewWriter(w) defer writer.Flush() // Start event startEvent := map[string]interface{}{ "type": "start", "timestamp": report.Metadata.StartedAt.Format(time.RFC3339), "agent_id": report.Summary.AgentID, "total_cases": report.Summary.Total, } if err := writeJSONLineToWriter(writer, startEvent); err != nil { return err } // Result events if report.Results != nil { for _, result := range report.Results { resultEvent := map[string]interface{}{ "type": "result", "id": result.ID, "status": result.Status, "duration_ms": result.DurationMs, } if result.Output != nil { resultEvent["output"] = result.Output } if result.Error != "" { resultEvent["error"] = result.Error } if result.Trace != nil { resultEvent["trace"] = result.Trace } if err := writeJSONLineToWriter(writer, resultEvent); err != nil { return err } } } // Stability results if report.StabilityResults != nil { for _, sr := range report.StabilityResults { stabilityEvent := map[string]interface{}{ "type": "stability", "id": sr.ID, "runs": sr.Runs, "passed": sr.Passed, "failed": sr.Failed, "pass_rate": sr.PassRate, "stable": sr.Stable, "stability_class": sr.StabilityClass, "avg_duration_ms": sr.AvgDurationMs, } if err := writeJSONLineToWriter(writer, stabilityEvent); err != nil { return err } } } // Summary event summaryEvent := map[string]interface{}{ "type": "summary", "total": report.Summary.Total, "passed": report.Summary.Passed, "failed": report.Summary.Failed, "skipped": report.Summary.Skipped, "errors": report.Summary.Errors, "timeouts": report.Summary.Timeouts, "duration_ms": report.Summary.DurationMs, } if report.Summary.RunsPerCase > 1 { summaryEvent["runs_per_case"] = report.Summary.RunsPerCase summaryEvent["total_runs"] = report.Summary.TotalRuns summaryEvent["overall_pass_rate"] = report.Summary.OverallPassRate summaryEvent["stable_cases"] = report.Summary.StableCases summaryEvent["unstable_cases"] = report.Summary.UnstableCases } return writeJSONLineToWriter(writer, summaryEvent) } // writeJSONLineToWriter writes a JSON line to the writer func writeJSONLineToWriter(writer *bufio.Writer, data interface{}) error { line, err := jsoniter.Marshal(data) if err != nil { return err } _, err = writer.Write(line) if err != nil { return err } _, err = writer.WriteString("\n") return err } // JSONReporter generates full JSON format reports type JSONReporter struct{} // NewJSONReporter creates a new JSON reporter func NewJSONReporter() *JSONReporter { return &JSONReporter{} } // Generate generates a JSON report func (r *JSONReporter) Generate(report *Report) error { return nil } // Write writes the report in JSON format func (r *JSONReporter) Write(report *Report, w io.Writer) error { encoder := jsoniter.NewEncoder(w) encoder.SetIndent("", " ") return encoder.Encode(report) } // MarkdownReporter generates Markdown format reports type MarkdownReporter struct{} // NewMarkdownReporter creates a new Markdown reporter func NewMarkdownReporter() *MarkdownReporter { return &MarkdownReporter{} } // Generate generates a Markdown report func (r *MarkdownReporter) Generate(report *Report) error { return nil } // Write writes the report in Markdown format func (r *MarkdownReporter) Write(report *Report, w io.Writer) error { var sb strings.Builder // Header sb.WriteString("# Agent Test Report\n\n") // Summary sb.WriteString("## Summary\n\n") sb.WriteString("| Metric | Value |\n") sb.WriteString("| ------ | ----- |\n") sb.WriteString(fmt.Sprintf("| Agent | %s |\n", report.Summary.AgentID)) if report.Summary.Connector != "" { sb.WriteString(fmt.Sprintf("| Connector | %s |\n", report.Summary.Connector)) } sb.WriteString(fmt.Sprintf("| Total | %d |\n", report.Summary.Total)) sb.WriteString(fmt.Sprintf("| Passed | %d |\n", report.Summary.Passed)) sb.WriteString(fmt.Sprintf("| Failed | %d |\n", report.Summary.Failed)) if report.Summary.Skipped > 0 { sb.WriteString(fmt.Sprintf("| Skipped | %d |\n", report.Summary.Skipped)) } if report.Summary.Errors > 0 { sb.WriteString(fmt.Sprintf("| Errors | %d |\n", report.Summary.Errors)) } if report.Summary.Timeouts > 0 { sb.WriteString(fmt.Sprintf("| Timeouts | %d |\n", report.Summary.Timeouts)) } passRate := float64(0) if report.Summary.Total > 0 { passRate = float64(report.Summary.Passed) / float64(report.Summary.Total) * 100 } sb.WriteString(fmt.Sprintf("| Pass Rate | %.1f%% |\n", passRate)) sb.WriteString(fmt.Sprintf("| Duration | %dms |\n", report.Summary.DurationMs)) sb.WriteString("\n") // Environment if report.Environment != nil { sb.WriteString("## Environment\n\n") sb.WriteString("| Setting | Value |\n") sb.WriteString("| ------- | ----- |\n") sb.WriteString(fmt.Sprintf("| User | %s |\n", report.Environment.UserID)) sb.WriteString(fmt.Sprintf("| Team | %s |\n", report.Environment.TeamID)) sb.WriteString(fmt.Sprintf("| Locale | %s |\n", report.Environment.Locale)) sb.WriteString("\n") } // Results sb.WriteString("## Results\n\n") if report.Results != nil { for _, result := range report.Results { statusIcon := "✅" switch result.Status { case StatusFailed: statusIcon = "❌" case StatusError: statusIcon = "💥" case StatusTimeout: statusIcon = "⏱️" case StatusSkipped: statusIcon = "⏭️" } sb.WriteString(fmt.Sprintf("### %s %s - %s (%dms)\n\n", statusIcon, result.ID, result.Status, result.DurationMs)) if result.Error != "" { sb.WriteString(fmt.Sprintf("**Error:** %s\n\n", result.Error)) } // Expand Trace for non-passed cases if result.Status != StatusPassed && result.Trace != nil { writeMarkdownTrace(&sb, result.Trace) } } } // Stability results if report.StabilityResults != nil { sb.WriteString("## Stability Analysis\n\n") sb.WriteString("| ID | Pass Rate | Runs | Status | Avg Duration |\n") sb.WriteString("| -- | --------- | ---- | ------ | ------------ |\n") for _, sr := range report.StabilityResults { status := string(sr.StabilityClass) sb.WriteString(fmt.Sprintf("| %s | %.0f%% | %d/%d | %s | %.0fms |\n", sr.ID, sr.PassRate, sr.Passed, sr.Runs, status, sr.AvgDurationMs)) } sb.WriteString("\n") } // Metadata sb.WriteString("## Metadata\n\n") sb.WriteString(fmt.Sprintf("- **Started:** %s\n", report.Metadata.StartedAt.Format(time.RFC3339))) sb.WriteString(fmt.Sprintf("- **Completed:** %s\n", report.Metadata.CompletedAt.Format(time.RFC3339))) if report.Metadata.InputFile != "" { sb.WriteString(fmt.Sprintf("- **Input File:** %s\n", report.Metadata.InputFile)) } if report.Metadata.OutputFile != "" { sb.WriteString(fmt.Sprintf("- **Output File:** %s\n", report.Metadata.OutputFile)) } _, err := w.Write([]byte(sb.String())) return err } // writeMarkdownTrace writes a Trace section in Markdown format. func writeMarkdownTrace(sb *strings.Builder, trace *Trace) { if trace.Completion != nil { sb.WriteString("**Completion**\n\n") if trace.Completion.Model != "" { sb.WriteString(fmt.Sprintf("- Model: `%s`\n", trace.Completion.Model)) } if trace.Completion.Refusal != "" { sb.WriteString(fmt.Sprintf("- Refusal: %s\n", trace.Completion.Refusal)) } if trace.Completion.ReasoningContent != "" { sb.WriteString(fmt.Sprintf("- Reasoning: %s\n", truncateString(trace.Completion.ReasoningContent, 200))) } sb.WriteString("\n") } if len(trace.ToolCalls) > 0 { sb.WriteString("**Tool Calls**\n\n") sb.WriteString("| Server | Tool | Status | Arguments |\n") sb.WriteString("| ------ | ---- | ------ | --------- |\n") for _, tc := range trace.ToolCalls { status := "OK" if tc.Error != "" { status = "ERR: " + truncateString(tc.Error, 40) } argsStr := "" if tc.Arguments != nil { if b, err := jsoniter.Marshal(tc.Arguments); err == nil { argsStr = truncateString(string(b), 60) } } sb.WriteString(fmt.Sprintf("| %s | %s | %s | `%s` |\n", tc.Server, tc.Tool, status, argsStr)) } sb.WriteString("\n") } } // HTMLReporter generates HTML format reports type HTMLReporter struct{} // NewHTMLReporter creates a new HTML reporter func NewHTMLReporter() *HTMLReporter { return &HTMLReporter{} } // Generate generates an HTML report func (r *HTMLReporter) Generate(report *Report) error { return nil } // Write writes the report in HTML format func (r *HTMLReporter) Write(report *Report, w io.Writer) error { funcMap := template.FuncMap{ "traceJSON": func(trace *Trace) string { if trace == nil { return "" } b, err := jsoniter.MarshalIndent(trace, "", " ") if err != nil { return err.Error() } return string(b) }, } tmpl, err := template.New("report").Funcs(funcMap).Parse(htmlTemplate) if err != nil { return fmt.Errorf("failed to parse HTML template: %w", err) } // Calculate pass rate passRate := float64(0) if report.Summary.Total > 0 { passRate = float64(report.Summary.Passed) / float64(report.Summary.Total) * 100 } data := map[string]interface{}{ "Report": report, "PassRate": passRate, } return tmpl.Execute(w, data) } // HTML template for reports const htmlTemplate = `
{{.Report.Summary.AgentID}} {{if .Report.Summary.Connector}}• {{.Report.Summary.Connector}}{{end}}
| ID | Status | Duration | Details |
|---|---|---|---|
| {{.ID}} | {{.Status}} | {{.DurationMs}}ms |
{{if .Error}} {{.Error}} {{end}}
{{if and (ne (printf "%s" .Status) "passed") .Trace}}Trace{{traceJSON .Trace}} |
| {{.ID}} | {{.StabilityClass}} | {{printf "%.0f" .AvgDurationMs}}ms avg | {{.Passed}}/{{.Runs}} passed ({{printf "%.0f" .PassRate}}%) |