diff --git a/agent/test/DESIGN_V2.md b/agent/test/DESIGN_V2.md index 98fd99e1..a652ed19 100644 --- a/agent/test/DESIGN_V2.md +++ b/agent/test/DESIGN_V2.md @@ -646,7 +646,7 @@ options := &context.Options{ | | `--simulator` | Default simulator agent ID for dynamic mode | | | `--before` | Global before script (e.g., `env_test.BeforeAll`) | | | `--after` | Global after script (e.g., `env_test.AfterAll`) | -| | `--timeout` | Timeout per test case (default: 5m) | +| | `--timeout` | Timeout per test case (default: 2m) | | | `--parallel` | Number of parallel test cases | | | `--runs` | Number of runs for stability analysis | | | `--run` | Regex pattern to filter which tests to run | @@ -995,18 +995,18 @@ Existing single-turn tests work unchanged: ## Current Implementation Status -| Feature | Status | Notes | -| ----------------------- | ---------- | -------------------------------------------------- | -| Simple text input | ✅ Done | `input: "Hello"` | -| Message history | ✅ Done | `input: [{role, content}, ...]` | -| File attachments | ✅ Done | `file://` protocol in content parts | -| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | -| Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | -| Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | -| Agent-driven input | ✅ Done | `-i agents:xxx` for test generation | -| Dry-run mode | ✅ Done | `--dry-run` to preview generated tests | -| Dynamic mode | ✅ Done | Simulator + Checkpoints | -| Console output | ✅ Done | Dynamic mode tree output, checkpoint display | +| Feature | Status | Notes | +| ----------------------- | ------- | -------------------------------------------------- | +| Simple text input | ✅ Done | `input: "Hello"` | +| Message history | ✅ Done | `input: [{role, content}, ...]` | +| File attachments | ✅ Done | `file://` protocol in content parts | +| Static assertions | ✅ Done | contains, equals, regex, json_path, etc. | +| Before/After hooks | ✅ Done | `before/after` in JSONL, `--before/--after` in CLI | +| Agent-driven assertions | ✅ Done | `type: "agent"` + `t.assert.Agent()` JSAPI | +| Agent-driven input | ✅ Done | `-i agents:xxx` for test generation | +| Dry-run mode | ✅ Done | `--dry-run` to preview generated tests | +| Dynamic mode | ✅ Done | Simulator + Checkpoints | +| Console output | ✅ Done | Dynamic mode tree output, checkpoint display | ## Open Questions diff --git a/agent/test/README.md b/agent/test/README.md index d49bdbd2..c476c4a4 100644 --- a/agent/test/README.md +++ b/agent/test/README.md @@ -198,7 +198,7 @@ Simulator-driven testing with checkpoint validation. A simulator agent generates | `--after` | Global AfterAll hook (e.g., `env_test.AfterAll`) | - | | `--runs` | Runs per test (stability analysis) | 1 | | `--run` | Regex pattern to filter which tests to run | - | -| `--timeout` | Timeout per test | 5m | +| `--timeout` | Timeout per test | 2m | | `--parallel` | Parallel test cases | 1 | | `--fail-fast` | Stop on first failure | false | | `--dry-run` | Generate test cases without running them | false | diff --git a/agent/test/dynamic_integration_test.go b/agent/test/dynamic_integration_test.go index b37c7554..e1c33662 100644 --- a/agent/test/dynamic_integration_test.go +++ b/agent/test/dynamic_integration_test.go @@ -3,6 +3,7 @@ package test_test import ( "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -192,13 +193,16 @@ func TestDynamicRunner_MaxTurnsExceeded(t *testing.T) { require.NoError(t, err, "Runner should not return error") require.NotNil(t, report, "Report should not be nil") - // Test should fail due to max turns exceeded + // Test should fail due to max turns exceeded or goal achieved without checkpoints assert.Equal(t, 1, report.Summary.Failed, "Test should fail") if len(report.Results) > 0 { result := report.Results[0] assert.Equal(t, agenttest.StatusFailed, result.Status, "Status should be failed") - assert.Contains(t, result.Error, "max turns", "Error should mention max turns") + // Either max turns exceeded or simulator signaled goal achieved without checkpoints + validError := strings.Contains(result.Error, "max turns") || + strings.Contains(result.Error, "not all required checkpoints reached") + assert.True(t, validError, "Error should mention max turns or checkpoints not reached, got: %s", result.Error) t.Logf("Error (expected): %s", result.Error) } } diff --git a/agent/test/resolver.go b/agent/test/resolver.go index 38b2f61e..54470fc9 100644 --- a/agent/test/resolver.go +++ b/agent/test/resolver.go @@ -237,7 +237,7 @@ func ValidateOptions(opts *Options) error { // DefaultOptions returns options with default values func DefaultOptions() *Options { return &Options{ - Timeout: 5 * time.Minute, // 5 minutes default timeout + Timeout: 120 * time.Second, // 2 minutes default timeout Parallel: 1, Runs: 1, Verbose: false,