Enhance README.md with Comprehensive Hook Documentation for Agent Testing
- Added detailed sections on Before/After hooks, including types, execution order, and function signatures. - Provided examples for common use cases such as database setup/teardown and conditional setup based on metadata. - Clarified parameters for hook functions, improving guidance for users on implementing hooks in their tests. - Enhanced overall documentation to support better understanding of the testing framework's capabilities.
This commit is contained in:
Max
parent
ea10a71050
commit
006dc8b3da
1 changed files with 170 additions and 5 deletions
|
|
@ -648,6 +648,31 @@ Supported types: images (jpg, png, gif, webp), audio (wav, mp3), documents (pdf,
|
|||
|
||||
## Before/After Hooks
|
||||
|
||||
Hooks allow you to run setup and teardown code before and after tests. Hook scripts must be placed in the agent's `src/` directory with `_test.ts` suffix.
|
||||
|
||||
### Hook Types
|
||||
|
||||
| Hook | Scope | When Called | Use Case |
|
||||
| ----------- | -------- | --------------------- | ------------------------------- |
|
||||
| `Before` | Per-test | Before each test case | Create test data, setup context |
|
||||
| `After` | Per-test | After each test case | Cleanup test data, log results |
|
||||
| `BeforeAll` | Global | Once before all tests | Database migration, init |
|
||||
| `AfterAll` | Global | Once after all tests | Global cleanup, report |
|
||||
|
||||
### Execution Order
|
||||
|
||||
```
|
||||
BeforeAll (global)
|
||||
├─ Before (test 1)
|
||||
│ └─ Test 1 execution
|
||||
│ └─ After (test 1)
|
||||
├─ Before (test 2)
|
||||
│ └─ Test 2 execution
|
||||
│ └─ After (test 2)
|
||||
└─ ...
|
||||
AfterAll (global)
|
||||
```
|
||||
|
||||
### Per-Test Hooks
|
||||
|
||||
Defined in JSONL, scripts located in agent's `src/` directory:
|
||||
|
|
@ -669,39 +694,179 @@ Via CLI flags:
|
|||
yao agent test -i tests/inputs.jsonl --before env_test.BeforeAll --after env_test.AfterAll
|
||||
```
|
||||
|
||||
### Hook Script Example
|
||||
### Hook Function Signatures
|
||||
|
||||
```typescript
|
||||
// assistants/expense/src/env_test.ts
|
||||
|
||||
/**
|
||||
* Before - Called before each test case
|
||||
* @param ctx - Agent context with user/team info
|
||||
* @param testCase - The test case about to run
|
||||
* @returns any - Data passed to After hook (optional)
|
||||
*/
|
||||
export function Before(ctx: Context, testCase: TestCase): any {
|
||||
// Setup: create test data
|
||||
const userId = Process("models.user.Create", { name: "Test User" });
|
||||
return { userId }; // Passed to After
|
||||
return { userId }; // This data is passed to After
|
||||
}
|
||||
|
||||
/**
|
||||
* After - Called after each test case (pass or fail)
|
||||
* @param ctx - Agent context
|
||||
* @param testCase - The test case that ran
|
||||
* @param result - Test result with status, output, duration
|
||||
* @param beforeData - Data returned from Before hook
|
||||
*/
|
||||
export function After(
|
||||
ctx: Context,
|
||||
testCase: TestCase,
|
||||
result: TestResult,
|
||||
beforeData: any
|
||||
) {
|
||||
// Cleanup: delete test data
|
||||
if (beforeData?.userId) {
|
||||
Process("models.user.Delete", beforeData.userId);
|
||||
}
|
||||
if (result.status === "failed") {
|
||||
console.log(`Test ${testCase.id} failed: ${result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* BeforeAll - Called once before all tests
|
||||
* @param ctx - Agent context
|
||||
* @param testCases - Array of all test cases
|
||||
* @returns any - Data passed to AfterAll hook (optional)
|
||||
*/
|
||||
export function BeforeAll(ctx: Context, testCases: TestCase[]): any {
|
||||
Process("models.migrate");
|
||||
return { initialized: true };
|
||||
return { initialized: true, count: testCases.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* AfterAll - Called once after all tests complete
|
||||
* @param ctx - Agent context
|
||||
* @param results - Array of all test results
|
||||
* @param beforeData - Data returned from BeforeAll hook
|
||||
*/
|
||||
export function AfterAll(ctx: Context, results: TestResult[], beforeData: any) {
|
||||
const passed = results.filter((r) => r.status === "passed").length;
|
||||
console.log(`Tests completed: ${passed}/${results.length} passed`);
|
||||
Process("models.cleanup");
|
||||
}
|
||||
```
|
||||
|
||||
### Hook Parameters
|
||||
|
||||
**Context** - Agent execution context:
|
||||
|
||||
```typescript
|
||||
interface Context {
|
||||
user_id: string; // Test user ID
|
||||
team_id: string; // Test team ID
|
||||
locale: string; // Locale (e.g., "en-us")
|
||||
metadata: object; // Custom metadata from test case
|
||||
}
|
||||
```
|
||||
|
||||
**TestCase** - Test case definition:
|
||||
|
||||
```typescript
|
||||
interface TestCase {
|
||||
id: string; // Test case ID
|
||||
input: any; // Test input (string, Message, or Message[])
|
||||
assert?: object; // Assertion rules
|
||||
expected?: any; // Expected output
|
||||
user?: string; // Override user ID
|
||||
team?: string; // Override team ID
|
||||
metadata?: object; // Custom metadata
|
||||
options?: object; // Context options
|
||||
timeout?: string; // Timeout (e.g., "30s")
|
||||
skip?: boolean; // Skip flag
|
||||
before?: string; // Before hook reference
|
||||
after?: string; // After hook reference
|
||||
}
|
||||
```
|
||||
|
||||
**TestResult** - Test execution result:
|
||||
|
||||
```typescript
|
||||
interface TestResult {
|
||||
id: string; // Test case ID
|
||||
status: string; // "passed" | "failed" | "error" | "skipped" | "timeout"
|
||||
input: any; // Actual input sent
|
||||
output: any; // Agent response
|
||||
expected?: any; // Expected output (if defined)
|
||||
error?: string; // Error message (if failed)
|
||||
duration_ms: number; // Execution time in milliseconds
|
||||
assertions?: object[]; // Assertion results
|
||||
}
|
||||
```
|
||||
|
||||
### Common Use Cases
|
||||
|
||||
**Database Setup/Teardown**:
|
||||
|
||||
```typescript
|
||||
export function Before(ctx: Context, testCase: TestCase): any {
|
||||
// Create test records
|
||||
const user = Process("models.user.Create", {
|
||||
name: "Test",
|
||||
email: "test@example.com",
|
||||
});
|
||||
const expense = Process("models.expense.Create", {
|
||||
user_id: user.id,
|
||||
amount: 100,
|
||||
});
|
||||
return { user, expense };
|
||||
}
|
||||
|
||||
export function After(
|
||||
ctx: Context,
|
||||
testCase: TestCase,
|
||||
result: TestResult,
|
||||
data: any
|
||||
) {
|
||||
// Clean up in reverse order
|
||||
if (data?.expense) Process("models.expense.Delete", data.expense.id);
|
||||
if (data?.user) Process("models.user.Delete", data.user.id);
|
||||
}
|
||||
```
|
||||
|
||||
**Conditional Setup Based on Metadata**:
|
||||
|
||||
```typescript
|
||||
export function Before(ctx: Context, testCase: TestCase): any {
|
||||
const scenario = testCase.metadata?.scenario || "default";
|
||||
|
||||
if (scenario === "empty_db") {
|
||||
Process("models.expense.DeleteAll");
|
||||
} else if (scenario === "with_data") {
|
||||
Process("scripts.tests.seed.LoadTestData");
|
||||
}
|
||||
|
||||
return { scenario };
|
||||
}
|
||||
```
|
||||
|
||||
**Logging and Debugging**:
|
||||
|
||||
```typescript
|
||||
export function After(
|
||||
ctx: Context,
|
||||
testCase: TestCase,
|
||||
result: TestResult,
|
||||
data: any
|
||||
) {
|
||||
if (result.status === "failed") {
|
||||
console.log("=== Test Failed ===");
|
||||
console.log("Test ID:", testCase.id);
|
||||
console.log("Input:", JSON.stringify(testCase.input));
|
||||
console.log("Output:", JSON.stringify(result.output));
|
||||
console.log("Error:", result.error);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Script Testing
|
||||
|
||||
Test agent handler scripts with the `t.assert` API:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue