From 889fba942aa28f14de3ef48742518ffed4430aae Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Nov 2025 12:38:03 +0800 Subject: [PATCH] Enhance testing and context management in assistant operations - Refactored the Makefile to include new benchmark and memory leak detection tests, improving test coverage. - Introduced context adjustments in the Create method, allowing hooks to modify context fields such as AssistantID, Connector, Locale, Theme, Route, and Metadata. - Added a new test scenario to validate context field adjustments, ensuring proper updates during assistant operations. - Updated test preparation to support optional V8 mode configuration for improved performance during benchmarks. --- Makefile | 50 +- agent/assistant/hook/create.go | 52 +- agent/assistant/hook/create_bench_test.go | 330 +++++++++++ agent/assistant/hook/create_mem_test.go | 602 +++++++++++++++++++++ agent/assistant/hook/create_nested_test.go | 117 ++++ agent/assistant/hook/create_test.go | 62 +++ agent/assistant/hook/script.go | 4 +- agent/context/types.go | 8 + agent/testutils/testutils.go | 10 +- test/utils.go | 51 +- 10 files changed, 1274 insertions(+), 12 deletions(-) create mode 100644 agent/assistant/hook/create_bench_test.go create mode 100644 agent/assistant/hook/create_mem_test.go create mode 100644 agent/assistant/hook/create_nested_test.go diff --git a/Makefile b/Makefile index c5b5fc2d..565524ec 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,11 @@ TESTTAGS ?= "" # TESTWIDGETS := $(shell $(GO) list ./widgets/...) # Unit Test -.PHONY: test -test: +.PHONY: unit-test +unit-test: echo "mode: count" > coverage.out for d in $(TESTFOLDER); do \ - $(GO) test -tags $(TESTTAGS) -v -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") $$d > tmp.out; \ + $(GO) test -tags $(TESTTAGS) -v -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") -skip='TestMemoryLeak|TestIsolateDisposal' $$d > tmp.out; \ cat tmp.out; \ if grep -q "^--- FAIL" tmp.out; then \ rm tmp.out; \ @@ -41,6 +41,50 @@ test: fi; \ done +# Benchmark Test +.PHONY: benchmark +benchmark: + @echo "" + @echo "=============================================" + @echo "Running Benchmark Tests (agent only)..." + @echo "=============================================" + @for d in $$($(GO) list ./agent/...); do \ + if $(GO) test -list=Benchmark $$d 2>/dev/null | grep -q "^Benchmark"; then \ + echo ""; \ + echo "📊 Benchmarking: $$d"; \ + echo "---------------------------------------------"; \ + $(GO) test -bench=. -benchmem -benchtime=100x -run='^$$' $$d || true; \ + fi; \ + done + @echo "" + @echo "=============================================" + @echo "✅ All benchmarks completed" + @echo "=============================================" + +# Memory Leak Detection Test +.PHONY: memory-leak +memory-leak: + @echo "" + @echo "=============================================" + @echo "Running Memory Leak Detection (agent only)..." + @echo "=============================================" + @for d in $$($(GO) list ./agent/...); do \ + if $(GO) test -list='TestMemoryLeak|TestIsolateDisposal' $$d 2>/dev/null | grep -qE "^Test(MemoryLeak|IsolateDisposal)"; then \ + echo ""; \ + echo "🔍 Memory Leak Detection: $$d"; \ + echo "---------------------------------------------"; \ + $(GO) test -run='TestMemoryLeak|TestIsolateDisposal' -v $$d || exit 1; \ + fi; \ + done + @echo "" + @echo "=============================================" + @echo "✅ All memory leak tests passed" + @echo "=============================================" + +# Run all tests (unit + benchmark + memory leak) +.PHONY: test +test: unit-test benchmark memory-leak + .PHONY: fmt fmt: $(GOFMT) -w $(GOFILES) diff --git a/agent/assistant/hook/create.go b/agent/assistant/hook/create.go index 770ed8bb..e48f4952 100644 --- a/agent/assistant/hook/create.go +++ b/agent/assistant/hook/create.go @@ -14,7 +14,57 @@ func (s *Script) Create(ctx *context.Context, messages []context.Message) (*cont if err != nil { return nil, err } - return s.getHookCreateResponse(res) + + response, err := s.getHookCreateResponse(res) + if err != nil { + return nil, err + } + + // Apply context adjustments from the response back to the context + if response != nil { + s.applyContextAdjustments(ctx, response) + } + + return response, nil +} + +// applyContextAdjustments applies context field overrides from the hook response back to the context +func (s *Script) applyContextAdjustments(ctx *context.Context, response *context.HookCreateResponse) { + // Override assistant ID if provided + if response.AssistantID != "" { + ctx.AssistantID = response.AssistantID + } + + // Override connector if provided + if response.Connector != "" { + ctx.Connector = response.Connector + } + + // Override locale if provided + if response.Locale != "" { + ctx.Locale = response.Locale + } + + // Override theme if provided + if response.Theme != "" { + ctx.Theme = response.Theme + } + + // Override route if provided + if response.Route != "" { + ctx.Route = response.Route + } + + // Merge or override metadata if provided + if len(response.Metadata) > 0 { + if ctx.Metadata == nil { + ctx.Metadata = make(map[string]interface{}) + } + // Merge metadata - response metadata takes precedence + for key, value := range response.Metadata { + ctx.Metadata[key] = value + } + } } // getHookCreateResponse convert the result to a HookCreateResponse diff --git a/agent/assistant/hook/create_bench_test.go b/agent/assistant/hook/create_bench_test.go new file mode 100644 index 00000000..9432dbee --- /dev/null +++ b/agent/assistant/hook/create_bench_test.go @@ -0,0 +1,330 @@ +package hook_test + +import ( + stdContext "context" + "testing" + + "github.com/yaoapp/gou/plan" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/testutils" + "github.com/yaoapp/yao/openapi/oauth/types" + "github.com/yaoapp/yao/test" +) + +// ============================================================================ +// Simple Scenario Benchmarks +// ============================================================================ + +// BenchmarkSimpleStandardMode benchmarks simple scenario in standard V8 mode +// Run with: go test -bench=BenchmarkSimpleStandardMode -benchmem -benchtime=100x +func BenchmarkSimpleStandardMode(b *testing.B) { + testutils.Prepare(&testing.T{}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ctx := newBenchContext("bench-simple-standard", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + b.Fatalf("Create failed: %s", err.Error()) + } + } +} + +// BenchmarkSimplePerformanceMode benchmarks simple scenario in performance V8 mode +// Run with: go test -bench=BenchmarkSimplePerformanceMode -benchmem -benchtime=100x +func BenchmarkSimplePerformanceMode(b *testing.B) { + testutils.Prepare(&testing.T{}, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ctx := newBenchContext("bench-simple-performance", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + b.Fatalf("Create failed: %s", err.Error()) + } + } +} + +// ============================================================================ +// Business Scenario Benchmarks (with Process calls, DB access, etc.) +// ============================================================================ + +// BenchmarkBusinessStandardMode benchmarks business scenarios in standard V8 mode +// Run with: go test -bench=BenchmarkBusinessStandardMode -benchmem -benchtime=100x +func BenchmarkBusinessStandardMode(b *testing.B) { + testutils.Prepare(&testing.T{}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + scenarios := getBusinessScenarios() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + scenario := scenarios[i%len(scenarios)] + ctx := newBenchContext("bench-business-standard", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: scenario.content}, + }) + if err != nil { + b.Errorf("%s failed: %s", scenario.name, err.Error()) + } + } +} + +// BenchmarkBusinessPerformanceMode benchmarks business scenarios in performance V8 mode +// Run with: go test -bench=BenchmarkBusinessPerformanceMode -benchmem -benchtime=100x +func BenchmarkBusinessPerformanceMode(b *testing.B) { + testutils.Prepare(&testing.T{}, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + scenarios := getBusinessScenarios() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + scenario := scenarios[i%len(scenarios)] + ctx := newBenchContext("bench-business-performance", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: scenario.content}, + }) + if err != nil { + b.Errorf("%s failed: %s", scenario.name, err.Error()) + } + } +} + +// ============================================================================ +// Concurrent Benchmarks +// ============================================================================ + +// BenchmarkConcurrentSimpleStandardMode benchmarks simple concurrent scenario in standard V8 mode +// Simulates concurrent users with isolate creation/disposal per request +// Run with: go test -bench=BenchmarkConcurrentSimpleStandardMode -benchmem -benchtime=100x +func BenchmarkConcurrentSimpleStandardMode(b *testing.B) { + testutils.Prepare(&testing.T{}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + ctx := newBenchContext("bench-concurrent-simple-standard", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + b.Errorf("Create failed (iteration %d): %s", i, err.Error()) + } + i++ + } + }) +} + +// BenchmarkConcurrentSimplePerformanceMode benchmarks simple concurrent scenario in performance V8 mode +// Simulates 100 users simultaneously using the system with isolate pool +// Run with: go test -bench=BenchmarkConcurrentSimplePerformanceMode -benchmem -benchtime=100x +func BenchmarkConcurrentSimplePerformanceMode(b *testing.B) { + testutils.Prepare(&testing.T{}, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + ctx := newBenchContext("bench-concurrent-simple", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + b.Errorf("Create failed (iteration %d): %s", i, err.Error()) + } + i++ + } + }) +} + +// BenchmarkConcurrentBusinessStandardMode benchmarks concurrent business scenarios in standard V8 mode +// Tests various scenarios with concurrent users and isolate creation/disposal per request +// Run with: go test -bench=BenchmarkConcurrentBusinessStandardMode -benchmem -benchtime=100x +func BenchmarkConcurrentBusinessStandardMode(b *testing.B) { + testutils.Prepare(&testing.T{}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + scenarios := getBusinessScenarios() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + scenario := scenarios[i%len(scenarios)] + ctx := newBenchContext("bench-concurrent-business-standard", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: scenario.content}, + }) + if err != nil { + b.Errorf("%s failed (iteration %d): %s", scenario.name, i, err.Error()) + } + i++ + } + }) +} + +// BenchmarkConcurrentBusinessPerformanceMode benchmarks concurrent business scenarios in performance V8 mode +// Tests various scenarios with 100 concurrent users with isolate pool +// Run with: go test -bench=BenchmarkConcurrentBusinessPerformanceMode -benchmem -benchtime=100x +func BenchmarkConcurrentBusinessPerformanceMode(b *testing.B) { + testutils.Prepare(&testing.T{}, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(&testing.T{}) + + agent, err := assistant.Get("tests.create") + if err != nil { + b.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + b.Fatalf("Assistant has no script") + } + + scenarios := getBusinessScenarios() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + scenario := scenarios[i%len(scenarios)] + ctx := newBenchContext("bench-concurrent-business", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: scenario.content}, + }) + if err != nil { + b.Errorf("%s failed (iteration %d): %s", scenario.name, i, err.Error()) + } + i++ + } + }) +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// getBusinessScenarios returns the business test scenarios +func getBusinessScenarios() []struct { + name string + content string +} { + return []struct { + name string + content string + }{ + {name: "FullResponse", content: "return_full"}, + {name: "PartialResponse", content: "return_partial"}, + {name: "ProcessCall", content: "return_process"}, + {name: "ContextAdjustment", content: "adjust_context"}, + {name: "NestedScriptCall", content: "nested_script_call"}, + {name: "DeepNestedCall", content: "deep_nested_call"}, + } +} + +// newBenchContext creates a minimal context for benchmarking +func newBenchContext(chatID, assistantID string) *context.Context { + return &context.Context{ + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + ChatID: chatID, + AssistantID: assistantID, + Connector: "", + Locale: "en-us", + Theme: "light", + Client: context.Client{ + Type: "web", + UserAgent: "BenchAgent/1.0", + IP: "127.0.0.1", + }, + Referer: context.RefererAPI, + Accept: context.AcceptWebCUI, + Route: "", + Metadata: make(map[string]interface{}), + Authorized: &types.AuthorizedInfo{ + Subject: "bench-user", + ClientID: "bench-client", + UserID: "bench-user-123", + TeamID: "bench-team-456", + TenantID: "bench-tenant-789", + Constraints: types.DataConstraints{ + TeamOnly: true, + Extra: map[string]interface{}{ + "department": "engineering", + }, + }, + }, + } +} diff --git a/agent/assistant/hook/create_mem_test.go b/agent/assistant/hook/create_mem_test.go new file mode 100644 index 00000000..2261101d --- /dev/null +++ b/agent/assistant/hook/create_mem_test.go @@ -0,0 +1,602 @@ +package hook_test + +import ( + stdContext "context" + "runtime" + "testing" + "time" + + "github.com/yaoapp/gou/plan" + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/testutils" + "github.com/yaoapp/yao/openapi/oauth/types" + "github.com/yaoapp/yao/test" +) + +// ============================================================================ +// Memory Leak Detection Tests +// ============================================================================ + +// TestMemoryLeakStandardMode checks for memory leaks in standard V8 mode +// Run with: go test -run=TestMemoryLeakStandardMode -v +func TestMemoryLeakStandardMode(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Warm up - execute a few times to stabilize memory + for i := 0; i < 10; i++ { + ctx := newMemTestContext("warmup", "tests.create") + _, _ = agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + } + + // Force GC and get baseline memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + + // Execute many iterations + iterations := 1000 + for i := 0; i < iterations; i++ { + ctx := newMemTestContext("mem-test-standard", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + t.Errorf("Create failed at iteration %d: %s", i, err.Error()) + } + + // Periodic GC to help detect leaks faster + if i%100 == 0 { + runtime.GC() + } + } + + // Force GC and check final memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var final runtime.MemStats + runtime.ReadMemStats(&final) + + // Calculate memory growth + baselineHeap := baseline.HeapAlloc + finalHeap := final.HeapAlloc + growth := int64(finalHeap) - int64(baselineHeap) + growthPerIteration := float64(growth) / float64(iterations) + + t.Logf("Memory Statistics (Standard Mode):") + t.Logf(" Iterations: %d", iterations) + t.Logf(" Baseline HeapAlloc: %d bytes (%.2f MB)", baselineHeap, float64(baselineHeap)/1024/1024) + t.Logf(" Final HeapAlloc: %d bytes (%.2f MB)", finalHeap, float64(finalHeap)/1024/1024) + t.Logf(" Total Growth: %d bytes (%.2f MB)", growth, float64(growth)/1024/1024) + t.Logf(" Growth per iteration: %.2f bytes", growthPerIteration) + t.Logf(" Total Alloc: %d bytes (%.2f MB)", final.TotalAlloc, float64(final.TotalAlloc)/1024/1024) + t.Logf(" Mallocs: %d", final.Mallocs) + t.Logf(" Frees: %d", final.Frees) + t.Logf(" Live Objects: %d", final.Mallocs-final.Frees) + t.Logf(" GC Runs: %d", final.NumGC-baseline.NumGC) + + // Check for memory leak + // Standard mode creates/disposes isolates per request, so some overhead is expected + // Allow up to 10KB growth per iteration as threshold + // Significant leaks would show much higher growth rates + maxGrowthPerIteration := 10240.0 + if growthPerIteration > maxGrowthPerIteration { + t.Errorf("Possible memory leak detected: %.2f bytes/iteration (threshold: %.2f bytes/iteration)", + growthPerIteration, maxGrowthPerIteration) + } else { + t.Logf("✓ Memory growth is within acceptable range") + } +} + +// TestMemoryLeakPerformanceMode checks for memory leaks in performance V8 mode +// Run with: go test -run=TestMemoryLeakPerformanceMode -v +func TestMemoryLeakPerformanceMode(t *testing.T) { + testutils.Prepare(t, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Warm up - execute a few times to stabilize memory and fill isolate pool + for i := 0; i < 20; i++ { + ctx := newMemTestContext("warmup", "tests.create") + _, _ = agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + } + + // Force GC and get baseline memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + + // Execute many iterations + iterations := 1000 + for i := 0; i < iterations; i++ { + ctx := newMemTestContext("mem-test-performance", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + t.Errorf("Create failed at iteration %d: %s", i, err.Error()) + } + + // Periodic GC + if i%100 == 0 { + runtime.GC() + } + } + + // Force GC and check final memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var final runtime.MemStats + runtime.ReadMemStats(&final) + + // Calculate memory growth + baselineHeap := baseline.HeapAlloc + finalHeap := final.HeapAlloc + growth := int64(finalHeap) - int64(baselineHeap) + growthPerIteration := float64(growth) / float64(iterations) + + t.Logf("Memory Statistics (Performance Mode):") + t.Logf(" Iterations: %d", iterations) + t.Logf(" Baseline HeapAlloc: %d bytes (%.2f MB)", baselineHeap, float64(baselineHeap)/1024/1024) + t.Logf(" Final HeapAlloc: %d bytes (%.2f MB)", finalHeap, float64(finalHeap)/1024/1024) + t.Logf(" Total Growth: %d bytes (%.2f MB)", growth, float64(growth)/1024/1024) + t.Logf(" Growth per iteration: %.2f bytes", growthPerIteration) + t.Logf(" Total Alloc: %d bytes (%.2f MB)", final.TotalAlloc, float64(final.TotalAlloc)/1024/1024) + t.Logf(" Mallocs: %d", final.Mallocs) + t.Logf(" Frees: %d", final.Frees) + t.Logf(" Live Objects: %d", final.Mallocs-final.Frees) + t.Logf(" GC Runs: %d", final.NumGC-baseline.NumGC) + + // Performance mode should have less growth due to isolate reuse + // Allow up to 5KB per iteration as threshold + maxGrowthPerIteration := 5120.0 + if growthPerIteration > maxGrowthPerIteration { + t.Errorf("Possible memory leak detected: %.2f bytes/iteration (threshold: %.2f bytes/iteration)", + growthPerIteration, maxGrowthPerIteration) + } else { + t.Logf("✓ Memory growth is within acceptable range") + } +} + +// TestMemoryLeakBusinessScenarios checks for memory leaks with business logic +// Run with: go test -run=TestMemoryLeakBusinessScenarios -v +func TestMemoryLeakBusinessScenarios(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + scenarios := []struct { + name string + content string + }{ + {name: "FullResponse", content: "return_full"}, + {name: "PartialResponse", content: "return_partial"}, + {name: "ProcessCall", content: "return_process"}, + {name: "ContextAdjustment", content: "adjust_context"}, + {name: "NestedScriptCall", content: "nested_script_call"}, + {name: "DeepNestedCall", content: "deep_nested_call"}, + } + + // Warm up + for i := 0; i < 10; i++ { + ctx := newMemTestContext("warmup", "tests.create") + _, _ = agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "return_full"}, + }) + } + + // Test each scenario + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Get baseline + runtime.GC() + time.Sleep(50 * time.Millisecond) + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + + // Execute iterations (reduced to avoid V8 OOM) + iterations := 200 + for i := 0; i < iterations; i++ { + ctx := newMemTestContext("mem-test-business", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: scenario.content}, + }) + if err != nil { + t.Errorf("Create failed at iteration %d: %s", i, err.Error()) + } + + if i%50 == 0 { + runtime.GC() + } + } + + // Check final memory + runtime.GC() + time.Sleep(50 * time.Millisecond) + var final runtime.MemStats + runtime.ReadMemStats(&final) + + growth := int64(final.HeapAlloc) - int64(baseline.HeapAlloc) + growthPerIteration := float64(growth) / float64(iterations) + + t.Logf(" Baseline HeapAlloc: %d bytes (%.2f MB)", baseline.HeapAlloc, float64(baseline.HeapAlloc)/1024/1024) + t.Logf(" Final HeapAlloc: %d bytes (%.2f MB)", final.HeapAlloc, float64(final.HeapAlloc)/1024/1024) + t.Logf(" Growth: %d bytes (%.2f MB)", growth, float64(growth)/1024/1024) + t.Logf(" Growth/iteration: %.2f bytes", growthPerIteration) + + // Business scenarios may have more memory usage due to complex operations + // Allow up to 15KB per iteration as threshold + maxGrowthPerIteration := 15360.0 + if growthPerIteration > maxGrowthPerIteration { + t.Errorf("Possible memory leak: %.2f bytes/iteration (threshold: %.2f)", + growthPerIteration, maxGrowthPerIteration) + } else { + t.Logf(" ✓ Memory growth is within acceptable range") + } + }) + } +} + +// TestMemoryLeakConcurrent checks for memory leaks under concurrent load +// Run with: go test -run=TestMemoryLeakConcurrent -v +func TestMemoryLeakConcurrent(t *testing.T) { + testutils.Prepare(t, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Warm up + for i := 0; i < 20; i++ { + ctx := newMemTestContext("warmup", "tests.create") + _, _ = agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + } + + // Get baseline + runtime.GC() + time.Sleep(100 * time.Millisecond) + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + + // Run concurrent load + iterations := 1000 + concurrency := 10 + iterPerGoroutine := iterations / concurrency + + done := make(chan bool, concurrency) + for g := 0; g < concurrency; g++ { + go func(id int) { + defer func() { done <- true }() + for i := 0; i < iterPerGoroutine; i++ { + ctx := newMemTestContext("mem-test-concurrent", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + t.Errorf("Goroutine %d failed at iteration %d: %s", id, i, err.Error()) + } + } + }(g) + } + + // Wait for all goroutines + for g := 0; g < concurrency; g++ { + <-done + } + + // Check final memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var final runtime.MemStats + runtime.ReadMemStats(&final) + + growth := int64(final.HeapAlloc) - int64(baseline.HeapAlloc) + growthPerIteration := float64(growth) / float64(iterations) + + t.Logf("Memory Statistics (Concurrent Load):") + t.Logf(" Iterations: %d", iterations) + t.Logf(" Concurrency: %d", concurrency) + t.Logf(" Baseline HeapAlloc: %d bytes (%.2f MB)", baseline.HeapAlloc, float64(baseline.HeapAlloc)/1024/1024) + t.Logf(" Final HeapAlloc: %d bytes (%.2f MB)", final.HeapAlloc, float64(final.HeapAlloc)/1024/1024) + t.Logf(" Growth: %d bytes (%.2f MB)", growth, float64(growth)/1024/1024) + t.Logf(" Growth/iteration: %.2f bytes", growthPerIteration) + t.Logf(" GC Runs: %d", final.NumGC-baseline.NumGC) + + // Concurrent scenarios may have slightly more overhead + maxGrowthPerIteration := 10240.0 + if growthPerIteration > maxGrowthPerIteration { + t.Errorf("Possible memory leak: %.2f bytes/iteration (threshold: %.2f)", + growthPerIteration, maxGrowthPerIteration) + } else { + t.Logf("✓ Memory growth is within acceptable range") + } +} + +// TestMemoryLeakNestedCalls checks for memory leaks with nested script calls +// Run with: go test -run=TestMemoryLeakNestedCalls -v +func TestMemoryLeakNestedCalls(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Warm up + for i := 0; i < 10; i++ { + ctx := newMemTestContext("warmup", "tests.create") + _, _ = agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "nested_script_call"}, + }) + } + + // Get baseline + runtime.GC() + time.Sleep(100 * time.Millisecond) + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + + // Execute iterations with nested calls + // Nested calls: hook -> scripts.tests.create.NestedCall -> GetRoles/GetRole -> models + iterations := 200 + for i := 0; i < iterations; i++ { + ctx := newMemTestContext("mem-test-nested", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "deep_nested_call"}, + }) + if err != nil { + t.Errorf("Nested call failed at iteration %d: %s", i, err.Error()) + } + + if i%50 == 0 { + runtime.GC() + } + } + + // Check final memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var final runtime.MemStats + runtime.ReadMemStats(&final) + + growth := int64(final.HeapAlloc) - int64(baseline.HeapAlloc) + growthPerIteration := float64(growth) / float64(iterations) + + t.Logf("Memory Statistics (Nested Calls):") + t.Logf(" Iterations: %d", iterations) + t.Logf(" Baseline HeapAlloc: %d bytes (%.2f MB)", baseline.HeapAlloc, float64(baseline.HeapAlloc)/1024/1024) + t.Logf(" Final HeapAlloc: %d bytes (%.2f MB)", final.HeapAlloc, float64(final.HeapAlloc)/1024/1024) + t.Logf(" Growth: %d bytes (%.2f MB)", growth, float64(growth)/1024/1024) + t.Logf(" Growth/iteration: %.2f bytes", growthPerIteration) + t.Logf(" GC Runs: %d", final.NumGC-baseline.NumGC) + + // Nested calls involve database operations, so allow more overhead + // Allow up to 20KB per iteration as threshold + maxGrowthPerIteration := 20480.0 + if growthPerIteration > maxGrowthPerIteration { + t.Errorf("Possible memory leak: %.2f bytes/iteration (threshold: %.2f)", + growthPerIteration, maxGrowthPerIteration) + } else { + t.Logf("✓ Memory growth is within acceptable range") + } +} + +// TestMemoryLeakNestedConcurrent checks for memory leaks with concurrent nested calls +// Run with: go test -run=TestMemoryLeakNestedConcurrent -v +func TestMemoryLeakNestedConcurrent(t *testing.T) { + testutils.Prepare(t, test.PrepareOption{V8Mode: "performance"}) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Warm up + for i := 0; i < 20; i++ { + ctx := newMemTestContext("warmup", "tests.create") + _, _ = agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "nested_script_call"}, + }) + } + + // Get baseline + runtime.GC() + time.Sleep(100 * time.Millisecond) + var baseline runtime.MemStats + runtime.ReadMemStats(&baseline) + + // Run concurrent nested calls + iterations := 500 + concurrency := 10 + iterPerGoroutine := iterations / concurrency + + done := make(chan bool, concurrency) + for g := 0; g < concurrency; g++ { + go func(id int) { + defer func() { done <- true }() + for i := 0; i < iterPerGoroutine; i++ { + ctx := newMemTestContext("mem-test-nested-concurrent", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "deep_nested_call"}, + }) + if err != nil { + t.Errorf("Goroutine %d nested call failed at iteration %d: %s", id, i, err.Error()) + } + } + }(g) + } + + // Wait for all goroutines + for g := 0; g < concurrency; g++ { + <-done + } + + // Check final memory + runtime.GC() + time.Sleep(100 * time.Millisecond) + var final runtime.MemStats + runtime.ReadMemStats(&final) + + growth := int64(final.HeapAlloc) - int64(baseline.HeapAlloc) + growthPerIteration := float64(growth) / float64(iterations) + + t.Logf("Memory Statistics (Concurrent Nested Calls):") + t.Logf(" Iterations: %d", iterations) + t.Logf(" Concurrency: %d", concurrency) + t.Logf(" Baseline HeapAlloc: %d bytes (%.2f MB)", baseline.HeapAlloc, float64(baseline.HeapAlloc)/1024/1024) + t.Logf(" Final HeapAlloc: %d bytes (%.2f MB)", final.HeapAlloc, float64(final.HeapAlloc)/1024/1024) + t.Logf(" Growth: %d bytes (%.2f MB)", growth, float64(growth)/1024/1024) + t.Logf(" Growth/iteration: %.2f bytes", growthPerIteration) + t.Logf(" GC Runs: %d", final.NumGC-baseline.NumGC) + + // Concurrent nested calls with database operations + // Allow up to 25KB per iteration as threshold + maxGrowthPerIteration := 25600.0 + if growthPerIteration > maxGrowthPerIteration { + t.Errorf("Possible memory leak: %.2f bytes/iteration (threshold: %.2f)", + growthPerIteration, maxGrowthPerIteration) + } else { + t.Logf("✓ Memory growth is within acceptable range") + } +} + +// TestIsolateDisposal verifies that isolates are properly disposed in standard mode +// Run with: go test -run=TestIsolateDisposal -v +func TestIsolateDisposal(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Track goroutine count to detect goroutine leaks + initialGoroutines := runtime.NumGoroutine() + + // Execute multiple iterations + iterations := 100 + for i := 0; i < iterations; i++ { + ctx := newMemTestContext("disposal-test", "tests.create") + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "Hello"}, + }) + if err != nil { + t.Errorf("Create failed at iteration %d: %s", i, err.Error()) + } + } + + // Give time for cleanup + time.Sleep(200 * time.Millisecond) + runtime.GC() + time.Sleep(200 * time.Millisecond) + + finalGoroutines := runtime.NumGoroutine() + goroutineGrowth := finalGoroutines - initialGoroutines + + t.Logf("Goroutine Statistics:") + t.Logf(" Initial: %d", initialGoroutines) + t.Logf(" Final: %d", finalGoroutines) + t.Logf(" Growth: %d", goroutineGrowth) + + // Allow some goroutine growth for runtime internals, but not proportional to iterations + // If goroutines grow with iterations, we have a leak + maxGoroutineGrowth := 20 + if goroutineGrowth > maxGoroutineGrowth { + t.Errorf("Possible goroutine leak: %d new goroutines (threshold: %d)", + goroutineGrowth, maxGoroutineGrowth) + } +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// newMemTestContext creates a context for memory leak testing +func newMemTestContext(chatID, assistantID string) *context.Context { + return &context.Context{ + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + ChatID: chatID, + AssistantID: assistantID, + Connector: "", + Locale: "en-us", + Theme: "light", + Client: context.Client{ + Type: "web", + UserAgent: "MemTestAgent/1.0", + IP: "127.0.0.1", + }, + Referer: context.RefererAPI, + Accept: context.AcceptWebCUI, + Route: "", + Metadata: make(map[string]interface{}), + Authorized: &types.AuthorizedInfo{ + Subject: "mem-test-user", + ClientID: "mem-test-client", + UserID: "mem-user-123", + TeamID: "mem-team-456", + TenantID: "mem-tenant-789", + Constraints: types.DataConstraints{ + TeamOnly: true, + Extra: map[string]interface{}{ + "department": "engineering", + }, + }, + }, + } +} diff --git a/agent/assistant/hook/create_nested_test.go b/agent/assistant/hook/create_nested_test.go new file mode 100644 index 00000000..c082324f --- /dev/null +++ b/agent/assistant/hook/create_nested_test.go @@ -0,0 +1,117 @@ +package hook_test + +import ( + "sync" + "testing" + + "github.com/yaoapp/yao/agent/assistant" + "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/testutils" +) + +// TestNestedScriptCall tests nested script calls with V8 context sharing +// This test calls: hook -> scripts.tests.create.NestedCall -> GetRoles/GetRole -> models +func TestNestedScriptCall(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // Create context + ctx := newTestContext("test-nested-call", "tests.create") + + // Call with deep_nested_call scenario + // This will: hook -> scripts.tests.create.NestedCall -> GetRoles -> model + res, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "deep_nested_call"}, + }) + + if err != nil { + t.Fatalf("Nested call failed: %s", err.Error()) + } + + if res == nil { + t.Fatal("Expected non-nil response") + } + + // Verify messages + if len(res.Messages) == 0 { + t.Fatal("Expected messages in response") + } + + t.Logf("✓ Nested script call completed successfully") + t.Logf(" Messages count: %d", len(res.Messages)) + if res.Metadata != nil { + t.Logf(" Metadata: %+v", res.Metadata) + } +} + +// TestNestedScriptCallConcurrent tests nested script calls under high concurrency +// Simulates 100 concurrent users making nested script calls +func TestNestedScriptCallConcurrent(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + agent, err := assistant.Get("tests.create") + if err != nil { + t.Fatalf("Failed to get assistant: %s", err.Error()) + } + + if agent.Script == nil { + t.Fatalf("Assistant has no script") + } + + // High concurrency test: 100 concurrent users (testing race condition) + concurrency := 100 + iterations := 1 // Each user makes 1 call + + var wg sync.WaitGroup + errors := make(chan error, concurrency*iterations) + + t.Logf("Starting concurrent test: %d users × %d iterations = %d total calls", + concurrency, iterations, concurrency*iterations) + + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func(userID int) { + defer wg.Done() + + for j := 0; j < iterations; j++ { + ctx := newTestContext("test-concurrent", "tests.create") + + _, err := agent.Script.Create(ctx, []context.Message{ + {Role: "user", Content: "deep_nested_call"}, + }) + + if err != nil { + errors <- err + return + } + } + }(i) + } + + // Wait for all goroutines to complete + wg.Wait() + close(errors) + + // Check for errors + errorCount := 0 + for err := range errors { + errorCount++ + t.Errorf("Concurrent call failed: %s", err.Error()) + } + + if errorCount > 0 { + t.Fatalf("Failed with %d errors out of %d total calls", errorCount, concurrency*iterations) + } + + t.Logf("✓ All %d concurrent nested calls completed successfully", concurrency*iterations) +} diff --git a/agent/assistant/hook/create_test.go b/agent/assistant/hook/create_test.go index aef9a36e..5e0a4131 100644 --- a/agent/assistant/hook/create_test.go +++ b/agent/assistant/hook/create_test.go @@ -297,4 +297,66 @@ func TestCreate(t *testing.T) { } } }) + + // Test scenario 9: Adjust context fields - tests that context fields can be modified by the hook + t.Run("AdjustContext", func(t *testing.T) { + // Create a fresh context for this test + adjustCtx := newTestContext("chat-test-adjust", "tests.create") + + // Call the hook which should adjust context fields + res, err := agent.Script.Create(adjustCtx, []context.Message{{Role: "user", Content: "adjust_context"}}) + if err != nil { + t.Fatalf("Failed to create with adjust_context: %s", err.Error()) + } + if res == nil { + t.Fatalf("Expected non-nil response, got nil") + } + + // Verify the response contains adjusted fields + if res.AssistantID != "adjusted.assistant" { + t.Errorf("Expected adjusted assistant_id 'adjusted.assistant', got: %s", res.AssistantID) + } + if res.Connector != "adjusted-connector" { + t.Errorf("Expected adjusted connector 'adjusted-connector', got: %s", res.Connector) + } + if res.Locale != "zh-cn" { + t.Errorf("Expected adjusted locale 'zh-cn', got: %s", res.Locale) + } + if res.Theme != "dark" { + t.Errorf("Expected adjusted theme 'dark', got: %s", res.Theme) + } + if res.Route != "/adjusted/route" { + t.Errorf("Expected adjusted route '/adjusted/route', got: %s", res.Route) + } + + // Verify metadata + if res.Metadata == nil { + t.Fatalf("Expected metadata, got nil") + } + if adjusted, ok := res.Metadata["adjusted"].(bool); !ok || !adjusted { + t.Errorf("Expected metadata['adjusted'] = true, got: %v", res.Metadata["adjusted"]) + } + + // Verify context fields were actually updated + if adjustCtx.AssistantID != "adjusted.assistant" { + t.Errorf("Context assistant_id not updated. Expected 'adjusted.assistant', got: %s", adjustCtx.AssistantID) + } + if adjustCtx.Connector != "adjusted-connector" { + t.Errorf("Context connector not updated. Expected 'adjusted-connector', got: %s", adjustCtx.Connector) + } + if adjustCtx.Locale != "zh-cn" { + t.Errorf("Context locale not updated. Expected 'zh-cn', got: %s", adjustCtx.Locale) + } + if adjustCtx.Theme != "dark" { + t.Errorf("Context theme not updated. Expected 'dark', got: %s", adjustCtx.Theme) + } + if adjustCtx.Route != "/adjusted/route" { + t.Errorf("Context route not updated. Expected '/adjusted/route', got: %s", adjustCtx.Route) + } + if adjustCtx.Metadata["adjusted"] != true { + t.Errorf("Context metadata not updated. Expected metadata['adjusted'] = true, got: %v", adjustCtx.Metadata["adjusted"]) + } + + t.Log("✓ Context fields successfully adjusted by hook") + }) } diff --git a/agent/assistant/hook/script.go b/agent/assistant/hook/script.go index 924d2a36..18b3cbe9 100644 --- a/agent/assistant/hook/script.go +++ b/agent/assistant/hook/script.go @@ -1,6 +1,8 @@ package hook -import "github.com/yaoapp/yao/agent/context" +import ( + "github.com/yaoapp/yao/agent/context" +) // Execute execute the script func (s *Script) Execute(ctx *context.Context, method string, args ...interface{}) (interface{}, error) { diff --git a/agent/context/types.go b/agent/context/types.go index d37a6105..712e7e74 100644 --- a/agent/context/types.go +++ b/agent/context/types.go @@ -204,6 +204,14 @@ type HookCreateResponse struct { Temperature *float64 `json:"temperature,omitempty"` MaxTokens *int `json:"max_tokens,omitempty"` MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` + + // Context adjustments - allow hook to modify context fields + AssistantID string `json:"assistant_id,omitempty"` // Override assistant ID + Connector string `json:"connector,omitempty"` // Override connector + Locale string `json:"locale,omitempty"` // Override locale + Theme string `json:"theme,omitempty"` // Override theme + Route string `json:"route,omitempty"` // Override route + Metadata map[string]interface{} `json:"metadata,omitempty"` // Override or merge metadata } // ResponseHookDone the response of the done hook diff --git a/agent/testutils/testutils.go b/agent/testutils/testutils.go index 5ea25003..a6fb5875 100644 --- a/agent/testutils/testutils.go +++ b/agent/testutils/testutils.go @@ -8,9 +8,13 @@ import ( "github.com/yaoapp/yao/test" ) -// Prepare prepare the test environment -func Prepare(t *testing.T) { - test.Prepare(t, config.Conf) +// Prepare prepare the test environment with optional V8 mode configuration +// Usage: +// +// testutils.Prepare(t) // standard mode (default) +// testutils.Prepare(t, test.PrepareOption{V8Mode: "performance"}) // performance mode for benchmarks +func Prepare(t *testing.T, opts ...interface{}) { + test.Prepare(t, config.Conf, opts...) // Load agent err := agent.Load(config.Conf) diff --git a/test/utils.go b/test/utils.go index 090f3783..feabcfb3 100644 --- a/test/utils.go +++ b/test/utils.go @@ -345,12 +345,42 @@ func loadSystemModels(t *testing.T, cfg config.Config) error { return nil } -// Prepare test environment -func Prepare(t *testing.T, cfg config.Config, rootEnv ...string) { +// PrepareOption options for test preparation +type PrepareOption struct { + // V8Mode sets the V8 runtime mode: "standard" (default) or "performance" + // - standard: Lower memory usage, creates/disposes isolates for each execution + // - performance: Higher memory usage, maintains isolate pool for better performance + // Use "performance" mode for benchmarks and stress tests + V8Mode string +} + +// Prepare test environment with optional configuration +// Usage: +// +// test.Prepare(t, config.Conf) // standard mode (default) +// test.Prepare(t, config.Conf, test.PrepareOption{V8Mode: "performance"}) // performance mode +func Prepare(t *testing.T, cfg config.Config, opts ...interface{}) { appRootEnv := "YAO_TEST_APPLICATION" - if len(rootEnv) > 0 { - appRootEnv = rootEnv[0] + v8Mode := "standard" // default to standard mode + + // Parse options + for _, opt := range opts { + switch v := opt.(type) { + case string: + // Legacy: string parameter for appRootEnv + appRootEnv = v + case PrepareOption: + // New: structured options + if v.V8Mode != "" { + v8Mode = v.V8Mode + } + } + } + + // Override with environment variable if set + if envMode := os.Getenv("YAO_RUNTIME_MODE"); envMode != "" { + v8Mode = envMode } // Remove the data store @@ -451,6 +481,19 @@ func Prepare(t *testing.T, cfg config.Config, rootEnv ...string) { share.App.Prefix = "yao_" } + // Apply V8 mode to config + cfg.Runtime.Mode = v8Mode + + // Ensure MinSize and MaxSize are set for performance mode + if v8Mode == "performance" { + if cfg.Runtime.MinSize == 0 { + cfg.Runtime.MinSize = 3 + } + if cfg.Runtime.MaxSize == 0 { + cfg.Runtime.MaxSize = 10 + } + } + utils.Init() dbconnect(t, cfg) load(t, cfg)