From 666c1e8d742bc624f237a1a45eb82876b271fd07 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 6 Dec 2025 17:38:21 +0800 Subject: [PATCH] Enhance content processing with forceUses configuration - Updated the Vision function and associated handlers to accept a forceUses parameter, allowing explicit control over the use of external tools regardless of model capabilities. - Modified the Assistant's BuildContent method to set the AssistantID in the context for better file tracking. - Enhanced the applyCreateResponseOptions method to merge and prioritize Uses and ForceUses configurations from the createResponse. - Added comprehensive tests to validate the adjustment of Uses and ForceUses configurations through hooks, ensuring correct application in content processing. --- agent/assistant/build.go | 30 ++ agent/assistant/build_content.go | 11 +- agent/assistant/hook/create_test.go | 126 ++++++ agent/content/audio.go | 2 +- agent/content/content.go | 31 +- agent/content/content_vision_test.go | 225 ++++++++++- agent/content/excel.go | 2 +- agent/content/fetch.go | 11 +- agent/content/image.go | 25 +- agent/content/image_test.go | 6 +- agent/content/interfaces.go | 3 +- agent/content/pdf.go | 2 +- agent/content/registry.go | 4 +- agent/content/text.go | 2 +- agent/content/text_test.go | 2 +- agent/content/tools.go | 216 +++++++--- agent/content/types.go | 1 + agent/content/word.go | 2 +- agent/context/jsapi.go | 90 +++++ agent/context/jsapi_space_test.go | 572 +++++++++++++++++++++++++++ agent/context/types.go | 6 + agent/context/types_llm.go | 6 + 22 files changed, 1291 insertions(+), 84 deletions(-) create mode 100644 agent/context/jsapi_space_test.go diff --git a/agent/assistant/build.go b/agent/assistant/build.go index 45adbce4..396091b4 100644 --- a/agent/assistant/build.go +++ b/agent/assistant/build.go @@ -500,9 +500,39 @@ func (ast *Assistant) applyCreateResponseOptions(options *context.CompletionOpti } } } + + // Uses configuration (merge with existing) + // createResponse.Uses has highest priority and overrides existing Uses + if createResponse.Uses != nil { + if options.Uses == nil { + options.Uses = createResponse.Uses + } else { + // Merge: createResponse.Uses overrides existing (only non-empty fields) + if createResponse.Uses.Vision != "" { + options.Uses.Vision = createResponse.Uses.Vision + } + if createResponse.Uses.Audio != "" { + options.Uses.Audio = createResponse.Uses.Audio + } + if createResponse.Uses.Search != "" { + options.Uses.Search = createResponse.Uses.Search + } + if createResponse.Uses.Fetch != "" { + options.Uses.Fetch = createResponse.Uses.Fetch + } + } + } + + // ForceUses configuration + // If hook specifies ForceUses, it takes priority + if createResponse.ForceUses != nil { + options.ForceUses = *createResponse.ForceUses + } } // getUses get the Uses configuration with priority: assistant.Uses > global settings +// Note: createResponse.Uses (applied in applyCreateResponseOptions) has even higher priority +// Final priority order: createResponse.Uses > assistant.Uses > global settings func (ast *Assistant) getUses() *context.Uses { // Priority 1: Assistant-specific Uses configuration if ast.Uses != nil { diff --git a/agent/assistant/build_content.go b/agent/assistant/build_content.go index ba254287..9a23eb7e 100644 --- a/agent/assistant/build_content.go +++ b/agent/assistant/build_content.go @@ -12,6 +12,12 @@ import ( // // This should be called after BuildRequest and before executing LLM call func (ast *Assistant) BuildContent(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, opts *context.Options) ([]context.Message, error) { + // Set AssistantID in context for file info tracking in Space + // This ensures hooks can access file information using the correct namespace + if ctx.AssistantID == "" { + ctx.AssistantID = ast.ID + } + // Get connector and capabilities _, capabilities, err := ast.GetConnector(ctx, opts) if err != nil { @@ -21,8 +27,11 @@ func (ast *Assistant) BuildContent(ctx *context.Context, messages []context.Mess // Get Uses configuration from options (already merged in BuildRequest) uses := options.Uses + // Get ForceUses configuration from options + forceUses := options.ForceUses + // Process content through Vision function - processedMessages, err := content.Vision(ctx, capabilities, messages, uses) + processedMessages, err := content.Vision(ctx, capabilities, messages, uses, forceUses) if err != nil { return nil, fmt.Errorf("failed to process content: %w", err) } diff --git a/agent/assistant/hook/create_test.go b/agent/assistant/hook/create_test.go index d9b2c4bd..e1f446c9 100644 --- a/agent/assistant/hook/create_test.go +++ b/agent/assistant/hook/create_test.go @@ -352,4 +352,130 @@ func TestCreate(t *testing.T) { t.Log("✓ Context fields successfully adjusted by hook") }) + + // Test scenario 10: Adjust uses configuration - tests that uses can be modified by the hook + t.Run("AdjustUses", func(t *testing.T) { + // Create a fresh context for this test + usesCtx := newTestContext("chat-test-uses", "tests.create") + + // Call the hook which should adjust uses configuration + res, _, err := agent.HookScript.Create(usesCtx, []context.Message{{Role: "user", Content: "adjust_uses"}}) + if err != nil { + t.Fatalf("Failed to create with adjust_uses: %s", err.Error()) + } + if res == nil { + t.Fatalf("Expected non-nil response, got nil") + } + + // Verify the response contains uses configuration + if res.Uses == nil { + t.Fatalf("Expected uses configuration, got nil") + } + + // Verify each uses field + if res.Uses.Vision != "mcp:vision-server" { + t.Errorf("Expected vision 'mcp:vision-server', got: %s", res.Uses.Vision) + } + if res.Uses.Audio != "mcp:audio-server" { + t.Errorf("Expected audio 'mcp:audio-server', got: %s", res.Uses.Audio) + } + if res.Uses.Search != "agent" { + t.Errorf("Expected search 'agent', got: %s", res.Uses.Search) + } + if res.Uses.Fetch != "mcp:fetch-server" { + t.Errorf("Expected fetch 'mcp:fetch-server', got: %s", res.Uses.Fetch) + } + + // Verify metadata + if res.Metadata == nil { + t.Fatalf("Expected metadata, got nil") + } + if usesAdjusted, ok := res.Metadata["uses_adjusted"].(bool); !ok || !usesAdjusted { + t.Errorf("Expected metadata['uses_adjusted'] = true, got: %v", res.Metadata["uses_adjusted"]) + } + + // Now test that BuildRequest properly applies the uses configuration + inputMessages := []context.Message{{Role: "user", Content: "test uses"}} + _, options, err := agent.BuildRequest(usesCtx, inputMessages, res) + if err != nil { + t.Fatalf("Failed to build request: %s", err.Error()) + } + + // Verify that options.Uses has the values from createResponse + if options.Uses == nil { + t.Fatalf("Expected options.Uses to be set, got nil") + } + if options.Uses.Vision != "mcp:vision-server" { + t.Errorf("Expected options.Uses.Vision 'mcp:vision-server', got: %s", options.Uses.Vision) + } + if options.Uses.Audio != "mcp:audio-server" { + t.Errorf("Expected options.Uses.Audio 'mcp:audio-server', got: %s", options.Uses.Audio) + } + if options.Uses.Search != "agent" { + t.Errorf("Expected options.Uses.Search 'agent', got: %s", options.Uses.Search) + } + if options.Uses.Fetch != "mcp:fetch-server" { + t.Errorf("Expected options.Uses.Fetch 'mcp:fetch-server', got: %s", options.Uses.Fetch) + } + + t.Log("✓ Uses configuration successfully adjusted by hook and applied to options") + }) + + // Test scenario 11: Adjust uses configuration with force_uses flag + t.Run("AdjustUsesForce", func(t *testing.T) { + // Create a fresh context for this test + usesCtx := newTestContext("chat-test-uses-force", "tests.create") + + // Call the hook which should adjust uses configuration and set force_uses + res, _, err := agent.HookScript.Create(usesCtx, []context.Message{{Role: "user", Content: "adjust_uses_force"}}) + if err != nil { + t.Fatalf("Failed to create with adjust_uses_force: %s", err.Error()) + } + if res == nil { + t.Fatalf("Expected non-nil response, got nil") + } + + // Verify the response contains uses configuration + if res.Uses == nil { + t.Fatalf("Expected uses configuration, got nil") + } + + // Verify uses fields + if res.Uses.Vision != "tests.vision-helper" { + t.Errorf("Expected vision 'tests.vision-helper', got: %s", res.Uses.Vision) + } + if res.Uses.Audio != "mcp:audio-server" { + t.Errorf("Expected audio 'mcp:audio-server', got: %s", res.Uses.Audio) + } + + // Verify force_uses flag + if res.ForceUses == nil { + t.Fatalf("Expected force_uses to be set, got nil") + } + if !*res.ForceUses { + t.Errorf("Expected force_uses to be true, got: %v", *res.ForceUses) + } + + // Verify metadata + if res.Metadata == nil { + t.Fatalf("Expected metadata, got nil") + } + if usesForced, ok := res.Metadata["uses_forced"].(bool); !ok || !usesForced { + t.Errorf("Expected metadata['uses_forced'] = true, got: %v", res.Metadata["uses_forced"]) + } + + // Now test that BuildRequest properly applies the force_uses flag + inputMessages := []context.Message{{Role: "user", Content: "test force uses"}} + _, options, err := agent.BuildRequest(usesCtx, inputMessages, res) + if err != nil { + t.Fatalf("Failed to build request: %s", err.Error()) + } + + // Verify that options.ForceUses is true + if !options.ForceUses { + t.Errorf("Expected options.ForceUses to be true, got: %v", options.ForceUses) + } + + t.Log("✓ Uses configuration with force_uses flag successfully adjusted by hook and applied to options") + }) } diff --git a/agent/content/audio.go b/agent/content/audio.go index dbdecb56..1b4c755c 100644 --- a/agent/content/audio.go +++ b/agent/content/audio.go @@ -20,7 +20,7 @@ func (h *AudioHandler) CanHandle(contentType string, fileType FileType) bool { // Logic similar to image: // 1. If model supports audio input -> convert to base64 format // 2. If model doesn't support audio -> use agent/MCP specified in uses.Audio -func (h *AudioHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +func (h *AudioHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { // TODO: Implement audio handling // 1. Check model audio capabilities // 2. If supported: diff --git a/agent/content/content.go b/agent/content/content.go index 8284cf5c..6dc6daf2 100644 --- a/agent/content/content.go +++ b/agent/content/content.go @@ -51,7 +51,12 @@ import ( // - MUST convert to type="text" // // Return: Messages with only standard LLM-compatible content types (text, image_url, input_audio) -func Vision(ctx *agentContext.Context, capabilities *openai.Capabilities, messages []agentContext.Message, uses *agentContext.Uses) ([]agentContext.Message, error) { +func Vision(ctx *agentContext.Context, capabilities *openai.Capabilities, messages []agentContext.Message, uses *agentContext.Uses, forceUses ...bool) ([]agentContext.Message, error) { + // Determine if we should force using Uses tools even when model has native capabilities + shouldForceUses := false + if len(forceUses) > 0 { + shouldForceUses = forceUses[0] + } // Initialize handlers and fetcher registry := NewRegistry() fetcher := NewFetcher() @@ -64,7 +69,7 @@ func Vision(ctx *agentContext.Context, capabilities *openai.Capabilities, messag processedMessages := make([]agentContext.Message, 0, len(messages)) for _, msg := range messages { - processedMsg, err := processMessage(ctx, &msg, capabilities, uses, registry, fetcher, processedFiles) + processedMsg, err := processMessage(ctx, &msg, capabilities, uses, shouldForceUses, registry, fetcher, processedFiles) if err != nil { // Log error but continue processing other messages // TODO: Add proper logging @@ -84,6 +89,7 @@ func processMessage( msg *agentContext.Message, capabilities *openai.Capabilities, uses *agentContext.Uses, + forceUses bool, registry *Registry, fetcher Fetcher, processedFiles map[string]string, @@ -99,10 +105,13 @@ func processMessage( return *msg, nil } + // Note: File information will be collected and stored in Space by CallAgentWithFileInfo + // when calling vision agents, using the agent ID as namespace prefix + // Process each content part processedParts := make([]agentContext.ContentPart, 0, len(parts)) for _, part := range parts { - processedPart, err := processContentPart(ctx, &part, capabilities, uses, registry, fetcher, processedFiles) + processedPart, err := processContentPart(ctx, &part, capabilities, uses, forceUses, registry, fetcher, processedFiles) if err != nil { // Log error and handle gracefully fmt.Printf("Warning: failed to process content part: %v\n", err) @@ -154,6 +163,7 @@ func processContentPart( part *agentContext.ContentPart, capabilities *openai.Capabilities, uses *agentContext.Uses, + forceUses bool, registry *Registry, fetcher Fetcher, processedFiles map[string]string, @@ -168,7 +178,7 @@ func processContentPart( case agentContext.ContentImageURL: // Image URL - check if it needs processing - return processImageURLContent(ctx, part, capabilities, uses, registry, fetcher, processedFiles) + return processImageURLContent(ctx, part, capabilities, uses, forceUses, registry, fetcher, processedFiles) case agentContext.ContentInputAudio: // Audio - check if it needs processing @@ -178,7 +188,7 @@ func processContentPart( // 2. Handle extended types - MUST convert to standard types switch part.Type { case agentContext.ContentFile: - return processFileContent(ctx, part, capabilities, uses, registry, fetcher, processedFiles) + return processFileContent(ctx, part, capabilities, uses, forceUses, registry, fetcher, processedFiles) case agentContext.ContentData: return processDataContent(ctx, part) @@ -195,6 +205,7 @@ func processFileContent( part *agentContext.ContentPart, capabilities *openai.Capabilities, uses *agentContext.Uses, + forceUses bool, registry *Registry, fetcher Fetcher, processedFiles map[string]string, @@ -230,13 +241,18 @@ func processFileContent( return nil, fmt.Errorf("failed to fetch content: %w", err) } + // Set filename from part if not already set + if info.Filename == "" && part.File.Filename != "" { + info.Filename = part.File.Filename + } + // Detect file type if not already set if info.FileType == FileTypeUnknown { info.FileType = DetectFileType(info.ContentType, part.File.Filename) } // Process with appropriate handler - result, err := registry.Handle(ctx, info, capabilities, uses) + result, err := registry.Handle(ctx, info, capabilities, uses, forceUses) if err != nil { return nil, fmt.Errorf("failed to handle content: %w", err) } @@ -259,6 +275,7 @@ func processImageURLContent( part *agentContext.ContentPart, capabilities *openai.Capabilities, uses *agentContext.Uses, + forceUses bool, registry *Registry, fetcher Fetcher, processedFiles map[string]string, @@ -305,7 +322,7 @@ func processImageURLContent( info.FileType = FileTypeImage // Process with image handler - result, err := registry.Handle(ctx, info, capabilities, uses) + result, err := registry.Handle(ctx, info, capabilities, uses, forceUses) if err != nil { return nil, fmt.Errorf("failed to handle image: %w", err) } diff --git a/agent/content/content_vision_test.go b/agent/content/content_vision_test.go index 837f2a8f..36060d6d 100644 --- a/agent/content/content_vision_test.go +++ b/agent/content/content_vision_test.go @@ -119,7 +119,7 @@ func main() { } // 3. Call Vision function - result, err := content.Vision(ctx, capabilities, messages, nil) + result, err := content.Vision(ctx, capabilities, messages, nil, false) if err != nil { t.Fatalf("Vision function failed: %v", err) } @@ -218,7 +218,7 @@ func TestVision_ImageWithVisionSupport(t *testing.T) { } // 3. Call Vision function (no uses needed for direct vision support) - result, err := content.Vision(ctx, capabilities, messages, nil) + result, err := content.Vision(ctx, capabilities, messages, nil, false) if err != nil { t.Fatalf("Vision function failed: %v", err) } @@ -318,7 +318,7 @@ func TestVision_ImageWithAgent(t *testing.T) { } // 3. Call Vision - should use agent since model doesn't support vision - result, err := content.Vision(ctx, capabilities, messages, uses) + result, err := content.Vision(ctx, capabilities, messages, uses, false) if err != nil { t.Fatalf("Vision function failed: %v", err) } @@ -408,7 +408,7 @@ func TestVision_CachedContent(t *testing.T) { } // 3. Call Vision - result, err := content.Vision(ctx, capabilities, messages, nil) + result, err := content.Vision(ctx, capabilities, messages, nil, false) if err != nil { t.Fatalf("Vision function failed: %v", err) } @@ -456,3 +456,220 @@ func TestVision_CachedContent(t *testing.T) { t.Logf("✓ Content successfully cached and reused: %d characters", len(cachedText)) } + +// TestVision_FileMetadataInSpace tests that file metadata is correctly passed to vision agent via ctx.Space +func TestVision_FileMetadataInSpace(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + // Setup test uploader + uploaderName := "test-vision-metadata" + manager := setupTestUploader(t, uploaderName) + defer cleanupTestUploader(uploaderName) + + // 1. Generate and upload a test image + imageData := generateTestImage(t) + + reader := strings.NewReader(string(imageData)) + fileHeader := &attachment.FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "test-metadata.png", + Size: int64(len(imageData)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "image/png") + + uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{ + Groups: []string{"vision", "test"}, + }) + if err != nil { + t.Fatalf("Failed to upload image: %v", err) + } + + t.Logf("Uploaded file: %s (ID: %s)", uploadedFile.Filename, uploadedFile.ID) + + // 2. Prepare Vision context - model doesn't support vision, use agent + ctx := agentContext.New(context.Background(), nil, "test") + + // Capabilities without vision support + capabilities := &openai.Capabilities{ + Vision: nil, // No vision support - force agent usage + } + + // Uses configuration with vision-helper agent + uses := &agentContext.Uses{ + Vision: "tests.vision-helper", // Use vision-helper agent that logs file metadata + } + + messages := []agentContext.Message{ + { + Role: "user", + Content: []agentContext.ContentPart{ + { + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: "__" + uploaderName + "://" + uploadedFile.ID, + }, + }, + }, + }, + } + + // 3. Call Vision - should pass file metadata to agent via Space + result, err := content.Vision(ctx, capabilities, messages, uses, false) + if err != nil { + t.Fatalf("Vision function failed: %v", err) + } + + if len(result) != 1 { + t.Fatalf("Expected 1 message, got %d", len(result)) + } + + // 4. Verify result contains file metadata validation info + contentParts, ok := result[0].Content.([]agentContext.ContentPart) + if !ok { + t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) + } + + if len(contentParts) != 1 { + t.Fatalf("Expected 1 content part, got %d", len(contentParts)) + } + + if contentParts[0].Type != agentContext.ContentText { + t.Errorf("Expected ContentText (from agent), got: %s", contentParts[0].Type) + } + + // The vision-helper assistant should have logged file metadata from Space + // We can verify this through the response text (which should contain the description) + if contentParts[0].Text == "" { + t.Error("Expected non-empty text from vision agent") + } + + t.Logf("✓ Vision agent processed image with file metadata") + t.Logf("Agent response: %s", contentParts[0].Text) + + // Note: The actual validation of Space data happens in the vision-helper's Next hook + // which logs the file metadata. In a real test, we would need to capture those logs + // or have the agent return structured data that we can verify. +} + +// TestVision_MultipleFilesMetadata tests file metadata handling with multiple attachments +func TestVision_MultipleFilesMetadata(t *testing.T) { + testutils.Prepare(t) + defer testutils.Clean(t) + + // Setup test uploader + uploaderName := "test-vision-multi" + manager := setupTestUploader(t, uploaderName) + defer cleanupTestUploader(uploaderName) + + // 1. Upload two test images + imageData1 := generateTestImage(t) + imageData2 := generateTestImage(t) + + // Upload first image + reader1 := strings.NewReader(string(imageData1)) + fileHeader1 := &attachment.FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "test-image-1.png", + Size: int64(len(imageData1)), + Header: make(map[string][]string), + }, + } + fileHeader1.Header.Set("Content-Type", "image/png") + + uploadedFile1, err := manager.Upload(context.Background(), fileHeader1, reader1, attachment.UploadOption{ + Groups: []string{"vision", "test"}, + }) + if err != nil { + t.Fatalf("Failed to upload first image: %v", err) + } + + // Upload second image + reader2 := strings.NewReader(string(imageData2)) + fileHeader2 := &attachment.FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "test-image-2.png", + Size: int64(len(imageData2)), + Header: make(map[string][]string), + }, + } + fileHeader2.Header.Set("Content-Type", "image/png") + + uploadedFile2, err := manager.Upload(context.Background(), fileHeader2, reader2, attachment.UploadOption{ + Groups: []string{"vision", "test"}, + }) + if err != nil { + t.Fatalf("Failed to upload second image: %v", err) + } + + t.Logf("Uploaded files: %s (ID: %s), %s (ID: %s)", + uploadedFile1.Filename, uploadedFile1.ID, + uploadedFile2.Filename, uploadedFile2.ID) + + // 2. Prepare Vision context with both images + ctx := agentContext.New(context.Background(), nil, "test") + + capabilities := &openai.Capabilities{ + Vision: nil, // No vision support + } + + uses := &agentContext.Uses{ + Vision: "tests.vision-helper", + } + + messages := []agentContext.Message{ + { + Role: "user", + Content: []agentContext.ContentPart{ + { + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: "__" + uploaderName + "://" + uploadedFile1.ID, + }, + }, + { + Type: agentContext.ContentImageURL, + ImageURL: &agentContext.ImageURL{ + URL: "__" + uploaderName + "://" + uploadedFile2.ID, + }, + }, + }, + }, + } + + // 3. Call Vision - should handle multiple files' metadata + result, err := content.Vision(ctx, capabilities, messages, uses, false) + if err != nil { + t.Fatalf("Vision function failed: %v", err) + } + + if len(result) != 1 { + t.Fatalf("Expected 1 message, got %d", len(result)) + } + + // 4. Verify both images were processed + contentParts, ok := result[0].Content.([]agentContext.ContentPart) + if !ok { + t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content) + } + + // Each image should be processed by the vision agent and return text + if len(contentParts) != 2 { + t.Fatalf("Expected 2 content parts (one per image), got %d", len(contentParts)) + } + + for i, part := range contentParts { + if part.Type != agentContext.ContentText { + t.Errorf("Part %d: expected ContentText, got: %s", i, part.Type) + } + if part.Text == "" { + t.Errorf("Part %d: expected non-empty text", i) + } + } + + t.Logf("✓ Multiple files processed with metadata tracking") + t.Logf("File 1 response: %s", contentParts[0].Text) + t.Logf("File 2 response: %s", contentParts[1].Text) +} diff --git a/agent/content/excel.go b/agent/content/excel.go index 61fb4440..24a8a94d 100644 --- a/agent/content/excel.go +++ b/agent/content/excel.go @@ -21,7 +21,7 @@ func (h *ExcelHandler) CanHandle(contentType string, fileType FileType) bool { } // Handle processes Excel spreadsheet content -func (h *ExcelHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +func (h *ExcelHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { // TODO: Implement Excel handling // 1. Extract data from .xlsx or .xls file // 2. Convert to text format (e.g., CSV-like or structured text) diff --git a/agent/content/fetch.go b/agent/content/fetch.go index dcff11a1..eaf35ab5 100644 --- a/agent/content/fetch.go +++ b/agent/content/fetch.go @@ -69,9 +69,14 @@ func (f *DefaultFetcher) fetchUploader(ctx *agentContext.Context, wrapper string // 5. Return Info with data return &Info{ - Data: data, - ContentType: file.ContentType, - FileType: DetectFileType(file.ContentType, file.Filename), + Data: data, + ContentType: file.ContentType, + Filename: file.Filename, + FileType: DetectFileType(file.ContentType, file.Filename), + URL: wrapper, + Source: SourceUploader, + UploaderName: uploaderName, + FileID: fileID, }, nil } diff --git a/agent/content/image.go b/agent/content/image.go index 7eb266f0..7f3ad0b3 100644 --- a/agent/content/image.go +++ b/agent/content/image.go @@ -18,9 +18,10 @@ func (h *ImageHandler) CanHandle(contentType string, fileType FileType) bool { // Handle processes image content // Logic: -// 1. If model supports vision -> convert to base64 or image_url format -// 2. If model doesn't support vision -> use agent/MCP specified in uses.Vision -func (h *ImageHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +// 1. If forceUses is true and uses.Vision is specified -> use vision tool regardless of model capability +// 2. If model supports vision and forceUses is false -> convert to base64 or image_url format +// 3. If model doesn't support vision -> use agent/MCP specified in uses.Vision +func (h *ImageHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { if len(info.Data) == 0 { return nil, fmt.Errorf("no image data to process") } @@ -32,6 +33,17 @@ func (h *ImageHandler) Handle(ctx *agentContext.Context, info *Info, capabilitie // Check if model supports vision supportsVision, visionFormat := agentContext.GetVisionSupport(capabilities) + // If forceUses is true and uses.Vision is specified, use vision tool regardless of model capability + if forceUses && uses != nil && uses.Vision != "" { + text, err := h.handleWithVisionAgent(ctx, info, uses.Vision) + if err != nil { + return nil, fmt.Errorf("failed to handle image with vision agent/MCP (forced): %w", err) + } + return &Result{ + Text: text, + }, nil + } + if supportsVision { // Model supports vision - return as image_url ContentPart contentPart, err := h.handleWithVisionModel(ctx, info, visionFormat) @@ -133,7 +145,7 @@ func (h *ImageHandler) callVisionAgent(ctx *agentContext.Context, agentID string Content: []agentContext.ContentPart{ { Type: agentContext.ContentText, - Text: "Please describe this image in detail.", + Text: "Please analyze this image.", }, { Type: agentContext.ContentImageURL, @@ -145,7 +157,10 @@ func (h *ImageHandler) callVisionAgent(ctx *agentContext.Context, agentID string }, } - return CallAgent(ctx, agentID, message) + // Call agent with file metadata in context + // File info (filename, file_id, etc.) will be available in ctx.Metadata["file_info"] + // This allows hooks (especially Next hook) to access and format file information + return CallAgentWithFileInfo(ctx, agentID, message, info) } // callMCPVisionTool calls an MCP vision tool to describe the image diff --git a/agent/content/image_test.go b/agent/content/image_test.go index fb0e7633..66d2d502 100644 --- a/agent/content/image_test.go +++ b/agent/content/image_test.go @@ -104,7 +104,7 @@ func TestImageHandler_Handle_WithVisionSupport(t *testing.T) { Data: pngData, } - result, err := handler.Handle(ctx, info, capabilities, nil) + result, err := handler.Handle(ctx, info, capabilities, nil, false) if err != nil { t.Fatalf("Handle() error = %v", err) } @@ -160,7 +160,7 @@ func TestImageHandler_Handle_WithoutVisionSupport(t *testing.T) { } // Should return error because no vision support and no tool - _, err := handler.Handle(ctx, info, capabilities, nil) + _, err := handler.Handle(ctx, info, capabilities, nil, false) if err == nil { t.Error("Expected error when no vision support and no tool specified") } @@ -185,7 +185,7 @@ func TestImageHandler_Handle_EmptyData(t *testing.T) { Data: []byte{}, // Empty data } - _, err := handler.Handle(ctx, info, capabilities, nil) + _, err := handler.Handle(ctx, info, capabilities, nil, false) if err == nil { t.Error("Expected error for empty image data") } diff --git a/agent/content/interfaces.go b/agent/content/interfaces.go index c1ed89c2..b6dc5a05 100644 --- a/agent/content/interfaces.go +++ b/agent/content/interfaces.go @@ -15,7 +15,8 @@ type Handler interface { // ctx: agent context (passed from Vision function) // capabilities: model capabilities (for vision/audio support detection) // uses: configuration for external tools (agents/MCP servers) - Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) + // forceUses: if true, force using Uses tools even when model has native capabilities + Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) } // Fetcher defines the interface for fetching content from different sources diff --git a/agent/content/pdf.go b/agent/content/pdf.go index 771d4879..5d14bb53 100644 --- a/agent/content/pdf.go +++ b/agent/content/pdf.go @@ -23,7 +23,7 @@ func (h *PDFHandler) CanHandle(contentType string, fileType FileType) bool { // 1. Check if uses.Vision is specified and supports PDF // 2. If yes, use vision tool to handle PDF (images + text) // 3. If no, extract text directly from PDF -func (h *PDFHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +func (h *PDFHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { // TODO: Implement PDF handling // 1. Check if vision tool supports PDF // 2. If yes: diff --git a/agent/content/registry.go b/agent/content/registry.go index fe3af0bd..77c98d23 100644 --- a/agent/content/registry.go +++ b/agent/content/registry.go @@ -37,11 +37,11 @@ func (r *Registry) GetHandler(contentType string, fileType FileType) Handler { } // Handle processes content using the appropriate handler -func (r *Registry) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +func (r *Registry) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { handler := r.GetHandler(info.ContentType, info.FileType) if handler == nil { return nil, fmt.Errorf("no handler found for content type: %s, file type: %s", info.ContentType, info.FileType) } - return handler.Handle(ctx, info, capabilities, uses) + return handler.Handle(ctx, info, capabilities, uses, forceUses) } diff --git a/agent/content/text.go b/agent/content/text.go index 5be8b693..6f64d03b 100644 --- a/agent/content/text.go +++ b/agent/content/text.go @@ -49,7 +49,7 @@ func (h *TextHandler) CanHandle(contentType string, fileType FileType) bool { } // Handle processes text content -func (h *TextHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +func (h *TextHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { if len(info.Data) == 0 { return nil, fmt.Errorf("no data to process") } diff --git a/agent/content/text_test.go b/agent/content/text_test.go index acc72912..029a4625 100644 --- a/agent/content/text_test.go +++ b/agent/content/text_test.go @@ -104,7 +104,7 @@ func TestTextHandler_Handle(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := handler.Handle(testCtx, tt.info, nil, nil) + result, err := handler.Handle(testCtx, tt.info, nil, nil, false) if (err != nil) != tt.wantErr { t.Errorf("Handle() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/agent/content/tools.go b/agent/content/tools.go index d1f2e9da..9a75436e 100644 --- a/agent/content/tools.go +++ b/agent/content/tools.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "fmt" + "sync" jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/mcp" @@ -19,6 +20,9 @@ type AgentCaller interface { // AgentGetterFunc is a function type that gets an agent by ID var AgentGetterFunc func(agentID string) (AgentCaller, error) +// fileInfoMutex protects concurrent access to files_info list in Space +var fileInfoMutex sync.Mutex + // CallAgent calls an agent to process content (vision, audio, etc.) // This is a generic function that can be used by any handler func CallAgent(ctx *agentContext.Context, agentID string, message agentContext.Message) (string, error) { @@ -51,75 +55,183 @@ func CallAgent(ctx *agentContext.Context, agentID string, message agentContext.M return extractTextFromAgentResponse(response) } +// CallAgentWithFileInfo calls an agent to process content with file metadata +// The file metadata is passed via ctx.Space for access by hooks (especially Next hook) +// Uses Space instead of Metadata to avoid creating context copies and ensure proper cleanup +// +// Space Keys (with agent ID as namespace prefix to avoid conflicts between different agents): +// - {agentID}:files_info - List of all files being processed by this agent (array) +// - {agentID}:current_file - Currently processing file (single object) +func CallAgentWithFileInfo(ctx *agentContext.Context, agentID string, message agentContext.Message, info *Info) (string, error) { + // Store file information in Space if available + if info != nil && ctx.Space != nil { + fileInfo := map[string]interface{}{ + "url": info.URL, + "filename": info.Filename, + "content_type": info.ContentType, + "file_type": string(info.FileType), + "source": string(info.Source), + } + + // Add uploader-specific information if available + if info.UploaderName != "" { + fileInfo["uploader_name"] = info.UploaderName + } + if info.FileID != "" { + fileInfo["file_id"] = info.FileID + } + + // Use agent ID as namespace prefix for Space keys + filesListKey := agentID + ":files_info" + currentFileKey := agentID + ":current_file" + + // Thread-safe: append current file to files list + fileInfoMutex.Lock() + var filesList []map[string]interface{} + if existing, err := ctx.Space.Get(filesListKey); err == nil { + // Convert existing data to []map[string]interface{} + if existingList, ok := existing.([]interface{}); ok { + for _, item := range existingList { + if itemMap, ok := item.(map[string]interface{}); ok { + filesList = append(filesList, itemMap) + } + } + } else if existingList, ok := existing.([]map[string]interface{}); ok { + filesList = existingList + } + } + // Append current file to list + filesList = append(filesList, fileInfo) + ctx.Space.Set(filesListKey, filesList) + fileInfoMutex.Unlock() + + // Store current file in Space + if err := ctx.Space.Set(currentFileKey, fileInfo); err != nil { + log.Trace("[Content] Failed to set current file info in Space: %v", err) + } + + // Ensure cleanup after agent call completes + defer func() { + // Clean up current file + if err := ctx.Space.Delete(currentFileKey); err != nil { + log.Trace("[Content] Failed to delete current file info from Space: %v", err) + } + // Clean up files list (reset for next call) + if err := ctx.Space.Delete(filesListKey); err != nil { + log.Trace("[Content] Failed to delete files list from Space: %v", err) + } + }() + } + + // Call the agent with the original context + return CallAgent(ctx, agentID, message) +} + // extractTextFromAgentResponse extracts text from agent response -// Handles two response formats: -// 1. Custom Hook response: if it's a string, return directly; otherwise JSON stringify -// 2. Standard response: extract from completion.content +// Handles two main response formats from agent.Stream(): +// +// 1. Standard Response (No Next Hook or Next Hook returns nil): +// Structure: { completion: { content: "text" | [...ContentPart] } } +// Action: Extract text from completion.content field +// +// 2. Next Hook Response with Custom Data: +// Structure: { next: } +// Action: +// - If next is string → return directly +// - If next is map/object → JSON stringify and return +// - This preserves the complete custom data structure from the hook +// +// Priority: +// 1. Check for "next" field (custom hook data) → return complete data +// 2. Check for "completion" field (standard LLM response) → extract text only +// 3. Fallback to direct string or JSON stringify func extractTextFromAgentResponse(response interface{}) (string, error) { if response == nil { return "", fmt.Errorf("agent returned nil response") } - // Try to parse as standard response format (has "completion" field with LLM result) - if responseMap, ok := response.(map[string]interface{}); ok { - // Check for completion field (standard LLM response) - if completion, hasCompletion := responseMap["completion"]; hasCompletion { - if completionMap, ok := completion.(map[string]interface{}); ok { - // Extract content from completion - if content, hasContent := completionMap["content"]; hasContent { - // Content can be string or structured - switch v := content.(type) { - case string: - return v, nil - case []interface{}: - // Handle multimodal content array - var text string - for _, part := range v { - if partMap, ok := part.(map[string]interface{}); ok { - if partType, _ := partMap["type"].(string); partType == "text" { - if textContent, ok := partMap["text"].(string); ok { - text += textContent - } + // First, try to convert to map if it's a struct + // agent.Stream() may return *agentContext.Response which needs to be converted + var responseMap map[string]interface{} + + // Check if it's already a map + if rm, ok := response.(map[string]interface{}); ok { + responseMap = rm + } else { + // Try to marshal and unmarshal to convert struct to map + jsonBytes, err := jsoniter.Marshal(response) + if err != nil { + // If it's a plain string, return directly + if responseStr, ok := response.(string); ok { + return responseStr, nil + } + return "", fmt.Errorf("failed to serialize agent response: %w", err) + } + + // Unmarshal to map + if err := jsoniter.Unmarshal(jsonBytes, &responseMap); err != nil { + // If unmarshal fails, return the JSON string + return string(jsonBytes), nil + } + } + + // Priority 1: Check for "next" field (custom hook data) + // If Next hook returns custom data, it's stored in the "next" field + // Return the complete custom data structure (preserve hook's intent) + if next, hasNext := responseMap["next"]; hasNext && next != nil { + // If next is a string, return directly + if nextStr, ok := next.(string); ok { + return nextStr, nil + } + // Otherwise, JSON stringify to preserve complete structure + jsonBytes, err := jsoniter.Marshal(next) + if err != nil { + return "", fmt.Errorf("failed to serialize next hook data: %w", err) + } + return string(jsonBytes), nil + } + + // Priority 2: Check for "completion" field (standard LLM response) + // Extract text content from the LLM completion + if completion, hasCompletion := responseMap["completion"]; hasCompletion && completion != nil { + if completionMap, ok := completion.(map[string]interface{}); ok { + // Extract content from completion + if content, hasContent := completionMap["content"]; hasContent { + // Content can be string or []ContentPart (multimodal) + switch v := content.(type) { + case string: + // Simple text content + return v, nil + case []interface{}: + // Multimodal content array - extract all text parts + var text string + for _, part := range v { + if partMap, ok := part.(map[string]interface{}); ok { + if partType, _ := partMap["type"].(string); partType == "text" { + if textContent, ok := partMap["text"].(string); ok { + text += textContent } } } - if text != "" { - return text, nil - } } + if text != "" { + return text, nil + } + // No text found in content parts + return "", fmt.Errorf("no text content found in completion content parts") } } } + } - // Check for data field (custom hook response with data wrapper) - if data, hasData := responseMap["data"]; hasData { - // If data is a string, return directly - if dataStr, ok := data.(string); ok { - return dataStr, nil - } - // Otherwise, JSON stringify - jsonBytes, err := jsoniter.Marshal(data) - if err != nil { - return "", fmt.Errorf("failed to serialize hook data response: %w", err) - } - return string(jsonBytes), nil - } - - // If the map itself looks like content, try to extract - // This handles cases where the response is the content directly - if content, hasContent := responseMap["content"]; hasContent { - if contentStr, ok := content.(string); ok { - return contentStr, nil - } + // Fallback: Try to find a "content" field directly (shouldn't happen normally) + if content, hasContent := responseMap["content"]; hasContent { + if contentStr, ok := content.(string); ok { + return contentStr, nil } } - // Custom Hook response: if it's a plain string, return directly - if responseStr, ok := response.(string); ok { - return responseStr, nil - } - - // Otherwise, JSON stringify the response + // Last resort: JSON stringify the entire response jsonBytes, err := jsoniter.Marshal(response) if err != nil { return "", fmt.Errorf("failed to serialize agent response: %w", err) diff --git a/agent/content/types.go b/agent/content/types.go index f0399854..b9d4fd70 100644 --- a/agent/content/types.go +++ b/agent/content/types.go @@ -55,6 +55,7 @@ type Info struct { FileType FileType // Type of the file ContentType string // MIME content type URL string // Original URL or file ID + Filename string // Original filename (if available) Data []byte // File data (if already fetched) // For uploader wrapper diff --git a/agent/content/word.go b/agent/content/word.go index d854e12f..9b438097 100644 --- a/agent/content/word.go +++ b/agent/content/word.go @@ -20,7 +20,7 @@ func (h *WordHandler) CanHandle(contentType string, fileType FileType) bool { } // Handle processes Word document content -func (h *WordHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses) (*Result, error) { +func (h *WordHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) { // TODO: Implement Word document handling // 1. Extract text from .docx or .doc file // 2. Preserve formatting information if needed diff --git a/agent/context/jsapi.go b/agent/context/jsapi.go index 2ecb77c4..4f6b5ea0 100644 --- a/agent/context/jsapi.go +++ b/agent/context/jsapi.go @@ -60,6 +60,8 @@ func (ctx *Context) NewObject(v8ctx *v8go.Context) (*v8go.Value, error) { // Set MCP object jsObject.Set("MCP", ctx.newMCPObject(v8ctx.Isolate())) + // Note: Space object will be set after instance creation (requires v8ctx) + // Create instance instance, err := jsObject.NewInstance(v8ctx) if err != nil { @@ -129,6 +131,13 @@ func (ctx *Context) NewObject(v8ctx *v8go.Context) (*v8go.Value, error) { } } + // Space object - create a JavaScript object with Get/Set/Delete methods + if ctx.Space != nil { + spaceObj := ctx.createSpaceObject(v8ctx) + obj.Set("space", spaceObj) + spaceObj.Release() + } + return instance.Value, nil } @@ -607,6 +616,87 @@ func (ctx *Context) endBlockMethod(iso *v8go.Isolate) *v8go.FunctionTemplate { }) } +// createSpaceObject creates a Space object for JavaScript access +// Space is a shared data space for passing data between requests and calls +func (ctx *Context) createSpaceObject(v8ctx *v8go.Context) *v8go.Value { + iso := v8ctx.Isolate() + spaceObj, _ := v8ctx.RunScript("({})", "space-init") + obj, _ := spaceObj.AsObject() + + // Get method: space.Get(key) + getFunc := v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + if ctx.Space == nil { + return v8go.Null(iso) + } + + if len(info.Args()) < 1 { + return bridge.JsException(info.Context(), "Get requires a key argument") + } + + key := info.Args()[0].String() + value, err := ctx.Space.Get(key) + if err != nil { + return v8go.Null(iso) + } + + jsValue, err := bridge.JsValue(info.Context(), value) + if err != nil { + return v8go.Null(iso) + } + + return jsValue + }) + getFuncVal := getFunc.GetFunction(v8ctx) + obj.Set("Get", getFuncVal.Value) + + // Set method: space.Set(key, value) + setFunc := v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + if ctx.Space == nil { + return bridge.JsException(info.Context(), "Space is not available") + } + + if len(info.Args()) < 2 { + return bridge.JsException(info.Context(), "Set requires key and value arguments") + } + + key := info.Args()[0].String() + value, err := bridge.GoValue(info.Args()[1], info.Context()) + if err != nil { + return bridge.JsException(info.Context(), "Failed to convert value: "+err.Error()) + } + + if err := ctx.Space.Set(key, value); err != nil { + return bridge.JsException(info.Context(), "Failed to set value: "+err.Error()) + } + + return v8go.Undefined(iso) + }) + setFuncVal := setFunc.GetFunction(v8ctx) + obj.Set("Set", setFuncVal.Value) + + // Delete method: space.Delete(key) + delFunc := v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value { + if ctx.Space == nil { + return bridge.JsException(info.Context(), "Space is not available") + } + + if len(info.Args()) < 1 { + return bridge.JsException(info.Context(), "Delete requires a key argument") + } + + key := info.Args()[0].String() + if err := ctx.Space.Delete(key); err != nil { + return bridge.JsException(info.Context(), "Failed to delete key: "+err.Error()) + } + + return v8go.Undefined(iso) + }) + delFuncVal := delFunc.GetFunction(v8ctx) + obj.Set("Delete", delFuncVal.Value) + + return spaceObj +} + // sendGroupMethod implements ctx.SendGroup(group) // Usage: ctx.SendGroup({ id: "group1", messages: [...] }) // Automatically generates IDs, sends group_start/group_end events, and flushes output diff --git a/agent/context/jsapi_space_test.go b/agent/context/jsapi_space_test.go new file mode 100644 index 00000000..6452c5da --- /dev/null +++ b/agent/context/jsapi_space_test.go @@ -0,0 +1,572 @@ +package context_test + +import ( + stdContext "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/plan" + v8 "github.com/yaoapp/gou/runtime/v8" + "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestSpaceSetAndGet tests ctx.space.Set and ctx.space.Get +func TestSpaceSetAndGet(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Set various types of values + ctx.space.Set("string_key", "hello world"); + ctx.space.Set("number_key", 42); + ctx.space.Set("boolean_key", true); + ctx.space.Set("object_key", { name: "test", value: 123 }); + ctx.space.Set("array_key", [1, 2, 3, 4, 5]); + + // Get values back + const str = ctx.space.Get("string_key"); + const num = ctx.space.Get("number_key"); + const bool = ctx.space.Get("boolean_key"); + const obj = ctx.space.Get("object_key"); + const arr = ctx.space.Get("array_key"); + + // Verify values + if (str !== "hello world") throw new Error("String mismatch"); + if (num !== 42) throw new Error("Number mismatch"); + if (bool !== true) throw new Error("Boolean mismatch"); + if (obj.name !== "test" || obj.value !== 123) throw new Error("Object mismatch"); + if (arr.length !== 5 || arr[0] !== 1) throw new Error("Array mismatch"); + + return { + success: true, + str: str, + num: num, + bool: bool, + obj: obj, + arr: arr + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + if !result["success"].(bool) { + t.Fatalf("Test failed: %v", result["error"]) + } + + assert.Equal(t, true, result["success"], "Space Set/Get should succeed") + assert.Equal(t, "hello world", result["str"], "String should match") + assert.Equal(t, float64(42), result["num"], "Number should match") + assert.Equal(t, true, result["bool"], "Boolean should match") +} + +// TestSpaceGetNonExistentKey tests getting a non-existent key +func TestSpaceGetNonExistentKey(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Get non-existent key should return null/undefined + const value = ctx.space.Get("non_existent_key"); + + return { + success: true, + value: value, + is_null: value === null, + is_undefined: value === undefined + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + assert.Equal(t, true, result["success"], "Get non-existent key should succeed") + // JavaScript null is returned as nil in Go + assert.True(t, result["is_null"].(bool) || result["is_undefined"].(bool), "Non-existent key should return null or undefined") +} + +// TestSpaceDelete tests ctx.space.Delete +func TestSpaceDelete(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Set a value + ctx.space.Set("delete_me", "temporary value"); + + // Verify it exists + const before = ctx.space.Get("delete_me"); + if (before !== "temporary value") throw new Error("Value not set correctly"); + + // Delete it + ctx.space.Delete("delete_me"); + + // Verify it's gone + const after = ctx.space.Get("delete_me"); + + return { + success: true, + before: before, + after: after, + is_deleted: after === null || after === undefined + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + if !result["success"].(bool) { + t.Fatalf("Test failed: %v", result["error"]) + } + + assert.Equal(t, true, result["success"], "Space Delete should succeed") + assert.Equal(t, "temporary value", result["before"], "Value should exist before delete") + assert.Equal(t, true, result["is_deleted"], "Value should be deleted") +} + +// TestSpaceDeleteNonExistentKey tests deleting a non-existent key +func TestSpaceDeleteNonExistentKey(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Delete non-existent key should not throw error + ctx.space.Delete("non_existent_key"); + + return { success: true }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + assert.Equal(t, true, result["success"], "Delete non-existent key should not throw error") +} + +// TestSpaceWithNamespace tests using Space with namespace prefixes (like agent IDs) +func TestSpaceWithNamespace(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Simulate namespace pattern used in voucher assistant + const agentID = "workers.voucher"; + + // Set files_info with namespace + const filesInfo = [ + { + file_id: "abc123", + filename: "test.png", + content_type: "image/png", + file_type: "image", + source: "uploader" + } + ]; + ctx.space.Set(agentID + ":files_info", filesInfo); + + // Set current_file with namespace + const currentFile = { + file_id: "abc123", + filename: "test.png", + content_type: "image/png" + }; + ctx.space.Set(agentID + ":current_file", currentFile); + + // Read back with namespace + const retrievedFiles = ctx.space.Get(agentID + ":files_info"); + const retrievedCurrent = ctx.space.Get(agentID + ":current_file"); + + // Verify + if (!Array.isArray(retrievedFiles)) throw new Error("files_info should be array"); + if (retrievedFiles.length !== 1) throw new Error("files_info length mismatch"); + if (retrievedFiles[0].file_id !== "abc123") throw new Error("file_id mismatch"); + if (retrievedCurrent.filename !== "test.png") throw new Error("filename mismatch"); + + return { + success: true, + files_count: retrievedFiles.length, + file_id: retrievedFiles[0].file_id, + current_filename: retrievedCurrent.filename + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + if !result["success"].(bool) { + t.Fatalf("Test failed: %v", result["error"]) + } + + assert.Equal(t, true, result["success"], "Namespace operations should succeed") + assert.Equal(t, float64(1), result["files_count"], "Should have 1 file") + assert.Equal(t, "abc123", result["file_id"], "File ID should match") + assert.Equal(t, "test.png", result["current_filename"], "Filename should match") +} + +// TestSpaceComplexData tests Space with complex nested data structures +func TestSpaceComplexData(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Complex nested structure + const complexData = { + metadata: { + assistant_id: "tests.vision-helper", + has_files_info: true, + files_count: 2 + }, + files_info: [ + { + file_id: "file1", + filename: "image1.png", + content_type: "image/png", + metadata: { + size: 1024, + created: Date.now() + } + }, + { + file_id: "file2", + filename: "image2.jpg", + content_type: "image/jpeg", + metadata: { + size: 2048, + created: Date.now() + } + } + ], + tags: ["vision", "test", "multi-file"] + }; + + ctx.space.Set("complex_data", complexData); + + // Retrieve and verify + const retrieved = ctx.space.Get("complex_data"); + + if (!retrieved) throw new Error("Data not retrieved"); + if (!retrieved.metadata) throw new Error("Metadata missing"); + if (retrieved.metadata.files_count !== 2) throw new Error("Files count mismatch"); + if (!Array.isArray(retrieved.files_info)) throw new Error("files_info not array"); + if (retrieved.files_info.length !== 2) throw new Error("files_info length mismatch"); + if (!Array.isArray(retrieved.tags)) throw new Error("tags not array"); + if (retrieved.tags[0] !== "vision") throw new Error("tags mismatch"); + + return { + success: true, + files_count: retrieved.files_info.length, + first_file_id: retrieved.files_info[0].file_id, + second_filename: retrieved.files_info[1].filename, + tags: retrieved.tags + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + if !result["success"].(bool) { + t.Fatalf("Test failed: %v", result["error"]) + } + + assert.Equal(t, true, result["success"], "Complex data operations should succeed") + assert.Equal(t, float64(2), result["files_count"], "Should have 2 files") + assert.Equal(t, "file1", result["first_file_id"], "First file ID should match") + assert.Equal(t, "image2.jpg", result["second_filename"], "Second filename should match") +} + +// TestSpaceOverwrite tests overwriting existing values +func TestSpaceOverwrite(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Set initial value + ctx.space.Set("counter", 1); + const first = ctx.space.Get("counter"); + + // Overwrite with new value + ctx.space.Set("counter", 2); + const second = ctx.space.Get("counter"); + + // Overwrite again + ctx.space.Set("counter", 3); + const third = ctx.space.Get("counter"); + + return { + success: true, + first: first, + second: second, + third: third + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + if !result["success"].(bool) { + t.Fatalf("Test failed: %v", result["error"]) + } + + assert.Equal(t, true, result["success"], "Overwrite operations should succeed") + assert.Equal(t, float64(1), result["first"], "First value should be 1") + assert.Equal(t, float64(2), result["second"], "Second value should be 2") + assert.Equal(t, float64(3), result["third"], "Third value should be 3") +} + +// TestSpaceNoSpace tests behavior when Space is nil +func TestSpaceNoSpace(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: nil, // No Space + } + + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // ctx.space should be undefined when Space is nil + const hasSpace = ctx.space !== undefined && ctx.space !== null; + + return { + success: true, + has_space: hasSpace + }; + } catch (error) { + return { success: false, error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + assert.Equal(t, true, result["success"], "Should handle nil Space gracefully") + assert.Equal(t, false, result["has_space"], "Should not have space when Space is nil") +} + +// TestSpaceErrorHandling tests error handling in Space methods +func TestSpaceErrorHandling(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + ctx := &context.Context{ + ChatID: "test-chat-id", + AssistantID: "test-assistant-id", + Locale: "en", + Context: stdContext.Background(), + Space: plan.NewMemorySharedSpace(), + } + + // Test Set without key + res, err := v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Set without proper arguments should throw + ctx.space.Set(); + return { success: false, error: "Should have thrown" }; + } catch (error) { + return { success: true, caught_error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok := res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + assert.Equal(t, true, result["success"], "Should catch Set error") + + // Test Get without key + res, err = v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Get without key should throw + ctx.space.Get(); + return { success: false, error: "Should have thrown" }; + } catch (error) { + return { success: true, caught_error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok = res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + assert.Equal(t, true, result["success"], "Should catch Get error") + + // Test Delete without key + res, err = v8.Call(v8.CallOptions{}, ` + function test(ctx) { + try { + // Delete without key should throw + ctx.space.Delete(); + return { success: false, error: "Should have thrown" }; + } catch (error) { + return { success: true, caught_error: error.message }; + } + }`, ctx) + + if err != nil { + t.Fatalf("Call failed: %v", err) + } + + result, ok = res.(map[string]interface{}) + if !ok { + t.Fatalf("Expected map result, got %T", res) + } + + assert.Equal(t, true, result["success"], "Should catch Delete error") +} diff --git a/agent/context/types.go b/agent/context/types.go index 15f6b605..8815c312 100644 --- a/agent/context/types.go +++ b/agent/context/types.go @@ -357,6 +357,12 @@ type HookCreateResponse struct { Theme string `json:"theme,omitempty"` // Override theme (session-level) Route string `json:"route,omitempty"` // Override route (session-level) Metadata map[string]interface{} `json:"metadata,omitempty"` // Override or merge metadata (session-level) + + // Uses configuration - allow hook to override wrapper configurations + Uses *Uses `json:"uses,omitempty"` // Override wrapper configurations for vision, audio, search, and fetch + + // ForceUses controls whether to force using Uses tools even when model has native capabilities + ForceUses *bool `json:"force_uses,omitempty"` // Force using Uses tools regardless of model capabilities } // NextHookPayload payload for the next hook diff --git a/agent/context/types_llm.go b/agent/context/types_llm.go index 541e51b8..ed5d09b2 100644 --- a/agent/context/types_llm.go +++ b/agent/context/types_llm.go @@ -69,6 +69,12 @@ type CompletionOptions struct { // User-specified tools for vision, audio, search, and fetch processing Uses *Uses `json:"uses,omitempty"` + // ForceUses controls whether to force using Uses tools even when model has native capabilities + // When true: Always use tools specified in Uses, ignore model's native multimodal capabilities + // When false (default): Use model's native capabilities if available, fallback to Uses tools + // This is useful when you want consistent behavior across different models or prefer specific tools + ForceUses bool `json:"force_uses,omitempty"` + // Audio configuration (for models that support audio output) Audio *AudioConfig `json:"audio,omitempty"`