diff --git a/agent/sandbox/claude/attachments_test.go b/agent/sandbox/claude/attachments_test.go new file mode 100644 index 00000000..b7caabff --- /dev/null +++ b/agent/sandbox/claude/attachments_test.go @@ -0,0 +1,316 @@ +package claude + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + agentContext "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestExtensionFromContentType(t *testing.T) { + tests := []struct { + contentType string + expected string + }{ + {"image/png", ".png"}, + {"image/jpeg", ".jpg"}, + {"image/gif", ".gif"}, + {"image/webp", ".webp"}, + {"image/svg+xml", ".svg"}, + {"application/pdf", ".pdf"}, + {"text/plain", ".txt"}, + {"text/html", ".html"}, + {"text/css", ".css"}, + {"text/javascript", ".js"}, + {"application/javascript", ".js"}, + {"application/json", ".json"}, + {"application/zip", ".zip"}, + {"application/octet-stream", ""}, + {"unknown/type", ""}, + } + + for _, tt := range tests { + t.Run(tt.contentType, func(t *testing.T) { + assert.Equal(t, tt.expected, extensionFromContentType(tt.contentType)) + }) + } +} + +func TestFormatFileSize(t *testing.T) { + tests := []struct { + bytes int + expected string + }{ + {0, "0B"}, + {100, "100B"}, + {1023, "1023B"}, + {1024, "1.0KB"}, + {1536, "1.5KB"}, + {10240, "10.0KB"}, + {1048576, "1.0MB"}, + {1572864, "1.5MB"}, + {10485760, "10.0MB"}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%d", tt.bytes), func(t *testing.T) { + assert.Equal(t, tt.expected, formatFileSize(tt.bytes)) + }) + } +} + +func TestPrepareAttachmentsPlainText(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager := createTestManager(t) + if manager == nil { + return + } + defer manager.Close() + + opts := &Options{ + Command: "claude", + Image: "alpine:latest", + UserID: "test-user", + ChatID: fmt.Sprintf("test-chat-att-plain-%d", time.Now().UnixNano()), + } + + exec, err := NewExecutor(manager, opts) + require.NoError(t, err) + defer exec.Close() + + ctx := context.Background() + + // Plain text messages should pass through unchanged + messages := []agentContext.Message{ + {Role: "system", Content: "You are a helpful assistant"}, + {Role: "user", Content: "Hello, world!"}, + {Role: "assistant", Content: "Hi there!"}, + {Role: "user", Content: "What is 1+1?"}, + } + + result, err := exec.prepareAttachments(ctx, messages) + require.NoError(t, err) + require.Len(t, result, 4) + + // Verify messages are unchanged + assert.Equal(t, "system", string(result[0].Role)) + assert.Equal(t, "You are a helpful assistant", result[0].Content) + assert.Equal(t, "Hello, world!", result[1].Content) + assert.Equal(t, "Hi there!", result[2].Content) + assert.Equal(t, "What is 1+1?", result[3].Content) +} + +func TestPrepareAttachmentsMultimodalNoWrapper(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager := createTestManager(t) + if manager == nil { + return + } + defer manager.Close() + + opts := &Options{ + Command: "claude", + Image: "alpine:latest", + UserID: "test-user", + ChatID: fmt.Sprintf("test-chat-att-nowrap-%d", time.Now().UnixNano()), + } + + exec, err := NewExecutor(manager, opts) + require.NoError(t, err) + defer exec.Close() + + ctx := context.Background() + + // Multimodal message with a non-wrapper URL (e.g. regular http URL) + // Should convert to text description but not try to resolve attachment + messages := []agentContext.Message{ + { + Role: "user", + Content: []interface{}{ + map[string]interface{}{"type": "text", "text": "Look at this"}, + map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{ + "url": "https://example.com/image.png", + "detail": "auto", + }, + }, + }, + }, + } + + result, err := exec.prepareAttachments(ctx, messages) + require.NoError(t, err) + require.Len(t, result, 1) + + // Content should be converted to text with URL reference + content, ok := result[0].Content.(string) + require.True(t, ok, "Content should be converted to string") + assert.Contains(t, content, "Look at this") + assert.Contains(t, content, "[Image: https://example.com/image.png]") +} + +func TestPrepareAttachmentsTextOnlyMultimodal(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager := createTestManager(t) + if manager == nil { + return + } + defer manager.Close() + + opts := &Options{ + Command: "claude", + Image: "alpine:latest", + UserID: "test-user", + ChatID: fmt.Sprintf("test-chat-att-textonly-%d", time.Now().UnixNano()), + } + + exec, err := NewExecutor(manager, opts) + require.NoError(t, err) + defer exec.Close() + + ctx := context.Background() + + // Multimodal message with only text parts + messages := []agentContext.Message{ + { + Role: "user", + Content: []interface{}{ + map[string]interface{}{"type": "text", "text": "Hello"}, + map[string]interface{}{"type": "text", "text": "World"}, + }, + }, + } + + result, err := exec.prepareAttachments(ctx, messages) + require.NoError(t, err) + require.Len(t, result, 1) + + // Should combine text parts + content, ok := result[0].Content.(string) + require.True(t, ok, "Content should be converted to string") + assert.Contains(t, content, "Hello") + assert.Contains(t, content, "World") +} + +func TestPrepareAttachmentsInvalidWrapperURL(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager := createTestManager(t) + if manager == nil { + return + } + defer manager.Close() + + opts := &Options{ + Command: "claude", + Image: "alpine:latest", + UserID: "test-user", + ChatID: fmt.Sprintf("test-chat-att-invalid-%d", time.Now().UnixNano()), + } + + exec, err := NewExecutor(manager, opts) + require.NoError(t, err) + defer exec.Close() + + ctx := context.Background() + + // Message with an attachment URL pointing to a non-existent manager + messages := []agentContext.Message{ + { + Role: "user", + Content: []interface{}{ + map[string]interface{}{"type": "text", "text": "See this image"}, + map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{ + "url": "__nonexistent.uploader://fakefile123", + "detail": "auto", + }, + }, + }, + }, + } + + result, err := exec.prepareAttachments(ctx, messages) + require.NoError(t, err) + require.Len(t, result, 1) + + // Should gracefully fallback to error text + content, ok := result[0].Content.(string) + require.True(t, ok, "Content should be converted to string") + assert.Contains(t, content, "See this image") + assert.Contains(t, content, "failed to load") +} + +func TestPrepareAttachmentsMixedRoles(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager := createTestManager(t) + if manager == nil { + return + } + defer manager.Close() + + opts := &Options{ + Command: "claude", + Image: "alpine:latest", + UserID: "test-user", + ChatID: fmt.Sprintf("test-chat-att-mixed-%d", time.Now().UnixNano()), + } + + exec, err := NewExecutor(manager, opts) + require.NoError(t, err) + defer exec.Close() + + ctx := context.Background() + + // Only user messages should be processed; system and assistant messages pass through + messages := []agentContext.Message{ + {Role: "system", Content: "System prompt"}, + { + Role: "user", + Content: []interface{}{ + map[string]interface{}{"type": "text", "text": "User message with image"}, + map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{ + "url": "https://example.com/photo.jpg", + "detail": "auto", + }, + }, + }, + }, + {Role: "assistant", Content: "I can see the photo"}, + {Role: "user", Content: "Thanks!"}, + } + + result, err := exec.prepareAttachments(ctx, messages) + require.NoError(t, err) + require.Len(t, result, 4) + + // System and assistant messages unchanged + assert.Equal(t, "System prompt", result[0].Content) + assert.Equal(t, "I can see the photo", result[2].Content) + assert.Equal(t, "Thanks!", result[3].Content) + + // User multimodal message converted + content, ok := result[1].Content.(string) + require.True(t, ok, "User multimodal content should be converted to string") + assert.Contains(t, content, "User message with image") + assert.Contains(t, content, "[Image: https://example.com/photo.jpg]") +} diff --git a/agent/sandbox/claude/command.go b/agent/sandbox/claude/command.go index a2bfc8fc..7736206f 100644 --- a/agent/sandbox/claude/command.go +++ b/agent/sandbox/claude/command.go @@ -34,6 +34,12 @@ The following tools are NOT available in this environment and you must NOT use t Focus on using the core tools: Bash, Read, Write, Edit, Glob, Grep, WebSearch, WebFetch. +## User Attachments + +User-uploaded files (images, documents, code files, etc.) are placed in /workspace/.attachments/ +When the user references an attached file, read it from this directory using the Read or Bash tool. +For image files, you can view them directly as Claude supports vision on local files. + ## GitHub CLI (gh) Usage When working with GitHub and a token is provided: diff --git a/agent/sandbox/claude/executor.go b/agent/sandbox/claude/executor.go index 10273830..53599f48 100644 --- a/agent/sandbox/claude/executor.go +++ b/agent/sandbox/claude/executor.go @@ -16,6 +16,7 @@ import ( agentContext "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" "github.com/yaoapp/yao/agent/output/message" + "github.com/yaoapp/yao/attachment" infraSandbox "github.com/yaoapp/yao/sandbox" "github.com/yaoapp/yao/sandbox/ipc" ) @@ -175,6 +176,15 @@ func (e *Executor) Stream(ctx *agentContext.Context, messages []agentContext.Mes return nil, fmt.Errorf("failed to prepare environment: %w", err) } + // Resolve attachment URLs and write files to container + // This converts __yao.attachment:// URLs to local file paths in /workspace/.attachments/ + if resolved, attErr := e.prepareAttachments(stdCtx, messages); attErr != nil { + // Non-fatal: log warning and continue with original messages + log.Printf("[sandbox] Warning: failed to prepare attachments: %v", attErr) + } else { + messages = resolved + } + // Check if we should skip Claude CLI execution // Skip if no prompts, no skills, and no MCP config if e.shouldSkipClaudeCLI() { @@ -407,6 +417,272 @@ func (e *Executor) copySkillsDirectory(ctx context.Context) error { return nil } +// prepareAttachments resolves __yao.attachment:// URLs in messages, +// writes the actual files to the container's /workspace/.attachments/ directory, +// and replaces the attachment content parts with text references to the file paths. +// This allows Claude CLI to read the files using its built-in Read/Bash tools. +func (e *Executor) prepareAttachments(ctx context.Context, messages []agentContext.Message) ([]agentContext.Message, error) { + // Track used filenames to handle duplicates + usedNames := make(map[string]int) + attachmentDir := e.workDir + "/.attachments" + dirCreated := false + hasAttachments := false + + result := make([]agentContext.Message, len(messages)) + copy(result, messages) + + for i, msg := range result { + if msg.Role != "user" { + continue + } + + // Handle content array (multimodal messages come as []interface{} from JSON) + parts, ok := msg.Content.([]interface{}) + if !ok { + // Try typed content parts + if typedParts, ok := msg.Content.([]agentContext.ContentPart); ok { + iparts := make([]interface{}, len(typedParts)) + for j, p := range typedParts { + // Convert to map for uniform handling + m := map[string]interface{}{"type": string(p.Type)} + if p.Text != "" { + m["text"] = p.Text + } + if p.ImageURL != nil { + m["image_url"] = map[string]interface{}{ + "url": p.ImageURL.URL, + "detail": string(p.ImageURL.Detail), + } + } + if p.File != nil { + m["file"] = map[string]interface{}{ + "url": p.File.URL, + "filename": p.File.Filename, + } + } + iparts[j] = m + } + parts = iparts + } else { + continue + } + } + + if len(parts) == 0 { + continue + } + + // Process each content part + var textParts []string + modified := false + + for _, item := range parts { + m, ok := item.(map[string]interface{}) + if !ok { + continue + } + + partType, _ := m["type"].(string) + + switch partType { + case "text": + if text, ok := m["text"].(string); ok && text != "" { + textParts = append(textParts, text) + } + + case "image_url": + imgData, _ := m["image_url"].(map[string]interface{}) + if imgData == nil { + continue + } + url, _ := imgData["url"].(string) + if url == "" { + continue + } + + uploaderName, fileID, isWrapper := attachment.Parse(url) + if !isWrapper { + // Not an attachment URL, keep as text reference + textParts = append(textParts, fmt.Sprintf("[Image: %s]", url)) + modified = true + continue + } + + // Resolve the attachment + ref, err := e.resolveAttachment(ctx, uploaderName, fileID, "", attachmentDir, usedNames, &dirCreated) + if err != nil { + log.Printf("[sandbox] Warning: failed to resolve image attachment %s: %v", fileID, err) + textParts = append(textParts, "[Attached image: failed to load]") + modified = true + continue + } + + textParts = append(textParts, ref) + hasAttachments = true + modified = true + + case "file": + fileData, _ := m["file"].(map[string]interface{}) + if fileData == nil { + continue + } + url, _ := fileData["url"].(string) + hintName, _ := fileData["filename"].(string) + if url == "" { + continue + } + + uploaderName, fileID, isWrapper := attachment.Parse(url) + if !isWrapper { + textParts = append(textParts, fmt.Sprintf("[File: %s]", url)) + modified = true + continue + } + + ref, err := e.resolveAttachment(ctx, uploaderName, fileID, hintName, attachmentDir, usedNames, &dirCreated) + if err != nil { + log.Printf("[sandbox] Warning: failed to resolve file attachment %s: %v", fileID, err) + textParts = append(textParts, "[Attached file: failed to load]") + modified = true + continue + } + + textParts = append(textParts, ref) + hasAttachments = true + modified = true + + default: + // Keep other types as-is (shouldn't happen normally) + continue + } + } + + if modified && len(textParts) > 0 { + newMsg := result[i] + newMsg.Content = strings.Join(textParts, "\n\n") + result[i] = newMsg + } + } + + if !hasAttachments { + return result, nil + } + + return result, nil +} + +// resolveAttachment reads an attachment from the attachment manager and writes it +// to the container's .attachments directory. Returns a text reference string. +func (e *Executor) resolveAttachment( + ctx context.Context, + uploaderName, fileID, hintName, attachmentDir string, + usedNames map[string]int, + dirCreated *bool, +) (string, error) { + // Get attachment manager + manager, exists := attachment.Managers[uploaderName] + if !exists { + return "", fmt.Errorf("attachment manager not found: %s", uploaderName) + } + + // Get file info + fileInfo, err := manager.Info(ctx, fileID) + if err != nil { + return "", fmt.Errorf("failed to get file info: %w", err) + } + + // Read file data + data, err := manager.Read(ctx, fileID) + if err != nil { + return "", fmt.Errorf("failed to read file: %w", err) + } + + // Determine filename + filename := fileInfo.Filename + if filename == "" && hintName != "" { + filename = hintName + } + if filename == "" { + // Fallback: use fileID with extension from content type + ext := extensionFromContentType(fileInfo.ContentType) + filename = fileID + ext + } + + // Handle duplicate filenames + baseName := filename + if count, exists := usedNames[baseName]; exists { + ext := filepath.Ext(filename) + name := strings.TrimSuffix(filename, ext) + filename = fmt.Sprintf("%s_%d%s", name, count+1, ext) + usedNames[baseName] = count + 1 + } else { + usedNames[baseName] = 0 + } + + // Create attachments directory if not yet created + if !*dirCreated { + if err := e.manager.WriteFile(ctx, e.containerName, attachmentDir+"/.keep", []byte("")); err != nil { + return "", fmt.Errorf("failed to create attachments directory: %w", err) + } + *dirCreated = true + } + + // Write file to container + containerPath := attachmentDir + "/" + filename + if err := e.manager.WriteFile(ctx, e.containerName, containerPath, data); err != nil { + return "", fmt.Errorf("failed to write file to container: %w", err) + } + + // Build human-readable size string + sizeStr := formatFileSize(fileInfo.Bytes) + + // Return text reference + return fmt.Sprintf("[Attached file: %s (%s, %s)]", containerPath, fileInfo.ContentType, sizeStr), nil +} + +// extensionFromContentType returns a file extension for a given content type +func extensionFromContentType(contentType string) string { + switch contentType { + case "image/png": + return ".png" + case "image/jpeg": + return ".jpg" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + case "image/svg+xml": + return ".svg" + case "application/pdf": + return ".pdf" + case "text/plain": + return ".txt" + case "text/html": + return ".html" + case "text/css": + return ".css" + case "text/javascript", "application/javascript": + return ".js" + case "application/json": + return ".json" + case "application/zip": + return ".zip" + default: + return "" + } +} + +// formatFileSize returns a human-readable file size string +func formatFileSize(bytes int) string { + if bytes < 1024 { + return fmt.Sprintf("%dB", bytes) + } + if bytes < 1024*1024 { + return fmt.Sprintf("%.1fKB", float64(bytes)/1024) + } + return fmt.Sprintf("%.1fMB", float64(bytes)/(1024*1024)) +} + // Execute runs the Claude CLI and returns the response func (e *Executor) Execute(ctx *agentContext.Context, messages []agentContext.Message) (*agentContext.CompletionResponse, error) { return e.Stream(ctx, messages, nil)