Add user attachment handling in Claude executor

- Introduce functionality to resolve and manage user-uploaded files in the sandbox environment.
- Implement `prepareAttachments` method to convert attachment URLs to local file paths and handle duplicates.
- Update message processing to replace attachment content with text references, allowing Claude CLI to access files using Read and Bash tools.
- Enhance documentation to inform users about the new attachment handling capabilities.

This change improves the interaction with user-uploaded files, enabling better integration within the Claude CLI environment.
This commit is contained in:
Max 2026-02-08 11:49:51 +08:00
parent 3bbc11604c
commit fea1ac0708
3 changed files with 598 additions and 0 deletions

View file

@ -0,0 +1,316 @@
package claude
import (
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestExtensionFromContentType(t *testing.T) {
tests := []struct {
contentType string
expected string
}{
{"image/png", ".png"},
{"image/jpeg", ".jpg"},
{"image/gif", ".gif"},
{"image/webp", ".webp"},
{"image/svg+xml", ".svg"},
{"application/pdf", ".pdf"},
{"text/plain", ".txt"},
{"text/html", ".html"},
{"text/css", ".css"},
{"text/javascript", ".js"},
{"application/javascript", ".js"},
{"application/json", ".json"},
{"application/zip", ".zip"},
{"application/octet-stream", ""},
{"unknown/type", ""},
}
for _, tt := range tests {
t.Run(tt.contentType, func(t *testing.T) {
assert.Equal(t, tt.expected, extensionFromContentType(tt.contentType))
})
}
}
func TestFormatFileSize(t *testing.T) {
tests := []struct {
bytes int
expected string
}{
{0, "0B"},
{100, "100B"},
{1023, "1023B"},
{1024, "1.0KB"},
{1536, "1.5KB"},
{10240, "10.0KB"},
{1048576, "1.0MB"},
{1572864, "1.5MB"},
{10485760, "10.0MB"},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("%d", tt.bytes), func(t *testing.T) {
assert.Equal(t, tt.expected, formatFileSize(tt.bytes))
})
}
}
func TestPrepareAttachmentsPlainText(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
manager := createTestManager(t)
if manager == nil {
return
}
defer manager.Close()
opts := &Options{
Command: "claude",
Image: "alpine:latest",
UserID: "test-user",
ChatID: fmt.Sprintf("test-chat-att-plain-%d", time.Now().UnixNano()),
}
exec, err := NewExecutor(manager, opts)
require.NoError(t, err)
defer exec.Close()
ctx := context.Background()
// Plain text messages should pass through unchanged
messages := []agentContext.Message{
{Role: "system", Content: "You are a helpful assistant"},
{Role: "user", Content: "Hello, world!"},
{Role: "assistant", Content: "Hi there!"},
{Role: "user", Content: "What is 1+1?"},
}
result, err := exec.prepareAttachments(ctx, messages)
require.NoError(t, err)
require.Len(t, result, 4)
// Verify messages are unchanged
assert.Equal(t, "system", string(result[0].Role))
assert.Equal(t, "You are a helpful assistant", result[0].Content)
assert.Equal(t, "Hello, world!", result[1].Content)
assert.Equal(t, "Hi there!", result[2].Content)
assert.Equal(t, "What is 1+1?", result[3].Content)
}
func TestPrepareAttachmentsMultimodalNoWrapper(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
manager := createTestManager(t)
if manager == nil {
return
}
defer manager.Close()
opts := &Options{
Command: "claude",
Image: "alpine:latest",
UserID: "test-user",
ChatID: fmt.Sprintf("test-chat-att-nowrap-%d", time.Now().UnixNano()),
}
exec, err := NewExecutor(manager, opts)
require.NoError(t, err)
defer exec.Close()
ctx := context.Background()
// Multimodal message with a non-wrapper URL (e.g. regular http URL)
// Should convert to text description but not try to resolve attachment
messages := []agentContext.Message{
{
Role: "user",
Content: []interface{}{
map[string]interface{}{"type": "text", "text": "Look at this"},
map[string]interface{}{
"type": "image_url",
"image_url": map[string]interface{}{
"url": "https://example.com/image.png",
"detail": "auto",
},
},
},
},
}
result, err := exec.prepareAttachments(ctx, messages)
require.NoError(t, err)
require.Len(t, result, 1)
// Content should be converted to text with URL reference
content, ok := result[0].Content.(string)
require.True(t, ok, "Content should be converted to string")
assert.Contains(t, content, "Look at this")
assert.Contains(t, content, "[Image: https://example.com/image.png]")
}
func TestPrepareAttachmentsTextOnlyMultimodal(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
manager := createTestManager(t)
if manager == nil {
return
}
defer manager.Close()
opts := &Options{
Command: "claude",
Image: "alpine:latest",
UserID: "test-user",
ChatID: fmt.Sprintf("test-chat-att-textonly-%d", time.Now().UnixNano()),
}
exec, err := NewExecutor(manager, opts)
require.NoError(t, err)
defer exec.Close()
ctx := context.Background()
// Multimodal message with only text parts
messages := []agentContext.Message{
{
Role: "user",
Content: []interface{}{
map[string]interface{}{"type": "text", "text": "Hello"},
map[string]interface{}{"type": "text", "text": "World"},
},
},
}
result, err := exec.prepareAttachments(ctx, messages)
require.NoError(t, err)
require.Len(t, result, 1)
// Should combine text parts
content, ok := result[0].Content.(string)
require.True(t, ok, "Content should be converted to string")
assert.Contains(t, content, "Hello")
assert.Contains(t, content, "World")
}
func TestPrepareAttachmentsInvalidWrapperURL(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
manager := createTestManager(t)
if manager == nil {
return
}
defer manager.Close()
opts := &Options{
Command: "claude",
Image: "alpine:latest",
UserID: "test-user",
ChatID: fmt.Sprintf("test-chat-att-invalid-%d", time.Now().UnixNano()),
}
exec, err := NewExecutor(manager, opts)
require.NoError(t, err)
defer exec.Close()
ctx := context.Background()
// Message with an attachment URL pointing to a non-existent manager
messages := []agentContext.Message{
{
Role: "user",
Content: []interface{}{
map[string]interface{}{"type": "text", "text": "See this image"},
map[string]interface{}{
"type": "image_url",
"image_url": map[string]interface{}{
"url": "__nonexistent.uploader://fakefile123",
"detail": "auto",
},
},
},
},
}
result, err := exec.prepareAttachments(ctx, messages)
require.NoError(t, err)
require.Len(t, result, 1)
// Should gracefully fallback to error text
content, ok := result[0].Content.(string)
require.True(t, ok, "Content should be converted to string")
assert.Contains(t, content, "See this image")
assert.Contains(t, content, "failed to load")
}
func TestPrepareAttachmentsMixedRoles(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
manager := createTestManager(t)
if manager == nil {
return
}
defer manager.Close()
opts := &Options{
Command: "claude",
Image: "alpine:latest",
UserID: "test-user",
ChatID: fmt.Sprintf("test-chat-att-mixed-%d", time.Now().UnixNano()),
}
exec, err := NewExecutor(manager, opts)
require.NoError(t, err)
defer exec.Close()
ctx := context.Background()
// Only user messages should be processed; system and assistant messages pass through
messages := []agentContext.Message{
{Role: "system", Content: "System prompt"},
{
Role: "user",
Content: []interface{}{
map[string]interface{}{"type": "text", "text": "User message with image"},
map[string]interface{}{
"type": "image_url",
"image_url": map[string]interface{}{
"url": "https://example.com/photo.jpg",
"detail": "auto",
},
},
},
},
{Role: "assistant", Content: "I can see the photo"},
{Role: "user", Content: "Thanks!"},
}
result, err := exec.prepareAttachments(ctx, messages)
require.NoError(t, err)
require.Len(t, result, 4)
// System and assistant messages unchanged
assert.Equal(t, "System prompt", result[0].Content)
assert.Equal(t, "I can see the photo", result[2].Content)
assert.Equal(t, "Thanks!", result[3].Content)
// User multimodal message converted
content, ok := result[1].Content.(string)
require.True(t, ok, "User multimodal content should be converted to string")
assert.Contains(t, content, "User message with image")
assert.Contains(t, content, "[Image: https://example.com/photo.jpg]")
}

View file

@ -34,6 +34,12 @@ The following tools are NOT available in this environment and you must NOT use t
Focus on using the core tools: Bash, Read, Write, Edit, Glob, Grep, WebSearch, WebFetch.
## User Attachments
User-uploaded files (images, documents, code files, etc.) are placed in /workspace/.attachments/
When the user references an attached file, read it from this directory using the Read or Bash tool.
For image files, you can view them directly as Claude supports vision on local files.
## GitHub CLI (gh) Usage
When working with GitHub and a token is provided:

View file

@ -16,6 +16,7 @@ import (
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/i18n"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/attachment"
infraSandbox "github.com/yaoapp/yao/sandbox"
"github.com/yaoapp/yao/sandbox/ipc"
)
@ -175,6 +176,15 @@ func (e *Executor) Stream(ctx *agentContext.Context, messages []agentContext.Mes
return nil, fmt.Errorf("failed to prepare environment: %w", err)
}
// Resolve attachment URLs and write files to container
// This converts __yao.attachment:// URLs to local file paths in /workspace/.attachments/
if resolved, attErr := e.prepareAttachments(stdCtx, messages); attErr != nil {
// Non-fatal: log warning and continue with original messages
log.Printf("[sandbox] Warning: failed to prepare attachments: %v", attErr)
} else {
messages = resolved
}
// Check if we should skip Claude CLI execution
// Skip if no prompts, no skills, and no MCP config
if e.shouldSkipClaudeCLI() {
@ -407,6 +417,272 @@ func (e *Executor) copySkillsDirectory(ctx context.Context) error {
return nil
}
// prepareAttachments resolves __yao.attachment:// URLs in messages,
// writes the actual files to the container's /workspace/.attachments/ directory,
// and replaces the attachment content parts with text references to the file paths.
// This allows Claude CLI to read the files using its built-in Read/Bash tools.
func (e *Executor) prepareAttachments(ctx context.Context, messages []agentContext.Message) ([]agentContext.Message, error) {
// Track used filenames to handle duplicates
usedNames := make(map[string]int)
attachmentDir := e.workDir + "/.attachments"
dirCreated := false
hasAttachments := false
result := make([]agentContext.Message, len(messages))
copy(result, messages)
for i, msg := range result {
if msg.Role != "user" {
continue
}
// Handle content array (multimodal messages come as []interface{} from JSON)
parts, ok := msg.Content.([]interface{})
if !ok {
// Try typed content parts
if typedParts, ok := msg.Content.([]agentContext.ContentPart); ok {
iparts := make([]interface{}, len(typedParts))
for j, p := range typedParts {
// Convert to map for uniform handling
m := map[string]interface{}{"type": string(p.Type)}
if p.Text != "" {
m["text"] = p.Text
}
if p.ImageURL != nil {
m["image_url"] = map[string]interface{}{
"url": p.ImageURL.URL,
"detail": string(p.ImageURL.Detail),
}
}
if p.File != nil {
m["file"] = map[string]interface{}{
"url": p.File.URL,
"filename": p.File.Filename,
}
}
iparts[j] = m
}
parts = iparts
} else {
continue
}
}
if len(parts) == 0 {
continue
}
// Process each content part
var textParts []string
modified := false
for _, item := range parts {
m, ok := item.(map[string]interface{})
if !ok {
continue
}
partType, _ := m["type"].(string)
switch partType {
case "text":
if text, ok := m["text"].(string); ok && text != "" {
textParts = append(textParts, text)
}
case "image_url":
imgData, _ := m["image_url"].(map[string]interface{})
if imgData == nil {
continue
}
url, _ := imgData["url"].(string)
if url == "" {
continue
}
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
// Not an attachment URL, keep as text reference
textParts = append(textParts, fmt.Sprintf("[Image: %s]", url))
modified = true
continue
}
// Resolve the attachment
ref, err := e.resolveAttachment(ctx, uploaderName, fileID, "", attachmentDir, usedNames, &dirCreated)
if err != nil {
log.Printf("[sandbox] Warning: failed to resolve image attachment %s: %v", fileID, err)
textParts = append(textParts, "[Attached image: failed to load]")
modified = true
continue
}
textParts = append(textParts, ref)
hasAttachments = true
modified = true
case "file":
fileData, _ := m["file"].(map[string]interface{})
if fileData == nil {
continue
}
url, _ := fileData["url"].(string)
hintName, _ := fileData["filename"].(string)
if url == "" {
continue
}
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
textParts = append(textParts, fmt.Sprintf("[File: %s]", url))
modified = true
continue
}
ref, err := e.resolveAttachment(ctx, uploaderName, fileID, hintName, attachmentDir, usedNames, &dirCreated)
if err != nil {
log.Printf("[sandbox] Warning: failed to resolve file attachment %s: %v", fileID, err)
textParts = append(textParts, "[Attached file: failed to load]")
modified = true
continue
}
textParts = append(textParts, ref)
hasAttachments = true
modified = true
default:
// Keep other types as-is (shouldn't happen normally)
continue
}
}
if modified && len(textParts) > 0 {
newMsg := result[i]
newMsg.Content = strings.Join(textParts, "\n\n")
result[i] = newMsg
}
}
if !hasAttachments {
return result, nil
}
return result, nil
}
// resolveAttachment reads an attachment from the attachment manager and writes it
// to the container's .attachments directory. Returns a text reference string.
func (e *Executor) resolveAttachment(
ctx context.Context,
uploaderName, fileID, hintName, attachmentDir string,
usedNames map[string]int,
dirCreated *bool,
) (string, error) {
// Get attachment manager
manager, exists := attachment.Managers[uploaderName]
if !exists {
return "", fmt.Errorf("attachment manager not found: %s", uploaderName)
}
// Get file info
fileInfo, err := manager.Info(ctx, fileID)
if err != nil {
return "", fmt.Errorf("failed to get file info: %w", err)
}
// Read file data
data, err := manager.Read(ctx, fileID)
if err != nil {
return "", fmt.Errorf("failed to read file: %w", err)
}
// Determine filename
filename := fileInfo.Filename
if filename == "" && hintName != "" {
filename = hintName
}
if filename == "" {
// Fallback: use fileID with extension from content type
ext := extensionFromContentType(fileInfo.ContentType)
filename = fileID + ext
}
// Handle duplicate filenames
baseName := filename
if count, exists := usedNames[baseName]; exists {
ext := filepath.Ext(filename)
name := strings.TrimSuffix(filename, ext)
filename = fmt.Sprintf("%s_%d%s", name, count+1, ext)
usedNames[baseName] = count + 1
} else {
usedNames[baseName] = 0
}
// Create attachments directory if not yet created
if !*dirCreated {
if err := e.manager.WriteFile(ctx, e.containerName, attachmentDir+"/.keep", []byte("")); err != nil {
return "", fmt.Errorf("failed to create attachments directory: %w", err)
}
*dirCreated = true
}
// Write file to container
containerPath := attachmentDir + "/" + filename
if err := e.manager.WriteFile(ctx, e.containerName, containerPath, data); err != nil {
return "", fmt.Errorf("failed to write file to container: %w", err)
}
// Build human-readable size string
sizeStr := formatFileSize(fileInfo.Bytes)
// Return text reference
return fmt.Sprintf("[Attached file: %s (%s, %s)]", containerPath, fileInfo.ContentType, sizeStr), nil
}
// extensionFromContentType returns a file extension for a given content type
func extensionFromContentType(contentType string) string {
switch contentType {
case "image/png":
return ".png"
case "image/jpeg":
return ".jpg"
case "image/gif":
return ".gif"
case "image/webp":
return ".webp"
case "image/svg+xml":
return ".svg"
case "application/pdf":
return ".pdf"
case "text/plain":
return ".txt"
case "text/html":
return ".html"
case "text/css":
return ".css"
case "text/javascript", "application/javascript":
return ".js"
case "application/json":
return ".json"
case "application/zip":
return ".zip"
default:
return ""
}
}
// formatFileSize returns a human-readable file size string
func formatFileSize(bytes int) string {
if bytes < 1024 {
return fmt.Sprintf("%dB", bytes)
}
if bytes < 1024*1024 {
return fmt.Sprintf("%.1fKB", float64(bytes)/1024)
}
return fmt.Sprintf("%.1fMB", float64(bytes)/(1024*1024))
}
// Execute runs the Claude CLI and returns the response
func (e *Executor) Execute(ctx *agentContext.Context, messages []agentContext.Message) (*agentContext.CompletionResponse, error) {
return e.Stream(ctx, messages, nil)