opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1401 from trheyi/main

Browse source 
Refactor Content Processing and Enhance Search Functionality
This commit is contained in:
Max 2025-12-29 18:59:04 +08:00 • committed by GitHub
commit 1b4036d053
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
39 changed files with 3319 additions and 3103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.github/workflows/pr-test.yml vendored
View file

@ -524,6 +524,17 @@ jobs:
- name: Setup Go Tools
run: make tools
- name: Install pdftoppm, mutool, imagemagick
run: |
sudo apt update
sudo apt install -y poppler-utils mupdf-tools imagemagick
- name: Test pdftoppm, mutool, imagemagick
run: |
pdftoppm -v
mutool -v
convert -version
- name: Setup ENV (SQLite)
run: |
mkdir -p ${{ github.WORKSPACE }}/../app/db

11
.github/workflows/unit-test.yml vendored
View file

@ -421,6 +421,17 @@ jobs:
- name: Setup Go Tools
run: make tools
- name: Install pdftoppm, mutool, imagemagick
run: |
sudo apt update
sudo apt install -y poppler-utils mupdf-tools imagemagick
- name: Test pdftoppm, mutool, imagemagick
run: |
pdftoppm -v
mutool -v
convert -version
- name: Setup ENV (SQLite)
run: |
mkdir -p ${{ github.WORKSPACE }}/../app/db

136
agent/assistant/build_content.go
View file

@ -4,6 +4,8 @@ import (
"fmt"
"github.com/yaoapp/yao/agent/content"
"github.com/yaoapp/yao/agent/content/text"
contentTypes "github.com/yaoapp/yao/agent/content/types"
"github.com/yaoapp/yao/agent/context"
)
@ -12,6 +14,12 @@ import (
//
// This should be called after BuildRequest and before executing LLM call
func (ast *Assistant) BuildContent(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, opts *context.Options) ([]context.Message, error) {
// Skip complex content parsing if requested (for internal calls like needsearch)
// Still convert file attachments to raw text
if opts != nil && opts.Skip != nil && opts.Skip.ContentParsing {
return convertFilesToText(ctx, messages), nil
}
// Set AssistantID in context for file info tracking in Space
// This ensures hooks can access file information using the correct namespace
if ctx.AssistantID == "" {
@ -23,19 +31,123 @@ func (ast *Assistant) BuildContent(ctx *context.Context, messages []context.Mess
if err != nil {
return nil, fmt.Errorf("failed to get connector: %w", err)
}
_ = connector // unused but needed for GetConnector call
// Get Uses configuration from options (already merged in BuildRequest)
uses := options.Uses
// Get ForceUses configuration from options
forceUses := options.ForceUses
// Process content through Vision function
processedMessages, err := content.Vision(ctx, capabilities, messages, uses, forceUses)
if err != nil {
return nil, fmt.Errorf("failed to process content: %w", err)
// Build parse options
parseOptions := &contentTypes.Options{
Capabilities: capabilities,
CompletionOptions: options,
Connector: connector,
StreamOptions: options.StreamOptions,
}
return processedMessages, nil
contentMessages, referenceContext, err := content.ParseUserInput(ctx, messages, parseOptions)
if err != nil {
return nil, fmt.Errorf("failed to parse content: %w", err)
}
// Inject reference context into messages
if referenceContext != nil {
contentMessages = ast.injectSearchContext(contentMessages, referenceContext)
}
return contentMessages, nil
}
// convertFilesToText converts file attachments in messages to raw text
// Used when SkipContentParsing is enabled - simple text extraction without vision/PDF processing
func convertFilesToText(ctx *context.Context, messages []context.Message) []context.Message {
result := make([]context.Message, 0, len(messages))
textHandler := text.New(nil)
for _, msg := range messages {
// Only process user messages
if msg.Role != context.RoleUser {
result = append(result, msg)
continue
}
// Handle content parts
parts, ok := msg.Content.([]context.ContentPart)
if !ok {
// Try []interface{} (from history/JSON)
if iparts, ok := msg.Content.([]interface{}); ok {
parts = convertInterfaceToParts(iparts)
}
}
if len(parts) == 0 {
result = append(result, msg)
continue
}
// Convert file parts to text
newParts := make([]context.ContentPart, 0, len(parts))
for _, part := range parts {
switch part.Type {
case context.ContentFile:
// Convert file to raw text
if part.File != nil && part.File.URL != "" {
textPart, _, err := textHandler.ParseRaw(ctx, part)
if err == nil {
newParts = append(newParts, textPart)
continue
}
}
newParts = append(newParts, part)
case context.ContentImageURL:
// Skip images - cannot convert to text without vision
continue
default:
newParts = append(newParts, part)
}
}
newMsg := msg
newMsg.Content = newParts
result = append(result, newMsg)
}
return result
}
// convertInterfaceToParts converts []interface{} to []ContentPart for file extraction
func convertInterfaceToParts(items []interface{}) []context.ContentPart {
parts := make([]context.ContentPart, 0, len(items))
for _, item := range items {
m, ok := item.(map[string]interface{})
if !ok {
continue
}
typeStr, _ := m["type"].(string)
part := context.ContentPart{
Type: context.ContentPartType(typeStr),
}
switch typeStr {
case "text":
if t, ok := m["text"].(string); ok {
part.Text = t
}
case "file":
if fileData, ok := m["file"].(map[string]interface{}); ok {
part.File = &context.FileAttachment{}
if url, ok := fileData["url"].(string); ok {
part.File.URL = url
}
if filename, ok := fileData["filename"].(string); ok {
part.File.Filename = filename
}
}
case "image_url":
part.Type = context.ContentImageURL
default:
continue
}
parts = append(parts, part)
}
return parts
}

14
agent/assistant/load.go
View file

@ -609,7 +609,15 @@ func loadMap(data map[string]interface{}) (*Assistant, error) {
// Search configuration (from package.yao search block)
// This contains search options like web.max_results, kb.threshold, citation.format, etc.
// Merge hierarchy: global config < assistant config
if v, ok := data["search"].(map[string]interface{}); ok {
switch v := data["search"].(type) {
case *searchTypes.Config:
assistant.Search = v
case searchTypes.Config:
assistant.Search = &v
case map[string]interface{}:
var assistantSearch searchTypes.Config
raw, err := jsoniter.Marshal(v)
if err != nil {
@ -621,8 +629,8 @@ func loadMap(data map[string]interface{}) (*Assistant, error) {
}
// Merge with global search config
assistant.Search = mergeSearchConfig(globalSearchConfig, &assistantSearch)
} else if globalSearchConfig != nil {
// No assistant-specific config, use global
default:
assistant.Search = globalSearchConfig
}

8
agent/assistant/next.go
View file

@ -65,6 +65,12 @@ func (ast *Assistant) handleDelegation(
// buildStandardResponse builds the standard agent response when no custom Next hook processing is needed
func (ast *Assistant) buildStandardResponse(npc *NextProcessContext) *agentContext.Response {
var next interface{} = nil
if npc.NextResponse != nil {
next = npc.NextResponse
}
return &agentContext.Response{
ContextID: npc.Context.ID,
RequestID: npc.Context.RequestID(),
@ -72,7 +78,7 @@ func (ast *Assistant) buildStandardResponse(npc *NextProcessContext) *agentConte
ChatID: npc.Context.ChatID,
AssistantID: ast.ID,
Create: npc.CreateResponse,
Next: npc.NextResponse,
Next: next,
Completion: npc.CompletionResponse,
Tools: npc.ToolCallResponses,
}

43
agent/assistant/search.go
View file

@ -31,6 +31,15 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.
return nil
}
// Check if search is skipped via ctx.Metadata["__disable_search"]
if ctx != nil && ctx.Metadata != nil {
disableSearch := getBool(ctx.Metadata, "__disable_search")
if disableSearch {
ctx.Logger.Debug("Auto search skipped by ctx.Metadata['__disable_search']")
return nil
}
}
// Check createResponse.Search field (highest priority from Create hook)
// Supports: bool | SearchIntent | nil
if createResponse != nil && createResponse.Search != nil {
@ -46,7 +55,7 @@ func (ast *Assistant) shouldAutoSearch(ctx *context.Context, messages []context.
}
// Get merged uses configuration
uses := ast.getMergedSearchUses(createResponse)
uses := ast.getMergedSearchUses(createResponse, opts)
// Check if search is explicitly disabled
if uses != nil && uses.Search == "disabled" {
@ -357,8 +366,9 @@ func (ast *Assistant) sendIntentDone(ctx *context.Context, loadingID string, nee
}
// getMergedSearchUses returns the merged uses configuration for search
// Priority: createResponse > assistant
func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResponse) *context.Uses {
// Priority: createResponse > options.Uses > assistant
func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResponse, opts ...*context.Options) *context.Uses {
// Start with assistant uses
var uses *context.Uses
if ast.Uses != nil {
@ -371,6 +381,31 @@ func (ast *Assistant) getMergedSearchUses(createResponse *context.HookCreateResp
}
}
// Override with options.Uses if provided (highest priority)
if len(opts) > 0 && opts[0] != nil && opts[0].Uses != nil {
if uses == nil {
uses = &context.Uses{}
}
if opts[0].Uses.Search != "" {
uses.Search = opts[0].Uses.Search
}
if opts[0].Uses.Web != "" {
uses.Web = opts[0].Uses.Web
}
if opts[0].Uses.Keyword != "" {
uses.Keyword = opts[0].Uses.Keyword
}
if opts[0].Uses.QueryDSL != "" {
uses.QueryDSL = opts[0].Uses.QueryDSL
}
if opts[0].Uses.Rerank != "" {
uses.Rerank = opts[0].Uses.Rerank
}
}
// Override with createResponse.Uses if provided (highest priority)
if createResponse != nil && createResponse.Uses != nil {
if uses == nil {
@ -405,7 +440,7 @@ func (ast *Assistant) executeAutoSearch(ctx *context.Context, messages []context
defer ctx.Logger.PhaseComplete("Search")
// Get merged uses configuration
uses := ast.getMergedSearchUses(createResponse)
uses := ast.getMergedSearchUses(createResponse, opts...)
// Convert to search.Uses
searchUses := &search.Uses{}

19
agent/assistant/utils.go
View file

@ -43,6 +43,25 @@ func getTimestamp(v interface{}) (int64, error) {
return 0, fmt.Errorf("invalid timestamp type %T", v)
}
// getBool gets bool from data map[string]interface{}, key string
func getBool(data map[string]interface{}, key string) bool {
switch v := data[key].(type) {
case bool:
return v
case int64:
return v != 0
case int:
return v != 0
case float64:
return v != 0
case string:
return v == "true" || v == "1" || v == "enabled" || v == "yes" || v == "on"
case nil:
return false
}
return false
}
// stringHash returns the sha256 hash of the string
func stringHash(v string) string {
h := sha256.New()

326
agent/content/README.md
View file

@ -1,326 +0,0 @@
# Content Processing Package
This package handles content transformation for multimodal messages in agent conversations. It is called **BEFORE** sending messages to the LLM and converts extended content types into standard LLM-compatible formats.
## ⚠️ Critical Design Principle
**Input**: Messages with extended content types (`file`, `data`, etc.)
**Output**: Messages with ONLY standard LLM-compatible types (`text`, `image_url`, `input_audio`)
The LLM should NEVER receive `type="file"` or `type="data"` content parts. These MUST be converted to `text` (or `image_url` for images if model supports vision).
## Architecture
```
Vision (main entry)
↓
Initialize processedFiles cache (map[fileID]text)
↓
processMessage (for each message)
↓
processContentPart (for each content part)
↓
Is uploader wrapper?
├── Yes → Check cache
│ ├── In cache? → Use cached text ✅
│ └── Not in cache → Try GetText(fileID) preview
│ ├── Has preview? → Use preview + cache ✅
│ └── No preview → Proceed to full processing ↓
└── No (HTTP/other) → Proceed to full processing ↓
↓
├── Fetch content (if needed)
│ ├── HTTP URL
│ └── Uploader Wrapper (__uploader://fileid)
↓
├── Determine Processing Strategy
│ ├── Model supports? → Format for model
│ └── Model doesn't support? → Use agent/MCP
↓
ProcessorRegistry
↓
├── ImageProcessor
├── AudioProcessor
├── PDFProcessor
├── WordProcessor
├── ExcelProcessor
└── TextProcessor
↓
Cache result (if uploader wrapper)
```
## Content Type Transformation
### Input → Output Mapping
| Input Type | Model Supports? | Output Type | Processing |
|------------|-----------------|-------------|------------|
| `text` | - | `text` | Pass through |
| `image_url` | ✅ Yes | `image_url` | Convert format if needed (base64/URL) |
| `image_url` | ❌ No | `text` | Use vision agent/MCP to describe |
| `input_audio` | ✅ Yes | `input_audio` | Keep as audio |
| `input_audio` | ❌ No | `text` | Transcribe using audio agent/MCP |
| `file` (image) | ✅ Yes | `image_url` | Same as image_url processing |
| `file` (image) | ❌ No | `text` | Use vision tool to describe |
| `file` (document) | - | `text` | Extract text from PDF/Word/Excel/etc |
| `data` | - | `text` | Fetch and format data sources |
### 1. Images and Audio
**If model supports (vision/audio capability):**
- Keep as multimodal content:
- `image_url`: Convert to appropriate format (OpenAI URL vs Claude base64)
- `input_audio`: Convert to base64 format
**If model doesn't support:**
- Convert to text:
- Use agent/MCP specified in `uses.Vision` or `uses.Audio`
- Extract text description or transcription
- Return as `type="text"` content
**HTTP URLs:**
- Fetch content first
- Then process the same way as above
### 2. Files (type="file")
**Critical**: All `type="file"` content MUST be converted to `text` or `image_url` (if image and model supports).
**Processing Steps:**
1. **Fetch file content**:
- Uploader wrapper: `__uploader://fileid` → Parse and fetch from attachment manager
- HTTP URL: Download from URL
2. **Detect file type** from content-type and magic bytes
3. **Process based on file type**:
| File Type | Output Type | Processing Method |
| ------------ | ----------- | -------------------------------------------------------------------------------------------------------- |
| **Image** | `image_url` or `text` | If model supports vision → `image_url`<br>If not → use vision tool → `text` |
| **PDF** | `text` | If `uses.Vision` supports PDF → use vision tool<br>Otherwise → extract text directly |
| **Word** | `text` | Extract text using Word document parser |
| **Excel** | `text` | Extract and format as readable table/CSV |
| **PPT** | `text` | Extract text and slide content |
| **CSV** | `text` | Format as readable table |
| **Text** | `text` | Read directly (with encoding detection) |
| **JSON/XML** | `text` | Pretty print for readability |
### 3. Data Sources (type="data")
**Critical**: All `type="data"` content MUST be converted to `text`.
**Processing Steps:**
1. **Parse DataContent.Sources** array
2. **Fetch data** from each source:
- `model`: Query data model
- `kb_collection`: Search knowledge base collection
- `kb_document`: Get document content
- `table`: Query database table
- `api`: Call API endpoint
- `mcp_resource`: Fetch MCP resource
3. **Format as readable text**:
- Tables: Format as markdown tables or CSV
- Documents: Include title and content
- JSON: Pretty print
4. **Return as** `type="text"` content
## Components
### Core Files
- **content.go** - Main entry point (`Vision` function)
- **types.go** - Type definitions and constants
- **interfaces.go** - Interface definitions
### Fetching
- **fetch.go** - Fetch content from HTTP or uploader
### Processors
- **processor.go** - Processor registry and routing
- **image.go** - Image processing
- **audio.go** - Audio processing
- **pdf.go** - PDF document processing
- **word.go** - Word document processing
- **excel.go** - Excel spreadsheet processing
- **text.go** - Plain text and CSV processing
## Frontend Message Format
The frontend (InputArea) sends messages in the following format:
### Image Attachments
```json
{
"type": "image_url",
"image_url": {
"url": "__yao.attachment://file_id",
"detail": "auto"
}
}
```
### File Attachments
```json
{
"type": "file",
"file": {
"url": "__yao.attachment://file_id",
"filename": "document.pdf"
}
}
```
The `url` field contains an uploader wrapper in the format `__uploader://fileid`.
## Data Structures
### ContentInfo
Holds information about content to be processed:
```go
type ContentInfo struct {
Source ContentSource // http, uploader, base64, local
FileType FileType // image, audio, pdf, word, excel, etc.
ContentType string // MIME type
URL string // Original URL or file ID
Data []byte // File data
// For uploader wrapper
UploaderName string
FileID string
}
```
### ProcessedContent
Result of content processing:
```go
type ProcessedContent struct {
Text string // Extracted text
ContentPart *context.ContentPart // For model input
Metadata map[string]interface{}
Error error
}
```
## Usage Example
```go
import (
"github.com/yaoapp/yao/agent/content"
"github.com/yaoapp/yao/agent/context"
)
// Process messages before sending to LLM
processedMessages, err := content.Vision(
ctx,
capabilities, // Model capabilities
messages, // Original messages
uses, // Tool specifications (vision, audio, etc.)
)
```
## Performance Optimization
### File Processing Cache
**Problem**: Same file (uploader wrapper) might appear in multiple messages or be referenced multiple times.
**Solution**: Three-level caching strategy:
1. **In-memory cache** (`processedFiles` map):
- Caches processed text for the duration of the Vision() call
- Key: file ID from uploader wrapper
- Value: extracted text content
2. **Attachment preview** (attachment.GetText with preview):
- Tries to get preview (first 2000 chars) from attachment manager
- If file was previously processed and saved, preview is available immediately
- Much faster than full file processing
3. **Full processing** (only if needed):
- Falls back to complete file processing if no cache/preview available
- Result is cached in memory and optionally saved to attachment manager
### Cache Flow
```go
// For uploader://file_id
1. Check processedFiles[file_id]
└── Found? → Return cached text ⚡ (fastest)
2. Not in cache → Call attachment.GetText(file_id, false) // preview only
└── Has preview? → Cache and return ⚡ (fast)
3. No preview → Process file fully 🔄 (slower)
└── Cache result in processedFiles
└── Optional: Save to attachment using SaveText for future use
```
### Benefits
- **Avoid duplicate processing**: Same file processed only once per Vision() call
- **Fast preview access**: Leverage pre-processed content from attachment manager
- **Reduced latency**: Especially important for large documents (PDFs, Word, Excel)
- **Resource efficient**: Less CPU/memory usage for repeated file references
## Implementation Status
### ✅ Completed
- [x] Package structure
- [x] Type definitions
- [x] Interface definitions
- [x] Skeleton functions with TODO comments
- [x] File processing cache infrastructure
- [x] Cache helper functions (tryGetCachedText, cacheProcessedText)
### 🚧 To Implement
- [ ] tryGetCachedText implementation (attachment.GetText integration)
- [ ] cacheProcessedText implementation (attachment.SaveText integration)
- [ ] HTTP fetching logic
- [ ] Uploader wrapper parsing and fetching
- [ ] Image processing (base64, vision API)
- [ ] Audio processing (transcription)
- [ ] PDF text extraction
- [ ] Word document parsing
- [ ] Excel spreadsheet parsing
- [ ] Text/CSV formatting
- [ ] Content part processing logic
- [ ] Model capability detection
- [ ] Agent/MCP tool invocation
## Configuration
Content processing behavior is controlled by:
1. **Model Capabilities** (`openai.Capabilities`)
- Determines if model can handle images/audio directly
- Specifies vision format (OpenAI vs Claude)
2. **Uses** (`context.Uses`)
```go
type Uses struct {
Vision string // "agent" or "mcp:server_id"
Audio string // "agent" or "mcp:server_id"
Search string
Fetch string
}
```
## Error Handling
- Errors during processing are logged but don't stop the entire pipeline
- Original content is kept if processing fails
- Graceful degradation: if advanced processing fails, fall back to simpler methods

61
agent/content/audio.go
View file

@ -1,61 +0,0 @@
package content
import (
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// AudioHandler handles audio content
type AudioHandler struct{}
// CanHandle checks if this handler can handle the content type
func (h *AudioHandler) CanHandle(contentType string, fileType FileType) bool {
return fileType == FileTypeAudio || strings.HasPrefix(contentType, "audio/")
}
// Handle processes audio content
// Logic similar to image:
// 1. If model supports audio input -> convert to base64 format
// 2. If model doesn't support audio -> use agent/MCP specified in uses.Audio
func (h *AudioHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
// TODO: Implement audio handling
// 1. Check model audio capabilities
// 2. If supported:
// - Encode audio as base64 with proper format
// 3. If not supported:
// - Call audio agent/MCP to transcribe audio to text
// 4. Return Result with text or ContentPart
return nil, fmt.Errorf("not implemented")
}
// handleWithAudioModel processes audio using model's audio capability
func (h *AudioHandler) handleWithAudioModel(ctx *agentContext.Context, info *Info) (*Result, error) {
// TODO: Implement audio model processing
// Format audio according to model's audio input format
return nil, fmt.Errorf("not implemented")
}
// handleWithAudioAgent processes audio using audio agent or MCP
func (h *AudioHandler) handleWithAudioAgent(ctx *agentContext.Context, info *Info, audioTool string) (string, error) {
// TODO: Implement audio agent/MCP processing
// 1. Parse audioTool (format: "agent" or "mcp:server_id")
// 2. Call appropriate tool to transcribe audio
// 3. Return transcribed text
return "", fmt.Errorf("not implemented")
}
// encodeAudioBase64 encodes audio data to base64 with proper format
func encodeAudioBase64(data []byte, contentType string) string {
// TODO: Implement audio base64 encoding
return ""
}
// detectAudioFormat detects audio format from content type or data
func detectAudioFormat(contentType string, data []byte) string {
// TODO: Implement audio format detection
// Return format like "wav", "mp3", "flac", etc.
return ""
}

619
agent/content/content.go
View file

@ -4,526 +4,150 @@ import (
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/agent/content/docx"
"github.com/yaoapp/yao/agent/content/image"
"github.com/yaoapp/yao/agent/content/pdf"
"github.com/yaoapp/yao/agent/content/pptx"
"github.com/yaoapp/yao/agent/content/text"
"github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/attachment"
searchTypes "github.com/yaoapp/yao/agent/search/types"
)
// Vision transforms extended content types to LLM-compatible formats
// This is the main entry point for content preprocessing before sending to LLM
//
// IMPORTANT: This function is called BEFORE sending messages to LLM (in agent.executeLLMStream)
// It must convert all extended content types to standard LLM-compatible types.
//
// Input Content Types (Extended):
// - type="text" -> Pass through (already standard)
// - type="image_url" -> Process based on model capability (may need base64 conversion or vision tool)
// - type="input_audio" -> Process based on model capability (may need transcription)
// - type="file" -> Convert to text or image_url (MUST be converted)
// - type="data" -> Convert to text (MUST be converted)
//
// Output Content Types (LLM-compatible only):
// - type="text" -> Text content
// - type="image_url" -> Image (only if model supports vision)
// - type="input_audio" -> Audio (only if model supports audio)
//
// Processing Logic:
// 1. For images (image_url):
// - If model supports vision -> keep as image_url (may convert URL to base64)
// - If model doesn't support -> use vision agent/MCP to extract text -> convert to type="text"
//
// 2. For audio (input_audio):
// - If model supports audio -> keep as input_audio
// - If model doesn't support -> use audio agent/MCP to transcribe -> convert to type="text"
//
// 3. For files (type="file"):
// - Parse uploader wrapper (__uploader://fileid) or fetch HTTP URL
// - Detect file type (PDF, Word, Excel, Image, etc.)
// - Process based on file type:
// - Images: same as image processing above
// - PDF: use vision tool if available, otherwise extract text -> type="text"
// - Word/Excel/PPT/CSV: extract text -> type="text"
// - MUST convert to type="text" or type="image_url" (if image and model supports)
//
// 4. For data (type="data"):
// - Fetch data from sources (models, KB, MCP resources, etc.)
// - Format as readable text
// - MUST convert to type="text"
//
// Return: Messages with only standard LLM-compatible content types (text, image_url, input_audio)
func Vision(ctx *agentContext.Context, capabilities *openai.Capabilities, messages []agentContext.Message, uses *agentContext.Uses, forceUses ...bool) ([]agentContext.Message, error) {
// Determine if we should force using Uses tools even when model has native capabilities
shouldForceUses := false
if len(forceUses) > 0 {
shouldForceUses = forceUses[0]
}
// Initialize handlers and fetcher
registry := NewRegistry()
fetcher := NewFetcher()
// Cache for processed files (uploader wrapper -> extracted text)
// Ensures each file is only processed once
processedFiles := make(map[string]string)
// Process each message
processedMessages := make([]agentContext.Message, 0, len(messages))
for _, msg := range messages {
processedMsg, err := processMessage(ctx, &msg, capabilities, uses, shouldForceUses, registry, fetcher, processedFiles)
if err != nil {
// Log error but continue processing other messages
// TODO: Add proper logging
fmt.Printf("Warning: failed to process message: %v\n", err)
processedMessages = append(processedMessages, msg) // Keep original on error
// ParseUserInput ParseUserInput
func ParseUserInput(ctx *agentContext.Context, messages []agentContext.Message, options *types.Options) ([]agentContext.Message, *searchTypes.ReferenceContext, error) {
var referenceContext *searchTypes.ReferenceContext = nil
var parsedMessages []agentContext.Message = make([]agentContext.Message, 0)
for _, message := range messages {
// Only process user messages (current or from history)
if message.Role != agentContext.RoleUser {
parsedMessages = append(parsedMessages, message)
continue
}
processedMessages = append(processedMessages, processedMsg)
}
return processedMessages, nil
}
// processMessage processes a single message and its content parts
func processMessage(
ctx *agentContext.Context,
msg *agentContext.Message,
capabilities *openai.Capabilities,
uses *agentContext.Uses,
forceUses bool,
registry *Registry,
fetcher Fetcher,
processedFiles map[string]string,
) (agentContext.Message, error) {
// If content is simple string, no processing needed
if _, ok := msg.GetContentAsString(); ok {
return *msg, nil
}
// Get content parts - try typed first, then convert from interface{}
parts, ok := msg.GetContentAsParts()
if !ok {
// Try to convert from []interface{} (common when loaded from history/JSON)
parts, ok = convertToContentParts(msg.Content)
if !ok {
return *msg, nil
}
}
// Note: File information will be collected and stored in Space by CallAgentWithFileInfo
// when calling vision agents, using the agent ID as namespace prefix
// Process each content part
processedParts := make([]agentContext.ContentPart, 0, len(parts))
for _, part := range parts {
processedPart, err := processContentPart(ctx, &part, capabilities, uses, forceUses, registry, fetcher, processedFiles)
// Parse user input message (Ignore errors)
parsedMessage, refs, err := parseUserInputMessage(ctx, message, options)
if err != nil {
// Log error and handle gracefully
fmt.Printf("Warning: failed to process content part: %v\n", err)
parsedMessages = append(parsedMessages, message)
log.Error("Failed to parse user input message: %v, %v", message.Content, err)
continue
}
parsedMessages = append(parsedMessages, parsedMessage)
// For image_url that failed to process, convert to text description
// This prevents sending unsupported multimodal content to non-vision models
if part.Type == agentContext.ContentImageURL {
processedParts = append(processedParts, agentContext.ContentPart{
Type: agentContext.ContentText,
Text: fmt.Sprintf("[Image processing failed: %s]", part.ImageURL.URL),
})
} else {
// For other types, keep original
processedParts = append(processedParts, part)
// Add reference to reference context
if refs != nil {
if referenceContext == nil {
referenceContext = &searchTypes.ReferenceContext{}
}
continue
}
// If handling returned text, convert to text part
if processedPart.Text != "" {
processedParts = append(processedParts, agentContext.ContentPart{
Type: agentContext.ContentText,
Text: processedPart.Text,
})
} else if processedPart.ContentPart != nil {
// Use the processed content part (e.g., base64 image)
processedParts = append(processedParts, *processedPart.ContentPart)
} else {
// Keep original if no handling result
processedParts = append(processedParts, part)
referenceContext.References = append(referenceContext.References, refs...)
}
}
// Return new message with processed content
return agentContext.Message{
Role: msg.Role,
Content: processedParts,
Name: msg.Name,
ToolCallID: msg.ToolCallID,
ToolCalls: msg.ToolCalls,
Refusal: msg.Refusal,
}, nil
return parsedMessages, referenceContext, nil
}
// processContentPart processes a single content part
// IMPORTANT: Must convert extended types (file, data) to standard types (text, image_url, input_audio)
func processContentPart(
ctx *agentContext.Context,
part *agentContext.ContentPart,
capabilities *openai.Capabilities,
uses *agentContext.Uses,
forceUses bool,
registry *Registry,
fetcher Fetcher,
processedFiles map[string]string,
) (*Result, error) {
// 1. Handle standard types - pass through
switch part.Type {
// parseUserInputMessage parse a user input message
func parseUserInputMessage(ctx *agentContext.Context, message agentContext.Message, options *types.Options) (agentContext.Message, []*searchTypes.Reference, error) {
// Context content type
switch content := message.Content.(type) {
case string:
return message, nil, nil
case []agentContext.ContentPart:
return parseContentParts(ctx, message, content, options)
case []interface{}:
// Handle content loaded from history/JSON ([]interface{} instead of []ContentPart)
parts, ok := convertToContentParts(content)
if !ok {
return message, nil, nil
}
return parseContentParts(ctx, message, parts, options)
}
return message, nil, fmt.Errorf("unsupported content type: %T", message.Content)
}
// parseContentParts parses content parts and returns the parsed message
func parseContentParts(ctx *agentContext.Context, message agentContext.Message, content []agentContext.ContentPart, options *types.Options) (agentContext.Message, []*searchTypes.Reference, error) {
allRefs := []*searchTypes.Reference{}
parts := make([]agentContext.ContentPart, 0, len(content))
for _, part := range content {
parsedPart, refs, err := parseContentPart(ctx, part, options)
if err != nil {
parts = append(parts, part)
continue
}
parts = append(parts, parsedPart)
if refs != nil {
allRefs = append(allRefs, refs...)
}
}
parsedMessage := message
parsedMessage.Content = parts
return parsedMessage, allRefs, nil
}
// parseContentPart parse a content part
func parseContentPart(ctx *agentContext.Context, content agentContext.ContentPart, options *types.Options) (agentContext.ContentPart, []*searchTypes.Reference, error) {
switch content.Type {
case agentContext.ContentText:
// Text is already standard, pass through
return &Result{
ContentPart: part,
}, nil
return content, nil, nil
case agentContext.ContentImageURL:
// Image URL - check if it needs processing
return processImageURLContent(ctx, part, capabilities, uses, forceUses, registry, fetcher, processedFiles)
return image.New(options).Parse(ctx, content)
case agentContext.ContentInputAudio:
// Audio - check if it needs processing
return processAudioContent(ctx, part, capabilities, uses, registry, fetcher, processedFiles)
}
return content, nil, nil
// 2. Handle extended types - MUST convert to standard types
switch part.Type {
case agentContext.ContentFile:
return processFileContent(ctx, part, capabilities, uses, forceUses, registry, fetcher, processedFiles)
return parseFileContent(ctx, content, options)
case agentContext.ContentData:
return processDataContent(ctx, part)
return content, nil, nil
default:
// Unknown type, return error
return nil, fmt.Errorf("unsupported content type: %s", part.Type)
return content, nil, fmt.Errorf("unsupported content part type: %s", content.Type)
}
}
// processFileContent processes file content with caching
func processFileContent(
ctx *agentContext.Context,
part *agentContext.ContentPart,
capabilities *openai.Capabilities,
uses *agentContext.Uses,
forceUses bool,
registry *Registry,
fetcher Fetcher,
processedFiles map[string]string,
) (*Result, error) {
if part.File == nil || part.File.URL == "" {
return nil, fmt.Errorf("file content part missing URL")
// parseFileContent parses file content based on file type
func parseFileContent(ctx *agentContext.Context, content agentContext.ContentPart, options *types.Options) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return content, nil, nil
}
url := part.File.URL
// Determine file type from filename
filename := strings.ToLower(content.File.Filename)
// Step 1: Try to get cached text (three-tier cache)
cachedText, found, err := tryGetCachedText(ctx, url, processedFiles)
if err != nil {
return nil, fmt.Errorf("failed to check cache: %w", err)
}
if found {
// Cache hit! Return as text
return &Result{
Text: cachedText,
}, nil
// Check file type and route to appropriate handler
switch {
case strings.HasSuffix(filename, ".pdf"):
return pdf.New(options).Parse(ctx, content)
case strings.HasSuffix(filename, ".docx"):
return docx.New(options).Parse(ctx, content)
case strings.HasSuffix(filename, ".pptx"):
return pptx.New(options).Parse(ctx, content)
case text.IsSupportedExtension(filename):
return text.New(options).Parse(ctx, content)
}
// Step 2: No cache, need to process the file
// Determine content source
source, sourceURL, err := determineContentSource(part)
if err != nil {
return nil, fmt.Errorf("failed to determine content source: %w", err)
}
// Fetch content
info, err := fetcher.Fetch(ctx, source, sourceURL)
if err != nil {
return nil, fmt.Errorf("failed to fetch content: %w", err)
}
// Set filename from part if not already set
if info.Filename == "" && part.File.Filename != "" {
info.Filename = part.File.Filename
}
// Detect file type if not already set
if info.FileType == FileTypeUnknown {
info.FileType = DetectFileType(info.ContentType, part.File.Filename)
}
// Process with appropriate handler
result, err := registry.Handle(ctx, info, capabilities, uses, forceUses)
if err != nil {
return nil, fmt.Errorf("failed to handle content: %w", err)
}
// Step 3: Cache the result if it's text
if result.Text != "" {
if cacheErr := cacheProcessedText(ctx, url, result.Text, processedFiles); cacheErr != nil {
// Log error but don't fail the request
fmt.Printf("Warning: failed to cache processed text: %v\n", cacheErr)
}
}
return result, nil
}
// processImageURLContent processes image_url content
// If URL is uploader wrapper or HTTP, fetch and process it
func processImageURLContent(
ctx *agentContext.Context,
part *agentContext.ContentPart,
capabilities *openai.Capabilities,
uses *agentContext.Uses,
forceUses bool,
registry *Registry,
fetcher Fetcher,
processedFiles map[string]string,
) (*Result, error) {
if part.ImageURL == nil || part.ImageURL.URL == "" {
return nil, fmt.Errorf("image_url content missing URL")
}
url := part.ImageURL.URL
// If it's a data URI (base64), pass through
if strings.HasPrefix(url, "data:") {
return &Result{
ContentPart: part,
}, nil
}
// Check model capabilities
supportsVision := false
if capabilities != nil && capabilities.Vision != nil {
// Vision can be bool or string (format)
switch v := capabilities.Vision.(type) {
case bool:
supportsVision = v
case string:
supportsVision = v != "" && v != "false" && v != "none"
}
}
// If model supports vision AND we're not forcing uses, pass through
if supportsVision && !forceUses {
return &Result{
ContentPart: part,
}, nil
}
// Model doesn't support vision OR forceUses is true
// Need to convert image to text
// If it's uploader wrapper or HTTP URL, need to process
// Check cache first
cachedText, found, err := tryGetCachedText(ctx, url, processedFiles)
if err != nil {
return nil, fmt.Errorf("failed to check cache: %w", err)
}
if found {
// Cache hit! Return as text
return &Result{
Text: cachedText,
}, nil
}
// Determine source
source, sourceURL, err := determineContentSource(part)
if err != nil {
return nil, fmt.Errorf("failed to determine content source: %w", err)
}
// Fetch content
info, err := fetcher.Fetch(ctx, source, sourceURL)
if err != nil {
return nil, fmt.Errorf("failed to fetch image: %w", err)
}
// Set file type as image
info.FileType = FileTypeImage
// Process with image handler
result, err := registry.Handle(ctx, info, capabilities, uses, forceUses)
if err != nil {
return nil, fmt.Errorf("failed to handle image: %w", err)
}
// Cache if result is text
if result.Text != "" {
if cacheErr := cacheProcessedText(ctx, url, result.Text, processedFiles); cacheErr != nil {
fmt.Printf("Warning: failed to cache processed text: %v\n", cacheErr)
}
}
return result, nil
}
// processAudioContent processes input_audio content
func processAudioContent(
ctx *agentContext.Context,
part *agentContext.ContentPart,
capabilities *openai.Capabilities,
uses *agentContext.Uses,
registry *Registry,
fetcher Fetcher,
processedFiles map[string]string,
) (*Result, error) {
if part.InputAudio == nil || part.InputAudio.Data == "" {
return nil, fmt.Errorf("input_audio content missing data")
}
// For now, pass through audio as-is
// TODO: Implement audio processing (transcription, etc.)
return &Result{
ContentPart: part,
}, nil
}
// processDataContent processes data content (converts to text)
func processDataContent(ctx *agentContext.Context, part *agentContext.ContentPart) (*Result, error) {
if part.Data == nil {
return nil, fmt.Errorf("data content part missing data")
}
// TODO: Implement data processing
// For now, just return error
return nil, fmt.Errorf("data content processing not implemented yet")
}
// determineContentSource determines where the content comes from
func determineContentSource(part *agentContext.ContentPart) (Source, string, error) {
var url string
// Extract URL based on content type
switch part.Type {
case agentContext.ContentFile:
if part.File == nil || part.File.URL == "" {
return "", "", fmt.Errorf("file content missing URL")
}
url = part.File.URL
case agentContext.ContentImageURL:
if part.ImageURL == nil || part.ImageURL.URL == "" {
return "", "", fmt.Errorf("image_url content missing URL")
}
url = part.ImageURL.URL
case agentContext.ContentInputAudio:
if part.InputAudio == nil || part.InputAudio.Data == "" {
return "", "", fmt.Errorf("input_audio content missing data")
}
// Audio data is base64, treat as base64 source
return SourceBase64, part.InputAudio.Data, nil
default:
return "", "", fmt.Errorf("unsupported content type for source detection: %s", part.Type)
}
// Determine source type based on URL format
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
return SourceHTTP, url, nil
}
if strings.HasPrefix(url, "__") {
// Uploader wrapper format: __uploader://fileid
return SourceUploader, url, nil
}
if strings.HasPrefix(url, "data:") {
// Data URI (base64)
return SourceBase64, url, nil
}
// Default to treating as uploader if no prefix matches
return SourceUploader, url, nil
}
// shouldProcessWithModel checks if content should be processed by the model directly
func shouldProcessWithModel(capabilities *openai.Capabilities, fileType FileType) (bool, agentContext.VisionFormat) {
// TODO: Implement model capability check
// For images: check if model supports vision
// For audio: check if model supports audio input
// Return whether to use model and the format to use
return false, agentContext.VisionFormatNone
}
// getToolForProcessing gets the agent/MCP tool to use for processing
func getToolForProcessing(uses *agentContext.Uses, fileType FileType) string {
// TODO: Implement tool selection
// Based on file type, return the appropriate tool from uses
// - Images -> uses.Vision
// - Audio -> uses.Audio
// - PDF (if vision available) -> uses.Vision
return ""
}
// tryGetCachedText checks if the URL is an uploader wrapper and tries to get cached text
// Returns (text, found, error)
func tryGetCachedText(ctx *agentContext.Context, url string, processedFiles map[string]string) (string, bool, error) {
// Parse URL to check if it's an uploader wrapper
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return "", false, nil // Not an uploader wrapper, no cache
}
// 1. Check in-memory cache for this Vision call
if text, ok := processedFiles[fileID]; ok {
return text, true, nil
}
// 2. Try attachment manager's content_preview (cross-call cache)
manager, exists := attachment.Managers[uploaderName]
if exists {
// GetText with fullContent=false to get preview (default)
text, err := manager.GetText(ctx.Context, fileID, false)
if err == nil && text != "" {
// Cache in-memory for this Vision call
processedFiles[fileID] = text
return text, true, nil
}
}
// No cache found
return "", false, nil
}
// cacheProcessedText caches the processed text for an uploader wrapper
func cacheProcessedText(ctx *agentContext.Context, url string, text string, processedFiles map[string]string) error {
// Parse URL to get uploader name and file ID
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return nil // Not an uploader wrapper, nothing to cache
}
// 1. Cache in-memory for this Vision call
processedFiles[fileID] = text
// 2. Save to attachment manager for future Vision calls
manager, exists := attachment.Managers[uploaderName]
if exists {
return manager.SaveText(ctx.Context, fileID, text)
}
return nil
// For unsupported file types, try to read as text
// This allows any file to be converted to text content
return text.New(options).ParseRaw(ctx, content)
}
// convertToContentParts converts []interface{} to []ContentPart
// This is needed when content is loaded from JSON/history and is []interface{} instead of []ContentPart
func convertToContentParts(content interface{}) ([]agentContext.ContentPart, bool) {
// Check if it's []interface{}
arr, ok := content.([]interface{})
if !ok {
return nil, false
}
parts := make([]agentContext.ContentPart, 0, len(arr))
for _, item := range arr {
func convertToContentParts(content []interface{}) ([]agentContext.ContentPart, bool) {
parts := make([]agentContext.ContentPart, 0, len(content))
for _, item := range content {
// Each item should be a map
m, ok := item.(map[string]interface{})
if !ok {
@ -578,6 +202,35 @@ func convertToContentParts(content interface{}) ([]agentContext.ContentPart, boo
part.InputAudio.Format = format
}
}
case "data":
if dataContent, ok := m["data"].(map[string]interface{}); ok {
part.Data = &agentContext.DataContent{}
if sources, ok := dataContent["sources"].([]interface{}); ok {
part.Data.Sources = make([]agentContext.DataSource, 0, len(sources))
for _, src := range sources {
if srcMap, ok := src.(map[string]interface{}); ok {
source := agentContext.DataSource{}
if t, ok := srcMap["type"].(string); ok {
source.Type = agentContext.DataSourceType(t)
}
if name, ok := srcMap["name"].(string); ok {
source.Name = name
}
if id, ok := srcMap["id"].(string); ok {
source.ID = id
}
if filters, ok := srcMap["filters"].(map[string]interface{}); ok {
source.Filters = filters
}
if metadata, ok := srcMap["metadata"].(map[string]interface{}); ok {
source.Metadata = metadata
}
part.Data.Sources = append(part.Data.Sources, source)
}
}
}
}
}
parts = append(parts, part)

675
agent/content/content_vision_test.go
View file

@ -1,675 +0,0 @@
package content_test
import (
"bytes"
"context"
"image"
"image/color"
"image/png"
"mime/multipart"
"strings"
"testing"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/yao/agent/content"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/testutils"
"github.com/yaoapp/yao/attachment"
)
// setupTestUploader creates and registers a test uploader manager
// The manager will be registered with "__" prefix as required by attachment.Parse
func setupTestUploader(t *testing.T, name string) attachment.FileManager {
// Register with __ prefix to match Parse behavior
managerName := "__" + name
manager, err := attachment.Register(managerName, "local", attachment.ManagerOption{
Driver: "local",
MaxSize: "10M",
AllowedTypes: []string{"text/*", "image/*", "application/*"},
Options: map[string]interface{}{
"path": "/tmp/test_vision_attachments_" + name,
},
})
if err != nil {
t.Fatalf("Failed to register attachment manager '%s': %v", managerName, err)
}
return manager
}
// cleanupTestUploader removes the test uploader from registry
func cleanupTestUploader(name string) {
delete(attachment.Managers, "__"+name)
}
// generateTestImage creates a valid PNG image (100x100 red square)
func generateTestImage(t *testing.T) []byte {
img := image.NewRGBA(image.Rect(0, 0, 100, 100))
red := color.RGBA{255, 0, 0, 255}
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
img.Set(x, y, red)
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("Failed to encode test image: %v", err)
}
return buf.Bytes()
}
// TestVision_TextFile tests Vision function with text/code file parsing
func TestVision_TextFile(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
// Setup test uploader
uploaderName := "test-vision-text"
manager := setupTestUploader(t, uploaderName)
defer cleanupTestUploader(uploaderName)
// 1. Create and upload a Go source file
testContent := `package main
import "fmt"
func main() {
fmt.Println("Hello, Vision Test!")
}
`
// Upload file
reader := strings.NewReader(testContent)
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "main.go",
Size: int64(len(testContent)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "text/x-go")
uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload file: %v", err)
}
t.Logf("Uploaded file ID: %s", uploadedFile.ID)
// 2. Prepare Vision context (text files don't need special capabilities)
ctx := agentContext.New(context.Background(), nil, "test")
capabilities := &openai.Capabilities{}
messages := []agentContext.Message{
{
Role: "user",
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "__" + uploaderName + "://" + uploadedFile.ID,
Filename: "main.go",
},
},
},
},
}
// 3. Call Vision function
result, err := content.Vision(ctx, capabilities, messages, nil, false)
if err != nil {
t.Fatalf("Vision function failed: %v", err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 message, got %d", len(result))
}
// 4. Verify result
contentParts, ok := result[0].Content.([]agentContext.ContentPart)
if !ok {
t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content)
}
if len(contentParts) != 1 {
t.Fatalf("Expected 1 content part, got %d", len(contentParts))
}
// Should be converted to text
if contentParts[0].Type != agentContext.ContentText {
t.Errorf("Expected ContentText type, got %s", contentParts[0].Type)
}
if !strings.Contains(contentParts[0].Text, "package main") {
t.Errorf("Expected text to contain 'package main', got: %s", contentParts[0].Text)
}
if !strings.Contains(contentParts[0].Text, "Hello, Vision Test!") {
t.Errorf("Expected text to contain 'Hello, Vision Test!', got: %s", contentParts[0].Text)
}
t.Logf("✓ Text file successfully parsed: %d characters", len(contentParts[0].Text))
}
// TestVision_ImageWithVisionSupport tests image processing with vision-capable model
func TestVision_ImageWithVisionSupport(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
// Setup test uploader
uploaderName := "test-vision-image"
manager := setupTestUploader(t, uploaderName)
defer cleanupTestUploader(uploaderName)
// 1. Create and upload a test image (1x1 red PNG)
imageData := []byte{
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A,
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41,
0x54, 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00,
0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D,
0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E,
0x44, 0xAE, 0x42, 0x60, 0x82,
}
// Upload image
reader := strings.NewReader(string(imageData))
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "test.png",
Size: int64(len(imageData)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "image/png")
uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload image: %v", err)
}
// 2. Prepare Vision context with vision-capable model
ctx := agentContext.New(context.Background(), nil, "test")
// Construct capabilities with vision support (OpenAI format)
capabilities := &openai.Capabilities{
Vision: agentContext.VisionFormatOpenAI, // OpenAI vision format
}
messages := []agentContext.Message{
{
Role: "user",
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: "__" + uploaderName + "://" + uploadedFile.ID,
},
},
},
},
}
// 3. Call Vision function (no uses needed for direct vision support)
result, err := content.Vision(ctx, capabilities, messages, nil, false)
if err != nil {
t.Fatalf("Vision function failed: %v", err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 message, got %d", len(result))
}
// 4. Verify result
contentParts, ok := result[0].Content.([]agentContext.ContentPart)
if !ok {
t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content)
}
if len(contentParts) != 1 {
t.Fatalf("Expected 1 content part, got %d", len(contentParts))
}
// If model supports vision, should be image_url with base64
if capabilities.Vision != nil {
if contentParts[0].Type != agentContext.ContentImageURL {
t.Errorf("Expected ContentImageURL type, got %s", contentParts[0].Type)
}
if contentParts[0].ImageURL == nil {
t.Fatal("Expected ImageURL to be set")
}
if !strings.Contains(contentParts[0].ImageURL.URL, "data:image/png;base64,") {
t.Errorf("Expected base64 data URI, got: %s", contentParts[0].ImageURL.URL)
}
t.Logf("✓ Image processed with vision support: %d bytes (base64)", len(contentParts[0].ImageURL.URL))
} else {
// If no vision support, should fall back to text (via agent/MCP)
t.Logf("ℹ Model doesn't support vision, result type: %s", contentParts[0].Type)
}
}
// TestVision_ImageWithAgent tests image processing with vision agent when model doesn't support vision
// Note: This test demonstrates the agent fallback mechanism when the model doesn't support vision
func TestVision_ImageWithAgent(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
// Setup test uploader
uploaderName := "test-vision-agent"
manager := setupTestUploader(t, uploaderName)
defer cleanupTestUploader(uploaderName)
// 1. Generate and upload a valid test image (100x100 red PNG)
imageData := generateTestImage(t)
reader := strings.NewReader(string(imageData))
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "test.png",
Size: int64(len(imageData)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "image/png")
uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload image: %v", err)
}
// 2. Prepare Vision context with proper setup
// Model does NOT support vision, but uses.Vision specifies a vision agent
ctx := agentContext.New(context.Background(), nil, "test")
// Capabilities without vision support
capabilities := &openai.Capabilities{
Vision: nil, // No vision support
}
// Uses configuration with vision agent
uses := &agentContext.Uses{
Vision: "tests.vision-helper", // Use vision-helper agent
}
messages := []agentContext.Message{
{
Role: "user",
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: "__" + uploaderName + "://" + uploadedFile.ID,
},
},
},
},
}
// 3. Call Vision - should use agent since model doesn't support vision
result, err := content.Vision(ctx, capabilities, messages, uses, false)
if err != nil {
t.Fatalf("Vision function failed: %v", err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 message, got %d", len(result))
}
// 4. Verify result is text (processed by vision agent)
contentParts, ok := result[0].Content.([]agentContext.ContentPart)
if !ok {
t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content)
}
if len(contentParts) != 1 {
t.Fatalf("Expected 1 content part, got %d", len(contentParts))
}
if contentParts[0].Type != agentContext.ContentText {
t.Errorf("Expected ContentText (from agent), got: %s", contentParts[0].Type)
}
if contentParts[0].Text == "" {
t.Error("Expected non-empty text from vision agent processing")
}
t.Logf("✓ Vision agent processed image to text: %d characters", len(contentParts[0].Text))
t.Logf("Agent response text:\n%s", contentParts[0].Text)
}
// TestVision_CachedContent tests that file content is cached and reused
func TestVision_CachedContent(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
// Setup test uploader
uploaderName := "test-vision-cache"
manager := setupTestUploader(t, uploaderName)
defer cleanupTestUploader(uploaderName)
// 1. Upload a text file
testContent := "Test content for caching verification"
reader := strings.NewReader(testContent)
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "cache-test.txt",
Size: int64(len(testContent)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "text/plain")
uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload file: %v", err)
}
// 2. Prepare Vision context with same file referenced twice
ctx := agentContext.New(context.Background(), nil, "test")
// Construct simple capabilities (text files don't need vision)
capabilities := &openai.Capabilities{}
messages := []agentContext.Message{
{
Role: "user",
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "__" + uploaderName + "://" + uploadedFile.ID,
Filename: "cache-test.txt",
},
},
{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "__" + uploaderName + "://" + uploadedFile.ID, // Same file
Filename: "cache-test.txt",
},
},
},
},
}
// 3. Call Vision
result, err := content.Vision(ctx, capabilities, messages, nil, false)
if err != nil {
t.Fatalf("Vision function failed: %v", err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 message, got %d", len(result))
}
// 4. Verify both file references were processed
contentParts, ok := result[0].Content.([]agentContext.ContentPart)
if !ok {
t.Fatalf("Expected content to be []ContentPart")
}
if len(contentParts) != 2 {
t.Fatalf("Expected 2 content parts (both files), got %d", len(contentParts))
}
// Both should be text with same content
if contentParts[0].Type != agentContext.ContentText {
t.Errorf("First part: expected ContentText, got %s", contentParts[0].Type)
}
if contentParts[1].Type != agentContext.ContentText {
t.Errorf("Second part: expected ContentText, got %s", contentParts[1].Type)
}
if !strings.Contains(contentParts[0].Text, testContent) {
t.Errorf("First part text doesn't contain expected content")
}
if !strings.Contains(contentParts[1].Text, testContent) {
t.Errorf("Second part text doesn't contain expected content")
}
// Verify content was cached (check attachment manager)
cachedText, err := manager.GetText(context.Background(), uploadedFile.ID)
if err != nil {
t.Fatalf("Failed to get cached text: %v", err)
}
if cachedText == "" {
t.Error("Expected content to be cached in attachment manager")
}
t.Logf("✓ Content successfully cached and reused: %d characters", len(cachedText))
}
// TestVision_FileMetadataInMemory tests that file metadata is correctly passed to vision agent via ctx.Memory.Context
func TestVision_FileMetadataInMemory(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
// Setup test uploader
uploaderName := "test-vision-metadata"
manager := setupTestUploader(t, uploaderName)
defer cleanupTestUploader(uploaderName)
// 1. Generate and upload a test image
imageData := generateTestImage(t)
reader := strings.NewReader(string(imageData))
fileHeader := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "test-metadata.png",
Size: int64(len(imageData)),
Header: make(map[string][]string),
},
}
fileHeader.Header.Set("Content-Type", "image/png")
uploadedFile, err := manager.Upload(context.Background(), fileHeader, reader, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload image: %v", err)
}
t.Logf("Uploaded file: %s (ID: %s)", uploadedFile.Filename, uploadedFile.ID)
// 2. Prepare Vision context - model doesn't support vision, use agent
ctx := agentContext.New(context.Background(), nil, "test")
// Capabilities without vision support
capabilities := &openai.Capabilities{
Vision: nil, // No vision support - force agent usage
}
// Uses configuration with vision-helper agent
uses := &agentContext.Uses{
Vision: "tests.vision-helper", // Use vision-helper agent that logs file metadata
}
messages := []agentContext.Message{
{
Role: "user",
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: "__" + uploaderName + "://" + uploadedFile.ID,
},
},
},
},
}
// 3. Call Vision - should pass file metadata to agent via Space
result, err := content.Vision(ctx, capabilities, messages, uses, false)
if err != nil {
t.Fatalf("Vision function failed: %v", err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 message, got %d", len(result))
}
// 4. Verify result contains file metadata validation info
contentParts, ok := result[0].Content.([]agentContext.ContentPart)
if !ok {
t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content)
}
if len(contentParts) != 1 {
t.Fatalf("Expected 1 content part, got %d", len(contentParts))
}
if contentParts[0].Type != agentContext.ContentText {
t.Errorf("Expected ContentText (from agent), got: %s", contentParts[0].Type)
}
// The vision-helper assistant should have logged file metadata from Space
// We can verify this through the response text (which should contain the description)
if contentParts[0].Text == "" {
t.Error("Expected non-empty text from vision agent")
}
t.Logf("✓ Vision agent processed image with file metadata")
t.Logf("Agent response: %s", contentParts[0].Text)
// Note: The actual validation of Space data happens in the vision-helper's Next hook
// which logs the file metadata. In a real test, we would need to capture those logs
// or have the agent return structured data that we can verify.
}
// TestVision_MultipleFilesMetadata tests file metadata handling with multiple attachments
func TestVision_MultipleFilesMetadata(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
// Setup test uploader
uploaderName := "test-vision-multi"
manager := setupTestUploader(t, uploaderName)
defer cleanupTestUploader(uploaderName)
// 1. Upload two test images
imageData1 := generateTestImage(t)
imageData2 := generateTestImage(t)
// Upload first image
reader1 := strings.NewReader(string(imageData1))
fileHeader1 := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "test-image-1.png",
Size: int64(len(imageData1)),
Header: make(map[string][]string),
},
}
fileHeader1.Header.Set("Content-Type", "image/png")
uploadedFile1, err := manager.Upload(context.Background(), fileHeader1, reader1, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload first image: %v", err)
}
// Upload second image
reader2 := strings.NewReader(string(imageData2))
fileHeader2 := &attachment.FileHeader{
FileHeader: &multipart.FileHeader{
Filename: "test-image-2.png",
Size: int64(len(imageData2)),
Header: make(map[string][]string),
},
}
fileHeader2.Header.Set("Content-Type", "image/png")
uploadedFile2, err := manager.Upload(context.Background(), fileHeader2, reader2, attachment.UploadOption{
Groups: []string{"vision", "test"},
})
if err != nil {
t.Fatalf("Failed to upload second image: %v", err)
}
t.Logf("Uploaded files: %s (ID: %s), %s (ID: %s)",
uploadedFile1.Filename, uploadedFile1.ID,
uploadedFile2.Filename, uploadedFile2.ID)
// 2. Prepare Vision context with both images
ctx := agentContext.New(context.Background(), nil, "test")
capabilities := &openai.Capabilities{
Vision: nil, // No vision support
}
uses := &agentContext.Uses{
Vision: "tests.vision-helper",
}
messages := []agentContext.Message{
{
Role: "user",
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: "__" + uploaderName + "://" + uploadedFile1.ID,
},
},
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: "__" + uploaderName + "://" + uploadedFile2.ID,
},
},
},
},
}
// 3. Call Vision - should handle multiple files' metadata
result, err := content.Vision(ctx, capabilities, messages, uses, false)
if err != nil {
t.Fatalf("Vision function failed: %v", err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 message, got %d", len(result))
}
// 4. Verify both images were processed
contentParts, ok := result[0].Content.([]agentContext.ContentPart)
if !ok {
t.Fatalf("Expected content to be []ContentPart, got %T", result[0].Content)
}
// Each image should be processed by the vision agent and return text
if len(contentParts) != 2 {
t.Fatalf("Expected 2 content parts (one per image), got %d", len(contentParts))
}
for i, part := range contentParts {
if part.Type != agentContext.ContentText {
t.Errorf("Part %d: expected ContentText, got: %s", i, part.Type)
}
if part.Text == "" {
t.Errorf("Part %d: expected non-empty text", i)
}
}
t.Logf("✓ Multiple files processed with metadata tracking")
t.Logf("File 1 response: %s", contentParts[0].Text)
t.Logf("File 2 response: %s", contentParts[1].Text)
}

143
agent/content/docx/docx.go Normal file
View file

@ -0,0 +1,143 @@
package docx
import (
"fmt"
"os"
"strings"
"github.com/yaoapp/gou/office"
"github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment"
)
// Docx handles DOCX content
type Docx struct {
options *types.Options
}
// New creates a new DOCX handler
func New(options *types.Options) *Docx {
return &Docx{options: options}
}
// Parse parses DOCX content and returns text
func (h *Docx) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return content, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
// Check cache first
cachedText, found, err := h.readFromCache(ctx, url)
if err == nil && found {
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
// Read DOCX file
data, err := h.readFile(ctx, url)
if err != nil {
return content, nil, fmt.Errorf("failed to read DOCX: %w", err)
}
// Parse DOCX using gou/office
parser := office.NewParser()
result, err := parser.Parse(data)
if err != nil {
return content, nil, fmt.Errorf("failed to parse DOCX: %w", err)
}
text := result.Markdown
if text == "" {
return content, nil, fmt.Errorf("no text content extracted from DOCX")
}
// Cache the result
if err := h.saveToCache(ctx, url, text); err != nil {
// Log warning but don't fail
fmt.Printf("Warning: failed to cache DOCX text: %v\n", err)
}
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: text,
}, nil, nil
}
// readFile reads DOCX content from various sources
func (h *Docx) readFile(ctx *agentContext.Context, url string) ([]byte, error) {
if strings.HasPrefix(url, "__") {
return h.readFromUploader(ctx, url)
}
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("HTTP URL fetch not implemented yet: %s", url)
}
// Try to read as local file path
if _, err := os.Stat(url); err == nil {
return os.ReadFile(url)
}
return nil, fmt.Errorf("unsupported DOCX source: %s", url)
}
// readFromUploader reads DOCX content from file uploader
func (h *Docx) readFromUploader(ctx *agentContext.Context, wrapper string) ([]byte, error) {
uploaderName, fileID, ok := attachment.Parse(wrapper)
if !ok {
return nil, fmt.Errorf("invalid uploader wrapper format: %s", wrapper)
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil, fmt.Errorf("uploader '%s' not found", uploaderName)
}
data, err := manager.Read(ctx.Context, fileID)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
return data, nil
}
// readFromCache reads cached text content for a DOCX
func (h *Docx) readFromCache(ctx *agentContext.Context, url string) (string, bool, error) {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return "", false, nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return "", false, nil
}
text, err := manager.GetText(ctx.Context, fileID, false)
if err == nil && text != "" {
return text, true, nil
}
return "", false, nil
}
// saveToCache saves processed text to cache
func (h *Docx) saveToCache(ctx *agentContext.Context, url string, text string) error {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil
}
return manager.SaveText(ctx.Context, fileID, text)
}

132
agent/content/docx/docx_test.go Normal file
View file

@ -0,0 +1,132 @@
package docx_test
import (
stdContext "context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/yao/agent/content/docx"
contentTypes "github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/agent/testutils"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
const testFilesDir = "assistants/tests/vision-helper/tests"
func newTestContext() *agentContext.Context {
authorized := &oauthTypes.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
}
ctx := agentContext.New(stdContext.Background(), authorized, "test-chat")
ctx.AssistantID = "test-assistant"
ctx.Locale = "en-us"
ctx.IDGenerator = message.NewIDGenerator()
return ctx
}
func newTestOptions() *contentTypes.Options {
return &contentTypes.Options{
Capabilities: &openai.Capabilities{},
}
}
func getTestFilePath(filename string) string {
yaoRoot := os.Getenv("YAO_TEST_APPLICATION")
if yaoRoot == "" {
yaoRoot = os.Getenv("YAO_ROOT")
}
return filepath.Join(yaoRoot, testFilesDir, filename)
}
// TestParseWithMissingURL tests parsing DOCX with missing URL
func TestParseWithMissingURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: nil,
}
handler := docx.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestParseWithLocalDocx tests parsing a local DOCX file
func TestParseWithLocalDocx(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
docxPath := getTestFilePath("docx.docx")
if _, err := os.Stat(docxPath); os.IsNotExist(err) {
t.Skipf("Test DOCX file not found: %s", docxPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: docxPath,
Filename: "docx.docx",
},
}
handler := docx.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("DOCX parse result (first 500 chars): %.500s...", result.Text)
}
// TestParseWithNonExistentFile tests parsing DOCX with non-existent file
func TestParseWithNonExistentFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "/non/existent/path/test.docx",
Filename: "test.docx",
},
}
handler := docx.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unsupported DOCX source")
}

48
agent/content/excel.go
View file

@ -1,48 +0,0 @@
package content
import (
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// ExcelHandler handles Microsoft Excel spreadsheets
type ExcelHandler struct{}
// CanHandle checks if this handler can handle the content type
func (h *ExcelHandler) CanHandle(contentType string, fileType FileType) bool {
return fileType == FileTypeExcel ||
contentType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ||
contentType == "application/vnd.ms-excel" ||
strings.Contains(contentType, "excel") ||
strings.Contains(contentType, "spreadsheet")
}
// Handle processes Excel spreadsheet content
func (h *ExcelHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
// TODO: Implement Excel handling
// 1. Extract data from .xlsx or .xls file
// 2. Convert to text format (e.g., CSV-like or structured text)
// 3. Handle multiple sheets
// 4. Return Result with formatted text
return nil, fmt.Errorf("not implemented")
}
// extractExcelText extracts text from Excel file
func extractExcelText(data []byte, contentType string) (string, error) {
// TODO: Implement Excel text extraction
// Handle both .xls (old format) and .xlsx (new format)
// Consider using libraries like:
// - github.com/360EntSecGroup-Skylar/excelize for .xlsx
// Format output as readable text or CSV
return "", fmt.Errorf("not implemented")
}
// formatExcelAsText formats Excel data as readable text
func formatExcelAsText(sheets map[string][][]string) string {
// TODO: Format multiple sheets into readable text
// Include sheet names, headers, and data
return ""
}

95
agent/content/fetch.go
View file

@ -1,95 +0,0 @@
package content
import (
"fmt"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/attachment"
)
// DefaultFetcher implements the Fetcher interface
type DefaultFetcher struct{}
// NewFetcher creates a new default fetcher
func NewFetcher() Fetcher {
return &DefaultFetcher{}
}
// Fetch retrieves content from HTTP URL or uploader wrapper
func (f *DefaultFetcher) Fetch(ctx *agentContext.Context, source Source, url string) (*Info, error) {
switch source {
case SourceHTTP:
return f.fetchHTTP(ctx, url)
case SourceUploader:
return f.fetchUploader(ctx, url)
default:
return nil, fmt.Errorf("unsupported source: %s", source)
}
}
// fetchHTTP fetches content from an HTTP(S) URL
func (f *DefaultFetcher) fetchHTTP(ctx *agentContext.Context, url string) (*Info, error) {
// TODO: Implement HTTP fetch logic
// 1. Download file from URL
// 2. Detect content type
// 3. Detect file type based on content type and extension
// 4. Return Info with data
return nil, fmt.Errorf("not implemented")
}
// fetchUploader fetches content from uploader wrapper (__uploader://fileid)
func (f *DefaultFetcher) fetchUploader(ctx *agentContext.Context, wrapper string) (*Info, error) {
// 1. Parse wrapper to get uploader name and file ID
uploaderName, fileID, ok := attachment.Parse(wrapper)
if !ok {
return nil, fmt.Errorf("invalid uploader wrapper format: %s", wrapper)
}
// 2. Get attachment manager
var manager attachment.FileManager
var exists bool
// Try to get manager by name
manager, exists = attachment.Managers[uploaderName]
if !exists {
return nil, fmt.Errorf("uploader '%s' not found", uploaderName)
}
// 3. Get file info
file, err := manager.Info(ctx.Context, fileID)
if err != nil {
return nil, fmt.Errorf("failed to get file info: %w", err)
}
// 4. Read file content
data, err := manager.Read(ctx.Context, fileID)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
// 5. Return Info with data
return &Info{
Data: data,
ContentType: file.ContentType,
Filename: file.Filename,
FileType: DetectFileType(file.ContentType, file.Filename),
URL: wrapper,
Source: SourceUploader,
UploaderName: uploaderName,
FileID: fileID,
}, nil
}
// parseUploaderWrapper parses uploader wrapper format: __uploader://fileid
func parseUploaderWrapper(wrapper string) (uploaderName, fileID string, err error) {
// TODO: Implement wrapper parsing
// Format: __uploader://fileid
return "", "", fmt.Errorf("not implemented")
}
// detectFileType detects file type from content type and data
func detectFileType(contentType string, data []byte) FileType {
// TODO: Implement file type detection
// Based on content type and magic bytes
return FileTypeUnknown
}

188
agent/content/image.go
View file

@ -1,188 +0,0 @@
package content
import (
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// ImageHandler handles image content
type ImageHandler struct{}
// CanHandle checks if this handler can handle the content type
func (h *ImageHandler) CanHandle(contentType string, fileType FileType) bool {
return fileType == FileTypeImage || strings.HasPrefix(contentType, "image/")
}
// Handle processes image content
// Logic:
// 1. If forceUses is true and uses.Vision is specified -> use vision tool regardless of model capability
// 2. If model supports vision and forceUses is false -> convert to base64 or image_url format
// 3. If model doesn't support vision -> use agent/MCP specified in uses.Vision
func (h *ImageHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
if len(info.Data) == 0 {
return nil, fmt.Errorf("no image data to process")
}
if capabilities == nil {
return nil, fmt.Errorf("no capabilities provided")
}
// Check if model supports vision
supportsVision, visionFormat := agentContext.GetVisionSupport(capabilities)
// If forceUses is true and uses.Vision is specified, use vision tool regardless of model capability
if forceUses && uses != nil && uses.Vision != "" {
text, err := h.handleWithVisionAgent(ctx, info, uses.Vision)
if err != nil {
return nil, fmt.Errorf("failed to handle image with vision agent/MCP (forced): %w", err)
}
return &Result{
Text: text,
}, nil
}
if supportsVision {
// Model supports vision - return as image_url ContentPart
contentPart, err := h.handleWithVisionModel(ctx, info, visionFormat)
if err != nil {
return nil, fmt.Errorf("failed to handle image with vision model: %w", err)
}
return &Result{
ContentPart: contentPart,
}, nil
}
// Model doesn't support vision - use vision agent/MCP
visionTool := ""
if uses != nil && uses.Vision != "" {
visionTool = uses.Vision
}
if visionTool == "" {
return nil, fmt.Errorf("model doesn't support vision and no vision tool specified in uses.Vision")
}
// Call vision agent/MCP to extract text
text, err := h.handleWithVisionAgent(ctx, info, visionTool)
if err != nil {
return nil, fmt.Errorf("failed to handle image with vision agent/MCP: %w", err)
}
return &Result{
Text: text,
}, nil
}
// handleWithVisionModel processes image using model's vision capability
func (h *ImageHandler) handleWithVisionModel(ctx *agentContext.Context, info *Info, format agentContext.VisionFormat) (*agentContext.ContentPart, error) {
// Encode image to base64
base64Data := encodeImageBase64(info.Data, info.ContentType)
// Format according to model's vision format
switch format {
case agentContext.VisionFormatOpenAI:
// OpenAI format: image_url with data URI
return &agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}, nil
case agentContext.VisionFormatClaude:
// Claude format: also uses image_url but may have different handling
// For now, use the same format as OpenAI
return &agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}, nil
case agentContext.VisionFormatDefault, "":
// Default format (when Vision: true) - use OpenAI format
return &agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}, nil
default:
return nil, fmt.Errorf("unsupported vision format: %s", format)
}
}
// handleWithVisionAgent processes image using vision agent or MCP
func (h *ImageHandler) handleWithVisionAgent(ctx *agentContext.Context, info *Info, visionTool string) (string, error) {
// Parse vision tool format
// Format can be:
// - "agent_id" (call agent)
// - "mcp:server_id" (call MCP tool)
if strings.HasPrefix(visionTool, "mcp:") {
// MCP tool
serverID := strings.TrimPrefix(visionTool, "mcp:")
return h.callMCPVisionTool(ctx, serverID, info)
}
// Agent call
return h.callVisionAgent(ctx, visionTool, info)
}
// callVisionAgent calls a vision agent to describe the image
func (h *ImageHandler) callVisionAgent(ctx *agentContext.Context, agentID string, info *Info) (string, error) {
// Prepare message with image
base64Data := EncodeToBase64DataURI(info.Data, info.ContentType)
message := agentContext.Message{
Role: agentContext.RoleUser,
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentText,
Text: "Please analyze this image.",
},
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
},
},
}
// Call agent with file metadata in context
// File info (filename, file_id, etc.) will be available in ctx.Metadata["file_info"]
// This allows hooks (especially Next hook) to access and format file information
return CallAgentWithFileInfo(ctx, agentID, message, info)
}
// callMCPVisionTool calls an MCP vision tool to describe the image
func (h *ImageHandler) callMCPVisionTool(ctx *agentContext.Context, serverID string, info *Info) (string, error) {
// Prepare base64 encoded image for MCP tool
base64Data := EncodeToBase64DataURI(info.Data, info.ContentType)
// Prepare arguments for MCP tool
arguments := map[string]interface{}{
"image": base64Data,
"content_type": info.ContentType,
}
// Call MCP tool (typically "describe_image" or similar)
return CallMCPTool(ctx, serverID, "describe_image", arguments)
}
// encodeImageBase64 encodes image data to base64 with data URI prefix
func encodeImageBase64(data []byte, contentType string) string {
// Use the common function
if contentType == "" {
contentType = "image/png" // default for images
}
return EncodeToBase64DataURI(data, contentType)
}

412
agent/content/image/image.go Normal file
View file

@ -0,0 +1,412 @@
package image
import (
"encoding/base64"
"fmt"
"strings"
"github.com/yaoapp/yao/agent/content/tools"
"github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/i18n"
"github.com/yaoapp/yao/agent/output/message"
searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment"
)
// Image handles image content
type Image struct {
options *types.Options
}
// New creates a new image handler
func New(options *types.Options) *Image {
return &Image{options: options}
}
// Parse parses image content
// Logic:
// 1. Check model capabilities first
// 2. If forceUses is true and uses.Vision is specified -> use vision tool regardless of model capability
// 3. If model supports vision -> pass through or convert to base64 format
// 4. If model doesn't support vision -> use vision agent/MCP to extract text
func (h *Image) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.ImageURL == nil || content.ImageURL.URL == "" {
return content, nil, fmt.Errorf("image_url content missing URL")
}
// Check model capabilities first
supportsVision, visionFormat := agentContext.GetVisionSupport(h.options.Capabilities)
// Check if we should force using Uses tools
forceUses := h.options.CompletionOptions != nil && h.options.CompletionOptions.ForceUses
// If forceUses is true and uses.Vision is specified, use vision tool regardless of model capability
if forceUses && h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" {
// Check cache first before calling agent
cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL)
if err == nil && found {
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
return h.agent(ctx, content)
}
// If model supports vision
if supportsVision {
url := content.ImageURL.URL
// If it's already a data URI (base64), pass through directly
if strings.HasPrefix(url, "data:") {
return content, nil, nil
}
// Convert to base64 format
return h.base64(ctx, content, visionFormat)
}
// Model doesn't support vision - check cache first, then use vision agent/MCP
// Try to get cached text (from attachment's content_preview)
cachedText, found, err := h.readFromCache(ctx, content.ImageURL.URL)
if err == nil && found {
// Cache hit! Return as text content
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
// No cache, try to use vision agent/MCP
if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil && h.options.CompletionOptions.Uses.Vision != "" {
return h.agent(ctx, content)
}
// No vision support and no vision tool specified, return error
return content, nil, fmt.Errorf("model doesn't support vision and no vision tool specified in uses.Vision")
}
// base64 encodes image content to base64 (for vision support)
func (h *Image) base64(ctx *agentContext.Context, content agentContext.ContentPart, format agentContext.VisionFormat) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.ImageURL == nil || content.ImageURL.URL == "" {
return content, nil, fmt.Errorf("image_url content missing URL")
}
url := content.ImageURL.URL
// Read image data from source
data, contentType, err := h.read(ctx, url)
if err != nil {
return content, nil, fmt.Errorf("failed to read image: %w", err)
}
// Encode to base64 data URI
base64Data := EncodeToBase64DataURI(data, contentType)
// Return as image_url ContentPart
return agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: content.ImageURL.Detail,
},
}, nil, nil
}
// read reads image content from various sources
func (h *Image) read(ctx *agentContext.Context, url string) ([]byte, string, error) {
// Determine source type and read accordingly
if strings.HasPrefix(url, "data:") {
// Data URI format: data:image/png;base64,xxxxx
return h.readFromDataURI(url)
}
if strings.HasPrefix(url, "__") {
// Uploader wrapper format: __uploader://fileid
return h.readFromUploader(ctx, url)
}
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
// HTTP URL - for now return error, can be implemented later
return nil, "", fmt.Errorf("HTTP URL fetch not implemented yet: %s", url)
}
// Unknown source
return nil, "", fmt.Errorf("unsupported image source: %s", url)
}
// readFromDataURI reads image content from a data URI
func (h *Image) readFromDataURI(dataURI string) ([]byte, string, error) {
// Parse data URI: data:image/png;base64,xxxxx
if !strings.HasPrefix(dataURI, "data:") {
return nil, "", fmt.Errorf("invalid data URI format")
}
// Find the comma separator
commaIndex := strings.Index(dataURI, ",")
if commaIndex == -1 {
return nil, "", fmt.Errorf("invalid data URI: missing comma separator")
}
// Extract metadata part (e.g., "image/png;base64")
metadata := dataURI[5:commaIndex] // Skip "data:"
base64Data := dataURI[commaIndex+1:]
// Parse content type
contentType := "image/png" // default
if strings.Contains(metadata, ";") {
parts := strings.Split(metadata, ";")
if len(parts) > 0 && parts[0] != "" {
contentType = parts[0]
}
} else if metadata != "" && metadata != "base64" {
contentType = metadata
}
// Decode base64 data
data, err := base64.StdEncoding.DecodeString(base64Data)
if err != nil {
return nil, "", fmt.Errorf("failed to decode base64 data: %w", err)
}
return data, contentType, nil
}
// readFromUploader reads image content from file uploader __uploader://fileid
func (h *Image) readFromUploader(ctx *agentContext.Context, wrapper string) ([]byte, string, error) {
// Parse wrapper to get uploader name and file ID
uploaderName, fileID, ok := attachment.Parse(wrapper)
if !ok {
return nil, "", fmt.Errorf("invalid uploader wrapper format: %s", wrapper)
}
// Get attachment manager
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil, "", fmt.Errorf("uploader '%s' not found", uploaderName)
}
// Get file info
file, err := manager.Info(ctx.Context, fileID)
if err != nil {
return nil, "", fmt.Errorf("failed to get file info: %w", err)
}
// Read file content
data, err := manager.Read(ctx.Context, fileID)
if err != nil {
return nil, "", fmt.Errorf("failed to read file: %w", err)
}
return data, file.ContentType, nil
}
// readFromCache reads cached text content for an image
func (h *Image) readFromCache(ctx *agentContext.Context, url string) (string, bool, error) {
// Parse URL to check if it's an uploader wrapper
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return "", false, nil // Not an uploader wrapper, no cache
}
// Try attachment manager's content_preview (cross-call cache)
manager, exists := attachment.Managers[uploaderName]
if !exists {
return "", false, nil
}
// GetText with fullContent=false to get preview (default)
text, err := manager.GetText(ctx.Context, fileID, false)
if err == nil && text != "" {
return text, true, nil
}
// No cache found
return "", false, nil
}
// saveToCache saves processed text to cache
func (h *Image) saveToCache(ctx *agentContext.Context, url string, text string) error {
// Parse URL to get uploader name and file ID
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return nil // Not an uploader wrapper, nothing to cache
}
// Save to attachment manager for future calls
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil
}
return manager.SaveText(ctx.Context, fileID, text)
}
// agent calls image agent to parse image content
// Note: Cache check is done in Parse() before calling this method
func (h *Image) agent(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.ImageURL == nil || content.ImageURL.URL == "" {
return content, nil, fmt.Errorf("image_url content missing URL")
}
url := content.ImageURL.URL
// Get vision tool from options
visionTool := ""
if h.options.CompletionOptions != nil && h.options.CompletionOptions.Uses != nil {
visionTool = h.options.CompletionOptions.Uses.Vision
}
if visionTool == "" {
return content, nil, fmt.Errorf("no vision tool specified in uses.Vision")
}
// Parse vision tool format
// Format can be:
// - "agent_id" (call agent)
// - "mcp:server_id" (call MCP tool)
var text string
var err error
if strings.HasPrefix(visionTool, "mcp:") {
// MCP tool
serverID := strings.TrimPrefix(visionTool, "mcp:")
text, err = h.callMCPVisionTool(ctx, serverID, content)
} else {
// Agent call
text, err = h.callVisionAgent(ctx, visionTool, content)
}
if err != nil {
return content, nil, fmt.Errorf("failed to process image with vision tool: %w", err)
}
// Cache the result
if cacheErr := h.saveToCache(ctx, url, text); cacheErr != nil {
// Log error but don't fail the request
fmt.Printf("Warning: failed to cache processed text: %v\n", cacheErr)
}
// Return as text content
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: text,
}, nil, nil
}
// callVisionAgent calls a vision agent to describe the image
func (h *Image) callVisionAgent(ctx *agentContext.Context, agentID string, content agentContext.ContentPart) (string, error) {
// Read image data and convert to base64
data, contentType, err := h.read(ctx, content.ImageURL.URL)
if err != nil {
return "", fmt.Errorf("failed to read image: %w", err)
}
base64Data := EncodeToBase64DataURI(data, contentType)
// Prepare message with image
message := agentContext.Message{
Role: agentContext.RoleUser,
Content: []agentContext.ContentPart{
{
Type: agentContext.ContentText,
Text: "Please analyze this image.",
},
{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
},
},
}
// Send loading message
loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing"))
// Call agent using the tools package
result, err := tools.CallAgent(ctx, agentID, message)
// Send done message
h.sendLoadingDone(ctx, loadingID)
return result, err
}
// callMCPVisionTool calls an MCP vision tool to describe the image
func (h *Image) callMCPVisionTool(ctx *agentContext.Context, serverID string, content agentContext.ContentPart) (string, error) {
// Read image data and convert to base64
data, contentType, err := h.read(ctx, content.ImageURL.URL)
if err != nil {
return "", fmt.Errorf("failed to read image: %w", err)
}
base64Data := EncodeToBase64DataURI(data, contentType)
// Prepare arguments for MCP tool
arguments := map[string]interface{}{
"image": base64Data,
"content_type": contentType,
}
// Send loading message
loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing"))
// Call MCP tool (typically "describe_image" or similar)
result, err := tools.CallMCPTool(ctx, serverID, "describe_image", arguments)
// Send done message
h.sendLoadingDone(ctx, loadingID)
return result, err
}
// sendLoading sends a loading message and returns the message ID
// Returns empty string if SilentLoading is enabled
func (h *Image) sendLoading(ctx *agentContext.Context, msg string) string {
// Skip loading message if SilentLoading is enabled (called from parent handler like PDF)
if h.options != nil && h.options.SilentLoading {
return ""
}
loadingMsg := &message.Message{
Type: message.TypeLoading,
Props: map[string]interface{}{
"message": msg,
},
}
msgID, err := ctx.SendStream(loadingMsg)
if err != nil {
return ""
}
return msgID
}
// sendLoadingDone marks the loading message as done
func (h *Image) sendLoadingDone(ctx *agentContext.Context, loadingID string) {
if loadingID == "" {
return
}
doneMsg := &message.Message{
MessageID: loadingID,
Delta: true,
DeltaAction: message.DeltaReplace,
Type: message.TypeLoading,
Props: map[string]interface{}{
"done": true,
},
}
ctx.Send(doneMsg)
}
// EncodeToBase64DataURI encodes data to base64 with data URI prefix
func EncodeToBase64DataURI(data []byte, contentType string) string {
if contentType == "" {
contentType = "image/png" // default for images
}
encoded := base64.StdEncoding.EncodeToString(data)
return fmt.Sprintf("data:%s;base64,%s", contentType, encoded)
}

374
agent/content/image/image_test.go Normal file
View file

@ -0,0 +1,374 @@
package image_test
import (
stdContext "context"
"encoding/base64"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/yao/agent/content/image"
contentTypes "github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/agent/testutils"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// newTestContext creates a Context for testing with commonly used fields pre-populated
func newTestContext(capabilities *openai.Capabilities) *agentContext.Context {
authorized := &oauthTypes.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
}
ctx := agentContext.New(stdContext.Background(), authorized, "test-chat")
ctx.AssistantID = "test-assistant"
ctx.Locale = "en-us"
ctx.Theme = "light"
ctx.Client = agentContext.Client{
Type: "web",
UserAgent: "TestAgent/1.0",
IP: "127.0.0.1",
}
ctx.Referer = agentContext.RefererAPI
ctx.Accept = agentContext.AcceptWebCUI
ctx.Route = ""
ctx.Metadata = make(map[string]interface{})
ctx.Capabilities = capabilities
ctx.IDGenerator = message.NewIDGenerator()
return ctx
}
// newTestOptions creates test options with the given capabilities
func newTestOptions(capabilities *openai.Capabilities, completionOptions *agentContext.CompletionOptions) *contentTypes.Options {
return &contentTypes.Options{
Capabilities: capabilities,
CompletionOptions: completionOptions,
}
}
// TestParseWithVisionSupport tests parsing image when model supports vision
func TestParseWithVisionSupport(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Create capabilities with vision support
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create test image content with data URI
base64Data := "data:image/png;base64," + base64.StdEncoding.EncodeToString(createTestPNG())
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}
handler := image.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentImageURL, result.Type)
assert.NotNil(t, result.ImageURL)
assert.Equal(t, base64Data, result.ImageURL.URL) // Should pass through unchanged
}
// TestParseWithoutVisionSupport tests parsing image when model doesn't support vision
func TestParseWithoutVisionSupport(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Create capabilities WITHOUT vision support
capabilities := &openai.Capabilities{
Vision: nil,
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create test image content
base64Data := "data:image/png;base64," + base64.StdEncoding.EncodeToString(createTestPNG())
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}
handler := image.New(options)
_, _, err := handler.Parse(ctx, content)
// Should return error because no vision support and no vision tool specified
assert.Error(t, err)
assert.Contains(t, err.Error(), "no vision tool specified")
}
// TestParseWithEmptyURL tests parsing image with empty URL
func TestParseWithEmptyURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with empty URL
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: "",
},
}
handler := image.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestParseWithNilImageURL tests parsing image with nil ImageURL
func TestParseWithNilImageURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with nil ImageURL
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: nil,
}
handler := image.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestEncodeToBase64DataURI tests base64 encoding
func TestEncodeToBase64DataURI(t *testing.T) {
tests := []struct {
name string
data []byte
contentType string
wantPrefix string
}{
{
name: "PNG image",
data: []byte{0x89, 0x50, 0x4E, 0x47},
contentType: "image/png",
wantPrefix: "data:image/png;base64,",
},
{
name: "JPEG image",
data: []byte{0xFF, 0xD8, 0xFF},
contentType: "image/jpeg",
wantPrefix: "data:image/jpeg;base64,",
},
{
name: "Empty content type defaults to PNG",
data: []byte{0x01, 0x02, 0x03},
contentType: "",
wantPrefix: "data:image/png;base64,",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := image.EncodeToBase64DataURI(tt.data, tt.contentType)
// Check prefix
assert.True(t, strings.HasPrefix(result, tt.wantPrefix))
// Verify base64 encoding by decoding
base64Part := result[len(tt.wantPrefix):]
decoded, err := base64.StdEncoding.DecodeString(base64Part)
assert.NoError(t, err)
// Verify decoded data matches original
assert.Equal(t, tt.data, decoded)
})
}
}
// TestParseDataURIPassthrough tests that data URI images pass through unchanged when vision is supported
func TestParseDataURIPassthrough(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create test image content with data URI
originalURL := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: originalURL,
Detail: agentContext.DetailHigh,
},
}
handler := image.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentImageURL, result.Type)
assert.NotNil(t, result.ImageURL)
assert.Equal(t, originalURL, result.ImageURL.URL)
assert.Equal(t, agentContext.DetailHigh, result.ImageURL.Detail)
}
// TestParseWithVisionAgent tests parsing image using a vision agent when model doesn't support vision
func TestParseWithVisionAgent(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Create capabilities WITHOUT vision support
capabilities := &openai.Capabilities{
Vision: nil, // Model doesn't support vision
}
// Configure to use vision agent
completionOptions := &agentContext.CompletionOptions{
Uses: &agentContext.Uses{
Vision: "tests.vision-test", // Use our test vision agent
},
}
options := newTestOptions(capabilities, completionOptions)
ctx := newTestContext(capabilities)
// Create test image content with data URI (1x1 red PNG)
base64Data := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}
handler := image.New(options)
result, refs, err := handler.Parse(ctx, content)
// Should succeed and return text content (image description from agent)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text) // Agent should return some description
t.Logf("Vision agent response: %s", result.Text)
}
// TestParseWithForceUsesVisionAgent tests forceUses flag with vision agent
func TestParseWithForceUsesVisionAgent(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Create capabilities WITH vision support
capabilities := &openai.Capabilities{
Vision: "openai", // Model supports vision
}
// Configure to FORCE use vision agent (even though model supports vision)
completionOptions := &agentContext.CompletionOptions{
ForceUses: true, // Force using the vision tool
Uses: &agentContext.Uses{
Vision: "tests.vision-test", // Use our test vision agent
},
}
options := newTestOptions(capabilities, completionOptions)
ctx := newTestContext(capabilities)
// Create test image content with data URI
base64Data := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
content := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}
handler := image.New(options)
result, refs, err := handler.Parse(ctx, content)
// Should succeed and return text content (forced to use agent even though model supports vision)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text) // Agent should return some description
t.Logf("Vision agent (forced) response: %s", result.Text)
}
// createTestPNG creates a minimal valid PNG image (1x1 red pixel)
func createTestPNG() []byte {
return []byte{
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG signature
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, // IHDR chunk
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, // 1x1 dimensions
0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, // IDAT chunk
0x54, 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00,
0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D,
0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, // IEND chunk
0x44, 0xAE, 0x42, 0x60, 0x82,
}
}

261
agent/content/image_test.go
View file

@ -1,261 +0,0 @@
package content
import (
stdContext "context"
"encoding/base64"
"os"
"strings"
"testing"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/openapi/oauth/types"
"github.com/yaoapp/yao/test"
)
func TestMain(m *testing.M) {
// Setup test environment
test.Prepare(nil, config.Conf)
defer test.Clean()
// Run tests
code := m.Run()
os.Exit(code)
}
// newTestContext creates a Context for testing with commonly used fields pre-populated
func newTestContext(capabilities *openai.Capabilities) *agentContext.Context {
authorized := &types.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
}
ctx := agentContext.New(stdContext.Background(), authorized, "test-chat")
ctx.AssistantID = "test-assistant"
ctx.Locale = "en-us"
ctx.Theme = "light"
ctx.Client = agentContext.Client{
Type: "web",
UserAgent: "TestAgent/1.0",
IP: "127.0.0.1",
}
ctx.Referer = agentContext.RefererAPI
ctx.Accept = agentContext.AcceptWebCUI
ctx.Route = ""
ctx.Metadata = make(map[string]interface{})
ctx.Capabilities = capabilities
return ctx
}
func TestImageHandler_CanHandle(t *testing.T) {
handler := &ImageHandler{}
tests := []struct {
name string
contentType string
fileType FileType
want bool
}{
{"PNG image", "image/png", FileTypeImage, true},
{"JPEG image", "image/jpeg", FileTypeImage, true},
{"GIF image", "image/gif", FileTypeImage, true},
{"WebP image", "image/webp", FileTypeImage, true},
{"Text (should not handle)", "text/plain", FileTypeText, false},
{"PDF (should not handle)", "application/pdf", FileTypePDF, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := handler.CanHandle(tt.contentType, tt.fileType)
if got != tt.want {
t.Errorf("CanHandle(%q, %q) = %v, want %v", tt.contentType, tt.fileType, got, tt.want)
}
})
}
}
func TestImageHandler_Handle_WithVisionSupport(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
handler := &ImageHandler{}
// Create a simple test image (1x1 red PNG)
pngData := createTestPNG()
// Create capabilities with vision support
capabilities := &openai.Capabilities{
Vision: "openai", // Vision is enabled with OpenAI format
}
// Create test context
ctx := newTestContext(capabilities)
info := &Info{
FileType: FileTypeImage,
ContentType: "image/png",
Data: pngData,
}
result, err := handler.Handle(ctx, info, capabilities, nil, false)
if err != nil {
t.Fatalf("Handle() error = %v", err)
}
if result == nil {
t.Fatal("Expected non-nil result")
}
if result.ContentPart == nil {
t.Fatal("Expected ContentPart for vision-supported model")
}
if result.ContentPart.Type != agentContext.ContentImageURL {
t.Errorf("Expected ContentPart type = %v, got %v", agentContext.ContentImageURL, result.ContentPart.Type)
}
if result.ContentPart.ImageURL == nil {
t.Fatal("Expected ImageURL to be set")
}
// Verify base64 encoding
if result.ContentPart.ImageURL.URL == "" {
t.Error("Expected non-empty URL")
}
// Should be data URI format
if len(result.ContentPart.ImageURL.URL) < 20 {
t.Error("Expected data URI to be longer")
}
}
func TestImageHandler_Handle_WithoutVisionSupport(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
handler := &ImageHandler{}
// Create a simple test image
pngData := createTestPNG()
// Create capabilities WITHOUT vision support
capabilities := &openai.Capabilities{
Vision: nil, // No vision support
}
// Create test context
ctx := newTestContext(capabilities)
info := &Info{
FileType: FileTypeImage,
ContentType: "image/png",
Data: pngData,
}
// Should return error because no vision support and no tool
_, err := handler.Handle(ctx, info, capabilities, nil, false)
if err == nil {
t.Error("Expected error when no vision support and no tool specified")
}
}
func TestImageHandler_Handle_EmptyData(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
handler := &ImageHandler{}
capabilities := &openai.Capabilities{
Vision: "openai",
}
// Create test context
ctx := newTestContext(capabilities)
info := &Info{
FileType: FileTypeImage,
ContentType: "image/png",
Data: []byte{}, // Empty data
}
_, err := handler.Handle(ctx, info, capabilities, nil, false)
if err == nil {
t.Error("Expected error for empty image data")
}
}
func TestEncodeImageBase64(t *testing.T) {
tests := []struct {
name string
data []byte
contentType string
wantPrefix string
}{
{
name: "PNG image",
data: []byte{0x89, 0x50, 0x4E, 0x47}, // PNG magic number
contentType: "image/png",
wantPrefix: "data:image/png;base64,",
},
{
name: "JPEG image",
data: []byte{0xFF, 0xD8, 0xFF}, // JPEG magic number
contentType: "image/jpeg",
wantPrefix: "data:image/jpeg;base64,",
},
{
name: "Empty content type defaults to PNG",
data: []byte{0x01, 0x02, 0x03},
contentType: "",
wantPrefix: "data:image/png;base64,",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := encodeImageBase64(tt.data, tt.contentType)
// Check prefix
if !strings.HasPrefix(result, tt.wantPrefix) {
t.Errorf("Expected prefix %q, got %q", tt.wantPrefix, result[:len(tt.wantPrefix)])
}
// Verify base64 encoding by decoding
base64Part := result[len(tt.wantPrefix):]
decoded, err := base64.StdEncoding.DecodeString(base64Part)
if err != nil {
t.Errorf("Failed to decode base64: %v", err)
}
// Verify decoded data matches original
if len(decoded) != len(tt.data) {
t.Errorf("Decoded length = %d, want %d", len(decoded), len(tt.data))
}
for i := range decoded {
if decoded[i] != tt.data[i] {
t.Errorf("Decoded byte[%d] = %x, want %x", i, decoded[i], tt.data[i])
}
}
})
}
}
// createTestPNG creates a minimal valid PNG image (1x1 red pixel)
func createTestPNG() []byte {
// This is a minimal valid 1x1 red PNG image
return []byte{
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG signature
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, // IHDR chunk
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, // 1x1 dimensions
0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, // IDAT chunk
0x54, 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00,
0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D,
0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, // IEND chunk
0x44, 0xAE, 0x42, 0x60, 0x82,
}
}

26
agent/content/interfaces.go
View file

@ -1,26 +0,0 @@
package content
import (
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// Handler defines the interface for handling different content types
// Converts content (images, documents, etc.) to text or standard formats
type Handler interface {
// CanHandle checks if this handler can handle the given content type
CanHandle(contentType string, fileType FileType) bool
// Handle converts the content and returns processed result
// ctx: agent context (passed from Vision function)
// capabilities: model capabilities (for vision/audio support detection)
// uses: configuration for external tools (agents/MCP servers)
// forceUses: if true, force using Uses tools even when model has native capabilities
Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error)
}
// Fetcher defines the interface for fetching content from different sources
type Fetcher interface {
// Fetch retrieves content from a URL or file ID
Fetch(ctx *agentContext.Context, source Source, url string) (*Info, error)
}

1
agent/content/link/link.go Normal file
View file

@ -0,0 +1 @@
package link

50
agent/content/pdf.go
View file

@ -1,50 +0,0 @@
package content
import (
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// PDFHandler handles PDF documents
type PDFHandler struct{}
// CanHandle checks if this handler can handle the content type
func (h *PDFHandler) CanHandle(contentType string, fileType FileType) bool {
return fileType == FileTypePDF ||
contentType == "application/pdf" ||
strings.Contains(contentType, "pdf")
}
// Handle processes PDF content
// Logic:
// 1. Check if uses.Vision is specified and supports PDF
// 2. If yes, use vision tool to handle PDF (images + text)
// 3. If no, extract text directly from PDF
func (h *PDFHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
// TODO: Implement PDF handling
// 1. Check if vision tool supports PDF
// 2. If yes:
// - Call vision tool to handle PDF (handles both text and images)
// 3. If no:
// - Extract text from PDF using default library
// 4. Return Result with extracted text
return nil, fmt.Errorf("not implemented")
}
// extractPDFText extracts text content from PDF
func extractPDFText(data []byte) (string, error) {
// TODO: Implement PDF text extraction
// Use a PDF library to extract text
// Consider preserving layout/structure
return "", fmt.Errorf("not implemented")
}
// handleWithVisionTool processes PDF using vision tool (for PDFs with images)
func handleWithVisionTool(ctx *agentContext.Context, data []byte, visionTool string) (string, error) {
// TODO: Implement vision tool PDF processing
// Some vision tools can handle PDF directly and extract both text and images
return "", fmt.Errorf("not implemented")
}

376
agent/content/pdf/pdf.go Normal file
View file

@ -0,0 +1,376 @@
package pdf
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
goupdf "github.com/yaoapp/gou/pdf"
"github.com/yaoapp/yao/agent/content/image"
"github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/i18n"
"github.com/yaoapp/yao/agent/output/message"
searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment"
kbTypes "github.com/yaoapp/yao/kb/types"
)
// PDF handles PDF content
type PDF struct {
options *types.Options
}
// New creates a new PDF handler
func New(options *types.Options) *PDF {
return &PDF{options: options}
}
// Parse parses PDF content by converting to images and processing each page
// Returns multiple ContentPart (one text part per page) combined into a single text part
func (h *PDF) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return content, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
// Check cache first
cachedText, found, err := h.readFromCache(ctx, url)
if err == nil && found {
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
// Convert PDF to images and process each page
return h.asImages(ctx, content)
}
// ParseMulti parses PDF content and returns multiple ContentParts (one per page)
// This is useful when you need separate parts for each page
func (h *PDF) ParseMulti(ctx *agentContext.Context, content agentContext.ContentPart) ([]agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return nil, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
// Check cache first - if cached, return as single text part
cachedText, found, err := h.readFromCache(ctx, url)
if err == nil && found {
return []agentContext.ContentPart{
{
Type: agentContext.ContentText,
Text: cachedText,
},
}, nil, nil
}
// Convert PDF to images and process each page
return h.asImagesMulti(ctx, content)
}
// asImages converts PDF to images and processes each page, returning combined result
func (h *PDF) asImages(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
parts, refs, err := h.asImagesMulti(ctx, content)
if err != nil {
return content, nil, err
}
if len(parts) == 0 {
return content, nil, fmt.Errorf("no pages extracted from PDF")
}
// Check if any parts are text (vision agent was used) or image_url (model supports vision)
hasTextParts := false
hasImageParts := false
for _, part := range parts {
if part.Type == agentContext.ContentText {
hasTextParts = true
} else if part.Type == agentContext.ContentImageURL {
hasImageParts = true
}
}
// If all parts are image_url (model supports vision), return the first image
// The caller should use ParseMulti to get all images
if hasImageParts && !hasTextParts {
return parts[0], refs, nil
}
// Combine all text parts into one
var combinedText strings.Builder
pageNum := 0
for _, part := range parts {
if part.Type == agentContext.ContentText && part.Text != "" {
pageNum++
if pageNum > 1 {
combinedText.WriteString("\n\n---\n\n") // Page separator
}
combinedText.WriteString(fmt.Sprintf("## Page %d\n\n", pageNum))
combinedText.WriteString(part.Text)
}
}
result := agentContext.ContentPart{
Type: agentContext.ContentText,
Text: combinedText.String(),
}
// Cache the combined result
if content.File != nil && content.File.URL != "" && combinedText.Len() > 0 {
h.saveToCache(ctx, content.File.URL, combinedText.String())
}
return result, refs, nil
}
// asImagesMulti converts PDF to images and processes each page separately
func (h *PDF) asImagesMulti(ctx *agentContext.Context, content agentContext.ContentPart) ([]agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return nil, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
// Read PDF file
pdfData, err := h.readPDF(ctx, url)
if err != nil {
return nil, nil, fmt.Errorf("failed to read PDF: %w", err)
}
// Create temporary file for PDF
tempDir := os.TempDir()
pdfPath := filepath.Join(tempDir, fmt.Sprintf("pdf_%d.pdf", time.Now().UnixNano()))
if err := os.WriteFile(pdfPath, pdfData, 0644); err != nil {
return nil, nil, fmt.Errorf("failed to write temp PDF: %w", err)
}
defer os.Remove(pdfPath)
// Get PDF processor with global config
processor, err := h.getPDFProcessor()
if err != nil {
return nil, nil, fmt.Errorf("failed to create PDF processor: %w", err)
}
// Create output directory for images
imagesDir := filepath.Join(tempDir, fmt.Sprintf("pdf_images_%d", time.Now().UnixNano()))
if err := os.MkdirAll(imagesDir, 0755); err != nil {
return nil, nil, fmt.Errorf("failed to create images directory: %w", err)
}
defer os.RemoveAll(imagesDir)
// Convert PDF to images
convertConfig := goupdf.ConvertConfig{
OutputDir: imagesDir,
OutputPrefix: "page",
Format: "png",
DPI: 150,
Quality: 90,
PageRange: "all",
}
imageFiles, err := processor.Convert(ctx.Context, pdfPath, convertConfig)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert PDF to images: %w", err)
}
if len(imageFiles) == 0 {
return nil, nil, fmt.Errorf("no pages extracted from PDF")
}
// Process each image using the image handler (with SilentLoading to suppress image loading messages)
imageOptions := *h.options // Copy options
imageOptions.SilentLoading = true
imageHandler := image.New(&imageOptions)
var parts []agentContext.ContentPart
var allRefs []*searchTypes.Reference
for i, imageFile := range imageFiles {
// Send loading message for this page
loadingMsg := fmt.Sprintf(i18n.T(ctx.Locale, "content.pdf.analyzing_page"), i+1, len(imageFiles))
loadingID := h.sendLoading(ctx, loadingMsg)
// Read image file
imageData, err := os.ReadFile(imageFile)
if err != nil {
h.sendLoadingDone(ctx, loadingID)
continue
}
// Convert to base64 data URI
base64Data := image.EncodeToBase64DataURI(imageData, "image/png")
// Create image content part
imagePart := agentContext.ContentPart{
Type: agentContext.ContentImageURL,
ImageURL: &agentContext.ImageURL{
URL: base64Data,
Detail: agentContext.DetailAuto,
},
}
// Parse image using image handler
parsedPart, refs, err := imageHandler.Parse(ctx, imagePart)
// Mark loading as done
h.sendLoadingDone(ctx, loadingID)
if err != nil {
// If parsing fails, skip this page
continue
}
parts = append(parts, parsedPart)
if refs != nil {
allRefs = append(allRefs, refs...)
}
}
if len(parts) == 0 {
return nil, nil, fmt.Errorf("failed to process any PDF pages")
}
return parts, allRefs, nil
}
// readPDF reads PDF content from various sources
func (h *PDF) readPDF(ctx *agentContext.Context, url string) ([]byte, error) {
if strings.HasPrefix(url, "__") {
// Uploader wrapper format: __uploader://fileid
return h.readFromUploader(ctx, url)
}
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("HTTP URL fetch not implemented yet: %s", url)
}
// Try to read as local file path
if _, err := os.Stat(url); err == nil {
return os.ReadFile(url)
}
return nil, fmt.Errorf("unsupported PDF source: %s", url)
}
// readFromUploader reads PDF content from file uploader
func (h *PDF) readFromUploader(ctx *agentContext.Context, wrapper string) ([]byte, error) {
uploaderName, fileID, ok := attachment.Parse(wrapper)
if !ok {
return nil, fmt.Errorf("invalid uploader wrapper format: %s", wrapper)
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil, fmt.Errorf("uploader '%s' not found", uploaderName)
}
data, err := manager.Read(ctx.Context, fileID)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
return data, nil
}
// readFromCache reads cached text content for a PDF
func (h *PDF) readFromCache(ctx *agentContext.Context, url string) (string, bool, error) {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return "", false, nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return "", false, nil
}
text, err := manager.GetText(ctx.Context, fileID, false)
if err == nil && text != "" {
return text, true, nil
}
return "", false, nil
}
// saveToCache saves processed text to cache
func (h *PDF) saveToCache(ctx *agentContext.Context, url string, text string) error {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil
}
return manager.SaveText(ctx.Context, fileID, text)
}
// getPDFProcessor creates a PDF processor using global KB config
func (h *PDF) getPDFProcessor() (*goupdf.PDF, error) {
globalPDF := kbTypes.GetGlobalPDF()
opts := goupdf.Options{
ConvertTool: goupdf.ToolPdftoppm, // default
ToolPath: "",
}
if globalPDF != nil {
if globalPDF.ConvertTool != "" {
switch globalPDF.ConvertTool {
case "pdftoppm":
opts.ConvertTool = goupdf.ToolPdftoppm
case "mutool":
opts.ConvertTool = goupdf.ToolMutool
case "imagemagick", "convert":
opts.ConvertTool = goupdf.ToolImageMagick
}
}
if globalPDF.ToolPath != "" {
opts.ToolPath = globalPDF.ToolPath
}
}
return goupdf.New(opts), nil
}
// sendLoading sends a loading message and returns the message ID
func (h *PDF) sendLoading(ctx *agentContext.Context, msg string) string {
loadingMsg := &message.Message{
Type: message.TypeLoading,
Props: map[string]interface{}{
"message": msg,
},
}
msgID, err := ctx.SendStream(loadingMsg)
if err != nil {
return ""
}
return msgID
}
// sendLoadingDone marks the loading message as done
func (h *PDF) sendLoadingDone(ctx *agentContext.Context, loadingID string) {
if loadingID == "" {
return
}
doneMsg := &message.Message{
MessageID: loadingID,
Delta: true,
DeltaAction: message.DeltaReplace,
Type: message.TypeLoading,
Props: map[string]interface{}{
"done": true,
},
}
ctx.Send(doneMsg)
}

362
agent/content/pdf/pdf_test.go Normal file
View file

@ -0,0 +1,362 @@
package pdf_test
import (
stdContext "context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/yao/agent/content/pdf"
contentTypes "github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/agent/testutils"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// Test files directory (relative to yao-dev-app)
const testFilesDir = "assistants/tests/vision-helper/tests"
// newTestContext creates a Context for testing with commonly used fields pre-populated
func newTestContext(capabilities *openai.Capabilities) *agentContext.Context {
authorized := &oauthTypes.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
}
ctx := agentContext.New(stdContext.Background(), authorized, "test-chat")
ctx.AssistantID = "test-assistant"
ctx.Locale = "en-us"
ctx.Theme = "light"
ctx.Client = agentContext.Client{
Type: "web",
UserAgent: "TestAgent/1.0",
IP: "127.0.0.1",
}
ctx.Referer = agentContext.RefererAPI
ctx.Accept = agentContext.AcceptWebCUI
ctx.Route = ""
ctx.Metadata = make(map[string]interface{})
ctx.Capabilities = capabilities
ctx.IDGenerator = message.NewIDGenerator()
return ctx
}
// newTestOptions creates test options with the given capabilities
func newTestOptions(capabilities *openai.Capabilities, completionOptions *agentContext.CompletionOptions) *contentTypes.Options {
return &contentTypes.Options{
Capabilities: capabilities,
CompletionOptions: completionOptions,
}
}
// getTestFilePath returns the full path to a test file
func getTestFilePath(filename string) string {
yaoRoot := os.Getenv("YAO_TEST_APPLICATION")
if yaoRoot == "" {
yaoRoot = os.Getenv("YAO_ROOT")
}
return filepath.Join(yaoRoot, testFilesDir, filename)
}
// TestParseWithMissingURL tests parsing PDF with missing URL
func TestParseWithMissingURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with nil File
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: nil,
}
handler := pdf.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestParseWithEmptyURL tests parsing PDF with empty URL
func TestParseWithEmptyURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with empty URL
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "",
Filename: "test.pdf",
},
}
handler := pdf.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestParseWithLocalPDFAndVisionSupport tests parsing a local PDF file when model supports vision
func TestParseWithLocalPDFAndVisionSupport(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Check if test file exists
pdfPath := getTestFilePath("test.pdf")
if _, err := os.Stat(pdfPath); os.IsNotExist(err) {
t.Skipf("Test PDF file not found: %s", pdfPath)
}
// Create capabilities with vision support
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with local file path
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: pdfPath,
Filename: "test.pdf",
},
}
handler := pdf.New(options)
result, refs, err := handler.Parse(ctx, content)
// Should succeed - PDF converted to images
assert.NoError(t, err)
assert.Nil(t, refs)
// When model supports vision, Parse returns the first image_url part
// Use ParseMulti to get all pages as separate image_url parts
assert.Equal(t, agentContext.ContentImageURL, result.Type)
assert.NotNil(t, result.ImageURL)
assert.NotEmpty(t, result.ImageURL.URL)
t.Logf("PDF parse result type: %s, URL prefix: %s...", result.Type, result.ImageURL.URL[:50])
}
// TestParseWithLocalPDFAndVisionAgent tests parsing a local PDF file using vision agent
func TestParseWithLocalPDFAndVisionAgent(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Check if test file exists
pdfPath := getTestFilePath("test.pdf")
if _, err := os.Stat(pdfPath); os.IsNotExist(err) {
t.Skipf("Test PDF file not found: %s", pdfPath)
}
// Create capabilities WITHOUT vision support
capabilities := &openai.Capabilities{
Vision: nil,
}
// Configure to use vision agent
completionOptions := &agentContext.CompletionOptions{
Uses: &agentContext.Uses{
Vision: "tests.vision-test", // Use our test vision agent
},
}
options := newTestOptions(capabilities, completionOptions)
ctx := newTestContext(capabilities)
// Create content with local file path
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: pdfPath,
Filename: "test.pdf",
},
}
handler := pdf.New(options)
result, refs, err := handler.Parse(ctx, content)
// Should succeed - PDF converted to images and processed by vision agent
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("PDF parse result (via vision agent): %s", result.Text)
}
// TestParseMultiWithLocalPDF tests ParseMulti which returns separate parts for each page
func TestParseMultiWithLocalPDF(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
// Check if test file exists
pdfPath := getTestFilePath("test.pdf")
if _, err := os.Stat(pdfPath); os.IsNotExist(err) {
t.Skipf("Test PDF file not found: %s", pdfPath)
}
// Create capabilities with vision support
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with local file path
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: pdfPath,
Filename: "test.pdf",
},
}
handler := pdf.New(options)
parts, refs, err := handler.ParseMulti(ctx, content)
// Should succeed and return at least one part (one per page)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.NotEmpty(t, parts)
t.Logf("PDF ParseMulti returned %d parts", len(parts))
// When model supports vision, each part should be image_url type
for i, part := range parts {
assert.Equal(t, agentContext.ContentImageURL, part.Type)
assert.NotNil(t, part.ImageURL)
t.Logf(" Part %d: type=%s, has URL=%v", i+1, part.Type, part.ImageURL != nil && part.ImageURL.URL != "")
}
}
// TestParseWithUnsupportedSource tests parsing PDF with unsupported source
func TestParseWithUnsupportedSource(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with HTTP URL (not implemented)
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "https://example.com/test.pdf",
Filename: "test.pdf",
},
}
handler := pdf.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "HTTP URL fetch not implemented")
}
// TestParseWithNonExistentFile tests parsing PDF with non-existent file
func TestParseWithNonExistentFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
options := newTestOptions(capabilities, nil)
ctx := newTestContext(capabilities)
// Create content with non-existent file
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "/non/existent/path/test.pdf",
Filename: "test.pdf",
},
}
handler := pdf.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unsupported PDF source")
}
// TestSilentLoadingOption tests that SilentLoading option is respected
func TestSilentLoadingOption(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
capabilities := &openai.Capabilities{
Vision: "openai",
}
// Create options with SilentLoading enabled
options := &contentTypes.Options{
Capabilities: capabilities,
SilentLoading: true,
}
// This test just verifies the option can be set
// The actual behavior is tested in the image handler tests
handler := pdf.New(options)
assert.NotNil(t, handler)
assert.True(t, options.SilentLoading)
}

143
agent/content/pptx/pptx.go Normal file
View file

@ -0,0 +1,143 @@
package pptx
import (
"fmt"
"os"
"strings"
"github.com/yaoapp/gou/office"
"github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment"
)
// Pptx handles PPTX content
type Pptx struct {
options *types.Options
}
// New creates a new PPTX handler
func New(options *types.Options) *Pptx {
return &Pptx{options: options}
}
// Parse parses PPTX content and returns text
func (h *Pptx) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return content, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
// Check cache first
cachedText, found, err := h.readFromCache(ctx, url)
if err == nil && found {
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
// Read PPTX file
data, err := h.readFile(ctx, url)
if err != nil {
return content, nil, fmt.Errorf("failed to read PPTX: %w", err)
}
// Parse PPTX using gou/office
parser := office.NewParser()
result, err := parser.Parse(data)
if err != nil {
return content, nil, fmt.Errorf("failed to parse PPTX: %w", err)
}
text := result.Markdown
if text == "" {
return content, nil, fmt.Errorf("no text content extracted from PPTX")
}
// Cache the result
if err := h.saveToCache(ctx, url, text); err != nil {
// Log warning but don't fail
fmt.Printf("Warning: failed to cache PPTX text: %v\n", err)
}
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: text,
}, nil, nil
}
// readFile reads PPTX content from various sources
func (h *Pptx) readFile(ctx *agentContext.Context, url string) ([]byte, error) {
if strings.HasPrefix(url, "__") {
return h.readFromUploader(ctx, url)
}
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("HTTP URL fetch not implemented yet: %s", url)
}
// Try to read as local file path
if _, err := os.Stat(url); err == nil {
return os.ReadFile(url)
}
return nil, fmt.Errorf("unsupported PPTX source: %s", url)
}
// readFromUploader reads PPTX content from file uploader
func (h *Pptx) readFromUploader(ctx *agentContext.Context, wrapper string) ([]byte, error) {
uploaderName, fileID, ok := attachment.Parse(wrapper)
if !ok {
return nil, fmt.Errorf("invalid uploader wrapper format: %s", wrapper)
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil, fmt.Errorf("uploader '%s' not found", uploaderName)
}
data, err := manager.Read(ctx.Context, fileID)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
return data, nil
}
// readFromCache reads cached text content for a PPTX
func (h *Pptx) readFromCache(ctx *agentContext.Context, url string) (string, bool, error) {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return "", false, nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return "", false, nil
}
text, err := manager.GetText(ctx.Context, fileID, false)
if err == nil && text != "" {
return text, true, nil
}
return "", false, nil
}
// saveToCache saves processed text to cache
func (h *Pptx) saveToCache(ctx *agentContext.Context, url string, text string) error {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil
}
return manager.SaveText(ctx.Context, fileID, text)
}

132
agent/content/pptx/pptx_test.go Normal file
View file

@ -0,0 +1,132 @@
package pptx_test
import (
stdContext "context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/yao/agent/content/pptx"
contentTypes "github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/agent/testutils"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
const testFilesDir = "assistants/tests/vision-helper/tests"
func newTestContext() *agentContext.Context {
authorized := &oauthTypes.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
}
ctx := agentContext.New(stdContext.Background(), authorized, "test-chat")
ctx.AssistantID = "test-assistant"
ctx.Locale = "en-us"
ctx.IDGenerator = message.NewIDGenerator()
return ctx
}
func newTestOptions() *contentTypes.Options {
return &contentTypes.Options{
Capabilities: &openai.Capabilities{},
}
}
func getTestFilePath(filename string) string {
yaoRoot := os.Getenv("YAO_TEST_APPLICATION")
if yaoRoot == "" {
yaoRoot = os.Getenv("YAO_ROOT")
}
return filepath.Join(yaoRoot, testFilesDir, filename)
}
// TestParseWithMissingURL tests parsing PPTX with missing URL
func TestParseWithMissingURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: nil,
}
handler := pptx.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestParseWithLocalPptx tests parsing a local PPTX file
func TestParseWithLocalPptx(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
pptxPath := getTestFilePath("pptx.pptx")
if _, err := os.Stat(pptxPath); os.IsNotExist(err) {
t.Skipf("Test PPTX file not found: %s", pptxPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: pptxPath,
Filename: "pptx.pptx",
},
}
handler := pptx.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("PPTX parse result (first 500 chars): %.500s...", result.Text)
}
// TestParseWithNonExistentFile tests parsing PPTX with non-existent file
func TestParseWithNonExistentFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "/non/existent/path/test.pptx",
Filename: "test.pptx",
},
}
handler := pptx.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unsupported PPTX source")
}

47
agent/content/registry.go
View file

@ -1,47 +0,0 @@
package content
import (
"fmt"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// Registry holds all registered content handlers
type Registry struct {
handlers []Handler
}
// NewRegistry creates a new handler registry with default handlers
func NewRegistry() *Registry {
return &Registry{
handlers: []Handler{
&ImageHandler{},
&AudioHandler{},
&PDFHandler{},
&WordHandler{},
&ExcelHandler{},
&TextHandler{},
},
}
}
// GetHandler finds the appropriate handler for the given content
func (r *Registry) GetHandler(contentType string, fileType FileType) Handler {
for _, handler := range r.handlers {
if handler.CanHandle(contentType, fileType) {
return handler
}
}
return nil
}
// Handle processes content using the appropriate handler
func (r *Registry) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
handler := r.GetHandler(info.ContentType, info.FileType)
if handler == nil {
return nil, fmt.Errorf("no handler found for content type: %s, file type: %s", info.ContentType, info.FileType)
}
return handler.Handle(ctx, info, capabilities, uses, forceUses)
}

123
agent/content/text.go
View file

@ -1,123 +0,0 @@
package content
import (
"encoding/json"
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// TextHandler handles plain text, code files, CSV, JSON, XML, Markdown, etc.
type TextHandler struct{}
// CanHandle checks if this handler can handle the content type
func (h *TextHandler) CanHandle(contentType string, fileType FileType) bool {
// Handle explicit text file types
if fileType == FileTypeText || fileType == FileTypeCSV || fileType == FileTypeJSON {
return true
}
// Handle text-based MIME types
if strings.HasPrefix(contentType, "text/") {
return true
}
// Handle common text-based content types
textContentTypes := []string{
"application/json",
"application/xml",
"application/javascript",
"application/typescript",
"application/x-yaml",
"application/yaml",
"application/toml",
"application/x-sh",
"application/x-python",
"application/x-ruby",
"application/x-perl",
}
for _, ct := range textContentTypes {
if contentType == ct || strings.Contains(contentType, ct) {
return true
}
}
return false
}
// Handle processes text content
func (h *TextHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
if len(info.Data) == 0 {
return nil, fmt.Errorf("no data to process")
}
var text string
var err error
// Handle different text formats
switch {
case info.FileType == FileTypeCSV || strings.Contains(info.ContentType, "csv"):
// Format CSV as readable text (for now, just return as-is, can enhance later)
text = string(info.Data)
case info.FileType == FileTypeJSON ||
info.ContentType == "application/json" ||
strings.Contains(info.ContentType, "json"):
// Pretty print JSON
text, err = formatJSONAsText(info.Data)
if err != nil {
// If JSON parsing fails, return raw text
text = string(info.Data)
}
case info.ContentType == "application/xml" ||
strings.Contains(info.ContentType, "xml"):
// For now, return XML as-is (can enhance formatting later)
text = string(info.Data)
default:
// Plain text, code files, markdown, etc.
text, err = readTextContent(info.Data, info.ContentType)
if err != nil {
return nil, fmt.Errorf("failed to read text content: %w", err)
}
}
return &Result{
Text: text,
}, nil
}
// readTextContent reads text content from data
func readTextContent(data []byte, contentType string) (string, error) {
// For now, assume UTF-8 encoding
// TODO: Add encoding detection if needed (e.g., using golang.org/x/text/encoding)
return string(data), nil
}
// formatCSVAsText formats CSV data as readable text
func formatCSVAsText(data []byte) (string, error) {
// TODO: Parse CSV and format as readable table
// Consider using encoding/csv package
// For now, just return as-is
return string(data), nil
}
// formatJSONAsText formats JSON data as readable text
func formatJSONAsText(data []byte) (string, error) {
// Pretty print JSON with indentation
var obj interface{}
if err := json.Unmarshal(data, &obj); err != nil {
return "", err
}
pretty, err := json.MarshalIndent(obj, "", " ")
if err != nil {
return "", err
}
return string(pretty), nil
}

352
agent/content/text/text.go Normal file
View file

@ -0,0 +1,352 @@
package text
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment"
)
// SupportedExtensions text file extensions
var SupportedExtensions = map[string]bool{
// Markdown
".md": true,
".markdown": true,
// Plain text
".txt": true,
// Code files
".go": true,
".ts": true,
".tsx": true,
".js": true,
".jsx": true,
".py": true,
".java": true,
".c": true,
".cpp": true,
".h": true,
".hpp": true,
".rs": true,
".rb": true,
".php": true,
".swift": true,
".kt": true,
".scala": true,
".sh": true,
".bash": true,
".zsh": true,
".fish": true,
".ps1": true,
".bat": true,
".cmd": true,
".sql": true,
".r": true,
".lua": true,
".perl": true,
".pl": true,
".groovy": true,
".dart": true,
".elm": true,
".ex": true,
".exs": true,
".erl": true,
".hs": true,
".clj": true,
".lisp": true,
".vim": true,
// Config files
".json": true,
".jsonc": true,
".yaml": true,
".yml": true,
".toml": true,
".ini": true,
".conf": true,
".cfg": true,
".env": true,
".yao": true,
// Web files
".html": true,
".htm": true,
".css": true,
".scss": true,
".sass": true,
".less": true,
".xml": true,
".svg": true,
// Documentation
".rst": true,
".tex": true,
".latex": true,
".org": true,
".adoc": true,
// Data files
".csv": true,
".tsv": true,
// Log files
".log": true,
}
// Text handles text file content
type Text struct {
options *types.Options
}
// New creates a new text handler
func New(options *types.Options) *Text {
return &Text{options: options}
}
// IsSupportedExtension checks if a file extension is supported
func IsSupportedExtension(filename string) bool {
ext := strings.ToLower(filepath.Ext(filename))
return SupportedExtensions[ext]
}
// Parse parses text file content and returns text
func (h *Text) Parse(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return content, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
filename := content.File.Filename
// Check cache first
cachedText, found, err := h.readFromCache(ctx, url)
if err == nil && found {
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
// Read text file
data, err := h.readFile(ctx, url)
if err != nil {
return content, nil, fmt.Errorf("failed to read text file: %w", err)
}
// Convert to string
text := string(data)
// Add file type context if it's a code file
ext := strings.ToLower(filepath.Ext(filename))
if isCodeFile(ext) {
// Wrap in markdown code block with language hint
lang := getLanguageFromExt(ext)
text = fmt.Sprintf("```%s\n%s\n```", lang, text)
}
// Cache the result
if err := h.saveToCache(ctx, url, text); err != nil {
// Log warning but don't fail
fmt.Printf("Warning: failed to cache text: %v\n", err)
}
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: text,
}, nil, nil
}
// ParseRaw parses any file as raw text content without code block wrapping
// This is used as a fallback for unsupported file types
func (h *Text) ParseRaw(ctx *agentContext.Context, content agentContext.ContentPart) (agentContext.ContentPart, []*searchTypes.Reference, error) {
if content.File == nil || content.File.URL == "" {
return content, nil, fmt.Errorf("file content missing URL")
}
url := content.File.URL
filename := content.File.Filename
// Check cache first
cachedText, found, err := h.readFromCache(ctx, url)
if err == nil && found {
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: cachedText,
}, nil, nil
}
// Read file
data, err := h.readFile(ctx, url)
if err != nil {
return content, nil, fmt.Errorf("failed to read file: %w", err)
}
// Convert to string directly (no code block wrapping)
text := string(data)
// Add filename as context
if filename != "" {
text = fmt.Sprintf("File: %s\n\n%s", filename, text)
}
// Cache the result
if err := h.saveToCache(ctx, url, text); err != nil {
// Log warning but don't fail
fmt.Printf("Warning: failed to cache text: %v\n", err)
}
return agentContext.ContentPart{
Type: agentContext.ContentText,
Text: text,
}, nil, nil
}
// readFile reads text content from various sources
func (h *Text) readFile(ctx *agentContext.Context, url string) ([]byte, error) {
if strings.HasPrefix(url, "__") {
return h.readFromUploader(ctx, url)
}
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("HTTP URL fetch not implemented yet: %s", url)
}
// Try to read as local file path
if _, err := os.Stat(url); err == nil {
return os.ReadFile(url)
}
return nil, fmt.Errorf("unsupported text file source: %s", url)
}
// readFromUploader reads text content from file uploader
func (h *Text) readFromUploader(ctx *agentContext.Context, wrapper string) ([]byte, error) {
uploaderName, fileID, ok := attachment.Parse(wrapper)
if !ok {
return nil, fmt.Errorf("invalid uploader wrapper format: %s", wrapper)
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil, fmt.Errorf("uploader '%s' not found", uploaderName)
}
data, err := manager.Read(ctx.Context, fileID)
if err != nil {
return nil, fmt.Errorf("failed to read file: %w", err)
}
return data, nil
}
// readFromCache reads cached text content
func (h *Text) readFromCache(ctx *agentContext.Context, url string) (string, bool, error) {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return "", false, nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return "", false, nil
}
text, err := manager.GetText(ctx.Context, fileID, false)
if err == nil && text != "" {
return text, true, nil
}
return "", false, nil
}
// saveToCache saves processed text to cache
func (h *Text) saveToCache(ctx *agentContext.Context, url string, text string) error {
uploaderName, fileID, isWrapper := attachment.Parse(url)
if !isWrapper {
return nil
}
manager, exists := attachment.Managers[uploaderName]
if !exists {
return nil
}
return manager.SaveText(ctx.Context, fileID, text)
}
// isCodeFile checks if the extension represents a code file
func isCodeFile(ext string) bool {
codeExts := map[string]bool{
".go": true, ".ts": true, ".tsx": true, ".js": true, ".jsx": true,
".py": true, ".java": true, ".c": true, ".cpp": true, ".h": true,
".hpp": true, ".rs": true, ".rb": true, ".php": true, ".swift": true,
".kt": true, ".scala": true, ".sh": true, ".bash": true, ".zsh": true,
".sql": true, ".r": true, ".lua": true, ".perl": true, ".pl": true,
".groovy": true, ".dart": true, ".elm": true, ".ex": true, ".exs": true,
".erl": true, ".hs": true, ".clj": true, ".lisp": true, ".vim": true,
}
return codeExts[ext]
}
// getLanguageFromExt returns the language name for markdown code block
func getLanguageFromExt(ext string) string {
langMap := map[string]string{
".go": "go",
".ts": "typescript",
".tsx": "tsx",
".js": "javascript",
".jsx": "jsx",
".py": "python",
".java": "java",
".c": "c",
".cpp": "cpp",
".h": "c",
".hpp": "cpp",
".rs": "rust",
".rb": "ruby",
".php": "php",
".swift": "swift",
".kt": "kotlin",
".scala": "scala",
".sh": "bash",
".bash": "bash",
".zsh": "zsh",
".fish": "fish",
".ps1": "powershell",
".bat": "batch",
".cmd": "batch",
".sql": "sql",
".r": "r",
".lua": "lua",
".perl": "perl",
".pl": "perl",
".groovy": "groovy",
".dart": "dart",
".elm": "elm",
".ex": "elixir",
".exs": "elixir",
".erl": "erlang",
".hs": "haskell",
".clj": "clojure",
".lisp": "lisp",
".vim": "vim",
".json": "json",
".jsonc": "jsonc",
".yaml": "yaml",
".yml": "yaml",
".toml": "toml",
".xml": "xml",
".html": "html",
".htm": "html",
".css": "css",
".scss": "scss",
".sass": "sass",
".less": "less",
".svg": "svg",
".yao": "json",
}
if lang, ok := langMap[ext]; ok {
return lang
}
return ""
}

340
agent/content/text/text_test.go Normal file
View file

@ -0,0 +1,340 @@
package text_test
import (
stdContext "context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/connector/openai"
"github.com/yaoapp/yao/agent/content/text"
contentTypes "github.com/yaoapp/yao/agent/content/types"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/agent/testutils"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
const testFilesDir = "assistants/tests/vision-helper/tests"
func newTestContext() *agentContext.Context {
authorized := &oauthTypes.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
}
ctx := agentContext.New(stdContext.Background(), authorized, "test-chat")
ctx.AssistantID = "test-assistant"
ctx.Locale = "en-us"
ctx.IDGenerator = message.NewIDGenerator()
return ctx
}
func newTestOptions() *contentTypes.Options {
return &contentTypes.Options{
Capabilities: &openai.Capabilities{},
}
}
func getTestFilePath(filename string) string {
yaoRoot := os.Getenv("YAO_TEST_APPLICATION")
if yaoRoot == "" {
yaoRoot = os.Getenv("YAO_ROOT")
}
return filepath.Join(yaoRoot, testFilesDir, filename)
}
// TestIsSupportedExtension tests the IsSupportedExtension function
func TestIsSupportedExtension(t *testing.T) {
// Supported extensions
assert.True(t, text.IsSupportedExtension("test.md"))
assert.True(t, text.IsSupportedExtension("test.txt"))
assert.True(t, text.IsSupportedExtension("test.go"))
assert.True(t, text.IsSupportedExtension("test.ts"))
assert.True(t, text.IsSupportedExtension("test.json"))
assert.True(t, text.IsSupportedExtension("test.jsonc"))
assert.True(t, text.IsSupportedExtension("test.yao"))
assert.True(t, text.IsSupportedExtension("test.yaml"))
assert.True(t, text.IsSupportedExtension("test.yml"))
assert.True(t, text.IsSupportedExtension("test.py"))
assert.True(t, text.IsSupportedExtension("test.js"))
assert.True(t, text.IsSupportedExtension("test.css"))
assert.True(t, text.IsSupportedExtension("test.html"))
// Unsupported extensions
assert.False(t, text.IsSupportedExtension("test.docx"))
assert.False(t, text.IsSupportedExtension("test.pptx"))
assert.False(t, text.IsSupportedExtension("test.pdf"))
assert.False(t, text.IsSupportedExtension("test.png"))
assert.False(t, text.IsSupportedExtension("test.jpg"))
assert.False(t, text.IsSupportedExtension("test.exe"))
assert.False(t, text.IsSupportedExtension("test.zip"))
}
// TestParseWithMissingURL tests parsing text with missing URL
func TestParseWithMissingURL(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: nil,
}
handler := text.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "missing URL")
}
// TestParseWithLocalTextFile tests parsing a local text file
func TestParseWithLocalTextFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
txtPath := getTestFilePath("text.txt")
if _, err := os.Stat(txtPath); os.IsNotExist(err) {
t.Skipf("Test text file not found: %s", txtPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: txtPath,
Filename: "text.txt",
},
}
handler := text.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("Text parse result: %s", result.Text)
}
// TestParseWithLocalMarkdownFile tests parsing a local markdown file
func TestParseWithLocalMarkdownFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
mdPath := getTestFilePath("test.md")
if _, err := os.Stat(mdPath); os.IsNotExist(err) {
t.Skipf("Test markdown file not found: %s", mdPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: mdPath,
Filename: "test.md",
},
}
handler := text.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("Markdown parse result: %s", result.Text)
}
// TestParseWithLocalCodeFile tests parsing a local code file (TypeScript)
func TestParseWithLocalCodeFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
tsPath := getTestFilePath("code.ts")
if _, err := os.Stat(tsPath); os.IsNotExist(err) {
t.Skipf("Test TypeScript file not found: %s", tsPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: tsPath,
Filename: "code.ts",
},
}
handler := text.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
// Code files should be wrapped in markdown code blocks
assert.True(t, strings.HasPrefix(result.Text, "```typescript"))
assert.True(t, strings.HasSuffix(strings.TrimSpace(result.Text), "```"))
t.Logf("Code parse result (first 500 chars): %.500s...", result.Text)
}
// TestParseWithLocalYaoFile tests parsing a local .yao file
func TestParseWithLocalYaoFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
yaoPath := getTestFilePath("hero.mod.yao")
if _, err := os.Stat(yaoPath); os.IsNotExist(err) {
t.Skipf("Test .yao file not found: %s", yaoPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: yaoPath,
Filename: "hero.mod.yao",
},
}
handler := text.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("Yao file parse result: %s", result.Text)
}
// TestParseWithLocalJsonFile tests parsing a local JSON file
func TestParseWithLocalJsonFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
jsonPath := getTestFilePath("test.json")
if _, err := os.Stat(jsonPath); os.IsNotExist(err) {
t.Skipf("Test JSON file not found: %s", jsonPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: jsonPath,
Filename: "test.json",
},
}
handler := text.New(options)
result, refs, err := handler.Parse(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
t.Logf("JSON parse result: %s", result.Text)
}
// TestParseWithNonExistentFile tests parsing text with non-existent file
func TestParseWithNonExistentFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: "/non/existent/path/test.txt",
Filename: "test.txt",
},
}
handler := text.New(options)
_, _, err := handler.Parse(ctx, content)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unsupported text file source")
}
// TestParseRawWithLocalFile tests ParseRaw with a local file
func TestParseRawWithLocalFile(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
testutils.Prepare(t)
defer testutils.Clean(t)
txtPath := getTestFilePath("text.txt")
if _, err := os.Stat(txtPath); os.IsNotExist(err) {
t.Skipf("Test text file not found: %s", txtPath)
}
options := newTestOptions()
ctx := newTestContext()
content := agentContext.ContentPart{
Type: agentContext.ContentFile,
File: &agentContext.FileAttachment{
URL: txtPath,
Filename: "text.txt",
},
}
handler := text.New(options)
result, refs, err := handler.ParseRaw(ctx, content)
assert.NoError(t, err)
assert.Nil(t, refs)
assert.Equal(t, agentContext.ContentText, result.Type)
assert.NotEmpty(t, result.Text)
// ParseRaw should include filename as context
assert.True(t, strings.HasPrefix(result.Text, "File: text.txt"))
t.Logf("ParseRaw result: %s", result.Text)
}

152
agent/content/text_test.go
View file

@ -1,152 +0,0 @@
package content
import (
"testing"
)
func TestTextHandler_CanHandle(t *testing.T) {
handler := &TextHandler{}
tests := []struct {
name string
contentType string
fileType FileType
want bool
}{
{"Plain text", "text/plain", FileTypeText, true},
{"Markdown", "text/markdown", FileTypeText, true},
{"HTML", "text/html", FileTypeText, true},
{"JSON", "application/json", FileTypeJSON, true},
{"JavaScript", "application/javascript", FileTypeText, true},
{"TypeScript", "application/typescript", FileTypeText, true},
{"YAML", "application/yaml", FileTypeText, true},
{"CSV", "text/csv", FileTypeCSV, true},
{"XML", "application/xml", FileTypeText, true},
{"PDF (should not handle)", "application/pdf", FileTypePDF, false},
{"Image (should not handle)", "image/png", FileTypeImage, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := handler.CanHandle(tt.contentType, tt.fileType)
if got != tt.want {
t.Errorf("CanHandle(%q, %q) = %v, want %v", tt.contentType, tt.fileType, got, tt.want)
}
})
}
}
func TestTextHandler_Handle(t *testing.T) {
handler := &TextHandler{}
tests := []struct {
name string
info *Info
wantErr bool
checkResult func(*testing.T, *Result)
}{
{
name: "Plain text",
info: &Info{
FileType: FileTypeText,
ContentType: "text/plain",
Data: []byte("Hello, World!"),
},
wantErr: false,
checkResult: func(t *testing.T, r *Result) {
if r.Text != "Hello, World!" {
t.Errorf("Expected 'Hello, World!', got %q", r.Text)
}
},
},
{
name: "JSON with pretty print",
info: &Info{
FileType: FileTypeJSON,
ContentType: "application/json",
Data: []byte(`{"name":"test","value":123}`),
},
wantErr: false,
checkResult: func(t *testing.T, r *Result) {
// Should be pretty printed
if len(r.Text) <= len(`{"name":"test","value":123}`) {
t.Errorf("JSON should be pretty printed, got: %q", r.Text)
}
},
},
{
name: "Code file (Go)",
info: &Info{
FileType: FileTypeText,
ContentType: "text/plain",
Data: []byte("package main\n\nfunc main() {\n\tprintln(\"Hello\")\n}"),
},
wantErr: false,
checkResult: func(t *testing.T, r *Result) {
if r.Text == "" {
t.Error("Expected non-empty text for Go code")
}
},
},
{
name: "Empty data",
info: &Info{
FileType: FileTypeText,
ContentType: "text/plain",
Data: []byte{},
},
wantErr: true,
},
}
// Create test context
testCtx := newTestContext(nil)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := handler.Handle(testCtx, tt.info, nil, nil, false)
if (err != nil) != tt.wantErr {
t.Errorf("Handle() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !tt.wantErr && tt.checkResult != nil {
tt.checkResult(t, result)
}
})
}
}
func TestDetectFileType(t *testing.T) {
tests := []struct {
name string
contentType string
filename string
want FileType
}{
{"Go file", "text/plain", "main.go", FileTypeText},
{"Python file", "text/plain", "script.py", FileTypeText},
{"JavaScript file", "application/javascript", "app.js", FileTypeText},
{"TypeScript file", "text/plain", "index.ts", FileTypeText},
{"Markdown file", "text/markdown", "README.md", FileTypeText},
{"JSON file", "application/json", "config.json", FileTypeJSON},
{"YAML file", "text/plain", "config.yml", FileTypeText},
{"PDF file", "application/pdf", "document.pdf", FileTypePDF},
{"Image file", "image/png", "photo.png", FileTypeImage},
{"CSV file", "text/csv", "data.csv", FileTypeCSV},
{"XML file", "application/xml", "config.xml", FileTypeXML},
{"Shell script", "text/plain", "script.sh", FileTypeText},
{"Dockerfile", "text/plain", "Dockerfile", FileTypeText},
{"gitignore", "text/plain", ".gitignore", FileTypeText},
{"HTML", "text/html", "index.html", FileTypeText},
{"CSS", "text/css", "styles.css", FileTypeText},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := DetectFileType(tt.contentType, tt.filename)
if got != tt.want {
t.Errorf("DetectFileType(%q, %q) = %v, want %v", tt.contentType, tt.filename, got, tt.want)
}
})
}
}

240
agent/content/tools.go
View file

@ -1,240 +0,0 @@
package content
import (
"context"
"encoding/base64"
"fmt"
"sync"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/mcp"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/agent/caller"
agentContext "github.com/yaoapp/yao/agent/context"
)
// fileInfoMutex protects concurrent access to files_info list in Space
var fileInfoMutex sync.Mutex
// CallAgent calls an agent to process content (vision, audio, etc.)
// This is a generic function that can be used by any handler
func CallAgent(ctx *agentContext.Context, agentID string, message agentContext.Message) (string, error) {
if caller.AgentGetterFunc == nil {
return "", fmt.Errorf("AgentGetterFunc not initialized")
}
// Load the agent by ID using the injected function
agent, err := caller.AgentGetterFunc(agentID)
if err != nil {
return "", fmt.Errorf("failed to load agent %s: %w", agentID, err)
}
// Call the agent with the message
messages := []agentContext.Message{message}
// Note: Connector is now in Options (call-level parameter), not Context
// For A2A calls, skip history and output (we only need the response data)
opts := &agentContext.Options{Skip: &agentContext.Skip{History: true, Output: true}} // Skip history and output
response, err := agent.Stream(ctx, messages, opts)
if err != nil {
return "", fmt.Errorf("failed to call agent %s: %w", agentID, err)
}
// Extract text from agent response
// Two formats are supported:
// 1. Custom Hook response (from Next hook) - response.Next
// 2. Standard Agent Stream response (LLM completion) - response.Completion
return extractTextFromResponse(response)
}
// CallAgentWithFileInfo calls an agent to process content with file metadata
// The file metadata is passed via ctx.Memory.Context for access by hooks (especially Next hook)
// Uses Memory.Context (request-scoped) to avoid creating context copies and ensure proper cleanup
//
// Memory Keys (with agent ID as namespace prefix to avoid conflicts between different agents):
// - {agentID}:files_info - List of all files being processed by this agent (array)
// - {agentID}:current_file - Currently processing file (single object)
func CallAgentWithFileInfo(ctx *agentContext.Context, agentID string, message agentContext.Message, info *Info) (string, error) {
// Store file information in Memory.Context if available
if info != nil && ctx.Memory != nil && ctx.Memory.Context != nil {
fileInfo := map[string]interface{}{
"url": info.URL,
"filename": info.Filename,
"content_type": info.ContentType,
"file_type": string(info.FileType),
"source": string(info.Source),
}
// Add uploader-specific information if available
if info.UploaderName != "" {
fileInfo["uploader_name"] = info.UploaderName
}
if info.FileID != "" {
fileInfo["file_id"] = info.FileID
}
// Use agent ID as namespace prefix for Memory keys
filesListKey := agentID + ":files_info"
currentFileKey := agentID + ":current_file"
// Thread-safe: append current file to files list
fileInfoMutex.Lock()
var filesList []map[string]interface{}
if existing, ok := ctx.Memory.Context.Get(filesListKey); ok {
// Convert existing data to []map[string]interface{}
if existingList, ok := existing.([]interface{}); ok {
for _, item := range existingList {
if itemMap, ok := item.(map[string]interface{}); ok {
filesList = append(filesList, itemMap)
}
}
} else if existingList, ok := existing.([]map[string]interface{}); ok {
filesList = existingList
}
}
// Append current file to list
filesList = append(filesList, fileInfo)
ctx.Memory.Context.Set(filesListKey, filesList, 0)
fileInfoMutex.Unlock()
// Store current file in Memory.Context
if err := ctx.Memory.Context.Set(currentFileKey, fileInfo, 0); err != nil {
log.Trace("[Content] Failed to set current file info in Memory.Context: %v", err)
}
// Ensure cleanup after agent call completes
defer func() {
// Clean up current file
if err := ctx.Memory.Context.Del(currentFileKey); err != nil {
log.Trace("[Content] Failed to delete current file info from Memory.Context: %v", err)
}
// Clean up files list (reset for next call)
if err := ctx.Memory.Context.Del(filesListKey); err != nil {
log.Trace("[Content] Failed to delete files list from Memory.Context: %v", err)
}
}()
}
// Call the agent with the original context
return CallAgent(ctx, agentID, message)
}
// extractTextFromResponse extracts text from agent response
// Now that agent.Stream() returns *agentContext.Response directly,
// we can access fields without type assertions or JSON conversion.
//
// Priority:
// 1. Check response.Next (custom hook data) → return complete data
// 2. Check response.Completion (standard LLM response) → extract text only
func extractTextFromResponse(response *agentContext.Response) (string, error) {
if response == nil {
return "", fmt.Errorf("agent returned nil response")
}
// Priority 1: Check Next field (custom hook data)
// If Next hook returns custom data, return the complete structure
if response.Next != nil {
// If next is a string, return directly
if nextStr, ok := response.Next.(string); ok {
return nextStr, nil
}
// Otherwise, JSON stringify to preserve complete structure
jsonBytes, err := jsoniter.Marshal(response.Next)
if err != nil {
return "", fmt.Errorf("failed to serialize next hook data: %w", err)
}
return string(jsonBytes), nil
}
// Priority 2: Check Completion field (standard LLM response)
// Extract text content from the LLM completion
if response.Completion != nil {
// Content can be string or []ContentPart (multimodal)
switch v := response.Completion.Content.(type) {
case string:
// Simple text content
return v, nil
case []interface{}:
// Multimodal content array - extract all text parts
var text string
for _, part := range v {
if partMap, ok := part.(map[string]interface{}); ok {
if partType, _ := partMap["type"].(string); partType == "text" {
if textContent, ok := partMap["text"].(string); ok {
text += textContent
}
}
}
}
if text != "" {
return text, nil
}
// No text found in content parts
return "", fmt.Errorf("no text content found in completion content parts")
}
}
// No content found
return "", fmt.Errorf("no content found in agent response")
}
// CallMCPTool calls an MCP tool to process content
// This is a generic function that can be used by any handler
func CallMCPTool(ctx *agentContext.Context, serverID string, toolName string, arguments map[string]interface{}) (string, error) {
// Get MCP context for cancellation/timeout control
mcpCtx := ctx.Context
if mcpCtx == nil {
mcpCtx = context.Background()
}
// Get MCP client
client, err := mcp.Select(serverID)
if err != nil {
return "", fmt.Errorf("failed to select MCP client '%s': %w", serverID, err)
}
// Call the tool
log.Trace("[Content] Calling MCP tool: %s (server: %s)", toolName, serverID)
callResult, err := client.CallTool(mcpCtx, toolName, arguments)
if err != nil {
return "", fmt.Errorf("MCP tool call failed: %w", err)
}
// Check if result is an error
if callResult.IsError {
return "", fmt.Errorf("MCP tool returned error: %v", callResult.Content)
}
// Extract text content from result
// callResult.Content is []ToolContent
var text string
for _, content := range callResult.Content {
if content.Type == "text" {
text += content.Text
}
// Can also handle other types like image, resource if needed
}
if text == "" {
// If no text content found, return error
return "", fmt.Errorf("MCP tool returned no text content")
}
return text, nil
}
// EncodeToBase64DataURI encodes data to base64 with data URI prefix
// This is useful for encoding images, audio, or other binary data
func EncodeToBase64DataURI(data []byte, contentType string) string {
// Ensure we have a valid content type
if contentType == "" {
contentType = "application/octet-stream" // default
}
// Encode to base64
encoded := base64.StdEncoding.EncodeToString(data)
// Return data URI format
return fmt.Sprintf("data:%s;base64,%s", contentType, encoded)
}

125
agent/content/tools/tools.go Normal file
View file

@ -0,0 +1,125 @@
package tools
import (
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/yaoapp/gou/mcp"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/agent/caller"
agentContext "github.com/yaoapp/yao/agent/context"
)
// CallAgent calls an agent to process content (vision, audio, etc.)
func CallAgent(ctx *agentContext.Context, agentID string, message agentContext.Message) (string, error) {
if caller.AgentGetterFunc == nil {
return "", fmt.Errorf("AgentGetterFunc not initialized")
}
// Load the agent by ID using the injected function
agent, err := caller.AgentGetterFunc(agentID)
if err != nil {
return "", fmt.Errorf("failed to load agent %s: %w", agentID, err)
}
// Call the agent with the message
messages := []agentContext.Message{message}
// For A2A calls, skip history and output (we only need the response data)
opts := &agentContext.Options{Skip: &agentContext.Skip{History: true, Output: true}}
response, err := agent.Stream(ctx, messages, opts)
if err != nil {
return "", fmt.Errorf("failed to call agent %s: %w", agentID, err)
}
// Extract text from agent response
return ExtractTextFromResponse(response)
}
// ExtractTextFromResponse extracts text from agent response
func ExtractTextFromResponse(response *agentContext.Response) (string, error) {
if response == nil {
return "", fmt.Errorf("agent returned nil response")
}
// Priority 1: Check Next field (custom hook data)
if response.Next != nil {
if nextStr, ok := response.Next.(string); ok {
return nextStr, nil
}
// Otherwise, JSON stringify to preserve complete structure
jsonBytes, err := jsoniter.Marshal(response.Next)
if err != nil {
return "", fmt.Errorf("failed to serialize next hook data: %w", err)
}
return string(jsonBytes), nil
}
// Priority 2: Check Completion field (standard LLM response)
if response.Completion != nil {
switch v := response.Completion.Content.(type) {
case string:
return v, nil
case []interface{}:
// Multimodal content array - extract all text parts
var text string
for _, part := range v {
if partMap, ok := part.(map[string]interface{}); ok {
if partType, _ := partMap["type"].(string); partType == "text" {
if textContent, ok := partMap["text"].(string); ok {
text += textContent
}
}
}
}
if text != "" {
return text, nil
}
return "", fmt.Errorf("no text content found in completion content parts")
}
}
return "", fmt.Errorf("no content found in agent response")
}
// CallMCPTool calls an MCP tool to process content
func CallMCPTool(ctx *agentContext.Context, serverID string, toolName string, arguments map[string]interface{}) (string, error) {
// Get MCP context for cancellation/timeout control
mcpCtx := ctx.Context
if mcpCtx == nil {
mcpCtx = context.Background()
}
// Get MCP client
client, err := mcp.Select(serverID)
if err != nil {
return "", fmt.Errorf("failed to select MCP client '%s': %w", serverID, err)
}
// Call the tool
log.Trace("[Content] Calling MCP tool: %s (server: %s)", toolName, serverID)
callResult, err := client.CallTool(mcpCtx, toolName, arguments)
if err != nil {
return "", fmt.Errorf("MCP tool call failed: %w", err)
}
// Check if result is an error
if callResult.IsError {
return "", fmt.Errorf("MCP tool returned error: %v", callResult.Content)
}
// Extract text content from result
var text string
for _, content := range callResult.Content {
if content.Type == "text" {
text += content.Text
}
}
if text == "" {
return "", fmt.Errorf("MCP tool returned no text content")
}
return text, nil
}

262
agent/content/types.go
View file

@ -1,262 +0,0 @@
package content
import "github.com/yaoapp/yao/agent/context"
// FileType represents the type of file content
type FileType string
const (
// Image types
FileTypeImage FileType = "image"
// Audio types
FileTypeAudio FileType = "audio"
// Document types
FileTypeText FileType = "text"
FileTypePDF FileType = "pdf"
FileTypeWord FileType = "word"
FileTypeExcel FileType = "excel"
FileTypePPT FileType = "ppt"
FileTypeCSV FileType = "csv"
// Data types
FileTypeJSON FileType = "json"
FileTypeXML FileType = "xml"
// Binary
FileTypeBinary FileType = "binary"
// Other
FileTypeUnknown FileType = "unknown"
)
// Source represents where the content comes from
type Source string
const (
SourceHTTP Source = "http" // HTTP(S) URL
SourceUploader Source = "uploader" // Uploader wrapper: __uploader://fileid
SourceBase64 Source = "base64" // Base64 encoded data
SourceLocal Source = "local" // Local file path
)
// Result represents the result of content handling
type Result struct {
Text string // Extracted text content
ContentPart *context.ContentPart // Processed ContentPart (for model input)
Metadata map[string]interface{} // Additional metadata
Error error // Error if handling failed
}
// Info holds information about a content part to be handled
type Info struct {
Source Source // Where the content comes from
FileType FileType // Type of the file
ContentType string // MIME content type
URL string // Original URL or file ID
Filename string // Original filename (if available)
Data []byte // File data (if already fetched)
// For uploader wrapper
UploaderName string // Uploader name from wrapper
FileID string // File ID from wrapper
}
// DetectFileType detects file type from content type, filename, and file extension
func DetectFileType(contentType, filename string) FileType {
// Check by content type first
switch {
case contentType == "application/pdf":
return FileTypePDF
case contentType == "application/json":
return FileTypeJSON
case contentType == "application/xml" || contentType == "text/xml":
return FileTypeXML
case contentType == "text/csv":
return FileTypeCSV
case contentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
contentType == "application/msword":
return FileTypeWord
case contentType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
contentType == "application/vnd.ms-excel":
return FileTypeExcel
case contentType == "application/vnd.openxmlformats-officedocument.presentationml.presentation",
contentType == "application/vnd.ms-powerpoint":
return FileTypePPT
}
// Check image types
if isImageContentType(contentType) {
return FileTypeImage
}
// Check audio types
if isAudioContentType(contentType) {
return FileTypeAudio
}
// Check text types
if isTextContentType(contentType) {
return FileTypeText
}
// If content type doesn't help, check file extension
if filename != "" {
if ext := getFileExtension(filename); ext != "" {
return detectTypeByExtension(ext)
}
}
return FileTypeUnknown
}
// isImageContentType checks if content type is an image
func isImageContentType(contentType string) bool {
return contentType != "" &&
(contentType == "image/png" ||
contentType == "image/jpeg" ||
contentType == "image/jpg" ||
contentType == "image/gif" ||
contentType == "image/webp" ||
contentType == "image/svg+xml" ||
contentType == "image/bmp")
}
// isAudioContentType checks if content type is audio
func isAudioContentType(contentType string) bool {
return contentType != "" &&
(contentType == "audio/mpeg" ||
contentType == "audio/mp3" ||
contentType == "audio/wav" ||
contentType == "audio/ogg" ||
contentType == "audio/flac" ||
contentType == "audio/aac")
}
// isTextContentType checks if content type is text-based
func isTextContentType(contentType string) bool {
if contentType == "" {
return false
}
// Common text MIME types
textTypes := []string{
"text/plain",
"text/html",
"text/css",
"text/javascript",
"text/markdown",
"text/x-markdown",
"application/javascript",
"application/typescript",
"application/x-yaml",
"application/yaml",
"application/toml",
"application/x-sh",
"application/x-python",
"application/x-ruby",
"application/x-perl",
"application/x-php",
"application/x-go",
}
for _, t := range textTypes {
if contentType == t {
return true
}
}
return false
}
// getFileExtension extracts file extension from filename (without dot)
func getFileExtension(filename string) string {
for i := len(filename) - 1; i >= 0; i-- {
if filename[i] == '.' {
return filename[i+1:]
}
if filename[i] == '/' || filename[i] == '\\' {
break
}
}
return ""
}
// detectTypeByExtension detects file type by file extension
func detectTypeByExtension(ext string) FileType {
// Normalize to lowercase
ext = toLower(ext)
// Image extensions
imageExts := []string{"png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "ico"}
for _, e := range imageExts {
if ext == e {
return FileTypeImage
}
}
// Audio extensions
audioExts := []string{"mp3", "wav", "ogg", "flac", "aac", "m4a"}
for _, e := range audioExts {
if ext == e {
return FileTypeAudio
}
}
// Document extensions
switch ext {
case "pdf":
return FileTypePDF
case "doc", "docx":
return FileTypeWord
case "xls", "xlsx":
return FileTypeExcel
case "ppt", "pptx":
return FileTypePPT
case "csv":
return FileTypeCSV
case "json":
return FileTypeJSON
case "xml":
return FileTypeXML
}
// Code and text file extensions (very comprehensive list)
textExts := []string{
"txt", "text", "md", "markdown", "rst",
// Programming languages
"go", "py", "js", "ts", "jsx", "tsx", "java", "c", "cpp", "h", "hpp",
"cs", "rb", "php", "pl", "swift", "kt", "rs", "scala", "clj",
// Web
"html", "htm", "css", "scss", "sass", "less",
// Config
"yaml", "yml", "toml", "ini", "conf", "config",
// Shell
"sh", "bash", "zsh", "fish",
// Data
"sql", "graphql", "proto",
// Others
"log", "gitignore", "env", "dockerfile",
}
for _, e := range textExts {
if ext == e {
return FileTypeText
}
}
return FileTypeUnknown
}
// toLower converts ASCII string to lowercase (simple version)
func toLower(s string) string {
result := make([]byte, len(s))
for i := 0; i < len(s); i++ {
c := s[i]
if c >= 'A' && c <= 'Z' {
c += 'a' - 'A'
}
result[i] = c
}
return string(result)
}

26
agent/content/types/types.go Normal file
View file

@ -0,0 +1,26 @@
package types
import (
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// Options represents the options for the content
type Options struct {
// Connector, Current connector instance
Connector connector.Connector
// Capabilities, Current capabilities instance
Capabilities *openai.Capabilities
// CompletionOptions, Current completion options instance
CompletionOptions *agentContext.CompletionOptions
// StreamOptions, Current stream options instance
StreamOptions *agentContext.StreamOptions
// SilentLoading, if true, suppress loading messages (used when called from parent handler)
SilentLoading bool
}

39
agent/content/word.go
View file

@ -1,39 +0,0 @@
package content
import (
"fmt"
"strings"
"github.com/yaoapp/gou/connector/openai"
agentContext "github.com/yaoapp/yao/agent/context"
)
// WordHandler handles Microsoft Word documents
type WordHandler struct{}
// CanHandle checks if this handler can handle the content type
func (h *WordHandler) CanHandle(contentType string, fileType FileType) bool {
return fileType == FileTypeWord ||
contentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
contentType == "application/msword" ||
strings.Contains(contentType, "word")
}
// Handle processes Word document content
func (h *WordHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
// TODO: Implement Word document handling
// 1. Extract text from .docx or .doc file
// 2. Preserve formatting information if needed
// 3. Return Result with extracted text
return nil, fmt.Errorf("not implemented")
}
// extractWordText extracts text from Word document
func extractWordText(data []byte, contentType string) (string, error) {
// TODO: Implement Word text extraction
// Handle both .doc (old format) and .docx (new format)
// Consider using libraries like:
// - github.com/unidoc/unioffice for .docx
// - Other libraries for .doc
return "", fmt.Errorf("not implemented")
}

14
agent/context/types.go
View file

@ -192,11 +192,12 @@ type AssistantInfo struct {
// Skip configuration for what to skip in this request
type Skip struct {
History bool `json:"history"` // Skip saving chat history (for internal calls like title/prompt generation)
Trace bool `json:"trace"` // Skip trace logging
Output bool `json:"output"` // Skip output to client (for internal A2A calls that only need response data)
Keyword bool `json:"keyword"` // Skip keyword extraction for web search (use raw query directly)
Search bool `json:"search"` // Skip auto search (for internal calls like needsearch intent detection)
History bool `json:"history"` // Skip saving chat history (for internal calls like title/prompt generation)
Trace bool `json:"trace"` // Skip trace logging
Output bool `json:"output"` // Skip output to client (for internal A2A calls that only need response data)
Keyword bool `json:"keyword"` // Skip keyword extraction for web search (use raw query directly)
Search bool `json:"search"` // Skip auto search (for internal calls like needsearch intent detection)
ContentParsing bool `json:"content_parsing"` // Skip content parsing (vision, PDF, docx, etc.), convert files to raw text directly
}
// MessageMetadata stores metadata for sent messages
@ -302,6 +303,9 @@ type Options struct {
// Agent mode, use to select the mode of the request, default is "chat"
Mode string `json:"mode,omitempty"` // Agent mode, use to select the mode of the request, default is "chat"
// Uses configuration, allow hook to override wrapper configurations for vision, audio, search, and fetch
Uses *Uses `json:"uses,omitempty"` // Uses configuration, allow hook to override wrapper configurations for vision, audio, search, and fetch
// Metadata for passing custom data to hooks (e.g., scenario selection)
Metadata map[string]any `json:"metadata,omitempty"` // Custom metadata passed to Create/Next hooks
}

18
agent/i18n/builtin.go
View file

@ -99,6 +99,12 @@ func init() {
"kb.chat.name": "Chat Knowledge Base",
"kb.chat.description": "Auto-created knowledge base collection for chat sessions",
// Content: content/image/image.go - Image processing messages
"content.image.analyzing": "Analyzing image...",
// Content: content/pdf/pdf.go - PDF processing messages
"content.pdf.analyzing_page": "Analyzing PDF page %d/%d...",
// Search: assistant/search.go - Output messages
"search.loading": "Searching...",
"search.success": "Found %d references",
@ -192,6 +198,12 @@ func init() {
"kb.chat.name": "聊天知识库",
"kb.chat.description": "自动为聊天会话创建的知识库集合",
// Content: content/image/image.go - Image processing messages
"content.image.analyzing": "正在分析图片...",
// Content: content/pdf/pdf.go - PDF processing messages
"content.pdf.analyzing_page": "正在分析 PDF 第 %d/%d 页...",
// Search: assistant/search.go - Output messages
"search.loading": "正在搜索...",
"search.success": "找到 %d 条参考资料",
@ -313,6 +325,12 @@ func init() {
"kb.chat.name": "聊天知识库",
"kb.chat.description": "自动为聊天会话创建的知识库集合",
// Content: content/image/image.go - Image processing messages
"content.image.analyzing": "正在分析图片...",
// Content: content/pdf/pdf.go - PDF processing messages
"content.pdf.analyzing_page": "正在分析 PDF 第 %d/%d 页...",
// Search: assistant/search.go - Output messages
"search.loading": "正在搜索...",
"search.success": "找到 %d 条参考资料",

18
agent/search/search.go
View file

@ -124,9 +124,23 @@ func (s *Searcher) parallelAll(ctx *context.Context, reqs []*types.Request) ([]*
wg.Add(1)
go func(idx int, r *types.Request) {
defer wg.Done()
result, _ := s.Search(ctx, r)
defer func() {
if err := recover(); err != nil {
mu.Lock()
results[idx] = &types.Result{Error: "search panic recovered"}
mu.Unlock()
}
}()
result, err := s.Search(ctx, r)
mu.Lock()
results[idx] = result
if err != nil {
results[idx] = &types.Result{Error: err.Error()}
} else if result == nil {
results[idx] = &types.Result{Error: "empty result"}
} else {
results[idx] = result
}
mu.Unlock()
}(i, req)
}