diff --git a/agent/llm/adapters/adapter.go b/agent/llm/adapters/adapter.go new file mode 100644 index 00000000..cf138971 --- /dev/null +++ b/agent/llm/adapters/adapter.go @@ -0,0 +1,64 @@ +package adapters + +import ( + "github.com/yaoapp/yao/agent/context" +) + +// CapabilityAdapter is the interface for capability-specific message and response processing +// Each adapter handles one capability dimension (tool calls, vision, audio, reasoning, etc.) +type CapabilityAdapter interface { + // Name returns the adapter name for debugging + Name() string + + // PreprocessMessages preprocesses messages before sending to LLM + // Returns modified messages or error + PreprocessMessages(messages []context.Message) ([]context.Message, error) + + // PreprocessOptions preprocesses completion options before sending to LLM + // Returns modified options or error + PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) + + // PostprocessResponse postprocesses the LLM response + // Returns modified response or error + PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) + + // ProcessStreamChunk processes a streaming chunk + // Returns modified chunk type and data, or error + ProcessStreamChunk(chunkType context.StreamChunkType, data []byte) (context.StreamChunkType, []byte, error) +} + +// BaseAdapter provides default implementations for CapabilityAdapter +// Adapters can embed this and override only the methods they need +type BaseAdapter struct { + name string +} + +// NewBaseAdapter creates a new base adapter +func NewBaseAdapter(name string) *BaseAdapter { + return &BaseAdapter{name: name} +} + +// Name returns the adapter name +func (a *BaseAdapter) Name() string { + return a.name +} + +// PreprocessMessages default implementation (no-op) +func (a *BaseAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) { + return messages, nil +} + +// PreprocessOptions default implementation (no-op) +func (a *BaseAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) { + return options, nil +} + +// PostprocessResponse default implementation (no-op) +func (a *BaseAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) { + return response, nil +} + +// ProcessStreamChunk default implementation (pass through) +func (a *BaseAdapter) ProcessStreamChunk(chunkType context.StreamChunkType, data []byte) (context.StreamChunkType, []byte, error) { + return chunkType, data, nil +} diff --git a/agent/llm/adapters/audio.go b/agent/llm/adapters/audio.go new file mode 100644 index 00000000..ec035abd --- /dev/null +++ b/agent/llm/adapters/audio.go @@ -0,0 +1,62 @@ +package adapters + +import ( + "github.com/yaoapp/yao/agent/context" +) + +// AudioAdapter handles audio capability +// If model doesn't support audio, it removes or converts audio content +type AudioAdapter struct { + *BaseAdapter + nativeSupport bool +} + +// NewAudioAdapter creates a new audio adapter +func NewAudioAdapter(nativeSupport bool) *AudioAdapter { + return &AudioAdapter{ + BaseAdapter: NewBaseAdapter("AudioAdapter"), + nativeSupport: nativeSupport, + } +} + +// PreprocessMessages removes or converts audio content if not supported +func (a *AudioAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) { + if a.nativeSupport { + // Native support, no preprocessing needed + return messages, nil + } + + // Process messages to remove audio content + processed := make([]context.Message, 0, len(messages)) + for _, msg := range messages { + processedMsg := msg + + // Handle multimodal content (array of ContentPart) + if contentParts, ok := msg.Content.([]context.ContentPart); ok { + filteredParts := make([]context.ContentPart, 0) + + for _, part := range contentParts { + // Skip audio content if not supported + if part.Type == context.ContentInputAudio { + // TODO: Optionally convert to transcription text if available + continue + } + filteredParts = append(filteredParts, part) + } + + // If all parts were filtered out, add placeholder text + if len(filteredParts) == 0 { + processedMsg.Content = "[Audio content not supported by this model]" + } else if len(filteredParts) == 1 && filteredParts[0].Type == context.ContentText { + // Single text part, convert to string + processedMsg.Content = filteredParts[0].Text + } else { + processedMsg.Content = filteredParts + } + } + + processed = append(processed, processedMsg) + } + + return processed, nil +} diff --git a/agent/llm/adapters/reasoning.go b/agent/llm/adapters/reasoning.go new file mode 100644 index 00000000..ba641032 --- /dev/null +++ b/agent/llm/adapters/reasoning.go @@ -0,0 +1,59 @@ +package adapters + +import ( + "github.com/yaoapp/yao/agent/context" +) + +// ReasoningFormat represents the reasoning content format +type ReasoningFormat string + +const ( + ReasoningFormatNone ReasoningFormat = "none" // No reasoning support + ReasoningFormatOpenAI ReasoningFormat = "openai-o1" // OpenAI o1 format + ReasoningFormatDeepSeek ReasoningFormat = "deepseek-r1" // DeepSeek R1 format + ReasoningFormatGPTThink ReasoningFormat = "gpt-think" // Future GPT with thinking +) + +// ReasoningAdapter handles reasoning content capability +// Parses reasoning_content from different model formats +type ReasoningAdapter struct { + *BaseAdapter + format ReasoningFormat +} + +// NewReasoningAdapter creates a new reasoning adapter +func NewReasoningAdapter(format ReasoningFormat) *ReasoningAdapter { + return &ReasoningAdapter{ + BaseAdapter: NewBaseAdapter("ReasoningAdapter"), + format: format, + } +} + +// ProcessStreamChunk processes streaming chunks with reasoning content +func (a *ReasoningAdapter) ProcessStreamChunk(chunkType context.StreamChunkType, data []byte) (context.StreamChunkType, []byte, error) { + if a.format == ReasoningFormatNone { + // No reasoning support, pass through + return chunkType, data, nil + } + + // TODO: Parse reasoning_content based on format + // - OpenAI o1: reasoning_content field in delta + // - DeepSeek R1: may have different format + // - Extract and emit as ChunkThinking + + return chunkType, data, nil +} + +// PostprocessResponse extracts reasoning content from the final response +func (a *ReasoningAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) { + if a.format == ReasoningFormatNone { + // No reasoning support + return response, nil + } + + // TODO: Extract reasoning content from response + // - Set response.ReasoningContent if present + // - Separate thinking from final answer + + return response, nil +} diff --git a/agent/llm/adapters/toolcall.go b/agent/llm/adapters/toolcall.go new file mode 100644 index 00000000..9e58b325 --- /dev/null +++ b/agent/llm/adapters/toolcall.go @@ -0,0 +1,67 @@ +package adapters + +import ( + "github.com/yaoapp/yao/agent/context" +) + +// ToolCallAdapter handles tool calling capability +// If model doesn't support native tool calls, it injects tool instructions into prompts +type ToolCallAdapter struct { + *BaseAdapter + nativeSupport bool +} + +// NewToolCallAdapter creates a new tool call adapter +func NewToolCallAdapter(nativeSupport bool) *ToolCallAdapter { + return &ToolCallAdapter{ + BaseAdapter: NewBaseAdapter("ToolCallAdapter"), + nativeSupport: nativeSupport, + } +} + +// PreprocessMessages injects tool calling instructions if not natively supported +func (a *ToolCallAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) { + if a.nativeSupport { + // Native support, no preprocessing needed + return messages, nil + } + + // TODO: Inject tool calling instructions into system prompt + // - Generate tool description prompt + // - Add to system message or create new system message + // - Include tool schemas and usage instructions + return messages, nil +} + +// PreprocessOptions removes tool-related options if not natively supported +func (a *ToolCallAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) { + if a.nativeSupport { + // Native support, keep options as-is + return options, nil + } + + if options == nil { + return options, nil + } + + // Remove tool parameters for non-native models + newOptions := *options + newOptions.Tools = nil + newOptions.ToolChoice = nil + return &newOptions, nil +} + +// PostprocessResponse extracts tool calls from text if not natively supported +func (a *ToolCallAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) { + if a.nativeSupport { + // Native support, response already has structured tool calls + return response, nil + } + + // TODO: Extract tool calls from text response + // - Look for JSON blocks or specific patterns + // - Parse tool name and arguments + // - Add to response.ToolCalls + return response, nil +} + diff --git a/agent/llm/adapters/vision.go b/agent/llm/adapters/vision.go new file mode 100644 index 00000000..0e746a65 --- /dev/null +++ b/agent/llm/adapters/vision.go @@ -0,0 +1,62 @@ +package adapters + +import ( + "github.com/yaoapp/yao/agent/context" +) + +// VisionAdapter handles vision (image) capability +// If model doesn't support vision, it removes or converts image content +type VisionAdapter struct { + *BaseAdapter + nativeSupport bool +} + +// NewVisionAdapter creates a new vision adapter +func NewVisionAdapter(nativeSupport bool) *VisionAdapter { + return &VisionAdapter{ + BaseAdapter: NewBaseAdapter("VisionAdapter"), + nativeSupport: nativeSupport, + } +} + +// PreprocessMessages removes or converts image content if not supported +func (a *VisionAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) { + if a.nativeSupport { + // Native support, no preprocessing needed + return messages, nil + } + + // Process messages to remove image content + processed := make([]context.Message, 0, len(messages)) + for _, msg := range messages { + processedMsg := msg + + // Handle multimodal content (array of ContentPart) + if contentParts, ok := msg.Content.([]context.ContentPart); ok { + filteredParts := make([]context.ContentPart, 0) + + for _, part := range contentParts { + // Skip image content if not supported + if part.Type == context.ContentImageURL { + // TODO: Optionally convert to text description + continue + } + filteredParts = append(filteredParts, part) + } + + // If all parts were filtered out, add placeholder text + if len(filteredParts) == 0 { + processedMsg.Content = "[Image content not supported by this model]" + } else if len(filteredParts) == 1 && filteredParts[0].Type == context.ContentText { + // Single text part, convert to string + processedMsg.Content = filteredParts[0].Text + } else { + processedMsg.Content = filteredParts + } + } + + processed = append(processed, processedMsg) + } + + return processed, nil +} diff --git a/agent/llm/providers/README.md b/agent/llm/providers/README.md index f21b4ca0..b9d3be2f 100644 --- a/agent/llm/providers/README.md +++ b/agent/llm/providers/README.md @@ -1,319 +1,335 @@ -# LLM Providers Architecture +# LLM Providers Architecture (New) ## Overview -This directory contains different LLM provider implementations, each optimized for specific model capabilities. +This directory contains LLM provider implementations using the **Capability Adapters** pattern. The new architecture separates API format handling from capability handling. -## Provider Selection Strategy +## Architecture Design -The `factory.SelectProvider()` function automatically selects the appropriate provider based on model capabilities: +``` +┌─────────────────────────────────────────────────┐ +│ LLM Provider (API Format) │ +│ - OpenAI-compatible │ +│ - Claude (TODO) │ +│ - Custom (TODO) │ +└──────────────┬──────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────┐ +│ Capability Adapters (Modular) │ +│ - ToolCallAdapter (native or prompt eng.) │ +│ - VisionAdapter (native or removal) │ +│ - AudioAdapter (native or removal) │ +│ - ReasoningAdapter (o1/R1/GPT-Think) │ +└─────────────────────────────────────────────────┘ +``` + +## Key Concepts + +### 1. Provider = API Format + +Providers handle the **API communication format**: +- OpenAI-compatible API (`/v1/chat/completions`) +- Claude API (TODO) +- Custom API formats (TODO) + +### 2. Adapters = Capabilities + +Adapters handle **model capabilities** independently: +- **ToolCallAdapter**: Tool calling (native or prompt engineering) +- **VisionAdapter**: Image input (native or removal/conversion) +- **AudioAdapter**: Audio input (native or removal/conversion) +- **ReasoningAdapter**: Reasoning content (o1/DeepSeek R1/GPT-4o thinking) + +## Provider Selection ```go -Priority 1: Reasoning models → reasoning.Provider -Priority 2: Native tool support → openai.Provider -Priority 3: Legacy models → legacy.Provider +// factory.go +func SelectProvider(conn connector.Connector, options *context.CompletionOptions) (LLM, error) { + apiFormat := DetectAPIFormat(conn) + + switch apiFormat { + case "openai": + // Adapters automatically configured based on capabilities + return openai.New(conn, options.Capabilities), nil + case "claude": + return claude.New(conn, options.Capabilities), nil + default: + return openai.New(conn, options.Capabilities), nil + } +} ``` -## Provider Types +## Directory Structure -### 1. Base Provider (`base/`) +``` +providers/ +├── factory.go # Provider selection based on API format +├── base/ # Common functionality +│ └── base.go +├── openai/ # OpenAI-compatible API provider +│ └── openai.go # Includes adapter integration +└── README.md # This file -**Purpose**: Common functionality shared across all providers +../adapters/ # Capability adapters (separate package) +├── adapter.go # Base interface +├── toolcall.go # Tool calling adapter +├── vision.go # Vision adapter +├── audio.go # Audio adapter +└── reasoning.go # Reasoning adapter +``` -**Features**: +## OpenAI Provider -- Message preprocessing -- Request body building -- Response parsing - -**Usage**: Embedded in all other providers - ---- - -### 2. OpenAI Provider (`openai/`) - -**Purpose**: OpenAI-compatible models with full feature support - -**Capabilities**: - -- ✅ Vision (image input) -- ✅ Native tool calls -- ✅ Streaming -- ✅ JSON mode - -**Models**: - -- GPT-4, GPT-4o, GPT-4-turbo -- GPT-3.5-turbo -- Claude (via OpenAI-compatible API) - ---- - -### 3. Reasoning Provider (`reasoning/`) - -**Purpose**: Reasoning models with special response format - -**Capabilities**: - -- ✅ Reasoning content (`reasoning_content` field) -- ✅ Thinking + Answer phases -- ⚠️ Tool calls support varies by model - -**Models**: - -- **OpenAI o1** (supports native tool calls) -- **DeepSeek R1** (no native tool calls, uses prompt engineering) - -**Special Handling**: +The OpenAI provider supports **all capabilities** through adapters: ```go -// DeepSeek R1 scenario -if !supportsNativeTools && hasTools { - // Inject tool instructions into prompt - messages = injectToolInstructions(messages, tools) - // Extract tool calls from text response - toolCalls = extractToolCallsFromText(response.Content) +type Provider struct { + *base.Provider + adapters []adapters.CapabilityAdapter +} + +func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider { + return &Provider{ + Provider: base.NewProvider(conn, capabilities), + adapters: buildAdapters(capabilities), // Auto-configured + } } ``` -**Response Format**: +### Adapter Pipeline -```json -{ - "content": "The answer is 42", - "reasoning_content": "Let me think... first we need to...", - "content_types": ["text", "reasoning"] -} +**Preprocessing** (before API call): +``` +Messages → ToolCallAdapter → VisionAdapter → AudioAdapter → API Request ``` ---- - -### 4. Legacy Provider (`legacy/`) - -**Purpose**: Older models without native tool calling - -**Capabilities**: - -- ✅ Text generation -- ⚠️ Tool calls via prompt engineering -- ❌ No native vision support -- ❌ No native tool API - -**Models**: - -- GPT-3 (davinci, curie) -- Older open-source models -- Custom models without tool API - -**Tool Call Flow**: - -1. Inject tool schemas into system prompt -2. Model returns tool call in text format (JSON) -3. Extract and parse tool calls from text -4. Execute tools -5. Continue conversation - ---- - -### 5. Vision Utils (`vision/`) - -**Purpose**: Vision-related preprocessing utilities - -**Functions**: - -- `PreprocessVisionMessages()` - Handle image content -- `ConvertImageToText()` - Convert images to descriptions (for non-vision models) -- `ValidateImageURL()` - Validate image URLs -- `ExtractImagesFromMessages()` - Extract all images from messages - -**Usage**: - -```go -// When model doesn't support vision -if !supportsVision { - messages = vision.PreprocessVisionMessages(messages, false) - // Images converted to text descriptions -} +**Streaming** (during API call): +``` +API Chunk → ReasoningAdapter → ToolCallAdapter → Output ``` ---- - -### 6. Audio Utils (`audio/`) - -**Purpose**: Audio-related preprocessing utilities - -**Functions**: - -- `PreprocessAudioMessages()` - Handle audio content -- `ConvertAudioToText()` - Convert audio to text transcription (for non-audio models) -- `ValidateAudioFormat()` - Validate audio format and encoding -- `ExtractAudioFromMessages()` - Extract all audio data from messages -- `RemoveAudioConfig()` - Remove audio configuration from options - -**Usage**: - -```go -// When model doesn't support audio -if !supportsAudio { - messages = audio.PreprocessAudioMessages(messages, false) - options = audio.RemoveAudioConfig(options) - // Audio converted to text transcriptions -} +**Postprocessing** (after API call): +``` +API Response → All Adapters → Final Response ``` ---- +## Model Examples -## Special Scenarios - -### Scenario 1: DeepSeek R1 (Reasoning + No Tool Support) - -**Provider**: `reasoning.Provider` - -**Handling**: - -```go -// Check if reasoning model supports tools -if !p.supportsNativeTools && len(options.Tools) > 0 { - // Use prompt engineering approach - messages = p.injectToolInstructions(messages, tools) - options = p.removeToolsFromOptions(options) -} - -// After getting response -if !p.supportsNativeTools { - toolCalls = p.extractToolCallsFromText(response.Content) -} -``` - -**Why reasoning provider?** - -- Primary characteristic is reasoning (special response format) -- Tool handling is secondary concern -- Reuses tool injection logic from legacy approach - ---- - -### Scenario 2: Legacy Model + Vision/Audio Request - -**Provider**: `legacy.Provider` - -**Handling**: - -```go -import ( - "github.com/yaoapp/yao/agent/llm/providers/vision" - "github.com/yaoapp/yao/agent/llm/providers/audio" -) - -// Preprocess to remove/convert vision content -if !supportsVision { - messages = vision.PreprocessVisionMessages(messages, false) - // Images converted to text: "[Image: description]" -} - -// Preprocess to remove/convert audio content -if !supportsAudio { - messages = audio.PreprocessAudioMessages(messages, false) - options = audio.RemoveAudioConfig(options) - // Audio converted to text: "[Audio transcription: ...]" -} -``` - ---- - -### Scenario 3: OpenAI o1 (Reasoning + Tool Support) - -**Provider**: `reasoning.Provider` - -**Handling**: - -```go -// o1 supports native tools, no special handling needed -if p.supportsNativeTools { - // Use standard OpenAI tool calling API -} -``` - ---- - -## Configuration Example - -In `connectors.yml`: +### Full-Featured Model (GPT-4o) ```yaml -# GPT-4o with all features +# connectors.yml gpt-4o: vision: true tool_calls: true audio: true - streaming: true - json: true - multimodal: true + reasoning: false +``` -# OpenAI o1 - reasoning with tool support +**Adapters created**: +- ToolCallAdapter(native=true) +- VisionAdapter(native=true) +- AudioAdapter(native=true) + +### Reasoning Model with Tools (OpenAI o1) + +```yaml o1-preview: reasoning: true tool_calls: true - streaming: true - -# DeepSeek R1 - reasoning without tool support -deepseek-reasoner: - reasoning: true - tool_calls: false # Will use prompt engineering - streaming: true - -# GPT-3 - legacy model -gpt-3.5-turbo-instruct: - tool_calls: false # Will use prompt engineering - vision: false # Will convert images to text - audio: false # Will convert audio to text - streaming: false - -# GPT-4 Vision only -gpt-4-vision: - vision: true - tool_calls: true - audio: false # No audio support - streaming: true ``` ---- +**Adapters created**: +- ToolCallAdapter(native=true) +- ReasoningAdapter(format=openai-o1) -## Adding a New Provider +### Reasoning Model without Tools (DeepSeek R1) -1. Create new directory: `providers/newprovider/` -2. Implement `LLM` interface: +```yaml +deepseek-reasoner: + reasoning: true + tool_calls: false +``` +**Adapters created**: +- ToolCallAdapter(native=false) → Uses prompt engineering +- ReasoningAdapter(format=deepseek-r1) + +### Legacy Model (GPT-3.5-instruct) + +```yaml +gpt-3.5-turbo-instruct: + tool_calls: false + vision: false + audio: false +``` + +**Adapters created**: +- ToolCallAdapter(native=false) → Prompt engineering +- VisionAdapter(native=false) → Removes images +- AudioAdapter(native=false) → Removes audio + +## Capability Adapters + +### ToolCallAdapter + +**When native=true**: +- Passes tool definitions to API +- Parses structured tool_calls from response + +**When native=false**: +- Injects tool schemas into system prompt +- Extracts tool calls from text response (JSON parsing) + +### VisionAdapter + +**When native=true**: +- Passes image URLs/data directly to API + +**When native=false**: +- Removes image content from messages +- Optionally converts to text descriptions + +### AudioAdapter + +**When native=true**: +- Passes audio data directly to API + +**When native=false**: +- Removes audio content from messages +- Optionally converts to text transcriptions + +### ReasoningAdapter + +Handles different reasoning formats: + +**OpenAI o1** (`reasoning_content` field): +```json +{ + "delta": { + "reasoning_content": "Let me think...", + "content": "The answer is 42" + } +} +``` + +**DeepSeek R1** (may have different format): +```json +{ + "delta": { + "content": "Let me think...The answer is 42" + } +} +``` + +**GPT-4o thinking** (future): +```json +{ + "delta": { + "thinking": "Let me think...", + "content": "The answer is 42" + } +} +``` + +## Adding New Capabilities + +1. Create new adapter in `../adapters/`: + ```go + type NewCapabilityAdapter struct { + *BaseAdapter + nativeSupport bool + } + ``` + +2. Implement CapabilityAdapter interface + +3. Add to `buildAdapters()` in `openai/openai.go`: + ```go + if cap.NewCapability != nil { + result = append(result, adapters.NewNewCapabilityAdapter(*cap.NewCapability)) + } + ``` + +## Adding New API Format Provider + +1. Create new directory: `providers/newapi/` + +2. Implement LLM interface: ```go type Provider struct { *base.Provider + adapters []adapters.CapabilityAdapter + } + + func (p *Provider) Stream(...) (*CompletionResponse, error) { + // Apply adapter preprocessing + // Make API call + // Apply adapter postprocessing } - - func (p *Provider) Stream(...) (*CompletionResponse, error) - func (p *Provider) Post(...) (*CompletionResponse, error) ``` -3. Update `factory.SelectProvider()` selection logic -4. Add capability flags to `ConnectorSetting` +3. Update `factory.go`: + ```go + case "newapi": + return newapi.New(conn, options.Capabilities), nil + ``` ---- +## Benefits of New Architecture -## Testing +1. **Separation of Concerns**: + - Providers handle API format + - Adapters handle capabilities -Each provider should have tests for: +2. **Code Reuse**: + - Same adapters work across different providers + - No duplication of capability logic -- Standard completion -- Streaming completion -- Tool calling (if supported) -- Vision input (if supported) -- Error handling -- Response parsing +3. **Easy Extension**: + - Add new capability = add one adapter + - Add new API = add one provider ---- +4. **Flexible Combinations**: + - Any provider can use any adapter combination + - Capabilities are composable -## Performance Considerations +5. **Clear Responsibility**: + - Each adapter handles exactly one capability dimension + - Easy to test and maintain + +## Testing Strategy + +### Unit Tests (per adapter) +- Test preprocessing logic +- Test postprocessing logic +- Test stream chunk processing + +### Integration Tests (per provider) +- Test with different adapter combinations +- Test full request/response flow +- Test error handling + +### End-to-End Tests +- Test real API calls with different models +- Verify capability detection +- Verify adapter selection + +## Migration Notes + +### Old Architecture → New Architecture + +**Before**: +``` +reasoning.Provider → Reasoning models (o1, R1) +openai.Provider → Full-featured models (GPT-4o) +legacy.Provider → Old models (GPT-3) +``` + +**After**: +``` +openai.Provider + adapters → ALL models +``` + +The same OpenAI provider now handles all cases through different adapter combinations. -- **Caching**: Consider caching connector instances -- **Pooling**: HTTP connection pooling for high throughput -- **Timeouts**: Configurable timeouts per provider -- **Retries**: Exponential backoff for transient errors diff --git a/agent/llm/providers/audio/audio.go b/agent/llm/providers/audio/audio.go deleted file mode 100644 index 9b5ea64b..00000000 --- a/agent/llm/providers/audio/audio.go +++ /dev/null @@ -1,59 +0,0 @@ -package audio - -import ( - "github.com/yaoapp/yao/agent/context" -) - -// PreprocessAudioMessages preprocess messages to handle audio content -// Removes or converts audio content for models that don't support it -func PreprocessAudioMessages(messages []context.Message, supportsAudio bool) []context.Message { - // TODO: Implement audio message preprocessing - // If supportsAudio is false: - // - Remove input_audio content parts - // - Convert to text-only messages - // - Optionally add audio transcriptions - // If supportsAudio is true: - // - Validate audio format - // - Ensure proper encoding - return messages -} - -// ConvertAudioToText convert audio content to text transcription -// Used when model doesn't support audio input -func ConvertAudioToText(audioData string) (string, error) { - // TODO: Implement audio to text conversion - // - Call speech-to-text API (Whisper, etc.) - // - Generate transcription - // - Return as text content - return "", nil -} - -// ValidateAudioFormat validate audio format and encoding -func ValidateAudioFormat(audioConfig *context.AudioConfig) error { - // TODO: Implement audio format validation - // - Check format (wav, mp3, etc.) - // - Validate encoding - // - Check sample rate - return nil -} - -// ExtractAudioFromMessages extract all audio data from messages -func ExtractAudioFromMessages(messages []context.Message) []string { - // TODO: Implement audio extraction - // - Iterate through messages - // - Find ContentPart with type="input_audio" - // - Collect all audio data - return nil -} - -// RemoveAudioConfig remove audio configuration from options -// Used when model doesn't support audio output -func RemoveAudioConfig(options *context.CompletionOptions) *context.CompletionOptions { - // TODO: Remove audio config from options - if options == nil { - return options - } - newOptions := *options - newOptions.Audio = nil - return &newOptions -} diff --git a/agent/llm/providers/factory.go b/agent/llm/providers/factory.go index 32635b58..e4e0b260 100644 --- a/agent/llm/providers/factory.go +++ b/agent/llm/providers/factory.go @@ -5,9 +5,7 @@ import ( "github.com/yaoapp/gou/connector" "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/llm/providers/legacy" "github.com/yaoapp/yao/agent/llm/providers/openai" - "github.com/yaoapp/yao/agent/llm/providers/reasoning" ) // LLM interface (copied to avoid import cycle) @@ -16,9 +14,9 @@ type LLM interface { Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) } -// SelectProvider select the appropriate provider based on connector and capabilities +// SelectProvider selects the appropriate provider based on API format and capabilities +// The new architecture uses capability adapters to handle different model features func SelectProvider(conn connector.Connector, options *context.CompletionOptions) (LLM, error) { - if options == nil { return nil, fmt.Errorf("options are required") } @@ -27,37 +25,68 @@ func SelectProvider(conn connector.Connector, options *context.CompletionOptions return nil, fmt.Errorf("capabilities are required") } - capabilities := options.Capabilities + // Detect API format + apiFormat := DetectAPIFormat(conn) - // return openai.New(conn, capabilities), nil + // Select provider based on API format + switch apiFormat { + case "openai": + // OpenAI-compatible API + // Capability adapters will handle: + // - Tool calling (native or prompt engineering) + // - Vision (native or removal) + // - Audio (native or removal) + // - Reasoning (o1, GPT-4o thinking, etc.) + return openai.New(conn, options.Capabilities), nil - // Priority 1: Reasoning models (special response format) - if capabilities.Reasoning != nil && *capabilities.Reasoning { - return reasoning.New(conn, capabilities), nil + case "claude": + // TODO: Implement Claude provider + // For now, use OpenAI provider (may have compatibility issues) + return openai.New(conn, options.Capabilities), nil + + default: + // Default to OpenAI-compatible provider + return openai.New(conn, options.Capabilities), nil } - - // Priority 2: Check if model supports native tool calls - if capabilities.ToolCalls != nil && *capabilities.ToolCalls { - // Use OpenAI-compatible provider (supports tools, vision, streaming) - return openai.New(conn, capabilities), nil - } - - // Priority 3: Legacy models (no native tool support) - // Will use prompt engineering for tool calls - return legacy.New(conn, capabilities), nil } -// DetectProvider detect provider type from connector -func DetectProvider(conn connector.Connector) string { - // TODO: Implement provider detection - // - Check connector type (Is(connector.OPENAI)) - // - Check connector settings - // - Determine provider type (openai, claude, deepseek, etc.) - +// DetectAPIFormat detects the API format from connector +func DetectAPIFormat(conn connector.Connector) string { + // Check connector type if conn.Is(connector.OPENAI) { return "openai" } + // Check connector settings for host URL + settings := conn.Setting() + if settings != nil { + if host, ok := settings["host"].(string); ok { + // Detect by host URL patterns + if contains(host, "anthropic.com") || contains(host, "claude") { + return "claude" + } + if contains(host, "deepseek.com") { + return "openai" // DeepSeek uses OpenAI-compatible API + } + } + } + // Default to OpenAI-compatible return "openai" } + +// contains checks if a string contains a substring (case-insensitive helper) +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && + (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || + findSubstring(s, substr))) +} + +func findSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/agent/llm/providers/legacy/legacy.go b/agent/llm/providers/legacy/legacy.go deleted file mode 100644 index a266452b..00000000 --- a/agent/llm/providers/legacy/legacy.go +++ /dev/null @@ -1,64 +0,0 @@ -package legacy - -import ( - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/llm/providers/base" -) - -// Provider legacy LLM provider (no native tool calling support) -// Implements tool calling via prompt engineering -type Provider struct { - *base.Provider -} - -// New create a new legacy provider -func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider { - return &Provider{ - Provider: base.NewProvider(conn, capabilities), - } -} - -// Stream stream completion from legacy model -func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) { - // TODO: Implement legacy model streaming - // - Preprocess messages (remove tool-specific fields, vision, audio) - // - Remove vision content (convert to text description) - // - Remove audio content (convert to text transcription) - // - Remove tool messages - // - Add tool calling instructions to system prompt if tools provided - // - Build request body without native tool parameters - // - Make streaming HTTP request - // - Parse response and detect tool calls from text - // - Extract tool calls using regex/JSON parsing - // - Call handler for each chunk - return nil, nil -} - -// Post post completion request to legacy model -func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) { - // TODO: Implement legacy model non-streaming completion - // - Preprocess messages - // - Add tool instructions to prompt - // - Make HTTP POST request - // - Parse response and extract tool calls from text - return nil, nil -} - -// InjectToolInstructions inject tool calling instructions into system prompt -func (p *Provider) InjectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message { - // TODO: Implement tool instruction injection - // - Generate tool description prompt - // - Add to system message or create new system message - // - Include tool schemas and usage instructions - return messages -} - -// ExtractToolCallsFromText extract tool calls from model's text response -func (p *Provider) ExtractToolCallsFromText(text string) []context.ToolCall { - // TODO: Implement tool call extraction - // - Look for JSON blocks or specific patterns - // - Parse tool name and arguments - // - Return structured tool calls - return nil -} diff --git a/agent/llm/providers/openai/openai.go b/agent/llm/providers/openai/openai.go index 3d50aed6..e270d9b1 100644 --- a/agent/llm/providers/openai/openai.go +++ b/agent/llm/providers/openai/openai.go @@ -11,6 +11,7 @@ import ( "github.com/yaoapp/gou/http" "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/llm/adapters" "github.com/yaoapp/yao/agent/llm/providers/base" "github.com/yaoapp/yao/utils/jsonschema" ) @@ -102,19 +103,64 @@ func (gt *groupTracker) endGroup(handler context.StreamFunc) { gt.toolCallInfo = nil } -// Provider OpenAI-compatible provider -// Supports: vision, tool calls, streaming, JSON mode +// Provider OpenAI-compatible provider with capability adapters +// Supports: vision, tool calls, streaming, JSON mode, reasoning type Provider struct { *base.Provider + adapters []adapters.CapabilityAdapter } -// New create a new OpenAI provider +// New create a new OpenAI provider with capability adapters func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider { return &Provider{ Provider: base.NewProvider(conn, capabilities), + adapters: buildAdapters(capabilities), } } +// buildAdapters builds capability adapters based on model capabilities +func buildAdapters(cap *context.ModelCapabilities) []adapters.CapabilityAdapter { + if cap == nil { + return []adapters.CapabilityAdapter{} + } + + result := make([]adapters.CapabilityAdapter, 0) + + // Tool call adapter + if cap.ToolCalls != nil { + result = append(result, adapters.NewToolCallAdapter(*cap.ToolCalls)) + } + + // Vision adapter + if cap.Vision != nil { + result = append(result, adapters.NewVisionAdapter(*cap.Vision)) + } + + // Audio adapter + if cap.Audio != nil { + result = append(result, adapters.NewAudioAdapter(*cap.Audio)) + } + + // Reasoning adapter + if cap.Reasoning != nil && *cap.Reasoning { + // Detect reasoning format based on capabilities + format := detectReasoningFormat(cap) + result = append(result, adapters.NewReasoningAdapter(format)) + } + + return result +} + +// detectReasoningFormat detects the reasoning format based on capabilities +func detectReasoningFormat(cap *context.ModelCapabilities) adapters.ReasoningFormat { + // TODO: Implement better detection logic + // For now, default to OpenAI o1 format if reasoning is supported + if cap.Reasoning != nil && *cap.Reasoning { + return adapters.ReasoningFormatOpenAI + } + return adapters.ReasoningFormatNone +} + // Stream stream completion from OpenAI API func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) { maxRetries := 3 diff --git a/agent/llm/providers/reasoning/reasoning.go b/agent/llm/providers/reasoning/reasoning.go deleted file mode 100644 index 15793d3f..00000000 --- a/agent/llm/providers/reasoning/reasoning.go +++ /dev/null @@ -1,115 +0,0 @@ -package reasoning - -import ( - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/llm/providers/base" -) - -// Provider reasoning model provider (o1, DeepSeek R1, etc.) -// Handles special response format with reasoning_content -// Note: Some reasoning models (e.g. DeepSeek R1) don't support native tool calls -type Provider struct { - *base.Provider - supportsNativeTools bool // Whether this reasoning model supports native tool calling -} - -// New create a new reasoning provider -func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider { - // Check if this reasoning model supports native tool calls - supportsTools := false - if capabilities != nil && capabilities.ToolCalls != nil && *capabilities.ToolCalls { - supportsTools = true - } - - return &Provider{ - Provider: base.NewProvider(conn, capabilities), - supportsNativeTools: supportsTools, - } -} - -// Stream stream completion from reasoning model -func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) { - // TODO: Implement reasoning model streaming - // - Preprocess messages (reasoning models have restrictions) - - // Handle tool calls based on model support - if !p.supportsNativeTools && options != nil && len(options.Tools) > 0 { - // Model doesn't support native tool calls (e.g. DeepSeek R1) - // Inject tool instructions into messages - messages = p.injectToolInstructions(messages, options.Tools) - // Remove tools from options to avoid API error - options = p.removeToolsFromOptions(options) - } - - // - Build request body (special parameters for reasoning) - // - Make streaming HTTP request - // - Parse SSE chunks with reasoning_content - // - Handle both thinking and answer phases - // - Call handler for each chunk - - // If tools were injected, extract tool calls from text response - // if !p.supportsNativeTools && hasTools { - // toolCalls = p.extractToolCallsFromText(response.Content) - // } - - // - Aggregate final response with reasoning content - return nil, nil -} - -// Post post completion request to reasoning model -func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) { - // TODO: Implement reasoning model non-streaming completion - // - Preprocess messages - // - Build request body - // - Make HTTP POST request - // - Parse response with reasoning_content field - // - Separate thinking from final answer - return nil, nil -} - -// ParseReasoningResponse parse response with reasoning content -// Handles both OpenAI o1 format and DeepSeek R1 format -func (p *Provider) ParseReasoningResponse(data []byte) (*context.CompletionResponse, error) { - // TODO: Implement reasoning response parsing - // - Detect format (OpenAI vs DeepSeek) - // - Extract reasoning_content - // - Extract final content - // - Set ContentTypes correctly (text + reasoning) - return nil, nil -} - -// injectToolInstructions inject tool calling instructions into messages -// Used for reasoning models that don't support native tool calls (e.g. DeepSeek R1) -func (p *Provider) injectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message { - // TODO: Implement tool instruction injection for reasoning models - // - Generate tool description prompt (optimized for reasoning models) - // - Add to system message or create new system message - // - Include tool schemas and usage instructions - // - Format should encourage reasoning about tool usage - return messages -} - -// extractToolCallsFromText extract tool calls from reasoning model's text response -// Used when model doesn't support native tool calls -func (p *Provider) extractToolCallsFromText(text string) []context.ToolCall { - // TODO: Implement tool call extraction from text - // - Look for JSON blocks or specific patterns - // - Parse tool name and arguments - // - Return structured tool calls - // - Handle reasoning model's specific output format - return nil -} - -// removeToolsFromOptions remove tool-related parameters from options -// Used when sending request to models that don't support native tool calls -func (p *Provider) removeToolsFromOptions(options *context.CompletionOptions) *context.CompletionOptions { - // TODO: Create a copy of options without tool parameters - // - Remove Tools field - // - Remove ToolChoice field - // - Keep other options intact - newOptions := *options - newOptions.Tools = nil - newOptions.ToolChoice = nil - return &newOptions -} diff --git a/agent/llm/providers/vision/vision.go b/agent/llm/providers/vision/vision.go deleted file mode 100644 index fd5e7b0e..00000000 --- a/agent/llm/providers/vision/vision.go +++ /dev/null @@ -1,47 +0,0 @@ -package vision - -import ( - "github.com/yaoapp/yao/agent/context" -) - -// PreprocessVisionMessages preprocess messages to handle vision content -// Removes or converts vision content for models that don't support it -func PreprocessVisionMessages(messages []context.Message, supportsVision bool) []context.Message { - // TODO: Implement vision message preprocessing - // If supportsVision is false: - // - Remove image_url content parts - // - Convert to text-only messages - // - Optionally add image descriptions from vision API - // If supportsVision is true: - // - Validate image URLs - // - Ensure proper format - return messages -} - -// ConvertImageToText convert image content to text description -// Used when model doesn't support vision -func ConvertImageToText(imageURL string) (string, error) { - // TODO: Implement image to text conversion - // - Call vision API (if configured) - // - Generate description - // - Return as text content - return "", nil -} - -// ValidateImageURL validate image URL format -func ValidateImageURL(imageURL string) error { - // TODO: Implement image URL validation - // - Check URL format - // - Validate image type - // - Check accessibility - return nil -} - -// ExtractImagesFromMessages extract all image URLs from messages -func ExtractImagesFromMessages(messages []context.Message) []string { - // TODO: Implement image extraction - // - Iterate through messages - // - Find ContentPart with type="image_url" - // - Collect all image URLs - return nil -}