Refactor LLM provider architecture to utilize capability adapters
- Removed legacy and audio providers, consolidating functionality into a new architecture that separates API format handling from capability management. - Updated the OpenAI provider to support capability adapters for tool calls, vision, audio, and reasoning, enhancing modularity and extensibility. - Introduced a new method for detecting API formats and streamlined the provider selection process. - Enhanced documentation to reflect the new architecture and clarify provider capabilities and usage.
This commit is contained in:
Max
parent
fa95f32db7
commit
3d149057fd
12 changed files with 699 additions and 579 deletions
64
agent/llm/adapters/adapter.go
Normal file
64
agent/llm/adapters/adapter.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package adapters
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// CapabilityAdapter is the interface for capability-specific message and response processing
|
||||
// Each adapter handles one capability dimension (tool calls, vision, audio, reasoning, etc.)
|
||||
type CapabilityAdapter interface {
|
||||
// Name returns the adapter name for debugging
|
||||
Name() string
|
||||
|
||||
// PreprocessMessages preprocesses messages before sending to LLM
|
||||
// Returns modified messages or error
|
||||
PreprocessMessages(messages []context.Message) ([]context.Message, error)
|
||||
|
||||
// PreprocessOptions preprocesses completion options before sending to LLM
|
||||
// Returns modified options or error
|
||||
PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error)
|
||||
|
||||
// PostprocessResponse postprocesses the LLM response
|
||||
// Returns modified response or error
|
||||
PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error)
|
||||
|
||||
// ProcessStreamChunk processes a streaming chunk
|
||||
// Returns modified chunk type and data, or error
|
||||
ProcessStreamChunk(chunkType context.StreamChunkType, data []byte) (context.StreamChunkType, []byte, error)
|
||||
}
|
||||
|
||||
// BaseAdapter provides default implementations for CapabilityAdapter
|
||||
// Adapters can embed this and override only the methods they need
|
||||
type BaseAdapter struct {
|
||||
name string
|
||||
}
|
||||
|
||||
// NewBaseAdapter creates a new base adapter
|
||||
func NewBaseAdapter(name string) *BaseAdapter {
|
||||
return &BaseAdapter{name: name}
|
||||
}
|
||||
|
||||
// Name returns the adapter name
|
||||
func (a *BaseAdapter) Name() string {
|
||||
return a.name
|
||||
}
|
||||
|
||||
// PreprocessMessages default implementation (no-op)
|
||||
func (a *BaseAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// PreprocessOptions default implementation (no-op)
|
||||
func (a *BaseAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) {
|
||||
return options, nil
|
||||
}
|
||||
|
||||
// PostprocessResponse default implementation (no-op)
|
||||
func (a *BaseAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) {
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// ProcessStreamChunk default implementation (pass through)
|
||||
func (a *BaseAdapter) ProcessStreamChunk(chunkType context.StreamChunkType, data []byte) (context.StreamChunkType, []byte, error) {
|
||||
return chunkType, data, nil
|
||||
}
|
||||
62
agent/llm/adapters/audio.go
Normal file
62
agent/llm/adapters/audio.go
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
package adapters
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// AudioAdapter handles audio capability
|
||||
// If model doesn't support audio, it removes or converts audio content
|
||||
type AudioAdapter struct {
|
||||
*BaseAdapter
|
||||
nativeSupport bool
|
||||
}
|
||||
|
||||
// NewAudioAdapter creates a new audio adapter
|
||||
func NewAudioAdapter(nativeSupport bool) *AudioAdapter {
|
||||
return &AudioAdapter{
|
||||
BaseAdapter: NewBaseAdapter("AudioAdapter"),
|
||||
nativeSupport: nativeSupport,
|
||||
}
|
||||
}
|
||||
|
||||
// PreprocessMessages removes or converts audio content if not supported
|
||||
func (a *AudioAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
|
||||
if a.nativeSupport {
|
||||
// Native support, no preprocessing needed
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// Process messages to remove audio content
|
||||
processed := make([]context.Message, 0, len(messages))
|
||||
for _, msg := range messages {
|
||||
processedMsg := msg
|
||||
|
||||
// Handle multimodal content (array of ContentPart)
|
||||
if contentParts, ok := msg.Content.([]context.ContentPart); ok {
|
||||
filteredParts := make([]context.ContentPart, 0)
|
||||
|
||||
for _, part := range contentParts {
|
||||
// Skip audio content if not supported
|
||||
if part.Type == context.ContentInputAudio {
|
||||
// TODO: Optionally convert to transcription text if available
|
||||
continue
|
||||
}
|
||||
filteredParts = append(filteredParts, part)
|
||||
}
|
||||
|
||||
// If all parts were filtered out, add placeholder text
|
||||
if len(filteredParts) == 0 {
|
||||
processedMsg.Content = "[Audio content not supported by this model]"
|
||||
} else if len(filteredParts) == 1 && filteredParts[0].Type == context.ContentText {
|
||||
// Single text part, convert to string
|
||||
processedMsg.Content = filteredParts[0].Text
|
||||
} else {
|
||||
processedMsg.Content = filteredParts
|
||||
}
|
||||
}
|
||||
|
||||
processed = append(processed, processedMsg)
|
||||
}
|
||||
|
||||
return processed, nil
|
||||
}
|
||||
59
agent/llm/adapters/reasoning.go
Normal file
59
agent/llm/adapters/reasoning.go
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
package adapters
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// ReasoningFormat represents the reasoning content format
|
||||
type ReasoningFormat string
|
||||
|
||||
const (
|
||||
ReasoningFormatNone ReasoningFormat = "none" // No reasoning support
|
||||
ReasoningFormatOpenAI ReasoningFormat = "openai-o1" // OpenAI o1 format
|
||||
ReasoningFormatDeepSeek ReasoningFormat = "deepseek-r1" // DeepSeek R1 format
|
||||
ReasoningFormatGPTThink ReasoningFormat = "gpt-think" // Future GPT with thinking
|
||||
)
|
||||
|
||||
// ReasoningAdapter handles reasoning content capability
|
||||
// Parses reasoning_content from different model formats
|
||||
type ReasoningAdapter struct {
|
||||
*BaseAdapter
|
||||
format ReasoningFormat
|
||||
}
|
||||
|
||||
// NewReasoningAdapter creates a new reasoning adapter
|
||||
func NewReasoningAdapter(format ReasoningFormat) *ReasoningAdapter {
|
||||
return &ReasoningAdapter{
|
||||
BaseAdapter: NewBaseAdapter("ReasoningAdapter"),
|
||||
format: format,
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessStreamChunk processes streaming chunks with reasoning content
|
||||
func (a *ReasoningAdapter) ProcessStreamChunk(chunkType context.StreamChunkType, data []byte) (context.StreamChunkType, []byte, error) {
|
||||
if a.format == ReasoningFormatNone {
|
||||
// No reasoning support, pass through
|
||||
return chunkType, data, nil
|
||||
}
|
||||
|
||||
// TODO: Parse reasoning_content based on format
|
||||
// - OpenAI o1: reasoning_content field in delta
|
||||
// - DeepSeek R1: may have different format
|
||||
// - Extract and emit as ChunkThinking
|
||||
|
||||
return chunkType, data, nil
|
||||
}
|
||||
|
||||
// PostprocessResponse extracts reasoning content from the final response
|
||||
func (a *ReasoningAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) {
|
||||
if a.format == ReasoningFormatNone {
|
||||
// No reasoning support
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// TODO: Extract reasoning content from response
|
||||
// - Set response.ReasoningContent if present
|
||||
// - Separate thinking from final answer
|
||||
|
||||
return response, nil
|
||||
}
|
||||
67
agent/llm/adapters/toolcall.go
Normal file
67
agent/llm/adapters/toolcall.go
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package adapters
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// ToolCallAdapter handles tool calling capability
|
||||
// If model doesn't support native tool calls, it injects tool instructions into prompts
|
||||
type ToolCallAdapter struct {
|
||||
*BaseAdapter
|
||||
nativeSupport bool
|
||||
}
|
||||
|
||||
// NewToolCallAdapter creates a new tool call adapter
|
||||
func NewToolCallAdapter(nativeSupport bool) *ToolCallAdapter {
|
||||
return &ToolCallAdapter{
|
||||
BaseAdapter: NewBaseAdapter("ToolCallAdapter"),
|
||||
nativeSupport: nativeSupport,
|
||||
}
|
||||
}
|
||||
|
||||
// PreprocessMessages injects tool calling instructions if not natively supported
|
||||
func (a *ToolCallAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
|
||||
if a.nativeSupport {
|
||||
// Native support, no preprocessing needed
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// TODO: Inject tool calling instructions into system prompt
|
||||
// - Generate tool description prompt
|
||||
// - Add to system message or create new system message
|
||||
// - Include tool schemas and usage instructions
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// PreprocessOptions removes tool-related options if not natively supported
|
||||
func (a *ToolCallAdapter) PreprocessOptions(options *context.CompletionOptions) (*context.CompletionOptions, error) {
|
||||
if a.nativeSupport {
|
||||
// Native support, keep options as-is
|
||||
return options, nil
|
||||
}
|
||||
|
||||
if options == nil {
|
||||
return options, nil
|
||||
}
|
||||
|
||||
// Remove tool parameters for non-native models
|
||||
newOptions := *options
|
||||
newOptions.Tools = nil
|
||||
newOptions.ToolChoice = nil
|
||||
return &newOptions, nil
|
||||
}
|
||||
|
||||
// PostprocessResponse extracts tool calls from text if not natively supported
|
||||
func (a *ToolCallAdapter) PostprocessResponse(response *context.CompletionResponse) (*context.CompletionResponse, error) {
|
||||
if a.nativeSupport {
|
||||
// Native support, response already has structured tool calls
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// TODO: Extract tool calls from text response
|
||||
// - Look for JSON blocks or specific patterns
|
||||
// - Parse tool name and arguments
|
||||
// - Add to response.ToolCalls
|
||||
return response, nil
|
||||
}
|
||||
|
||||
62
agent/llm/adapters/vision.go
Normal file
62
agent/llm/adapters/vision.go
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
package adapters
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// VisionAdapter handles vision (image) capability
|
||||
// If model doesn't support vision, it removes or converts image content
|
||||
type VisionAdapter struct {
|
||||
*BaseAdapter
|
||||
nativeSupport bool
|
||||
}
|
||||
|
||||
// NewVisionAdapter creates a new vision adapter
|
||||
func NewVisionAdapter(nativeSupport bool) *VisionAdapter {
|
||||
return &VisionAdapter{
|
||||
BaseAdapter: NewBaseAdapter("VisionAdapter"),
|
||||
nativeSupport: nativeSupport,
|
||||
}
|
||||
}
|
||||
|
||||
// PreprocessMessages removes or converts image content if not supported
|
||||
func (a *VisionAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
|
||||
if a.nativeSupport {
|
||||
// Native support, no preprocessing needed
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// Process messages to remove image content
|
||||
processed := make([]context.Message, 0, len(messages))
|
||||
for _, msg := range messages {
|
||||
processedMsg := msg
|
||||
|
||||
// Handle multimodal content (array of ContentPart)
|
||||
if contentParts, ok := msg.Content.([]context.ContentPart); ok {
|
||||
filteredParts := make([]context.ContentPart, 0)
|
||||
|
||||
for _, part := range contentParts {
|
||||
// Skip image content if not supported
|
||||
if part.Type == context.ContentImageURL {
|
||||
// TODO: Optionally convert to text description
|
||||
continue
|
||||
}
|
||||
filteredParts = append(filteredParts, part)
|
||||
}
|
||||
|
||||
// If all parts were filtered out, add placeholder text
|
||||
if len(filteredParts) == 0 {
|
||||
processedMsg.Content = "[Image content not supported by this model]"
|
||||
} else if len(filteredParts) == 1 && filteredParts[0].Type == context.ContentText {
|
||||
// Single text part, convert to string
|
||||
processedMsg.Content = filteredParts[0].Text
|
||||
} else {
|
||||
processedMsg.Content = filteredParts
|
||||
}
|
||||
}
|
||||
|
||||
processed = append(processed, processedMsg)
|
||||
}
|
||||
|
||||
return processed, nil
|
||||
}
|
||||
|
|
@ -1,319 +1,335 @@
|
|||
# LLM Providers Architecture
|
||||
# LLM Providers Architecture (New)
|
||||
|
||||
## Overview
|
||||
|
||||
This directory contains different LLM provider implementations, each optimized for specific model capabilities.
|
||||
This directory contains LLM provider implementations using the **Capability Adapters** pattern. The new architecture separates API format handling from capability handling.
|
||||
|
||||
## Provider Selection Strategy
|
||||
## Architecture Design
|
||||
|
||||
The `factory.SelectProvider()` function automatically selects the appropriate provider based on model capabilities:
|
||||
```
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ LLM Provider (API Format) │
|
||||
│ - OpenAI-compatible │
|
||||
│ - Claude (TODO) │
|
||||
│ - Custom (TODO) │
|
||||
└──────────────┬──────────────────────────────────┘
|
||||
│
|
||||
↓
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ Capability Adapters (Modular) │
|
||||
│ - ToolCallAdapter (native or prompt eng.) │
|
||||
│ - VisionAdapter (native or removal) │
|
||||
│ - AudioAdapter (native or removal) │
|
||||
│ - ReasoningAdapter (o1/R1/GPT-Think) │
|
||||
└─────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Key Concepts
|
||||
|
||||
### 1. Provider = API Format
|
||||
|
||||
Providers handle the **API communication format**:
|
||||
- OpenAI-compatible API (`/v1/chat/completions`)
|
||||
- Claude API (TODO)
|
||||
- Custom API formats (TODO)
|
||||
|
||||
### 2. Adapters = Capabilities
|
||||
|
||||
Adapters handle **model capabilities** independently:
|
||||
- **ToolCallAdapter**: Tool calling (native or prompt engineering)
|
||||
- **VisionAdapter**: Image input (native or removal/conversion)
|
||||
- **AudioAdapter**: Audio input (native or removal/conversion)
|
||||
- **ReasoningAdapter**: Reasoning content (o1/DeepSeek R1/GPT-4o thinking)
|
||||
|
||||
## Provider Selection
|
||||
|
||||
```go
|
||||
Priority 1: Reasoning models → reasoning.Provider
|
||||
Priority 2: Native tool support → openai.Provider
|
||||
Priority 3: Legacy models → legacy.Provider
|
||||
// factory.go
|
||||
func SelectProvider(conn connector.Connector, options *context.CompletionOptions) (LLM, error) {
|
||||
apiFormat := DetectAPIFormat(conn)
|
||||
|
||||
switch apiFormat {
|
||||
case "openai":
|
||||
// Adapters automatically configured based on capabilities
|
||||
return openai.New(conn, options.Capabilities), nil
|
||||
case "claude":
|
||||
return claude.New(conn, options.Capabilities), nil
|
||||
default:
|
||||
return openai.New(conn, options.Capabilities), nil
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Provider Types
|
||||
## Directory Structure
|
||||
|
||||
### 1. Base Provider (`base/`)
|
||||
```
|
||||
providers/
|
||||
├── factory.go # Provider selection based on API format
|
||||
├── base/ # Common functionality
|
||||
│ └── base.go
|
||||
├── openai/ # OpenAI-compatible API provider
|
||||
│ └── openai.go # Includes adapter integration
|
||||
└── README.md # This file
|
||||
|
||||
**Purpose**: Common functionality shared across all providers
|
||||
../adapters/ # Capability adapters (separate package)
|
||||
├── adapter.go # Base interface
|
||||
├── toolcall.go # Tool calling adapter
|
||||
├── vision.go # Vision adapter
|
||||
├── audio.go # Audio adapter
|
||||
└── reasoning.go # Reasoning adapter
|
||||
```
|
||||
|
||||
**Features**:
|
||||
## OpenAI Provider
|
||||
|
||||
- Message preprocessing
|
||||
- Request body building
|
||||
- Response parsing
|
||||
|
||||
**Usage**: Embedded in all other providers
|
||||
|
||||
---
|
||||
|
||||
### 2. OpenAI Provider (`openai/`)
|
||||
|
||||
**Purpose**: OpenAI-compatible models with full feature support
|
||||
|
||||
**Capabilities**:
|
||||
|
||||
- ✅ Vision (image input)
|
||||
- ✅ Native tool calls
|
||||
- ✅ Streaming
|
||||
- ✅ JSON mode
|
||||
|
||||
**Models**:
|
||||
|
||||
- GPT-4, GPT-4o, GPT-4-turbo
|
||||
- GPT-3.5-turbo
|
||||
- Claude (via OpenAI-compatible API)
|
||||
|
||||
---
|
||||
|
||||
### 3. Reasoning Provider (`reasoning/`)
|
||||
|
||||
**Purpose**: Reasoning models with special response format
|
||||
|
||||
**Capabilities**:
|
||||
|
||||
- ✅ Reasoning content (`reasoning_content` field)
|
||||
- ✅ Thinking + Answer phases
|
||||
- ⚠️ Tool calls support varies by model
|
||||
|
||||
**Models**:
|
||||
|
||||
- **OpenAI o1** (supports native tool calls)
|
||||
- **DeepSeek R1** (no native tool calls, uses prompt engineering)
|
||||
|
||||
**Special Handling**:
|
||||
The OpenAI provider supports **all capabilities** through adapters:
|
||||
|
||||
```go
|
||||
// DeepSeek R1 scenario
|
||||
if !supportsNativeTools && hasTools {
|
||||
// Inject tool instructions into prompt
|
||||
messages = injectToolInstructions(messages, tools)
|
||||
// Extract tool calls from text response
|
||||
toolCalls = extractToolCallsFromText(response.Content)
|
||||
type Provider struct {
|
||||
*base.Provider
|
||||
adapters []adapters.CapabilityAdapter
|
||||
}
|
||||
|
||||
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||
return &Provider{
|
||||
Provider: base.NewProvider(conn, capabilities),
|
||||
adapters: buildAdapters(capabilities), // Auto-configured
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Response Format**:
|
||||
### Adapter Pipeline
|
||||
|
||||
```json
|
||||
{
|
||||
"content": "The answer is 42",
|
||||
"reasoning_content": "Let me think... first we need to...",
|
||||
"content_types": ["text", "reasoning"]
|
||||
}
|
||||
**Preprocessing** (before API call):
|
||||
```
|
||||
Messages → ToolCallAdapter → VisionAdapter → AudioAdapter → API Request
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. Legacy Provider (`legacy/`)
|
||||
|
||||
**Purpose**: Older models without native tool calling
|
||||
|
||||
**Capabilities**:
|
||||
|
||||
- ✅ Text generation
|
||||
- ⚠️ Tool calls via prompt engineering
|
||||
- ❌ No native vision support
|
||||
- ❌ No native tool API
|
||||
|
||||
**Models**:
|
||||
|
||||
- GPT-3 (davinci, curie)
|
||||
- Older open-source models
|
||||
- Custom models without tool API
|
||||
|
||||
**Tool Call Flow**:
|
||||
|
||||
1. Inject tool schemas into system prompt
|
||||
2. Model returns tool call in text format (JSON)
|
||||
3. Extract and parse tool calls from text
|
||||
4. Execute tools
|
||||
5. Continue conversation
|
||||
|
||||
---
|
||||
|
||||
### 5. Vision Utils (`vision/`)
|
||||
|
||||
**Purpose**: Vision-related preprocessing utilities
|
||||
|
||||
**Functions**:
|
||||
|
||||
- `PreprocessVisionMessages()` - Handle image content
|
||||
- `ConvertImageToText()` - Convert images to descriptions (for non-vision models)
|
||||
- `ValidateImageURL()` - Validate image URLs
|
||||
- `ExtractImagesFromMessages()` - Extract all images from messages
|
||||
|
||||
**Usage**:
|
||||
|
||||
```go
|
||||
// When model doesn't support vision
|
||||
if !supportsVision {
|
||||
messages = vision.PreprocessVisionMessages(messages, false)
|
||||
// Images converted to text descriptions
|
||||
}
|
||||
**Streaming** (during API call):
|
||||
```
|
||||
API Chunk → ReasoningAdapter → ToolCallAdapter → Output
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 6. Audio Utils (`audio/`)
|
||||
|
||||
**Purpose**: Audio-related preprocessing utilities
|
||||
|
||||
**Functions**:
|
||||
|
||||
- `PreprocessAudioMessages()` - Handle audio content
|
||||
- `ConvertAudioToText()` - Convert audio to text transcription (for non-audio models)
|
||||
- `ValidateAudioFormat()` - Validate audio format and encoding
|
||||
- `ExtractAudioFromMessages()` - Extract all audio data from messages
|
||||
- `RemoveAudioConfig()` - Remove audio configuration from options
|
||||
|
||||
**Usage**:
|
||||
|
||||
```go
|
||||
// When model doesn't support audio
|
||||
if !supportsAudio {
|
||||
messages = audio.PreprocessAudioMessages(messages, false)
|
||||
options = audio.RemoveAudioConfig(options)
|
||||
// Audio converted to text transcriptions
|
||||
}
|
||||
**Postprocessing** (after API call):
|
||||
```
|
||||
API Response → All Adapters → Final Response
|
||||
```
|
||||
|
||||
---
|
||||
## Model Examples
|
||||
|
||||
## Special Scenarios
|
||||
|
||||
### Scenario 1: DeepSeek R1 (Reasoning + No Tool Support)
|
||||
|
||||
**Provider**: `reasoning.Provider`
|
||||
|
||||
**Handling**:
|
||||
|
||||
```go
|
||||
// Check if reasoning model supports tools
|
||||
if !p.supportsNativeTools && len(options.Tools) > 0 {
|
||||
// Use prompt engineering approach
|
||||
messages = p.injectToolInstructions(messages, tools)
|
||||
options = p.removeToolsFromOptions(options)
|
||||
}
|
||||
|
||||
// After getting response
|
||||
if !p.supportsNativeTools {
|
||||
toolCalls = p.extractToolCallsFromText(response.Content)
|
||||
}
|
||||
```
|
||||
|
||||
**Why reasoning provider?**
|
||||
|
||||
- Primary characteristic is reasoning (special response format)
|
||||
- Tool handling is secondary concern
|
||||
- Reuses tool injection logic from legacy approach
|
||||
|
||||
---
|
||||
|
||||
### Scenario 2: Legacy Model + Vision/Audio Request
|
||||
|
||||
**Provider**: `legacy.Provider`
|
||||
|
||||
**Handling**:
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/llm/providers/vision"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/audio"
|
||||
)
|
||||
|
||||
// Preprocess to remove/convert vision content
|
||||
if !supportsVision {
|
||||
messages = vision.PreprocessVisionMessages(messages, false)
|
||||
// Images converted to text: "[Image: description]"
|
||||
}
|
||||
|
||||
// Preprocess to remove/convert audio content
|
||||
if !supportsAudio {
|
||||
messages = audio.PreprocessAudioMessages(messages, false)
|
||||
options = audio.RemoveAudioConfig(options)
|
||||
// Audio converted to text: "[Audio transcription: ...]"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Scenario 3: OpenAI o1 (Reasoning + Tool Support)
|
||||
|
||||
**Provider**: `reasoning.Provider`
|
||||
|
||||
**Handling**:
|
||||
|
||||
```go
|
||||
// o1 supports native tools, no special handling needed
|
||||
if p.supportsNativeTools {
|
||||
// Use standard OpenAI tool calling API
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration Example
|
||||
|
||||
In `connectors.yml`:
|
||||
### Full-Featured Model (GPT-4o)
|
||||
|
||||
```yaml
|
||||
# GPT-4o with all features
|
||||
# connectors.yml
|
||||
gpt-4o:
|
||||
vision: true
|
||||
tool_calls: true
|
||||
audio: true
|
||||
streaming: true
|
||||
json: true
|
||||
multimodal: true
|
||||
reasoning: false
|
||||
```
|
||||
|
||||
# OpenAI o1 - reasoning with tool support
|
||||
**Adapters created**:
|
||||
- ToolCallAdapter(native=true)
|
||||
- VisionAdapter(native=true)
|
||||
- AudioAdapter(native=true)
|
||||
|
||||
### Reasoning Model with Tools (OpenAI o1)
|
||||
|
||||
```yaml
|
||||
o1-preview:
|
||||
reasoning: true
|
||||
tool_calls: true
|
||||
streaming: true
|
||||
|
||||
# DeepSeek R1 - reasoning without tool support
|
||||
deepseek-reasoner:
|
||||
reasoning: true
|
||||
tool_calls: false # Will use prompt engineering
|
||||
streaming: true
|
||||
|
||||
# GPT-3 - legacy model
|
||||
gpt-3.5-turbo-instruct:
|
||||
tool_calls: false # Will use prompt engineering
|
||||
vision: false # Will convert images to text
|
||||
audio: false # Will convert audio to text
|
||||
streaming: false
|
||||
|
||||
# GPT-4 Vision only
|
||||
gpt-4-vision:
|
||||
vision: true
|
||||
tool_calls: true
|
||||
audio: false # No audio support
|
||||
streaming: true
|
||||
```
|
||||
|
||||
---
|
||||
**Adapters created**:
|
||||
- ToolCallAdapter(native=true)
|
||||
- ReasoningAdapter(format=openai-o1)
|
||||
|
||||
## Adding a New Provider
|
||||
### Reasoning Model without Tools (DeepSeek R1)
|
||||
|
||||
1. Create new directory: `providers/newprovider/`
|
||||
2. Implement `LLM` interface:
|
||||
```yaml
|
||||
deepseek-reasoner:
|
||||
reasoning: true
|
||||
tool_calls: false
|
||||
```
|
||||
|
||||
**Adapters created**:
|
||||
- ToolCallAdapter(native=false) → Uses prompt engineering
|
||||
- ReasoningAdapter(format=deepseek-r1)
|
||||
|
||||
### Legacy Model (GPT-3.5-instruct)
|
||||
|
||||
```yaml
|
||||
gpt-3.5-turbo-instruct:
|
||||
tool_calls: false
|
||||
vision: false
|
||||
audio: false
|
||||
```
|
||||
|
||||
**Adapters created**:
|
||||
- ToolCallAdapter(native=false) → Prompt engineering
|
||||
- VisionAdapter(native=false) → Removes images
|
||||
- AudioAdapter(native=false) → Removes audio
|
||||
|
||||
## Capability Adapters
|
||||
|
||||
### ToolCallAdapter
|
||||
|
||||
**When native=true**:
|
||||
- Passes tool definitions to API
|
||||
- Parses structured tool_calls from response
|
||||
|
||||
**When native=false**:
|
||||
- Injects tool schemas into system prompt
|
||||
- Extracts tool calls from text response (JSON parsing)
|
||||
|
||||
### VisionAdapter
|
||||
|
||||
**When native=true**:
|
||||
- Passes image URLs/data directly to API
|
||||
|
||||
**When native=false**:
|
||||
- Removes image content from messages
|
||||
- Optionally converts to text descriptions
|
||||
|
||||
### AudioAdapter
|
||||
|
||||
**When native=true**:
|
||||
- Passes audio data directly to API
|
||||
|
||||
**When native=false**:
|
||||
- Removes audio content from messages
|
||||
- Optionally converts to text transcriptions
|
||||
|
||||
### ReasoningAdapter
|
||||
|
||||
Handles different reasoning formats:
|
||||
|
||||
**OpenAI o1** (`reasoning_content` field):
|
||||
```json
|
||||
{
|
||||
"delta": {
|
||||
"reasoning_content": "Let me think...",
|
||||
"content": "The answer is 42"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**DeepSeek R1** (may have different format):
|
||||
```json
|
||||
{
|
||||
"delta": {
|
||||
"content": "<think>Let me think...</think>The answer is 42"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**GPT-4o thinking** (future):
|
||||
```json
|
||||
{
|
||||
"delta": {
|
||||
"thinking": "Let me think...",
|
||||
"content": "The answer is 42"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Adding New Capabilities
|
||||
|
||||
1. Create new adapter in `../adapters/`:
|
||||
```go
|
||||
type NewCapabilityAdapter struct {
|
||||
*BaseAdapter
|
||||
nativeSupport bool
|
||||
}
|
||||
```
|
||||
|
||||
2. Implement CapabilityAdapter interface
|
||||
|
||||
3. Add to `buildAdapters()` in `openai/openai.go`:
|
||||
```go
|
||||
if cap.NewCapability != nil {
|
||||
result = append(result, adapters.NewNewCapabilityAdapter(*cap.NewCapability))
|
||||
}
|
||||
```
|
||||
|
||||
## Adding New API Format Provider
|
||||
|
||||
1. Create new directory: `providers/newapi/`
|
||||
|
||||
2. Implement LLM interface:
|
||||
```go
|
||||
type Provider struct {
|
||||
*base.Provider
|
||||
adapters []adapters.CapabilityAdapter
|
||||
}
|
||||
|
||||
func (p *Provider) Stream(...) (*CompletionResponse, error) {
|
||||
// Apply adapter preprocessing
|
||||
// Make API call
|
||||
// Apply adapter postprocessing
|
||||
}
|
||||
|
||||
func (p *Provider) Stream(...) (*CompletionResponse, error)
|
||||
func (p *Provider) Post(...) (*CompletionResponse, error)
|
||||
```
|
||||
|
||||
3. Update `factory.SelectProvider()` selection logic
|
||||
4. Add capability flags to `ConnectorSetting`
|
||||
3. Update `factory.go`:
|
||||
```go
|
||||
case "newapi":
|
||||
return newapi.New(conn, options.Capabilities), nil
|
||||
```
|
||||
|
||||
---
|
||||
## Benefits of New Architecture
|
||||
|
||||
## Testing
|
||||
1. **Separation of Concerns**:
|
||||
- Providers handle API format
|
||||
- Adapters handle capabilities
|
||||
|
||||
Each provider should have tests for:
|
||||
2. **Code Reuse**:
|
||||
- Same adapters work across different providers
|
||||
- No duplication of capability logic
|
||||
|
||||
- Standard completion
|
||||
- Streaming completion
|
||||
- Tool calling (if supported)
|
||||
- Vision input (if supported)
|
||||
- Error handling
|
||||
- Response parsing
|
||||
3. **Easy Extension**:
|
||||
- Add new capability = add one adapter
|
||||
- Add new API = add one provider
|
||||
|
||||
---
|
||||
4. **Flexible Combinations**:
|
||||
- Any provider can use any adapter combination
|
||||
- Capabilities are composable
|
||||
|
||||
## Performance Considerations
|
||||
5. **Clear Responsibility**:
|
||||
- Each adapter handles exactly one capability dimension
|
||||
- Easy to test and maintain
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests (per adapter)
|
||||
- Test preprocessing logic
|
||||
- Test postprocessing logic
|
||||
- Test stream chunk processing
|
||||
|
||||
### Integration Tests (per provider)
|
||||
- Test with different adapter combinations
|
||||
- Test full request/response flow
|
||||
- Test error handling
|
||||
|
||||
### End-to-End Tests
|
||||
- Test real API calls with different models
|
||||
- Verify capability detection
|
||||
- Verify adapter selection
|
||||
|
||||
## Migration Notes
|
||||
|
||||
### Old Architecture → New Architecture
|
||||
|
||||
**Before**:
|
||||
```
|
||||
reasoning.Provider → Reasoning models (o1, R1)
|
||||
openai.Provider → Full-featured models (GPT-4o)
|
||||
legacy.Provider → Old models (GPT-3)
|
||||
```
|
||||
|
||||
**After**:
|
||||
```
|
||||
openai.Provider + adapters → ALL models
|
||||
```
|
||||
|
||||
The same OpenAI provider now handles all cases through different adapter combinations.
|
||||
|
||||
- **Caching**: Consider caching connector instances
|
||||
- **Pooling**: HTTP connection pooling for high throughput
|
||||
- **Timeouts**: Configurable timeouts per provider
|
||||
- **Retries**: Exponential backoff for transient errors
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
package audio
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// PreprocessAudioMessages preprocess messages to handle audio content
|
||||
// Removes or converts audio content for models that don't support it
|
||||
func PreprocessAudioMessages(messages []context.Message, supportsAudio bool) []context.Message {
|
||||
// TODO: Implement audio message preprocessing
|
||||
// If supportsAudio is false:
|
||||
// - Remove input_audio content parts
|
||||
// - Convert to text-only messages
|
||||
// - Optionally add audio transcriptions
|
||||
// If supportsAudio is true:
|
||||
// - Validate audio format
|
||||
// - Ensure proper encoding
|
||||
return messages
|
||||
}
|
||||
|
||||
// ConvertAudioToText convert audio content to text transcription
|
||||
// Used when model doesn't support audio input
|
||||
func ConvertAudioToText(audioData string) (string, error) {
|
||||
// TODO: Implement audio to text conversion
|
||||
// - Call speech-to-text API (Whisper, etc.)
|
||||
// - Generate transcription
|
||||
// - Return as text content
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ValidateAudioFormat validate audio format and encoding
|
||||
func ValidateAudioFormat(audioConfig *context.AudioConfig) error {
|
||||
// TODO: Implement audio format validation
|
||||
// - Check format (wav, mp3, etc.)
|
||||
// - Validate encoding
|
||||
// - Check sample rate
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractAudioFromMessages extract all audio data from messages
|
||||
func ExtractAudioFromMessages(messages []context.Message) []string {
|
||||
// TODO: Implement audio extraction
|
||||
// - Iterate through messages
|
||||
// - Find ContentPart with type="input_audio"
|
||||
// - Collect all audio data
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveAudioConfig remove audio configuration from options
|
||||
// Used when model doesn't support audio output
|
||||
func RemoveAudioConfig(options *context.CompletionOptions) *context.CompletionOptions {
|
||||
// TODO: Remove audio config from options
|
||||
if options == nil {
|
||||
return options
|
||||
}
|
||||
newOptions := *options
|
||||
newOptions.Audio = nil
|
||||
return &newOptions
|
||||
}
|
||||
|
|
@ -5,9 +5,7 @@ import (
|
|||
|
||||
"github.com/yaoapp/gou/connector"
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/legacy"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/openai"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/reasoning"
|
||||
)
|
||||
|
||||
// LLM interface (copied to avoid import cycle)
|
||||
|
|
@ -16,9 +14,9 @@ type LLM interface {
|
|||
Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error)
|
||||
}
|
||||
|
||||
// SelectProvider select the appropriate provider based on connector and capabilities
|
||||
// SelectProvider selects the appropriate provider based on API format and capabilities
|
||||
// The new architecture uses capability adapters to handle different model features
|
||||
func SelectProvider(conn connector.Connector, options *context.CompletionOptions) (LLM, error) {
|
||||
|
||||
if options == nil {
|
||||
return nil, fmt.Errorf("options are required")
|
||||
}
|
||||
|
|
@ -27,37 +25,68 @@ func SelectProvider(conn connector.Connector, options *context.CompletionOptions
|
|||
return nil, fmt.Errorf("capabilities are required")
|
||||
}
|
||||
|
||||
capabilities := options.Capabilities
|
||||
// Detect API format
|
||||
apiFormat := DetectAPIFormat(conn)
|
||||
|
||||
// return openai.New(conn, capabilities), nil
|
||||
// Select provider based on API format
|
||||
switch apiFormat {
|
||||
case "openai":
|
||||
// OpenAI-compatible API
|
||||
// Capability adapters will handle:
|
||||
// - Tool calling (native or prompt engineering)
|
||||
// - Vision (native or removal)
|
||||
// - Audio (native or removal)
|
||||
// - Reasoning (o1, GPT-4o thinking, etc.)
|
||||
return openai.New(conn, options.Capabilities), nil
|
||||
|
||||
// Priority 1: Reasoning models (special response format)
|
||||
if capabilities.Reasoning != nil && *capabilities.Reasoning {
|
||||
return reasoning.New(conn, capabilities), nil
|
||||
case "claude":
|
||||
// TODO: Implement Claude provider
|
||||
// For now, use OpenAI provider (may have compatibility issues)
|
||||
return openai.New(conn, options.Capabilities), nil
|
||||
|
||||
default:
|
||||
// Default to OpenAI-compatible provider
|
||||
return openai.New(conn, options.Capabilities), nil
|
||||
}
|
||||
|
||||
// Priority 2: Check if model supports native tool calls
|
||||
if capabilities.ToolCalls != nil && *capabilities.ToolCalls {
|
||||
// Use OpenAI-compatible provider (supports tools, vision, streaming)
|
||||
return openai.New(conn, capabilities), nil
|
||||
}
|
||||
|
||||
// Priority 3: Legacy models (no native tool support)
|
||||
// Will use prompt engineering for tool calls
|
||||
return legacy.New(conn, capabilities), nil
|
||||
}
|
||||
|
||||
// DetectProvider detect provider type from connector
|
||||
func DetectProvider(conn connector.Connector) string {
|
||||
// TODO: Implement provider detection
|
||||
// - Check connector type (Is(connector.OPENAI))
|
||||
// - Check connector settings
|
||||
// - Determine provider type (openai, claude, deepseek, etc.)
|
||||
|
||||
// DetectAPIFormat detects the API format from connector
|
||||
func DetectAPIFormat(conn connector.Connector) string {
|
||||
// Check connector type
|
||||
if conn.Is(connector.OPENAI) {
|
||||
return "openai"
|
||||
}
|
||||
|
||||
// Check connector settings for host URL
|
||||
settings := conn.Setting()
|
||||
if settings != nil {
|
||||
if host, ok := settings["host"].(string); ok {
|
||||
// Detect by host URL patterns
|
||||
if contains(host, "anthropic.com") || contains(host, "claude") {
|
||||
return "claude"
|
||||
}
|
||||
if contains(host, "deepseek.com") {
|
||||
return "openai" // DeepSeek uses OpenAI-compatible API
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Default to OpenAI-compatible
|
||||
return "openai"
|
||||
}
|
||||
|
||||
// contains checks if a string contains a substring (case-insensitive helper)
|
||||
func contains(s, substr string) bool {
|
||||
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) &&
|
||||
(s[:len(substr)] == substr || s[len(s)-len(substr):] == substr ||
|
||||
findSubstring(s, substr)))
|
||||
}
|
||||
|
||||
func findSubstring(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
package legacy
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/gou/connector"
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/base"
|
||||
)
|
||||
|
||||
// Provider legacy LLM provider (no native tool calling support)
|
||||
// Implements tool calling via prompt engineering
|
||||
type Provider struct {
|
||||
*base.Provider
|
||||
}
|
||||
|
||||
// New create a new legacy provider
|
||||
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||
return &Provider{
|
||||
Provider: base.NewProvider(conn, capabilities),
|
||||
}
|
||||
}
|
||||
|
||||
// Stream stream completion from legacy model
|
||||
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
|
||||
// TODO: Implement legacy model streaming
|
||||
// - Preprocess messages (remove tool-specific fields, vision, audio)
|
||||
// - Remove vision content (convert to text description)
|
||||
// - Remove audio content (convert to text transcription)
|
||||
// - Remove tool messages
|
||||
// - Add tool calling instructions to system prompt if tools provided
|
||||
// - Build request body without native tool parameters
|
||||
// - Make streaming HTTP request
|
||||
// - Parse response and detect tool calls from text
|
||||
// - Extract tool calls using regex/JSON parsing
|
||||
// - Call handler for each chunk
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Post post completion request to legacy model
|
||||
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
|
||||
// TODO: Implement legacy model non-streaming completion
|
||||
// - Preprocess messages
|
||||
// - Add tool instructions to prompt
|
||||
// - Make HTTP POST request
|
||||
// - Parse response and extract tool calls from text
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// InjectToolInstructions inject tool calling instructions into system prompt
|
||||
func (p *Provider) InjectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message {
|
||||
// TODO: Implement tool instruction injection
|
||||
// - Generate tool description prompt
|
||||
// - Add to system message or create new system message
|
||||
// - Include tool schemas and usage instructions
|
||||
return messages
|
||||
}
|
||||
|
||||
// ExtractToolCallsFromText extract tool calls from model's text response
|
||||
func (p *Provider) ExtractToolCallsFromText(text string) []context.ToolCall {
|
||||
// TODO: Implement tool call extraction
|
||||
// - Look for JSON blocks or specific patterns
|
||||
// - Parse tool name and arguments
|
||||
// - Return structured tool calls
|
||||
return nil
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"github.com/yaoapp/gou/http"
|
||||
"github.com/yaoapp/kun/log"
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/llm/adapters"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/base"
|
||||
"github.com/yaoapp/yao/utils/jsonschema"
|
||||
)
|
||||
|
|
@ -102,19 +103,64 @@ func (gt *groupTracker) endGroup(handler context.StreamFunc) {
|
|||
gt.toolCallInfo = nil
|
||||
}
|
||||
|
||||
// Provider OpenAI-compatible provider
|
||||
// Supports: vision, tool calls, streaming, JSON mode
|
||||
// Provider OpenAI-compatible provider with capability adapters
|
||||
// Supports: vision, tool calls, streaming, JSON mode, reasoning
|
||||
type Provider struct {
|
||||
*base.Provider
|
||||
adapters []adapters.CapabilityAdapter
|
||||
}
|
||||
|
||||
// New create a new OpenAI provider
|
||||
// New create a new OpenAI provider with capability adapters
|
||||
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||
return &Provider{
|
||||
Provider: base.NewProvider(conn, capabilities),
|
||||
adapters: buildAdapters(capabilities),
|
||||
}
|
||||
}
|
||||
|
||||
// buildAdapters builds capability adapters based on model capabilities
|
||||
func buildAdapters(cap *context.ModelCapabilities) []adapters.CapabilityAdapter {
|
||||
if cap == nil {
|
||||
return []adapters.CapabilityAdapter{}
|
||||
}
|
||||
|
||||
result := make([]adapters.CapabilityAdapter, 0)
|
||||
|
||||
// Tool call adapter
|
||||
if cap.ToolCalls != nil {
|
||||
result = append(result, adapters.NewToolCallAdapter(*cap.ToolCalls))
|
||||
}
|
||||
|
||||
// Vision adapter
|
||||
if cap.Vision != nil {
|
||||
result = append(result, adapters.NewVisionAdapter(*cap.Vision))
|
||||
}
|
||||
|
||||
// Audio adapter
|
||||
if cap.Audio != nil {
|
||||
result = append(result, adapters.NewAudioAdapter(*cap.Audio))
|
||||
}
|
||||
|
||||
// Reasoning adapter
|
||||
if cap.Reasoning != nil && *cap.Reasoning {
|
||||
// Detect reasoning format based on capabilities
|
||||
format := detectReasoningFormat(cap)
|
||||
result = append(result, adapters.NewReasoningAdapter(format))
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// detectReasoningFormat detects the reasoning format based on capabilities
|
||||
func detectReasoningFormat(cap *context.ModelCapabilities) adapters.ReasoningFormat {
|
||||
// TODO: Implement better detection logic
|
||||
// For now, default to OpenAI o1 format if reasoning is supported
|
||||
if cap.Reasoning != nil && *cap.Reasoning {
|
||||
return adapters.ReasoningFormatOpenAI
|
||||
}
|
||||
return adapters.ReasoningFormatNone
|
||||
}
|
||||
|
||||
// Stream stream completion from OpenAI API
|
||||
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
|
||||
maxRetries := 3
|
||||
|
|
|
|||
|
|
@ -1,115 +0,0 @@
|
|||
package reasoning
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/gou/connector"
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
"github.com/yaoapp/yao/agent/llm/providers/base"
|
||||
)
|
||||
|
||||
// Provider reasoning model provider (o1, DeepSeek R1, etc.)
|
||||
// Handles special response format with reasoning_content
|
||||
// Note: Some reasoning models (e.g. DeepSeek R1) don't support native tool calls
|
||||
type Provider struct {
|
||||
*base.Provider
|
||||
supportsNativeTools bool // Whether this reasoning model supports native tool calling
|
||||
}
|
||||
|
||||
// New create a new reasoning provider
|
||||
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||
// Check if this reasoning model supports native tool calls
|
||||
supportsTools := false
|
||||
if capabilities != nil && capabilities.ToolCalls != nil && *capabilities.ToolCalls {
|
||||
supportsTools = true
|
||||
}
|
||||
|
||||
return &Provider{
|
||||
Provider: base.NewProvider(conn, capabilities),
|
||||
supportsNativeTools: supportsTools,
|
||||
}
|
||||
}
|
||||
|
||||
// Stream stream completion from reasoning model
|
||||
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
|
||||
// TODO: Implement reasoning model streaming
|
||||
// - Preprocess messages (reasoning models have restrictions)
|
||||
|
||||
// Handle tool calls based on model support
|
||||
if !p.supportsNativeTools && options != nil && len(options.Tools) > 0 {
|
||||
// Model doesn't support native tool calls (e.g. DeepSeek R1)
|
||||
// Inject tool instructions into messages
|
||||
messages = p.injectToolInstructions(messages, options.Tools)
|
||||
// Remove tools from options to avoid API error
|
||||
options = p.removeToolsFromOptions(options)
|
||||
}
|
||||
|
||||
// - Build request body (special parameters for reasoning)
|
||||
// - Make streaming HTTP request
|
||||
// - Parse SSE chunks with reasoning_content
|
||||
// - Handle both thinking and answer phases
|
||||
// - Call handler for each chunk
|
||||
|
||||
// If tools were injected, extract tool calls from text response
|
||||
// if !p.supportsNativeTools && hasTools {
|
||||
// toolCalls = p.extractToolCallsFromText(response.Content)
|
||||
// }
|
||||
|
||||
// - Aggregate final response with reasoning content
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Post post completion request to reasoning model
|
||||
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
|
||||
// TODO: Implement reasoning model non-streaming completion
|
||||
// - Preprocess messages
|
||||
// - Build request body
|
||||
// - Make HTTP POST request
|
||||
// - Parse response with reasoning_content field
|
||||
// - Separate thinking from final answer
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ParseReasoningResponse parse response with reasoning content
|
||||
// Handles both OpenAI o1 format and DeepSeek R1 format
|
||||
func (p *Provider) ParseReasoningResponse(data []byte) (*context.CompletionResponse, error) {
|
||||
// TODO: Implement reasoning response parsing
|
||||
// - Detect format (OpenAI vs DeepSeek)
|
||||
// - Extract reasoning_content
|
||||
// - Extract final content
|
||||
// - Set ContentTypes correctly (text + reasoning)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// injectToolInstructions inject tool calling instructions into messages
|
||||
// Used for reasoning models that don't support native tool calls (e.g. DeepSeek R1)
|
||||
func (p *Provider) injectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message {
|
||||
// TODO: Implement tool instruction injection for reasoning models
|
||||
// - Generate tool description prompt (optimized for reasoning models)
|
||||
// - Add to system message or create new system message
|
||||
// - Include tool schemas and usage instructions
|
||||
// - Format should encourage reasoning about tool usage
|
||||
return messages
|
||||
}
|
||||
|
||||
// extractToolCallsFromText extract tool calls from reasoning model's text response
|
||||
// Used when model doesn't support native tool calls
|
||||
func (p *Provider) extractToolCallsFromText(text string) []context.ToolCall {
|
||||
// TODO: Implement tool call extraction from text
|
||||
// - Look for JSON blocks or specific patterns
|
||||
// - Parse tool name and arguments
|
||||
// - Return structured tool calls
|
||||
// - Handle reasoning model's specific output format
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeToolsFromOptions remove tool-related parameters from options
|
||||
// Used when sending request to models that don't support native tool calls
|
||||
func (p *Provider) removeToolsFromOptions(options *context.CompletionOptions) *context.CompletionOptions {
|
||||
// TODO: Create a copy of options without tool parameters
|
||||
// - Remove Tools field
|
||||
// - Remove ToolChoice field
|
||||
// - Keep other options intact
|
||||
newOptions := *options
|
||||
newOptions.Tools = nil
|
||||
newOptions.ToolChoice = nil
|
||||
return &newOptions
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
package vision
|
||||
|
||||
import (
|
||||
"github.com/yaoapp/yao/agent/context"
|
||||
)
|
||||
|
||||
// PreprocessVisionMessages preprocess messages to handle vision content
|
||||
// Removes or converts vision content for models that don't support it
|
||||
func PreprocessVisionMessages(messages []context.Message, supportsVision bool) []context.Message {
|
||||
// TODO: Implement vision message preprocessing
|
||||
// If supportsVision is false:
|
||||
// - Remove image_url content parts
|
||||
// - Convert to text-only messages
|
||||
// - Optionally add image descriptions from vision API
|
||||
// If supportsVision is true:
|
||||
// - Validate image URLs
|
||||
// - Ensure proper format
|
||||
return messages
|
||||
}
|
||||
|
||||
// ConvertImageToText convert image content to text description
|
||||
// Used when model doesn't support vision
|
||||
func ConvertImageToText(imageURL string) (string, error) {
|
||||
// TODO: Implement image to text conversion
|
||||
// - Call vision API (if configured)
|
||||
// - Generate description
|
||||
// - Return as text content
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ValidateImageURL validate image URL format
|
||||
func ValidateImageURL(imageURL string) error {
|
||||
// TODO: Implement image URL validation
|
||||
// - Check URL format
|
||||
// - Validate image type
|
||||
// - Check accessibility
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractImagesFromMessages extract all image URLs from messages
|
||||
func ExtractImagesFromMessages(messages []context.Message) []string {
|
||||
// TODO: Implement image extraction
|
||||
// - Iterate through messages
|
||||
// - Find ContentPart with type="image_url"
|
||||
// - Collect all image URLs
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue