666c1e8d74
- Updated the Vision function and associated handlers to accept a forceUses parameter, allowing explicit control over the use of external tools regardless of model capabilities. - Modified the Assistant's BuildContent method to set the AssistantID in the context for better file tracking. - Enhanced the applyCreateResponseOptions method to merge and prioritize Uses and ForceUses configurations from the createResponse. - Added comprehensive tests to validate the adjustment of Uses and ForceUses configurations through hooks, ensuring correct application in content processing.
61 lines
2.2 KiB
Go
61 lines
2.2 KiB
Go
package content
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/yaoapp/gou/connector/openai"
|
|
agentContext "github.com/yaoapp/yao/agent/context"
|
|
)
|
|
|
|
// AudioHandler handles audio content
|
|
type AudioHandler struct{}
|
|
|
|
// CanHandle checks if this handler can handle the content type
|
|
func (h *AudioHandler) CanHandle(contentType string, fileType FileType) bool {
|
|
return fileType == FileTypeAudio || strings.HasPrefix(contentType, "audio/")
|
|
}
|
|
|
|
// Handle processes audio content
|
|
// Logic similar to image:
|
|
// 1. If model supports audio input -> convert to base64 format
|
|
// 2. If model doesn't support audio -> use agent/MCP specified in uses.Audio
|
|
func (h *AudioHandler) Handle(ctx *agentContext.Context, info *Info, capabilities *openai.Capabilities, uses *agentContext.Uses, forceUses bool) (*Result, error) {
|
|
// TODO: Implement audio handling
|
|
// 1. Check model audio capabilities
|
|
// 2. If supported:
|
|
// - Encode audio as base64 with proper format
|
|
// 3. If not supported:
|
|
// - Call audio agent/MCP to transcribe audio to text
|
|
// 4. Return Result with text or ContentPart
|
|
return nil, fmt.Errorf("not implemented")
|
|
}
|
|
|
|
// handleWithAudioModel processes audio using model's audio capability
|
|
func (h *AudioHandler) handleWithAudioModel(ctx *agentContext.Context, info *Info) (*Result, error) {
|
|
// TODO: Implement audio model processing
|
|
// Format audio according to model's audio input format
|
|
return nil, fmt.Errorf("not implemented")
|
|
}
|
|
|
|
// handleWithAudioAgent processes audio using audio agent or MCP
|
|
func (h *AudioHandler) handleWithAudioAgent(ctx *agentContext.Context, info *Info, audioTool string) (string, error) {
|
|
// TODO: Implement audio agent/MCP processing
|
|
// 1. Parse audioTool (format: "agent" or "mcp:server_id")
|
|
// 2. Call appropriate tool to transcribe audio
|
|
// 3. Return transcribed text
|
|
return "", fmt.Errorf("not implemented")
|
|
}
|
|
|
|
// encodeAudioBase64 encodes audio data to base64 with proper format
|
|
func encodeAudioBase64(data []byte, contentType string) string {
|
|
// TODO: Implement audio base64 encoding
|
|
return ""
|
|
}
|
|
|
|
// detectAudioFormat detects audio format from content type or data
|
|
func detectAudioFormat(contentType string, data []byte) string {
|
|
// TODO: Implement audio format detection
|
|
// Return format like "wav", "mp3", "flac", etc.
|
|
return ""
|
|
}
|