Implement global uses configuration and enhance assistant capabilities
- Added global uses configuration to the assistant, allowing for centralized management of vision, audio, search, and fetch settings. - Updated the Assistant struct and related methods to support the new Uses configuration, improving flexibility in assistant operations. - Refactored the Stream method to utilize the new CompletionResponse type, enhancing response handling. - Introduced new methods for building requests and managing capabilities, streamlining the assistant's interaction with various connectors.
This commit is contained in:
Max
parent
889fba942a
commit
124bd38f7a
31 changed files with 2166 additions and 178 deletions
|
|
@ -1,12 +1,16 @@
|
||||||
package assistant
|
package assistant
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
"github.com/yaoapp/yao/agent/context"
|
"github.com/yaoapp/yao/agent/context"
|
||||||
"github.com/yaoapp/yao/agent/llm"
|
"github.com/yaoapp/yao/agent/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Stream stream the agent
|
// Stream stream the agent
|
||||||
func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Message, handler context.StreamFunc) (*context.Response, error) {
|
// handler is optional, if not provided, a default handler will be used
|
||||||
|
func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Message, handler ...context.StreamFunc) (*context.Response, error) {
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
|
@ -31,27 +35,44 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = createResponse // createResponse is available for further processing
|
|
||||||
|
|
||||||
var completionOptions *llm.CompletionOptions // default is nil
|
var completionOptions *context.CompletionOptions // default is nil
|
||||||
|
|
||||||
// LLM Call Stream ( Optional )
|
// LLM Call Stream ( Optional )
|
||||||
var completionMessages []context.Message
|
var completionMessages []context.Message
|
||||||
var completionResponse *context.ResponseCompletion
|
var completionResponse *context.CompletionResponse
|
||||||
if ast.Prompts != nil || ast.MCP != nil {
|
if ast.Prompts != nil || ast.MCP != nil {
|
||||||
llm, err := llm.New(ast.GetConnector(ctx))
|
// Build the LLM request first
|
||||||
|
completionMessages, completionOptions, err = ast.BuildRequest(ctx, inputMessages, createResponse)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build the LLM request
|
// Get connector object and capabilities
|
||||||
completionMessages, completionOptions, err = ast.BuildLLMRequest(ctx, inputMessages, createResponse)
|
conn, capabilities, err := ast.GetConnector(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set capabilities in options if not already set
|
||||||
|
if completionOptions.Capabilities == nil && capabilities != nil {
|
||||||
|
completionOptions.Capabilities = capabilities
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create LLM instance with connector and options
|
||||||
|
llmInstance, err := llm.New(conn, completionOptions)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use provided handler or default handler
|
||||||
|
streamHandler := llm.DefaultStreamHandler(ctx)
|
||||||
|
if len(handler) > 0 && handler[0] != nil {
|
||||||
|
streamHandler = handler[0]
|
||||||
|
}
|
||||||
|
|
||||||
// Call the LLM Completion Stream
|
// Call the LLM Completion Stream
|
||||||
completionResponse, err = llm.Stream(ctx, completionMessages, completionOptions, handler)
|
completionResponse, err = llmInstance.Stream(ctx, completionMessages, completionOptions, streamHandler)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -81,17 +102,328 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
|
||||||
return &context.Response{Create: createResponse, Done: doneResponse, Completion: completionResponse}, nil
|
return &context.Response{Create: createResponse, Done: doneResponse, Completion: completionResponse}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetConnector get the connector from the context
|
// GetConnector get the connector object, capabilities, and error with priority: createResponse > ctx > ast
|
||||||
func (ast *Assistant) GetConnector(ctx *context.Context) string {
|
// Note: createResponse.Connector is already applied to ctx.Connector by applyContextAdjustments in create.go
|
||||||
|
// Returns: (connector, capabilities, error)
|
||||||
|
func (ast *Assistant) GetConnector(ctx *context.Context) (connector.Connector, *context.ModelCapabilities, error) {
|
||||||
|
// Determine connector ID with priority
|
||||||
|
connectorID := ast.Connector
|
||||||
if ctx.Connector != "" {
|
if ctx.Connector != "" {
|
||||||
return ctx.Connector
|
connectorID = ctx.Connector
|
||||||
}
|
}
|
||||||
return ast.Connector
|
|
||||||
|
// If empty, return error
|
||||||
|
if connectorID == "" {
|
||||||
|
return nil, nil, fmt.Errorf("connector not specified")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load gou connector
|
||||||
|
conn, err := connector.Select(connectorID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get connector capabilities from settings
|
||||||
|
capabilities := ast.getConnectorCapabilities(connectorID)
|
||||||
|
|
||||||
|
return conn, capabilities, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildLLMRequest build the LLM request
|
// getConnectorCapabilities get the capabilities of a connector from settings
|
||||||
func (ast *Assistant) BuildLLMRequest(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse) ([]context.Message, *llm.CompletionOptions, error) {
|
func (ast *Assistant) getConnectorCapabilities(connectorID string) *context.ModelCapabilities {
|
||||||
return messages, nil, nil
|
// Get connector setting from global settings
|
||||||
|
setting, exists := connectorSettings[connectorID]
|
||||||
|
if !exists {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert ConnectorSetting to ModelCapabilities
|
||||||
|
capabilities := &context.ModelCapabilities{}
|
||||||
|
|
||||||
|
if setting.Vision {
|
||||||
|
v := true
|
||||||
|
capabilities.Vision = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle both Tools (deprecated) and ToolCalls
|
||||||
|
if setting.ToolCalls || setting.Tools {
|
||||||
|
v := true
|
||||||
|
capabilities.ToolCalls = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
if setting.Audio {
|
||||||
|
v := true
|
||||||
|
capabilities.Audio = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
if setting.Reasoning {
|
||||||
|
v := true
|
||||||
|
capabilities.Reasoning = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
if setting.Streaming {
|
||||||
|
v := true
|
||||||
|
capabilities.Streaming = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
if setting.JSON {
|
||||||
|
v := true
|
||||||
|
capabilities.JSON = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
if setting.Multimodal {
|
||||||
|
v := true
|
||||||
|
capabilities.Multimodal = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
return capabilities
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildRequest build the LLM request
|
||||||
|
func (ast *Assistant) BuildRequest(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse) ([]context.Message, *context.CompletionOptions, error) {
|
||||||
|
// Build final messages with proper priority
|
||||||
|
finalMessages, err := ast.buildMessages(ctx, messages, createResponse)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build completion options from createResponse and ctx
|
||||||
|
options := ast.buildCompletionOptions(ctx, createResponse)
|
||||||
|
|
||||||
|
return finalMessages, options, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildMessages builds the final message list with proper priority
|
||||||
|
// Priority: createResponse.Messages > input messages
|
||||||
|
// If createResponse is nil or has no messages, use input messages
|
||||||
|
func (ast *Assistant) buildMessages(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse) ([]context.Message, error) {
|
||||||
|
// If createResponse is nil or has no messages, return input messages as-is
|
||||||
|
if createResponse == nil || len(createResponse.Messages) == 0 {
|
||||||
|
return messages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createResponse.Messages takes highest priority
|
||||||
|
// Return them directly as they override everything
|
||||||
|
return createResponse.Messages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildCompletionOptions builds completion options from multiple sources
|
||||||
|
// Priority (lowest to highest, later overrides earlier): ast > ctx > createResponse
|
||||||
|
// The priority means: if createResponse has a value, use it; else use ctx; else use ast
|
||||||
|
func (ast *Assistant) buildCompletionOptions(ctx *context.Context, createResponse *context.HookCreateResponse) *context.CompletionOptions {
|
||||||
|
options := &context.CompletionOptions{}
|
||||||
|
|
||||||
|
// Layer 1 (base): Apply ast - Assistant configuration
|
||||||
|
ast.applyAssistantOptions(options)
|
||||||
|
|
||||||
|
// Layer 2 (middle): Apply ctx - Context configuration (overrides ast)
|
||||||
|
ast.applyContextOptions(options, ctx)
|
||||||
|
|
||||||
|
// Layer 3 (highest): Apply createResponse - Hook configuration (overrides all)
|
||||||
|
if createResponse != nil {
|
||||||
|
ast.applyCreateResponseOptions(options, createResponse)
|
||||||
|
}
|
||||||
|
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyAssistantOptions applies options from ast.Options to CompletionOptions
|
||||||
|
// ast.Options can contain any OpenAI API parameters (temperature, top_p, stop, etc.)
|
||||||
|
func (ast *Assistant) applyAssistantOptions(options *context.CompletionOptions) {
|
||||||
|
if ast.Options == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Temperature
|
||||||
|
if v, ok := ast.Options["temperature"].(float64); ok {
|
||||||
|
options.Temperature = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxTokens
|
||||||
|
if v, ok := ast.Options["max_tokens"].(float64); ok {
|
||||||
|
intVal := int(v)
|
||||||
|
options.MaxTokens = &intVal
|
||||||
|
} else if v, ok := ast.Options["max_tokens"].(int); ok {
|
||||||
|
options.MaxTokens = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxCompletionTokens
|
||||||
|
if v, ok := ast.Options["max_completion_tokens"].(float64); ok {
|
||||||
|
intVal := int(v)
|
||||||
|
options.MaxCompletionTokens = &intVal
|
||||||
|
} else if v, ok := ast.Options["max_completion_tokens"].(int); ok {
|
||||||
|
options.MaxCompletionTokens = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// TopP
|
||||||
|
if v, ok := ast.Options["top_p"].(float64); ok {
|
||||||
|
options.TopP = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// N (number of choices)
|
||||||
|
if v, ok := ast.Options["n"].(float64); ok {
|
||||||
|
intVal := int(v)
|
||||||
|
options.N = &intVal
|
||||||
|
} else if v, ok := ast.Options["n"].(int); ok {
|
||||||
|
options.N = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop sequences (can be string or []string)
|
||||||
|
if v, ok := ast.Options["stop"]; ok {
|
||||||
|
options.Stop = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// PresencePenalty
|
||||||
|
if v, ok := ast.Options["presence_penalty"].(float64); ok {
|
||||||
|
options.PresencePenalty = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// FrequencyPenalty
|
||||||
|
if v, ok := ast.Options["frequency_penalty"].(float64); ok {
|
||||||
|
options.FrequencyPenalty = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// LogitBias
|
||||||
|
if v, ok := ast.Options["logit_bias"].(map[string]interface{}); ok {
|
||||||
|
logitBias := make(map[string]float64)
|
||||||
|
for key, val := range v {
|
||||||
|
if fval, ok := val.(float64); ok {
|
||||||
|
logitBias[key] = fval
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(logitBias) > 0 {
|
||||||
|
options.LogitBias = logitBias
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// User
|
||||||
|
if v, ok := ast.Options["user"].(string); ok {
|
||||||
|
options.User = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResponseFormat
|
||||||
|
if v, ok := ast.Options["response_format"].(map[string]interface{}); ok {
|
||||||
|
options.ResponseFormat = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed
|
||||||
|
if v, ok := ast.Options["seed"].(float64); ok {
|
||||||
|
intVal := int(v)
|
||||||
|
options.Seed = &intVal
|
||||||
|
} else if v, ok := ast.Options["seed"].(int); ok {
|
||||||
|
options.Seed = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tools
|
||||||
|
if v, ok := ast.Options["tools"].([]interface{}); ok {
|
||||||
|
tools := make([]map[string]interface{}, 0, len(v))
|
||||||
|
for _, tool := range v {
|
||||||
|
if toolMap, ok := tool.(map[string]interface{}); ok {
|
||||||
|
tools = append(tools, toolMap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(tools) > 0 {
|
||||||
|
options.Tools = tools
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToolChoice
|
||||||
|
if v, ok := ast.Options["tool_choice"]; ok {
|
||||||
|
options.ToolChoice = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream
|
||||||
|
if v, ok := ast.Options["stream"].(bool); ok {
|
||||||
|
options.Stream = &v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyContextOptions applies options from ctx to CompletionOptions
|
||||||
|
// ctx provides Route and Metadata for CUI context
|
||||||
|
func (ast *Assistant) applyContextOptions(options *context.CompletionOptions, ctx *context.Context) {
|
||||||
|
// Set Route and Metadata from ctx
|
||||||
|
options.Route = ctx.Route
|
||||||
|
options.Metadata = ctx.Metadata
|
||||||
|
|
||||||
|
// Set wrapper configurations (assistant.Uses has priority over global settings)
|
||||||
|
// These can be overridden by createResponse
|
||||||
|
if visionWrapper := ast.getVisionWrapper(); visionWrapper != "" {
|
||||||
|
options.VisionWrapper = visionWrapper
|
||||||
|
}
|
||||||
|
if audioWrapper := ast.getAudioWrapper(); audioWrapper != "" {
|
||||||
|
options.AudioWrapper = audioWrapper
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyCreateResponseOptions applies options from createResponse to CompletionOptions
|
||||||
|
// createResponse takes highest priority and overrides any previous settings
|
||||||
|
func (ast *Assistant) applyCreateResponseOptions(options *context.CompletionOptions, createResponse *context.HookCreateResponse) {
|
||||||
|
// Audio configuration
|
||||||
|
if createResponse.Audio != nil {
|
||||||
|
options.Audio = createResponse.Audio
|
||||||
|
}
|
||||||
|
|
||||||
|
// Temperature
|
||||||
|
if createResponse.Temperature != nil {
|
||||||
|
options.Temperature = createResponse.Temperature
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxTokens
|
||||||
|
if createResponse.MaxTokens != nil {
|
||||||
|
options.MaxTokens = createResponse.MaxTokens
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxCompletionTokens
|
||||||
|
if createResponse.MaxCompletionTokens != nil {
|
||||||
|
options.MaxCompletionTokens = createResponse.MaxCompletionTokens
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route
|
||||||
|
if createResponse.Route != "" {
|
||||||
|
options.Route = createResponse.Route
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metadata (merge with existing)
|
||||||
|
if createResponse.Metadata != nil {
|
||||||
|
if options.Metadata == nil {
|
||||||
|
options.Metadata = createResponse.Metadata
|
||||||
|
} else {
|
||||||
|
// Merge: createResponse.Metadata overrides existing
|
||||||
|
for key, value := range createResponse.Metadata {
|
||||||
|
options.Metadata[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// getVisionWrapper get the vision wrapper with priority: assistant.Uses > global settings
|
||||||
|
func (ast *Assistant) getVisionWrapper() string {
|
||||||
|
// Priority 1: Assistant-specific Uses configuration
|
||||||
|
if ast.Uses != nil && ast.Uses.Vision != "" {
|
||||||
|
return ast.Uses.Vision
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Global settings from globalUses
|
||||||
|
if globalUses != nil && globalUses.Vision != "" {
|
||||||
|
return globalUses.Vision
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// getAudioWrapper get the audio wrapper with priority: assistant.Uses > global settings
|
||||||
|
func (ast *Assistant) getAudioWrapper() string {
|
||||||
|
// Priority 1: Assistant-specific Uses configuration
|
||||||
|
if ast.Uses != nil && ast.Uses.Audio != "" {
|
||||||
|
return ast.Uses.Audio
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Global settings from globalUses
|
||||||
|
if globalUses != nil && globalUses.Audio != "" {
|
||||||
|
return globalUses.Audio
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithHistory with the history messages
|
// WithHistory with the history messages
|
||||||
|
|
|
||||||
262
agent/assistant/agent_test.go
Normal file
262
agent/assistant/agent_test.go
Normal file
|
|
@ -0,0 +1,262 @@
|
||||||
|
package assistant_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
stdContext "context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/yaoapp/gou/plan"
|
||||||
|
"github.com/yaoapp/yao/agent/assistant"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/testutils"
|
||||||
|
"github.com/yaoapp/yao/openapi/oauth/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newTestContext creates a Context for testing with commonly used fields pre-populated
|
||||||
|
func newTestContext(chatID, assistantID string) *context.Context {
|
||||||
|
return &context.Context{
|
||||||
|
Context: stdContext.Background(),
|
||||||
|
Space: plan.NewMemorySharedSpace(),
|
||||||
|
ChatID: chatID,
|
||||||
|
AssistantID: assistantID,
|
||||||
|
Connector: "",
|
||||||
|
Locale: "en-us",
|
||||||
|
Theme: "light",
|
||||||
|
Client: context.Client{
|
||||||
|
Type: "web",
|
||||||
|
UserAgent: "TestAgent/1.0",
|
||||||
|
IP: "127.0.0.1",
|
||||||
|
},
|
||||||
|
Referer: context.RefererAPI,
|
||||||
|
Accept: context.AcceptWebCUI,
|
||||||
|
Route: "/test/route",
|
||||||
|
Metadata: map[string]interface{}{
|
||||||
|
"test": "context_metadata",
|
||||||
|
},
|
||||||
|
Authorized: &types.AuthorizedInfo{
|
||||||
|
Subject: "test-user",
|
||||||
|
ClientID: "test-client-id",
|
||||||
|
UserID: "test-user-123",
|
||||||
|
TeamID: "test-team-456",
|
||||||
|
TenantID: "test-tenant-789",
|
||||||
|
SessionID: "test-session-id",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBuildRequest tests the BuildRequest function
|
||||||
|
func TestBuildRequest(t *testing.T) {
|
||||||
|
testutils.Prepare(t)
|
||||||
|
defer testutils.Clean(t)
|
||||||
|
|
||||||
|
agent, err := assistant.Get("tests.buildrequest")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get tests.buildrequest assistant: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
if agent.Script == nil {
|
||||||
|
t.Fatalf("The tests.buildrequest assistant has no script")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := newTestContext("chat-test-buildrequest", "tests.buildrequest")
|
||||||
|
|
||||||
|
// Test 1: No override from hook - should use ast.Options and ctx values
|
||||||
|
t.Run("NoOverride", func(t *testing.T) {
|
||||||
|
inputMessages := []context.Message{{Role: "user", Content: "no_override"}}
|
||||||
|
|
||||||
|
// Call Create hook
|
||||||
|
createResponse, err := agent.Script.Create(ctx, inputMessages)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to call Create hook: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build LLM request
|
||||||
|
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to build LLM request: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify options - should use ast.Options values
|
||||||
|
if options.Temperature == nil {
|
||||||
|
t.Error("Expected temperature from ast.Options, got nil")
|
||||||
|
} else if *options.Temperature != 0.5 {
|
||||||
|
t.Errorf("Expected temperature 0.5 from ast.Options, got: %f", *options.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.MaxTokens == nil {
|
||||||
|
t.Error("Expected max_tokens from ast.Options, got nil")
|
||||||
|
} else if *options.MaxTokens != 1000 {
|
||||||
|
t.Errorf("Expected max_tokens 1000 from ast.Options, got: %d", *options.MaxTokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.TopP == nil {
|
||||||
|
t.Error("Expected top_p from ast.Options, got nil")
|
||||||
|
} else if *options.TopP != 0.9 {
|
||||||
|
t.Errorf("Expected top_p 0.9 from ast.Options, got: %f", *options.TopP)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify ctx values
|
||||||
|
if options.Route != "/test/route" {
|
||||||
|
t.Errorf("Expected route '/test/route' from ctx, got: %s", options.Route)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.Metadata == nil {
|
||||||
|
t.Error("Expected metadata from ctx, got nil")
|
||||||
|
} else if options.Metadata["test"] != "context_metadata" {
|
||||||
|
t.Errorf("Expected metadata from ctx, got: %v", options.Metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ No override: ast.Options and ctx values used correctly")
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 2: Override temperature - hook value should take priority
|
||||||
|
t.Run("OverrideTemperature", func(t *testing.T) {
|
||||||
|
inputMessages := []context.Message{{Role: "user", Content: "override_temperature"}}
|
||||||
|
|
||||||
|
createResponse, err := agent.Script.Create(ctx, inputMessages)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to call Create hook: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to build LLM request: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify temperature override
|
||||||
|
if options.Temperature == nil {
|
||||||
|
t.Error("Expected temperature, got nil")
|
||||||
|
} else if *options.Temperature != 0.9 {
|
||||||
|
t.Errorf("Expected temperature 0.9 from hook, got: %f", *options.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Other values should still come from ast.Options
|
||||||
|
if options.MaxTokens == nil {
|
||||||
|
t.Error("Expected max_tokens from ast.Options, got nil")
|
||||||
|
} else if *options.MaxTokens != 1000 {
|
||||||
|
t.Errorf("Expected max_tokens 1000 from ast.Options, got: %d", *options.MaxTokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ Temperature override: hook value takes priority over ast.Options")
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 3: Override all - all hook values should take priority
|
||||||
|
t.Run("OverrideAll", func(t *testing.T) {
|
||||||
|
inputMessages := []context.Message{{Role: "user", Content: "override_all"}}
|
||||||
|
|
||||||
|
createResponse, err := agent.Script.Create(ctx, inputMessages)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to call Create hook: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to build LLM request: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all overrides
|
||||||
|
if options.Temperature == nil || *options.Temperature != 0.8 {
|
||||||
|
t.Errorf("Expected temperature 0.8 from hook, got: %v", options.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.MaxTokens == nil || *options.MaxTokens != 2000 {
|
||||||
|
t.Errorf("Expected max_tokens 2000 from hook, got: %v", options.MaxTokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.MaxCompletionTokens == nil || *options.MaxCompletionTokens != 1800 {
|
||||||
|
t.Errorf("Expected max_completion_tokens 1800 from hook, got: %v", options.MaxCompletionTokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.Audio == nil {
|
||||||
|
t.Error("Expected audio from hook, got nil")
|
||||||
|
} else {
|
||||||
|
if options.Audio.Voice != "alloy" {
|
||||||
|
t.Errorf("Expected voice 'alloy', got: %s", options.Audio.Voice)
|
||||||
|
}
|
||||||
|
if options.Audio.Format != "mp3" {
|
||||||
|
t.Errorf("Expected format 'mp3', got: %s", options.Audio.Format)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.Route != "/hook/route" {
|
||||||
|
t.Errorf("Expected route '/hook/route' from hook, got: %s", options.Route)
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.Metadata == nil {
|
||||||
|
t.Error("Expected metadata from hook, got nil")
|
||||||
|
} else {
|
||||||
|
if options.Metadata["source"] != "hook" {
|
||||||
|
t.Errorf("Expected metadata['source'] = 'hook', got: %v", options.Metadata["source"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ Override all: all hook values take priority")
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 4: Override route and metadata - tests CUI context priority
|
||||||
|
t.Run("OverrideRouteMetadata", func(t *testing.T) {
|
||||||
|
inputMessages := []context.Message{{Role: "user", Content: "override_route_metadata"}}
|
||||||
|
|
||||||
|
createResponse, err := agent.Script.Create(ctx, inputMessages)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to call Create hook: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to build LLM request: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify route override
|
||||||
|
if options.Route != "/custom/route" {
|
||||||
|
t.Errorf("Expected route '/custom/route' from hook, got: %s", options.Route)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify metadata merge (ctx metadata should be merged with hook metadata)
|
||||||
|
if options.Metadata == nil {
|
||||||
|
t.Error("Expected metadata, got nil")
|
||||||
|
} else {
|
||||||
|
// Hook metadata should be present
|
||||||
|
if options.Metadata["custom"] != true {
|
||||||
|
t.Errorf("Expected metadata['custom'] = true from hook, got: %v", options.Metadata["custom"])
|
||||||
|
}
|
||||||
|
if options.Metadata["hook_data"] != "test" {
|
||||||
|
t.Errorf("Expected metadata['hook_data'] = 'test' from hook, got: %v", options.Metadata["hook_data"])
|
||||||
|
}
|
||||||
|
// Original ctx metadata should still be there (merged)
|
||||||
|
if options.Metadata["test"] != "context_metadata" {
|
||||||
|
t.Errorf("Expected original ctx metadata to be preserved, got: %v", options.Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Other values should still come from ast.Options
|
||||||
|
if options.Temperature == nil || *options.Temperature != 0.5 {
|
||||||
|
t.Errorf("Expected temperature 0.5 from ast.Options, got: %v", options.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ Route and metadata override: hook values take priority, metadata merged")
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 5: Nil createResponse - should use ast.Options and ctx values
|
||||||
|
t.Run("NilCreateResponse", func(t *testing.T) {
|
||||||
|
// Create a fresh context for this test
|
||||||
|
freshCtx := newTestContext("chat-test-nil", "tests.buildrequest")
|
||||||
|
inputMessages := []context.Message{{Role: "user", Content: "test message"}}
|
||||||
|
|
||||||
|
_, options, err := agent.BuildRequest(freshCtx, inputMessages, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to build LLM request: %s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should use ast.Options values
|
||||||
|
if options.Temperature == nil || *options.Temperature != 0.5 {
|
||||||
|
t.Errorf("Expected temperature 0.5 from ast.Options, got: %v", options.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should use ctx values
|
||||||
|
if options.Route != "/test/route" {
|
||||||
|
t.Errorf("Expected route '/test/route' from ctx, got: %s", options.Route)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✓ Nil createResponse: ast.Options and ctx values used")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -5,6 +5,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// Done done hook
|
// Done done hook
|
||||||
func (s *Script) Done(ctx *context.Context, inputMessages []context.Message, completionResponse *context.ResponseCompletion, mcpResponse *context.ResponseHookMCP) (*context.ResponseHookDone, error) {
|
func (s *Script) Done(ctx *context.Context, inputMessages []context.Message, completionResponse *context.CompletionResponse, mcpResponse *context.ResponseHookMCP) (*context.ResponseHookDone, error) {
|
||||||
return &context.ResponseHookDone{}, nil
|
return &context.ResponseHookDone{}, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
package hook
|
package hook
|
||||||
|
|
||||||
import "github.com/yaoapp/yao/agent/context"
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
)
|
||||||
|
|
||||||
// Failback failback hook
|
// Failback failback hook
|
||||||
func (s *Script) Failback(ctx *context.Context, inputMessages []context.Message, completionResponse *context.ResponseCompletion) (*context.ResponseHookFailback, error) {
|
func (s *Script) Failback(ctx *context.Context, inputMessages []context.Message, completionResponse *context.CompletionResponse) (*context.ResponseHookFailback, error) {
|
||||||
return &context.ResponseHookFailback{}, nil
|
return &context.ResponseHookFailback{}, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,7 @@ var search interface{} = nil
|
||||||
var connectorSettings map[string]ConnectorSetting = map[string]ConnectorSetting{}
|
var connectorSettings map[string]ConnectorSetting = map[string]ConnectorSetting{}
|
||||||
var vision *agentvision.Vision = nil
|
var vision *agentvision.Vision = nil
|
||||||
var defaultConnector string = "" // default connector
|
var defaultConnector string = "" // default connector
|
||||||
|
var globalUses *store.Uses = nil // global uses configuration from agent.yml
|
||||||
|
|
||||||
// LoadBuiltIn load the built-in assistants
|
// LoadBuiltIn load the built-in assistants
|
||||||
func LoadBuiltIn() error {
|
func LoadBuiltIn() error {
|
||||||
|
|
@ -145,6 +146,11 @@ func SetConnector(c string) {
|
||||||
defaultConnector = c
|
defaultConnector = c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetGlobalUses set the global uses configuration
|
||||||
|
func SetGlobalUses(uses *store.Uses) {
|
||||||
|
globalUses = uses
|
||||||
|
}
|
||||||
|
|
||||||
// SetCache set the cache
|
// SetCache set the cache
|
||||||
func SetCache(capacity int) {
|
func SetCache(capacity int) {
|
||||||
ClearCache()
|
ClearCache()
|
||||||
|
|
|
||||||
|
|
@ -109,9 +109,16 @@ type Assistant struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConnectorSetting the connector setting
|
// ConnectorSetting the connector setting
|
||||||
|
// Defines the capabilities of a connector/model
|
||||||
type ConnectorSetting struct {
|
type ConnectorSetting struct {
|
||||||
Vision bool `json:"vision,omitempty" yaml:"vision,omitempty"`
|
Vision bool `json:"vision,omitempty" yaml:"vision,omitempty"` // Supports vision/image input
|
||||||
Tools bool `json:"tools,omitempty" yaml:"tools,omitempty"`
|
Tools bool `json:"tools,omitempty" yaml:"tools,omitempty"` // Supports tool/function calling (deprecated, use ToolCalls)
|
||||||
|
ToolCalls bool `json:"tool_calls,omitempty" yaml:"tool_calls,omitempty"` // Supports tool/function calling
|
||||||
|
Audio bool `json:"audio,omitempty" yaml:"audio,omitempty"` // Supports audio input/output
|
||||||
|
Reasoning bool `json:"reasoning,omitempty" yaml:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1)
|
||||||
|
Streaming bool `json:"streaming,omitempty" yaml:"streaming,omitempty"` // Supports streaming responses
|
||||||
|
JSON bool `json:"json,omitempty" yaml:"json,omitempty"` // Supports JSON mode
|
||||||
|
Multimodal bool `json:"multimodal,omitempty" yaml:"multimodal,omitempty"` // Supports multimodal input
|
||||||
}
|
}
|
||||||
|
|
||||||
// VisionCapableModels list of LLM models that support vision capabilities
|
// VisionCapableModels list of LLM models that support vision capabilities
|
||||||
|
|
|
||||||
|
|
@ -188,7 +188,7 @@ type Response struct {
|
||||||
MCP *ResponseHookMCP `json:"mcp,omitempty"`
|
MCP *ResponseHookMCP `json:"mcp,omitempty"`
|
||||||
Done *ResponseHookDone `json:"done,omitempty"`
|
Done *ResponseHookDone `json:"done,omitempty"`
|
||||||
Failback *ResponseHookFailback `json:"failback,omitempty"`
|
Failback *ResponseHookFailback `json:"failback,omitempty"`
|
||||||
Completion *ResponseCompletion `json:"completion,omitempty"`
|
Completion *CompletionResponse `json:"completion,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// HookCreateResponse the response of the create hook
|
// HookCreateResponse the response of the create hook
|
||||||
|
|
@ -223,9 +223,6 @@ type ResponseHookMCP struct{}
|
||||||
// ResponseHookFailback the response of the failback hook
|
// ResponseHookFailback the response of the failback hook
|
||||||
type ResponseHookFailback struct{}
|
type ResponseHookFailback struct{}
|
||||||
|
|
||||||
// ResponseCompletion the response of the completion
|
|
||||||
type ResponseCompletion struct{}
|
|
||||||
|
|
||||||
// Message Structure ( OpenAI Chat Completion Input Message Structure, https://platform.openai.com/docs/api-reference/chat/create#chat/create-messages )
|
// Message Structure ( OpenAI Chat Completion Input Message Structure, https://platform.openai.com/docs/api-reference/chat/create#chat/create-messages )
|
||||||
// ===============================
|
// ===============================
|
||||||
|
|
||||||
|
|
|
||||||
146
agent/context/types_llm.go
Normal file
146
agent/context/types_llm.go
Normal file
|
|
@ -0,0 +1,146 @@
|
||||||
|
package context
|
||||||
|
|
||||||
|
// ModelCapabilities defines the capabilities of a language model
|
||||||
|
// Used by LLM to select appropriate provider and validate requests
|
||||||
|
type ModelCapabilities struct {
|
||||||
|
Vision *bool `json:"vision,omitempty"` // Supports vision/image input
|
||||||
|
ToolCalls *bool `json:"tool_calls,omitempty"` // Supports tool/function calling
|
||||||
|
Audio *bool `json:"audio,omitempty"` // Supports audio input/output
|
||||||
|
Reasoning *bool `json:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1)
|
||||||
|
Streaming *bool `json:"streaming,omitempty"` // Supports streaming responses
|
||||||
|
JSON *bool `json:"json,omitempty"` // Supports JSON mode
|
||||||
|
Multimodal *bool `json:"multimodal,omitempty"` // Supports multimodal input (text + images + audio)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompletionOptions the completion request options
|
||||||
|
// These options are extracted from HookCreateResponse and Context, then passed to the LLM connector
|
||||||
|
// Compatible with OpenAI Chat Completion API: https://platform.openai.com/docs/api-reference/chat/create
|
||||||
|
type CompletionOptions struct {
|
||||||
|
// Model capabilities (used by LLM to select appropriate provider)
|
||||||
|
// nil means capabilities are not specified/checked
|
||||||
|
Capabilities *ModelCapabilities `json:"capabilities,omitempty"`
|
||||||
|
|
||||||
|
// Wrapper configurations for vision and audio processing
|
||||||
|
// Format: "agent" (default) or "mcp:mcp_server_id"
|
||||||
|
VisionWrapper string `json:"vision_wrapper,omitempty"` // Vision processing wrapper (for image/video description)
|
||||||
|
AudioWrapper string `json:"audio_wrapper,omitempty"` // Audio processing wrapper (for speech-to-text/text-to-speech)
|
||||||
|
|
||||||
|
// Audio configuration (for models that support audio output)
|
||||||
|
Audio *AudioConfig `json:"audio,omitempty"`
|
||||||
|
|
||||||
|
// Generation parameters
|
||||||
|
Temperature *float64 `json:"temperature,omitempty"` // Sampling temperature (0-2), defaults to 1
|
||||||
|
MaxTokens *int `json:"max_tokens,omitempty"` // Maximum tokens to generate (deprecated, use MaxCompletionTokens)
|
||||||
|
MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` // Maximum tokens in completion
|
||||||
|
TopP *float64 `json:"top_p,omitempty"` // Nucleus sampling parameter (0-1), alternative to temperature
|
||||||
|
N *int `json:"n,omitempty"` // Number of chat completion choices to generate
|
||||||
|
|
||||||
|
// Control parameters
|
||||||
|
Stop interface{} `json:"stop,omitempty"` // Up to 4 sequences where the API will stop generating (string or []string)
|
||||||
|
PresencePenalty *float64 `json:"presence_penalty,omitempty"` // Presence penalty (-2.0 to 2.0)
|
||||||
|
FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` // Frequency penalty (-2.0 to 2.0)
|
||||||
|
LogitBias map[string]float64 `json:"logit_bias,omitempty"` // Modify likelihood of specified tokens appearing
|
||||||
|
|
||||||
|
// User and response format
|
||||||
|
User string `json:"user,omitempty"` // Unique identifier representing end-user
|
||||||
|
ResponseFormat map[string]interface{} `json:"response_format,omitempty"` // Format of the response (e.g., {"type": "json_object"})
|
||||||
|
Seed *int `json:"seed,omitempty"` // Seed for deterministic sampling
|
||||||
|
|
||||||
|
// Tool calling
|
||||||
|
Tools []map[string]interface{} `json:"tools,omitempty"` // List of tools the model may call
|
||||||
|
ToolChoice interface{} `json:"tool_choice,omitempty"` // Controls which tool is called ("none", "auto", "required", or specific tool)
|
||||||
|
|
||||||
|
// Streaming configuration
|
||||||
|
Stream *bool `json:"stream,omitempty"` // If true, stream partial message deltas
|
||||||
|
StreamOptions *StreamOptions `json:"stream_options,omitempty"` // Options for streaming response
|
||||||
|
|
||||||
|
// CUI Context information (from Context)
|
||||||
|
Route string `json:"route,omitempty"` // Route of the request for CUI context
|
||||||
|
Metadata map[string]interface{} `json:"metadata,omitempty"` // Metadata to pass to the page for CUI context
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompletionResponse represents the unified completion response
|
||||||
|
// Compatible with OpenAI chat completion response format
|
||||||
|
type CompletionResponse struct {
|
||||||
|
// Response metadata
|
||||||
|
ID string `json:"id"` // Unique identifier for the completion
|
||||||
|
Object string `json:"object"` // Object type (e.g., "chat.completion")
|
||||||
|
Created int64 `json:"created"` // Unix timestamp of creation
|
||||||
|
Model string `json:"model"` // Model used for completion
|
||||||
|
|
||||||
|
// Completion content (these fields can coexist)
|
||||||
|
Content string `json:"content"` // Text content (regular response text)
|
||||||
|
ReasoningContent string `json:"reasoning_content,omitempty"` // Reasoning/thinking content (for o1, DeepSeek R1, etc.)
|
||||||
|
ToolCalls []ToolCallResult `json:"tool_calls,omitempty"` // Tool calls made by the model
|
||||||
|
Refusal string `json:"refusal,omitempty"` // Refusal message if model refused to answer
|
||||||
|
ContentTypes []ContentType `json:"content_types"` // Types of content present (can have multiple simultaneously)
|
||||||
|
|
||||||
|
// Raw response data
|
||||||
|
Raw interface{} `json:"raw,omitempty"` // Original raw response from the LLM provider (for debugging and special cases)
|
||||||
|
|
||||||
|
// Completion metadata
|
||||||
|
FinishReason string `json:"finish_reason"` // Reason for completion (stop, length, tool_calls, content_filter, etc.)
|
||||||
|
|
||||||
|
// Usage statistics
|
||||||
|
Usage *UsageInfo `json:"usage,omitempty"` // Token usage statistics
|
||||||
|
|
||||||
|
// Additional metadata
|
||||||
|
SystemFingerprint string `json:"system_fingerprint,omitempty"` // System fingerprint for reproducibility
|
||||||
|
Metadata map[string]interface{} `json:"metadata,omitempty"` // Additional metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContentType represents the type of content in the response
|
||||||
|
// A response can contain multiple content types simultaneously
|
||||||
|
type ContentType string
|
||||||
|
|
||||||
|
// Content type constants - a response can have multiple types simultaneously
|
||||||
|
// For example: text + reasoning, or text + tool_call, or all three
|
||||||
|
const (
|
||||||
|
ContentTypeText ContentType = "text" // Regular text content
|
||||||
|
ContentTypeReasoning ContentType = "reasoning" // Reasoning/thinking content (o1, DeepSeek R1, etc.)
|
||||||
|
ContentTypeToolCall ContentType = "tool_call" // Tool/function call
|
||||||
|
ContentTypeRefusal ContentType = "refusal" // Model refused to answer
|
||||||
|
ContentTypeEmpty ContentType = "empty" // Empty response (no content)
|
||||||
|
)
|
||||||
|
|
||||||
|
// UsageInfo represents token usage statistics
|
||||||
|
type UsageInfo struct {
|
||||||
|
PromptTokens int `json:"prompt_tokens"` // Tokens in the prompt
|
||||||
|
CompletionTokens int `json:"completion_tokens"` // Tokens in the completion
|
||||||
|
TotalTokens int `json:"total_tokens"` // Total tokens used
|
||||||
|
|
||||||
|
// Detailed token breakdown (for models with reasoning)
|
||||||
|
PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"` // Detailed prompt token breakdown
|
||||||
|
CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"` // Detailed completion token breakdown
|
||||||
|
}
|
||||||
|
|
||||||
|
// TokenDetails provides detailed token usage breakdown
|
||||||
|
type TokenDetails struct {
|
||||||
|
CachedTokens int `json:"cached_tokens,omitempty"` // Tokens from cache
|
||||||
|
ReasoningTokens int `json:"reasoning_tokens,omitempty"` // Tokens used for reasoning/thinking
|
||||||
|
AudioTokens int `json:"audio_tokens,omitempty"` // Tokens used for audio
|
||||||
|
TextTokens int `json:"text_tokens,omitempty"` // Tokens used for text
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToolCallResult represents a tool call result in the completion
|
||||||
|
type ToolCallResult struct {
|
||||||
|
ID string `json:"id"` // Tool call ID
|
||||||
|
Type string `json:"type"` // Tool call type (usually "function")
|
||||||
|
Function FunctionCallResult `json:"function"` // Function call details
|
||||||
|
}
|
||||||
|
|
||||||
|
// FunctionCallResult represents a function call result
|
||||||
|
type FunctionCallResult struct {
|
||||||
|
Name string `json:"name"` // Function name
|
||||||
|
Arguments string `json:"arguments"` // Function arguments as JSON string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FinishReason constants
|
||||||
|
const (
|
||||||
|
FinishReasonStop = "stop" // Natural stop point
|
||||||
|
FinishReasonLength = "length" // Max tokens reached
|
||||||
|
FinishReasonToolCalls = "tool_calls" // Tool calls made
|
||||||
|
FinishReasonContentFilter = "content_filter" // Content filtered
|
||||||
|
FinishReasonFunctionCall = "function_call" // Function call (deprecated)
|
||||||
|
FinishReasonError = "error" // Error occurred
|
||||||
|
)
|
||||||
49
agent/context/types_wrapper.go
Normal file
49
agent/context/types_wrapper.go
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
package context
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// WrapperType represents the type of wrapper for processing
|
||||||
|
type WrapperType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
WrapperTypeAgent WrapperType = "agent" // Use agent for processing
|
||||||
|
WrapperTypeMCP WrapperType = "mcp" // Use MCP server for processing
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParseWrapper parses a wrapper string and returns the type and ID
|
||||||
|
// Format: "agent" or "mcp:mcp_server_id"
|
||||||
|
func ParseWrapper(wrapper string) (WrapperType, string) {
|
||||||
|
if wrapper == "" || wrapper == "agent" {
|
||||||
|
return WrapperTypeAgent, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.HasPrefix(wrapper, "mcp:") {
|
||||||
|
mcpID := strings.TrimPrefix(wrapper, "mcp:")
|
||||||
|
return WrapperTypeMCP, mcpID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default to agent if format is unknown
|
||||||
|
return WrapperTypeAgent, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsAgentWrapper checks if the wrapper is an agent wrapper
|
||||||
|
func IsAgentWrapper(wrapper string) bool {
|
||||||
|
wrapperType, _ := ParseWrapper(wrapper)
|
||||||
|
return wrapperType == WrapperTypeAgent
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsMCPWrapper checks if the wrapper is an MCP wrapper
|
||||||
|
func IsMCPWrapper(wrapper string) bool {
|
||||||
|
wrapperType, _ := ParseWrapper(wrapper)
|
||||||
|
return wrapperType == WrapperTypeMCP
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMCPServerID extracts the MCP server ID from wrapper string
|
||||||
|
// Returns empty string if not an MCP wrapper
|
||||||
|
func GetMCPServerID(wrapper string) string {
|
||||||
|
wrapperType, id := ParseWrapper(wrapper)
|
||||||
|
if wrapperType == WrapperTypeMCP {
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
51
agent/llm/handlers/handlers.go
Normal file
51
agent/llm/handlers/handlers.go
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Handler interface for stream handlers
|
||||||
|
type Handler interface {
|
||||||
|
OnChunk(chunk *StreamChunk) error
|
||||||
|
OnComplete() error
|
||||||
|
OnError(err error) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDefaultHandler creates a default handler that sends chunks via context
|
||||||
|
func NewDefaultHandler(ctx *context.Context) Handler {
|
||||||
|
return &DefaultHandler{
|
||||||
|
ctx: ctx,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultHandler default stream handler implementation
|
||||||
|
type DefaultHandler struct {
|
||||||
|
ctx *context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
// OnChunk handles a streaming chunk
|
||||||
|
func (h *DefaultHandler) OnChunk(chunk *StreamChunk) error {
|
||||||
|
// TODO: Implement chunk handling
|
||||||
|
// - Send chunk via ctx
|
||||||
|
// - Handle different chunk types
|
||||||
|
// - Aggregate content for final response
|
||||||
|
return SendStreamChunk(h.ctx, chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OnComplete handles stream completion
|
||||||
|
func (h *DefaultHandler) OnComplete() error {
|
||||||
|
// TODO: Implement completion handling
|
||||||
|
// - Send final message
|
||||||
|
// - Close stream
|
||||||
|
// - Return aggregated response
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// OnError handles stream errors
|
||||||
|
func (h *DefaultHandler) OnError(err error) error {
|
||||||
|
// TODO: Implement error handling
|
||||||
|
// - Send error message to client
|
||||||
|
// - Log error
|
||||||
|
// - Clean up resources
|
||||||
|
return err
|
||||||
|
}
|
||||||
86
agent/llm/handlers/stream.go
Normal file
86
agent/llm/handlers/stream.go
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultStreamHandler creates a default stream handler that sends messages via context
|
||||||
|
// This handler is used when no custom handler is provided
|
||||||
|
func DefaultStreamHandler(ctx *context.Context) context.StreamFunc {
|
||||||
|
return func(data []byte) int {
|
||||||
|
// TODO: Implement default stream handling
|
||||||
|
// - Parse streaming chunk data
|
||||||
|
// - Extract content from chunk
|
||||||
|
// - Send message via ctx (SSE, WebSocket, etc.)
|
||||||
|
// - Handle different chunk types (content, tool_calls, reasoning)
|
||||||
|
// - Return 1 to continue streaming, 0 to stop
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SendStreamChunk sends a stream chunk via context
|
||||||
|
// Used internally by DefaultStreamHandler
|
||||||
|
func SendStreamChunk(ctx *context.Context, chunk *StreamChunk) error {
|
||||||
|
// TODO: Implement sending stream chunk
|
||||||
|
// - Format chunk for transport (SSE, WebSocket)
|
||||||
|
// - Send via ctx's connection
|
||||||
|
// - Handle errors and retries
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StreamChunk represents a parsed streaming chunk
|
||||||
|
type StreamChunk struct {
|
||||||
|
Type ChunkType `json:"type"` // Type of chunk (content, reasoning, tool_call, etc.)
|
||||||
|
Content string `json:"content,omitempty"` // Text content
|
||||||
|
|
||||||
|
// For reasoning chunks
|
||||||
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||||
|
|
||||||
|
// For tool call chunks
|
||||||
|
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||||
|
ToolCallFunction string `json:"tool_call_function,omitempty"`
|
||||||
|
ToolCallArgs string `json:"tool_call_args,omitempty"`
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
Done bool `json:"done"` // Whether this is the final chunk
|
||||||
|
FinishReason string `json:"finish_reason,omitempty"` // Reason for completion (if done)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkType represents the type of streaming chunk
|
||||||
|
type ChunkType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ChunkTypeContent ChunkType = "content" // Regular text content
|
||||||
|
ChunkTypeReasoning ChunkType = "reasoning" // Reasoning/thinking content
|
||||||
|
ChunkTypeToolCall ChunkType = "tool_call" // Tool call chunk
|
||||||
|
ChunkTypeDone ChunkType = "done" // Final chunk (completion)
|
||||||
|
ChunkTypeError ChunkType = "error" // Error chunk
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParseStreamChunk parses raw streaming data into StreamChunk
|
||||||
|
func ParseStreamChunk(data []byte) (*StreamChunk, error) {
|
||||||
|
// TODO: Implement stream chunk parsing
|
||||||
|
// - Parse SSE format (data: {...})
|
||||||
|
// - Handle different provider formats (OpenAI, DeepSeek, etc.)
|
||||||
|
// - Extract content, reasoning, tool calls
|
||||||
|
// - Detect completion (done: true)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatSSE formats a StreamChunk as Server-Sent Events format
|
||||||
|
func FormatSSE(chunk *StreamChunk) string {
|
||||||
|
// TODO: Implement SSE formatting
|
||||||
|
// - Format as "data: {...}\n\n"
|
||||||
|
// - Handle special cases (done, error)
|
||||||
|
// - Ensure proper JSON encoding
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatWebSocket formats a StreamChunk as WebSocket message
|
||||||
|
func FormatWebSocket(chunk *StreamChunk) []byte {
|
||||||
|
// TODO: Implement WebSocket formatting
|
||||||
|
// - Format as JSON message
|
||||||
|
// - Add message type/metadata
|
||||||
|
// - Handle binary vs text frames
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
@ -4,6 +4,6 @@ import "github.com/yaoapp/yao/agent/context"
|
||||||
|
|
||||||
// LLM the LLM interface
|
// LLM the LLM interface
|
||||||
type LLM interface {
|
type LLM interface {
|
||||||
Stream(ctx *context.Context, messages []context.Message, options *CompletionOptions, handler context.StreamFunc) (*context.ResponseCompletion, error)
|
Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error)
|
||||||
Post(ctx *context.Context, messages []context.Message, options *CompletionOptions) (*context.ResponseCompletion, error)
|
Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,15 @@
|
||||||
package llm
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers"
|
||||||
|
)
|
||||||
|
|
||||||
// New create a new LLM instance
|
// New create a new LLM instance
|
||||||
func New(connector string) (LLM, error) {
|
// conn: connector object from connector.Select()
|
||||||
return nil, nil
|
// options: completion options containing capabilities and other settings
|
||||||
|
func New(conn connector.Connector, options *context.CompletionOptions) (LLM, error) {
|
||||||
|
// Select appropriate provider based on capabilities
|
||||||
|
return providers.SelectProvider(conn, options)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
319
agent/llm/providers/README.md
Normal file
319
agent/llm/providers/README.md
Normal file
|
|
@ -0,0 +1,319 @@
|
||||||
|
# LLM Providers Architecture
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
This directory contains different LLM provider implementations, each optimized for specific model capabilities.
|
||||||
|
|
||||||
|
## Provider Selection Strategy
|
||||||
|
|
||||||
|
The `factory.SelectProvider()` function automatically selects the appropriate provider based on model capabilities:
|
||||||
|
|
||||||
|
```go
|
||||||
|
Priority 1: Reasoning models → reasoning.Provider
|
||||||
|
Priority 2: Native tool support → openai.Provider
|
||||||
|
Priority 3: Legacy models → legacy.Provider
|
||||||
|
```
|
||||||
|
|
||||||
|
## Provider Types
|
||||||
|
|
||||||
|
### 1. Base Provider (`base/`)
|
||||||
|
|
||||||
|
**Purpose**: Common functionality shared across all providers
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
|
||||||
|
- Message preprocessing
|
||||||
|
- Request body building
|
||||||
|
- Response parsing
|
||||||
|
|
||||||
|
**Usage**: Embedded in all other providers
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. OpenAI Provider (`openai/`)
|
||||||
|
|
||||||
|
**Purpose**: OpenAI-compatible models with full feature support
|
||||||
|
|
||||||
|
**Capabilities**:
|
||||||
|
|
||||||
|
- ✅ Vision (image input)
|
||||||
|
- ✅ Native tool calls
|
||||||
|
- ✅ Streaming
|
||||||
|
- ✅ JSON mode
|
||||||
|
|
||||||
|
**Models**:
|
||||||
|
|
||||||
|
- GPT-4, GPT-4o, GPT-4-turbo
|
||||||
|
- GPT-3.5-turbo
|
||||||
|
- Claude (via OpenAI-compatible API)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. Reasoning Provider (`reasoning/`)
|
||||||
|
|
||||||
|
**Purpose**: Reasoning models with special response format
|
||||||
|
|
||||||
|
**Capabilities**:
|
||||||
|
|
||||||
|
- ✅ Reasoning content (`reasoning_content` field)
|
||||||
|
- ✅ Thinking + Answer phases
|
||||||
|
- ⚠️ Tool calls support varies by model
|
||||||
|
|
||||||
|
**Models**:
|
||||||
|
|
||||||
|
- **OpenAI o1** (supports native tool calls)
|
||||||
|
- **DeepSeek R1** (no native tool calls, uses prompt engineering)
|
||||||
|
|
||||||
|
**Special Handling**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// DeepSeek R1 scenario
|
||||||
|
if !supportsNativeTools && hasTools {
|
||||||
|
// Inject tool instructions into prompt
|
||||||
|
messages = injectToolInstructions(messages, tools)
|
||||||
|
// Extract tool calls from text response
|
||||||
|
toolCalls = extractToolCallsFromText(response.Content)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response Format**:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"content": "The answer is 42",
|
||||||
|
"reasoning_content": "Let me think... first we need to...",
|
||||||
|
"content_types": ["text", "reasoning"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. Legacy Provider (`legacy/`)
|
||||||
|
|
||||||
|
**Purpose**: Older models without native tool calling
|
||||||
|
|
||||||
|
**Capabilities**:
|
||||||
|
|
||||||
|
- ✅ Text generation
|
||||||
|
- ⚠️ Tool calls via prompt engineering
|
||||||
|
- ❌ No native vision support
|
||||||
|
- ❌ No native tool API
|
||||||
|
|
||||||
|
**Models**:
|
||||||
|
|
||||||
|
- GPT-3 (davinci, curie)
|
||||||
|
- Older open-source models
|
||||||
|
- Custom models without tool API
|
||||||
|
|
||||||
|
**Tool Call Flow**:
|
||||||
|
|
||||||
|
1. Inject tool schemas into system prompt
|
||||||
|
2. Model returns tool call in text format (JSON)
|
||||||
|
3. Extract and parse tool calls from text
|
||||||
|
4. Execute tools
|
||||||
|
5. Continue conversation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. Vision Utils (`vision/`)
|
||||||
|
|
||||||
|
**Purpose**: Vision-related preprocessing utilities
|
||||||
|
|
||||||
|
**Functions**:
|
||||||
|
|
||||||
|
- `PreprocessVisionMessages()` - Handle image content
|
||||||
|
- `ConvertImageToText()` - Convert images to descriptions (for non-vision models)
|
||||||
|
- `ValidateImageURL()` - Validate image URLs
|
||||||
|
- `ExtractImagesFromMessages()` - Extract all images from messages
|
||||||
|
|
||||||
|
**Usage**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// When model doesn't support vision
|
||||||
|
if !supportsVision {
|
||||||
|
messages = vision.PreprocessVisionMessages(messages, false)
|
||||||
|
// Images converted to text descriptions
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Audio Utils (`audio/`)
|
||||||
|
|
||||||
|
**Purpose**: Audio-related preprocessing utilities
|
||||||
|
|
||||||
|
**Functions**:
|
||||||
|
|
||||||
|
- `PreprocessAudioMessages()` - Handle audio content
|
||||||
|
- `ConvertAudioToText()` - Convert audio to text transcription (for non-audio models)
|
||||||
|
- `ValidateAudioFormat()` - Validate audio format and encoding
|
||||||
|
- `ExtractAudioFromMessages()` - Extract all audio data from messages
|
||||||
|
- `RemoveAudioConfig()` - Remove audio configuration from options
|
||||||
|
|
||||||
|
**Usage**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// When model doesn't support audio
|
||||||
|
if !supportsAudio {
|
||||||
|
messages = audio.PreprocessAudioMessages(messages, false)
|
||||||
|
options = audio.RemoveAudioConfig(options)
|
||||||
|
// Audio converted to text transcriptions
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Special Scenarios
|
||||||
|
|
||||||
|
### Scenario 1: DeepSeek R1 (Reasoning + No Tool Support)
|
||||||
|
|
||||||
|
**Provider**: `reasoning.Provider`
|
||||||
|
|
||||||
|
**Handling**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Check if reasoning model supports tools
|
||||||
|
if !p.supportsNativeTools && len(options.Tools) > 0 {
|
||||||
|
// Use prompt engineering approach
|
||||||
|
messages = p.injectToolInstructions(messages, tools)
|
||||||
|
options = p.removeToolsFromOptions(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
// After getting response
|
||||||
|
if !p.supportsNativeTools {
|
||||||
|
toolCalls = p.extractToolCallsFromText(response.Content)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Why reasoning provider?**
|
||||||
|
|
||||||
|
- Primary characteristic is reasoning (special response format)
|
||||||
|
- Tool handling is secondary concern
|
||||||
|
- Reuses tool injection logic from legacy approach
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Scenario 2: Legacy Model + Vision/Audio Request
|
||||||
|
|
||||||
|
**Provider**: `legacy.Provider`
|
||||||
|
|
||||||
|
**Handling**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/vision"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/audio"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Preprocess to remove/convert vision content
|
||||||
|
if !supportsVision {
|
||||||
|
messages = vision.PreprocessVisionMessages(messages, false)
|
||||||
|
// Images converted to text: "[Image: description]"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preprocess to remove/convert audio content
|
||||||
|
if !supportsAudio {
|
||||||
|
messages = audio.PreprocessAudioMessages(messages, false)
|
||||||
|
options = audio.RemoveAudioConfig(options)
|
||||||
|
// Audio converted to text: "[Audio transcription: ...]"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Scenario 3: OpenAI o1 (Reasoning + Tool Support)
|
||||||
|
|
||||||
|
**Provider**: `reasoning.Provider`
|
||||||
|
|
||||||
|
**Handling**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// o1 supports native tools, no special handling needed
|
||||||
|
if p.supportsNativeTools {
|
||||||
|
// Use standard OpenAI tool calling API
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Configuration Example
|
||||||
|
|
||||||
|
In `connectors.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# GPT-4o with all features
|
||||||
|
gpt-4o:
|
||||||
|
vision: true
|
||||||
|
tool_calls: true
|
||||||
|
audio: true
|
||||||
|
streaming: true
|
||||||
|
json: true
|
||||||
|
multimodal: true
|
||||||
|
|
||||||
|
# OpenAI o1 - reasoning with tool support
|
||||||
|
o1-preview:
|
||||||
|
reasoning: true
|
||||||
|
tool_calls: true
|
||||||
|
streaming: true
|
||||||
|
|
||||||
|
# DeepSeek R1 - reasoning without tool support
|
||||||
|
deepseek-reasoner:
|
||||||
|
reasoning: true
|
||||||
|
tool_calls: false # Will use prompt engineering
|
||||||
|
streaming: true
|
||||||
|
|
||||||
|
# GPT-3 - legacy model
|
||||||
|
gpt-3.5-turbo-instruct:
|
||||||
|
tool_calls: false # Will use prompt engineering
|
||||||
|
vision: false # Will convert images to text
|
||||||
|
audio: false # Will convert audio to text
|
||||||
|
streaming: false
|
||||||
|
|
||||||
|
# GPT-4 Vision only
|
||||||
|
gpt-4-vision:
|
||||||
|
vision: true
|
||||||
|
tool_calls: true
|
||||||
|
audio: false # No audio support
|
||||||
|
streaming: true
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Adding a New Provider
|
||||||
|
|
||||||
|
1. Create new directory: `providers/newprovider/`
|
||||||
|
2. Implement `LLM` interface:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Provider struct {
|
||||||
|
*base.Provider
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Provider) Stream(...) (*CompletionResponse, error)
|
||||||
|
func (p *Provider) Post(...) (*CompletionResponse, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Update `factory.SelectProvider()` selection logic
|
||||||
|
4. Add capability flags to `ConnectorSetting`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
Each provider should have tests for:
|
||||||
|
|
||||||
|
- Standard completion
|
||||||
|
- Streaming completion
|
||||||
|
- Tool calling (if supported)
|
||||||
|
- Vision input (if supported)
|
||||||
|
- Error handling
|
||||||
|
- Response parsing
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Performance Considerations
|
||||||
|
|
||||||
|
- **Caching**: Consider caching connector instances
|
||||||
|
- **Pooling**: HTTP connection pooling for high throughput
|
||||||
|
- **Timeouts**: Configurable timeouts per provider
|
||||||
|
- **Retries**: Exponential backoff for transient errors
|
||||||
59
agent/llm/providers/audio/audio.go
Normal file
59
agent/llm/providers/audio/audio.go
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreprocessAudioMessages preprocess messages to handle audio content
|
||||||
|
// Removes or converts audio content for models that don't support it
|
||||||
|
func PreprocessAudioMessages(messages []context.Message, supportsAudio bool) []context.Message {
|
||||||
|
// TODO: Implement audio message preprocessing
|
||||||
|
// If supportsAudio is false:
|
||||||
|
// - Remove input_audio content parts
|
||||||
|
// - Convert to text-only messages
|
||||||
|
// - Optionally add audio transcriptions
|
||||||
|
// If supportsAudio is true:
|
||||||
|
// - Validate audio format
|
||||||
|
// - Ensure proper encoding
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConvertAudioToText convert audio content to text transcription
|
||||||
|
// Used when model doesn't support audio input
|
||||||
|
func ConvertAudioToText(audioData string) (string, error) {
|
||||||
|
// TODO: Implement audio to text conversion
|
||||||
|
// - Call speech-to-text API (Whisper, etc.)
|
||||||
|
// - Generate transcription
|
||||||
|
// - Return as text content
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateAudioFormat validate audio format and encoding
|
||||||
|
func ValidateAudioFormat(audioConfig *context.AudioConfig) error {
|
||||||
|
// TODO: Implement audio format validation
|
||||||
|
// - Check format (wav, mp3, etc.)
|
||||||
|
// - Validate encoding
|
||||||
|
// - Check sample rate
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractAudioFromMessages extract all audio data from messages
|
||||||
|
func ExtractAudioFromMessages(messages []context.Message) []string {
|
||||||
|
// TODO: Implement audio extraction
|
||||||
|
// - Iterate through messages
|
||||||
|
// - Find ContentPart with type="input_audio"
|
||||||
|
// - Collect all audio data
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveAudioConfig remove audio configuration from options
|
||||||
|
// Used when model doesn't support audio output
|
||||||
|
func RemoveAudioConfig(options *context.CompletionOptions) *context.CompletionOptions {
|
||||||
|
// TODO: Remove audio config from options
|
||||||
|
if options == nil {
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
newOptions := *options
|
||||||
|
newOptions.Audio = nil
|
||||||
|
return &newOptions
|
||||||
|
}
|
||||||
65
agent/llm/providers/base/base.go
Normal file
65
agent/llm/providers/base/base.go
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
package base
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Provider base provider implementation
|
||||||
|
// Provides common functionality for all LLM providers
|
||||||
|
type Provider struct {
|
||||||
|
Connector connector.Connector
|
||||||
|
Capabilities *context.ModelCapabilities
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewProvider create a new base provider
|
||||||
|
func NewProvider(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||||
|
return &Provider{
|
||||||
|
Connector: conn,
|
||||||
|
Capabilities: capabilities,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PreprocessMessages preprocess messages before sending to LLM
|
||||||
|
// Handles vision messages, audio messages, tool messages, etc.
|
||||||
|
func (p *Provider) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
|
||||||
|
// TODO: Implement message preprocessing
|
||||||
|
// - Remove vision content if not supported
|
||||||
|
// - Remove audio content if not supported
|
||||||
|
// - Convert tool messages if needed
|
||||||
|
// - Validate message format
|
||||||
|
return messages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportsVision check if this provider supports vision
|
||||||
|
func (p *Provider) SupportsVision() bool {
|
||||||
|
return p.Capabilities != nil && p.Capabilities.Vision != nil && *p.Capabilities.Vision
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportsAudio check if this provider supports audio
|
||||||
|
func (p *Provider) SupportsAudio() bool {
|
||||||
|
return p.Capabilities != nil && p.Capabilities.Audio != nil && *p.Capabilities.Audio
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportsTools check if this provider supports tool calls
|
||||||
|
func (p *Provider) SupportsTools() bool {
|
||||||
|
return p.Capabilities != nil && p.Capabilities.ToolCalls != nil && *p.Capabilities.ToolCalls
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildRequestBody build the request body for the LLM API
|
||||||
|
func (p *Provider) BuildRequestBody(messages []context.Message, options *context.CompletionOptions) (map[string]interface{}, error) {
|
||||||
|
// TODO: Implement request body building
|
||||||
|
// - Convert messages to API format
|
||||||
|
// - Apply options (temperature, max_tokens, etc.)
|
||||||
|
// - Add model-specific parameters
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseResponse parse the response from LLM API
|
||||||
|
func (p *Provider) ParseResponse(data []byte, isStreaming bool) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement response parsing
|
||||||
|
// - Parse JSON response
|
||||||
|
// - Extract content, tool calls, reasoning, etc.
|
||||||
|
// - Handle streaming chunks
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
56
agent/llm/providers/factory.go
Normal file
56
agent/llm/providers/factory.go
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
package providers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/legacy"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/openai"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/reasoning"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LLM interface (copied to avoid import cycle)
|
||||||
|
type LLM interface {
|
||||||
|
Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error)
|
||||||
|
Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectProvider select the appropriate provider based on connector and capabilities
|
||||||
|
func SelectProvider(conn connector.Connector, options *context.CompletionOptions) (LLM, error) {
|
||||||
|
if options == nil || options.Capabilities == nil {
|
||||||
|
return nil, fmt.Errorf("options and capabilities are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
capabilities := options.Capabilities
|
||||||
|
|
||||||
|
// Priority 1: Reasoning models (special response format)
|
||||||
|
if capabilities.Reasoning != nil && *capabilities.Reasoning {
|
||||||
|
return reasoning.New(conn, capabilities), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Check if model supports native tool calls
|
||||||
|
if capabilities.ToolCalls != nil && *capabilities.ToolCalls {
|
||||||
|
// Use OpenAI-compatible provider (supports tools, vision, streaming)
|
||||||
|
return openai.New(conn, capabilities), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: Legacy models (no native tool support)
|
||||||
|
// Will use prompt engineering for tool calls
|
||||||
|
return legacy.New(conn, capabilities), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectProvider detect provider type from connector
|
||||||
|
func DetectProvider(conn connector.Connector) string {
|
||||||
|
// TODO: Implement provider detection
|
||||||
|
// - Check connector type (Is(connector.OPENAI))
|
||||||
|
// - Check connector settings
|
||||||
|
// - Determine provider type (openai, claude, deepseek, etc.)
|
||||||
|
|
||||||
|
if conn.Is(connector.OPENAI) {
|
||||||
|
return "openai"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default to OpenAI-compatible
|
||||||
|
return "openai"
|
||||||
|
}
|
||||||
64
agent/llm/providers/legacy/legacy.go
Normal file
64
agent/llm/providers/legacy/legacy.go
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
package legacy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Provider legacy LLM provider (no native tool calling support)
|
||||||
|
// Implements tool calling via prompt engineering
|
||||||
|
type Provider struct {
|
||||||
|
*base.Provider
|
||||||
|
}
|
||||||
|
|
||||||
|
// New create a new legacy provider
|
||||||
|
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||||
|
return &Provider{
|
||||||
|
Provider: base.NewProvider(conn, capabilities),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream stream completion from legacy model
|
||||||
|
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement legacy model streaming
|
||||||
|
// - Preprocess messages (remove tool-specific fields, vision, audio)
|
||||||
|
// - Remove vision content (convert to text description)
|
||||||
|
// - Remove audio content (convert to text transcription)
|
||||||
|
// - Remove tool messages
|
||||||
|
// - Add tool calling instructions to system prompt if tools provided
|
||||||
|
// - Build request body without native tool parameters
|
||||||
|
// - Make streaming HTTP request
|
||||||
|
// - Parse response and detect tool calls from text
|
||||||
|
// - Extract tool calls using regex/JSON parsing
|
||||||
|
// - Call handler for each chunk
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Post post completion request to legacy model
|
||||||
|
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement legacy model non-streaming completion
|
||||||
|
// - Preprocess messages
|
||||||
|
// - Add tool instructions to prompt
|
||||||
|
// - Make HTTP POST request
|
||||||
|
// - Parse response and extract tool calls from text
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InjectToolInstructions inject tool calling instructions into system prompt
|
||||||
|
func (p *Provider) InjectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message {
|
||||||
|
// TODO: Implement tool instruction injection
|
||||||
|
// - Generate tool description prompt
|
||||||
|
// - Add to system message or create new system message
|
||||||
|
// - Include tool schemas and usage instructions
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractToolCallsFromText extract tool calls from model's text response
|
||||||
|
func (p *Provider) ExtractToolCallsFromText(text string) []context.ToolCallResult {
|
||||||
|
// TODO: Implement tool call extraction
|
||||||
|
// - Look for JSON blocks or specific patterns
|
||||||
|
// - Parse tool name and arguments
|
||||||
|
// - Return structured tool calls
|
||||||
|
return nil
|
||||||
|
}
|
||||||
50
agent/llm/providers/openai/openai.go
Normal file
50
agent/llm/providers/openai/openai.go
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
package openai
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Provider OpenAI-compatible provider
|
||||||
|
// Supports: vision, tool calls, streaming, JSON mode
|
||||||
|
type Provider struct {
|
||||||
|
*base.Provider
|
||||||
|
}
|
||||||
|
|
||||||
|
// New create a new OpenAI provider
|
||||||
|
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||||
|
return &Provider{
|
||||||
|
Provider: base.NewProvider(conn, capabilities),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream stream completion from OpenAI API
|
||||||
|
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement OpenAI streaming
|
||||||
|
// - Preprocess messages (vision, audio, tools)
|
||||||
|
// - Remove vision content if not supported
|
||||||
|
// - Remove audio content if not supported
|
||||||
|
// - Convert to text where needed
|
||||||
|
// - Build request body
|
||||||
|
// - Make streaming HTTP request
|
||||||
|
// - Parse SSE chunks
|
||||||
|
// - Call handler for each chunk
|
||||||
|
// - Aggregate final response
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Post post completion request to OpenAI API
|
||||||
|
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement OpenAI non-streaming completion
|
||||||
|
// - Preprocess messages
|
||||||
|
// - Build request body
|
||||||
|
// - Make HTTP POST request
|
||||||
|
// - Parse response
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportsAudio check if this provider supports audio
|
||||||
|
func (p *Provider) SupportsAudio() bool {
|
||||||
|
return p.Capabilities != nil && p.Capabilities.Audio != nil && *p.Capabilities.Audio
|
||||||
|
}
|
||||||
115
agent/llm/providers/reasoning/reasoning.go
Normal file
115
agent/llm/providers/reasoning/reasoning.go
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
package reasoning
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/gou/connector"
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/providers/base"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Provider reasoning model provider (o1, DeepSeek R1, etc.)
|
||||||
|
// Handles special response format with reasoning_content
|
||||||
|
// Note: Some reasoning models (e.g. DeepSeek R1) don't support native tool calls
|
||||||
|
type Provider struct {
|
||||||
|
*base.Provider
|
||||||
|
supportsNativeTools bool // Whether this reasoning model supports native tool calling
|
||||||
|
}
|
||||||
|
|
||||||
|
// New create a new reasoning provider
|
||||||
|
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
|
||||||
|
// Check if this reasoning model supports native tool calls
|
||||||
|
supportsTools := false
|
||||||
|
if capabilities != nil && capabilities.ToolCalls != nil && *capabilities.ToolCalls {
|
||||||
|
supportsTools = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Provider{
|
||||||
|
Provider: base.NewProvider(conn, capabilities),
|
||||||
|
supportsNativeTools: supportsTools,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream stream completion from reasoning model
|
||||||
|
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement reasoning model streaming
|
||||||
|
// - Preprocess messages (reasoning models have restrictions)
|
||||||
|
|
||||||
|
// Handle tool calls based on model support
|
||||||
|
if !p.supportsNativeTools && options != nil && len(options.Tools) > 0 {
|
||||||
|
// Model doesn't support native tool calls (e.g. DeepSeek R1)
|
||||||
|
// Inject tool instructions into messages
|
||||||
|
messages = p.injectToolInstructions(messages, options.Tools)
|
||||||
|
// Remove tools from options to avoid API error
|
||||||
|
options = p.removeToolsFromOptions(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
// - Build request body (special parameters for reasoning)
|
||||||
|
// - Make streaming HTTP request
|
||||||
|
// - Parse SSE chunks with reasoning_content
|
||||||
|
// - Handle both thinking and answer phases
|
||||||
|
// - Call handler for each chunk
|
||||||
|
|
||||||
|
// If tools were injected, extract tool calls from text response
|
||||||
|
// if !p.supportsNativeTools && hasTools {
|
||||||
|
// toolCalls = p.extractToolCallsFromText(response.Content)
|
||||||
|
// }
|
||||||
|
|
||||||
|
// - Aggregate final response with reasoning content
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Post post completion request to reasoning model
|
||||||
|
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement reasoning model non-streaming completion
|
||||||
|
// - Preprocess messages
|
||||||
|
// - Build request body
|
||||||
|
// - Make HTTP POST request
|
||||||
|
// - Parse response with reasoning_content field
|
||||||
|
// - Separate thinking from final answer
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseReasoningResponse parse response with reasoning content
|
||||||
|
// Handles both OpenAI o1 format and DeepSeek R1 format
|
||||||
|
func (p *Provider) ParseReasoningResponse(data []byte) (*context.CompletionResponse, error) {
|
||||||
|
// TODO: Implement reasoning response parsing
|
||||||
|
// - Detect format (OpenAI vs DeepSeek)
|
||||||
|
// - Extract reasoning_content
|
||||||
|
// - Extract final content
|
||||||
|
// - Set ContentTypes correctly (text + reasoning)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// injectToolInstructions inject tool calling instructions into messages
|
||||||
|
// Used for reasoning models that don't support native tool calls (e.g. DeepSeek R1)
|
||||||
|
func (p *Provider) injectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message {
|
||||||
|
// TODO: Implement tool instruction injection for reasoning models
|
||||||
|
// - Generate tool description prompt (optimized for reasoning models)
|
||||||
|
// - Add to system message or create new system message
|
||||||
|
// - Include tool schemas and usage instructions
|
||||||
|
// - Format should encourage reasoning about tool usage
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractToolCallsFromText extract tool calls from reasoning model's text response
|
||||||
|
// Used when model doesn't support native tool calls
|
||||||
|
func (p *Provider) extractToolCallsFromText(text string) []context.ToolCallResult {
|
||||||
|
// TODO: Implement tool call extraction from text
|
||||||
|
// - Look for JSON blocks or specific patterns
|
||||||
|
// - Parse tool name and arguments
|
||||||
|
// - Return structured tool calls
|
||||||
|
// - Handle reasoning model's specific output format
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeToolsFromOptions remove tool-related parameters from options
|
||||||
|
// Used when sending request to models that don't support native tool calls
|
||||||
|
func (p *Provider) removeToolsFromOptions(options *context.CompletionOptions) *context.CompletionOptions {
|
||||||
|
// TODO: Create a copy of options without tool parameters
|
||||||
|
// - Remove Tools field
|
||||||
|
// - Remove ToolChoice field
|
||||||
|
// - Keep other options intact
|
||||||
|
newOptions := *options
|
||||||
|
newOptions.Tools = nil
|
||||||
|
newOptions.ToolChoice = nil
|
||||||
|
return &newOptions
|
||||||
|
}
|
||||||
47
agent/llm/providers/vision/vision.go
Normal file
47
agent/llm/providers/vision/vision.go
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
package vision
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreprocessVisionMessages preprocess messages to handle vision content
|
||||||
|
// Removes or converts vision content for models that don't support it
|
||||||
|
func PreprocessVisionMessages(messages []context.Message, supportsVision bool) []context.Message {
|
||||||
|
// TODO: Implement vision message preprocessing
|
||||||
|
// If supportsVision is false:
|
||||||
|
// - Remove image_url content parts
|
||||||
|
// - Convert to text-only messages
|
||||||
|
// - Optionally add image descriptions from vision API
|
||||||
|
// If supportsVision is true:
|
||||||
|
// - Validate image URLs
|
||||||
|
// - Ensure proper format
|
||||||
|
return messages
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConvertImageToText convert image content to text description
|
||||||
|
// Used when model doesn't support vision
|
||||||
|
func ConvertImageToText(imageURL string) (string, error) {
|
||||||
|
// TODO: Implement image to text conversion
|
||||||
|
// - Call vision API (if configured)
|
||||||
|
// - Generate description
|
||||||
|
// - Return as text content
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateImageURL validate image URL format
|
||||||
|
func ValidateImageURL(imageURL string) error {
|
||||||
|
// TODO: Implement image URL validation
|
||||||
|
// - Check URL format
|
||||||
|
// - Validate image type
|
||||||
|
// - Check accessibility
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractImagesFromMessages extract all image URLs from messages
|
||||||
|
func ExtractImagesFromMessages(messages []context.Message) []string {
|
||||||
|
// TODO: Implement image extraction
|
||||||
|
// - Iterate through messages
|
||||||
|
// - Find ContentPart with type="image_url"
|
||||||
|
// - Collect all image URLs
|
||||||
|
return nil
|
||||||
|
}
|
||||||
12
agent/llm/stream.go
Normal file
12
agent/llm/stream.go
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/llm/handlers"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultStreamHandler creates a default stream handler
|
||||||
|
// This is a convenience function that wraps handlers.DefaultStreamHandler
|
||||||
|
func DefaultStreamHandler(ctx *context.Context) context.StreamFunc {
|
||||||
|
return handlers.DefaultStreamHandler(ctx)
|
||||||
|
}
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
package llm
|
|
||||||
|
|
||||||
// CompletionOptions the completion request
|
|
||||||
type CompletionOptions struct{}
|
|
||||||
|
|
@ -171,6 +171,17 @@ func initAssistant() error {
|
||||||
assistant.SetVision(api.Agent.DSL.Vision)
|
assistant.SetVision(api.Agent.DSL.Vision)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set global Uses configuration
|
||||||
|
if api.Agent.DSL.Use != nil {
|
||||||
|
globalUses := &store.Uses{
|
||||||
|
Vision: api.Agent.DSL.Use.Vision,
|
||||||
|
Audio: api.Agent.DSL.Use.Audio,
|
||||||
|
Search: api.Agent.DSL.Use.Search,
|
||||||
|
Fetch: api.Agent.DSL.Use.Fetch,
|
||||||
|
}
|
||||||
|
assistant.SetGlobalUses(globalUses)
|
||||||
|
}
|
||||||
|
|
||||||
if api.Agent.DSL.Connectors != nil {
|
if api.Agent.DSL.Connectors != nil {
|
||||||
assistant.SetConnectorSettings(api.Agent.DSL.Connectors)
|
assistant.SetConnectorSettings(api.Agent.DSL.Connectors)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -173,6 +173,15 @@ func ToMySQLTime(v interface{}) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Uses the wrapper configurations for assistant
|
||||||
|
// Used to specify which assistant or MCP server to use for vision, audio, etc.
|
||||||
|
type Uses struct {
|
||||||
|
Vision string `json:"vision,omitempty"` // Vision processing wrapper. Format: "agent" or "mcp:mcp_server_id"
|
||||||
|
Audio string `json:"audio,omitempty"` // Audio processing wrapper. Format: "agent" or "mcp:mcp_server_id"
|
||||||
|
Search string `json:"search,omitempty"` // Search wrapper. Format: "agent" or "mcp:mcp_server_id"
|
||||||
|
Fetch string `json:"fetch,omitempty"` // Fetch wrapper. Format: "agent" or "mcp:mcp_server_id"
|
||||||
|
}
|
||||||
|
|
||||||
// ToAssistantModel converts various types to AssistantModel
|
// ToAssistantModel converts various types to AssistantModel
|
||||||
func ToAssistantModel(v interface{}) (*AssistantModel, error) {
|
func ToAssistantModel(v interface{}) (*AssistantModel, error) {
|
||||||
if v == nil {
|
if v == nil {
|
||||||
|
|
|
||||||
|
|
@ -155,6 +155,7 @@ type AssistantModel struct {
|
||||||
Workflow *Workflow `json:"workflow,omitempty"` // Workflow configuration
|
Workflow *Workflow `json:"workflow,omitempty"` // Workflow configuration
|
||||||
Placeholder *Placeholder `json:"placeholder,omitempty"` // Assistant Placeholder
|
Placeholder *Placeholder `json:"placeholder,omitempty"` // Assistant Placeholder
|
||||||
Locales i18n.Map `json:"locales,omitempty"` // Assistant Locales
|
Locales i18n.Map `json:"locales,omitempty"` // Assistant Locales
|
||||||
|
Uses *Uses `json:"uses,omitempty"` // Assistant-specific wrapper configurations for vision, audio, etc. If not set, use global settings
|
||||||
CreatedAt int64 `json:"created_at"` // Creation timestamp
|
CreatedAt int64 `json:"created_at"` // Creation timestamp
|
||||||
UpdatedAt int64 `json:"updated_at"` // Last update timestamp
|
UpdatedAt int64 `json:"updated_at"` // Last update timestamp
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -158,6 +158,7 @@ func (conv *Xun) SaveAssistant(assistant *types.AssistantModel) (string, error)
|
||||||
"tools": assistant.Tools,
|
"tools": assistant.Tools,
|
||||||
"placeholder": assistant.Placeholder,
|
"placeholder": assistant.Placeholder,
|
||||||
"locales": assistant.Locales,
|
"locales": assistant.Locales,
|
||||||
|
"uses": assistant.Uses,
|
||||||
}
|
}
|
||||||
|
|
||||||
for field, value := range jsonFields {
|
for field, value := range jsonFields {
|
||||||
|
|
@ -216,7 +217,7 @@ func (conv *Xun) UpdateAssistant(assistantID string, updates map[string]interfac
|
||||||
data := make(map[string]interface{})
|
data := make(map[string]interface{})
|
||||||
|
|
||||||
// List of fields that need JSON marshaling
|
// List of fields that need JSON marshaling
|
||||||
jsonFields := []string{"options", "tags", "prompts", "kb", "mcp", "workflow", "tools", "placeholder", "locales"}
|
jsonFields := []string{"options", "tags", "prompts", "kb", "mcp", "workflow", "tools", "placeholder", "locales", "uses"}
|
||||||
jsonFieldSet := make(map[string]bool)
|
jsonFieldSet := make(map[string]bool)
|
||||||
for _, field := range jsonFields {
|
for _, field := range jsonFields {
|
||||||
jsonFieldSet[field] = true
|
jsonFieldSet[field] = true
|
||||||
|
|
@ -416,7 +417,7 @@ func (conv *Xun) GetAssistants(filter types.AssistantFilter, locale ...string) (
|
||||||
|
|
||||||
// Convert rows to types.AssistantModel slice
|
// Convert rows to types.AssistantModel slice
|
||||||
assistants := make([]*types.AssistantModel, 0, len(rows))
|
assistants := make([]*types.AssistantModel, 0, len(rows))
|
||||||
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales"}
|
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales", "uses"}
|
||||||
|
|
||||||
for _, row := range rows {
|
for _, row := range rows {
|
||||||
data := row.ToMap()
|
data := row.ToMap()
|
||||||
|
|
@ -473,7 +474,7 @@ func (conv *Xun) GetAssistant(assistantID string, locale ...string) (*types.Assi
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse JSON fields
|
// Parse JSON fields
|
||||||
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales"}
|
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales", "uses"}
|
||||||
conv.parseJSONFields(data, jsonFields)
|
conv.parseJSONFields(data, jsonFields)
|
||||||
|
|
||||||
// Convert map to types.AssistantModel
|
// Convert map to types.AssistantModel
|
||||||
|
|
@ -578,6 +579,16 @@ func (conv *Xun) GetAssistant(assistantID string, locale ...string) (*types.Assi
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if uses, has := data["uses"]; has && uses != nil {
|
||||||
|
raw, err := jsoniter.Marshal(uses)
|
||||||
|
if err == nil {
|
||||||
|
var u types.Uses
|
||||||
|
if err := jsoniter.Unmarshal(raw, &u); err == nil {
|
||||||
|
model.Uses = &u
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Apply i18n translation if locale is provided
|
// Apply i18n translation if locale is provided
|
||||||
if len(locale) > 0 && locale[0] != "" {
|
if len(locale) > 0 && locale[0] != "" {
|
||||||
conv.translate(model, assistantID, locale[0])
|
conv.translate(model, assistantID, locale[0])
|
||||||
|
|
|
||||||
|
|
@ -197,6 +197,125 @@ func TestSaveAssistant(t *testing.T) {
|
||||||
t.Errorf("Expected 3 tags, got %d", len(retrieved.Tags))
|
t.Errorf("Expected 3 tags, got %d", len(retrieved.Tags))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("UsesConfiguration", func(t *testing.T) {
|
||||||
|
// Test assistant with Uses configuration
|
||||||
|
assistant := &types.AssistantModel{
|
||||||
|
Name: "Uses Test Assistant",
|
||||||
|
Type: "assistant",
|
||||||
|
Connector: "openai",
|
||||||
|
Share: "private",
|
||||||
|
Uses: &types.Uses{
|
||||||
|
Vision: "mcp:vision-server",
|
||||||
|
Audio: "agent",
|
||||||
|
Search: "mcp:search-server",
|
||||||
|
Fetch: "agent",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := store.SaveAssistant(assistant)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to save assistant with uses: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retrieve and verify uses configuration
|
||||||
|
retrieved, err := store.GetAssistant(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to retrieve assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses == nil {
|
||||||
|
t.Fatal("Expected uses to be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Vision != "mcp:vision-server" {
|
||||||
|
t.Errorf("Expected vision 'mcp:vision-server', got '%s'", retrieved.Uses.Vision)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Audio != "agent" {
|
||||||
|
t.Errorf("Expected audio 'agent', got '%s'", retrieved.Uses.Audio)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Search != "mcp:search-server" {
|
||||||
|
t.Errorf("Expected search 'mcp:search-server', got '%s'", retrieved.Uses.Search)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Fetch != "agent" {
|
||||||
|
t.Errorf("Expected fetch 'agent', got '%s'", retrieved.Uses.Fetch)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Successfully saved and retrieved assistant with uses configuration")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("NilUses", func(t *testing.T) {
|
||||||
|
// Test assistant without Uses configuration
|
||||||
|
assistant := &types.AssistantModel{
|
||||||
|
Name: "No Uses Assistant",
|
||||||
|
Type: "assistant",
|
||||||
|
Connector: "openai",
|
||||||
|
Share: "private",
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := store.SaveAssistant(assistant)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to save assistant without uses: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retrieve and verify uses is nil
|
||||||
|
retrieved, err := store.GetAssistant(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to retrieve assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses != nil {
|
||||||
|
t.Errorf("Expected uses to be nil, got %+v", retrieved.Uses)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("PartialUsesConfiguration", func(t *testing.T) {
|
||||||
|
// Test assistant with partial Uses configuration
|
||||||
|
assistant := &types.AssistantModel{
|
||||||
|
Name: "Partial Uses Assistant",
|
||||||
|
Type: "assistant",
|
||||||
|
Connector: "openai",
|
||||||
|
Share: "private",
|
||||||
|
Uses: &types.Uses{
|
||||||
|
Vision: "mcp:vision-only",
|
||||||
|
// Audio, Search, Fetch not set
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := store.SaveAssistant(assistant)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to save assistant with partial uses: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retrieve and verify
|
||||||
|
retrieved, err := store.GetAssistant(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to retrieve assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses == nil {
|
||||||
|
t.Fatal("Expected uses to be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Vision != "mcp:vision-only" {
|
||||||
|
t.Errorf("Expected vision 'mcp:vision-only', got '%s'", retrieved.Uses.Vision)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Audio != "" {
|
||||||
|
t.Errorf("Expected audio to be empty, got '%s'", retrieved.Uses.Audio)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Search != "" {
|
||||||
|
t.Errorf("Expected search to be empty, got '%s'", retrieved.Uses.Search)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Fetch != "" {
|
||||||
|
t.Errorf("Expected fetch to be empty, got '%s'", retrieved.Uses.Fetch)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeleteAssistant tests deleting a single assistant
|
// TestDeleteAssistant tests deleting a single assistant
|
||||||
|
|
@ -1953,6 +2072,105 @@ func TestUpdateAssistant(t *testing.T) {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("UpdateUses", func(t *testing.T) {
|
||||||
|
// Create assistant without uses
|
||||||
|
assistant := &types.AssistantModel{
|
||||||
|
Name: "Uses Update Test",
|
||||||
|
Type: "assistant",
|
||||||
|
Connector: "openai",
|
||||||
|
Share: "private",
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := store.SaveAssistant(assistant)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update with uses configuration
|
||||||
|
updates := map[string]interface{}{
|
||||||
|
"uses": &types.Uses{
|
||||||
|
Vision: "mcp:new-vision",
|
||||||
|
Audio: "mcp:new-audio",
|
||||||
|
Search: "agent",
|
||||||
|
Fetch: "mcp:fetch-server",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err = store.UpdateAssistant(id, updates)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to update uses: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify updates
|
||||||
|
retrieved, err := store.GetAssistant(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to retrieve assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses == nil {
|
||||||
|
t.Fatal("Expected uses to be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved.Uses.Vision != "mcp:new-vision" {
|
||||||
|
t.Errorf("Expected vision 'mcp:new-vision', got '%s'", retrieved.Uses.Vision)
|
||||||
|
}
|
||||||
|
if retrieved.Uses.Audio != "mcp:new-audio" {
|
||||||
|
t.Errorf("Expected audio 'mcp:new-audio', got '%s'", retrieved.Uses.Audio)
|
||||||
|
}
|
||||||
|
if retrieved.Uses.Search != "agent" {
|
||||||
|
t.Errorf("Expected search 'agent', got '%s'", retrieved.Uses.Search)
|
||||||
|
}
|
||||||
|
if retrieved.Uses.Fetch != "mcp:fetch-server" {
|
||||||
|
t.Errorf("Expected fetch 'mcp:fetch-server', got '%s'", retrieved.Uses.Fetch)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update to change uses
|
||||||
|
updates2 := map[string]interface{}{
|
||||||
|
"uses": &types.Uses{
|
||||||
|
Vision: "agent",
|
||||||
|
Audio: "agent",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err = store.UpdateAssistant(id, updates2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to update uses again: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify second update
|
||||||
|
retrieved2, err := store.GetAssistant(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to retrieve assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved2.Uses.Vision != "agent" {
|
||||||
|
t.Errorf("Expected vision 'agent', got '%s'", retrieved2.Uses.Vision)
|
||||||
|
}
|
||||||
|
if retrieved2.Uses.Audio != "agent" {
|
||||||
|
t.Errorf("Expected audio 'agent', got '%s'", retrieved2.Uses.Audio)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update to remove uses (set to nil)
|
||||||
|
updates3 := map[string]interface{}{
|
||||||
|
"uses": nil,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = store.UpdateAssistant(id, updates3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to set uses to nil: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify uses is nil
|
||||||
|
retrieved3, err := store.GetAssistant(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to retrieve assistant: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if retrieved3.Uses != nil {
|
||||||
|
t.Errorf("Expected uses to be nil, got %+v", retrieved3.Uses)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("UpdatePermissionFields", func(t *testing.T) {
|
t.Run("UpdatePermissionFields", func(t *testing.T) {
|
||||||
// Create assistant with permission fields
|
// Create assistant with permission fields
|
||||||
assistant := &types.AssistantModel{
|
assistant := &types.AssistantModel{
|
||||||
|
|
|
||||||
|
|
@ -44,7 +44,8 @@ type Use struct {
|
||||||
Default string `json:"default,omitempty" yaml:"default,omitempty"` // The default assistant to use
|
Default string `json:"default,omitempty" yaml:"default,omitempty"` // The default assistant to use
|
||||||
Title string `json:"title,omitempty" yaml:"title,omitempty"` // The assistant for generating the topic title.
|
Title string `json:"title,omitempty" yaml:"title,omitempty"` // The assistant for generating the topic title.
|
||||||
Prompt string `json:"prompt,omitempty" yaml:"prompt,omitempty"` // The assistant for generating the prompt.
|
Prompt string `json:"prompt,omitempty" yaml:"prompt,omitempty"` // The assistant for generating the prompt.
|
||||||
Vision string `json:"vision,omitempty" yaml:"vision,omitempty"` // The assistant for generating the image/video description, if the assistant enable the vision and model not support vision, use the vision model to describe the image/video, and return the messages with the image/video's description.
|
Vision string `json:"vision,omitempty" yaml:"vision,omitempty"` // The assistant for generating the image/video description, if the assistant enable the vision and model not support vision, use the vision model to describe the image/video, and return the messages with the image/video's description. Format: "agent" or "mcp:mcp_server_id"
|
||||||
|
Audio string `json:"audio,omitempty" yaml:"audio,omitempty"` // The assistant for processing audio (speech-to-text, text-to-speech). If the model doesn't support audio, use this to convert audio to text. Format: "agent" or "mcp:mcp_server_id"
|
||||||
Search string `json:"search,omitempty" yaml:"search,omitempty"` // The assistant for searching the knowledge, global web search. If not set, and the assistant enable the knowledge, it will search the result from the knowledge automatically.
|
Search string `json:"search,omitempty" yaml:"search,omitempty"` // The assistant for searching the knowledge, global web search. If not set, and the assistant enable the knowledge, it will search the result from the knowledge automatically.
|
||||||
Fetch string `json:"fetch,omitempty" yaml:"fetch,omitempty"` // The assistant for fetching the http/https/ftp/sftp/etc. file, and return the file's content. if not set, use the http process to fetch the file.
|
Fetch string `json:"fetch,omitempty" yaml:"fetch,omitempty"` // The assistant for fetching the http/https/ftp/sftp/etc. file, and return the file's content. if not set, use the http process to fetch the file.
|
||||||
}
|
}
|
||||||
|
|
|
||||||
284
data/bindata.go
284
data/bindata.go
File diff suppressed because it is too large
Load diff
|
|
@ -184,6 +184,13 @@
|
||||||
"comment": "Assistant i18n locales",
|
"comment": "Assistant i18n locales",
|
||||||
"nullable": true
|
"nullable": true
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "uses",
|
||||||
|
"type": "json",
|
||||||
|
"label": "Uses",
|
||||||
|
"comment": "Assistant-specific wrapper configurations for vision, audio, etc. If not set, use global settings",
|
||||||
|
"nullable": true
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "automated",
|
"name": "automated",
|
||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue