opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement global uses configuration and enhance assistant capabilities

Browse source 
- Added global uses configuration to the assistant, allowing for centralized management of vision, audio, search, and fetch settings.
- Updated the Assistant struct and related methods to support the new Uses configuration, improving flexibility in assistant operations.
- Refactored the Stream method to utilize the new CompletionResponse type, enhancing response handling.
- Introduced new methods for building requests and managing capabilities, streamlining the assistant's interaction with various connectors.
This commit is contained in:
Max 2025-11-14 19:17:21 +08:00
parent 889fba942a
commit 124bd38f7a
31 changed files with 2166 additions and 178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

362
agent/assistant/agent.go
View file

@ -1,12 +1,16 @@
package assistant
import (
"fmt"
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm"
)
// Stream stream the agent
func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Message, handler context.StreamFunc) (*context.Response, error) {
// handler is optional, if not provided, a default handler will be used
func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Message, handler ...context.StreamFunc) (*context.Response, error) {
var err error
@ -31,27 +35,44 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
return nil, err
}
}
_ = createResponse // createResponse is available for further processing
var completionOptions *llm.CompletionOptions // default is nil
var completionOptions *context.CompletionOptions // default is nil
// LLM Call Stream ( Optional )
var completionMessages []context.Message
var completionResponse *context.ResponseCompletion
var completionResponse *context.CompletionResponse
if ast.Prompts != nil || ast.MCP != nil {
llm, err := llm.New(ast.GetConnector(ctx))
// Build the LLM request first
completionMessages, completionOptions, err = ast.BuildRequest(ctx, inputMessages, createResponse)
if err != nil {
return nil, err
}
// Build the LLM request
completionMessages, completionOptions, err = ast.BuildLLMRequest(ctx, inputMessages, createResponse)
// Get connector object and capabilities
conn, capabilities, err := ast.GetConnector(ctx)
if err != nil {
return nil, err
}
// Set capabilities in options if not already set
if completionOptions.Capabilities == nil && capabilities != nil {
completionOptions.Capabilities = capabilities
}
// Create LLM instance with connector and options
llmInstance, err := llm.New(conn, completionOptions)
if err != nil {
return nil, err
}
// Use provided handler or default handler
streamHandler := llm.DefaultStreamHandler(ctx)
if len(handler) > 0 && handler[0] != nil {
streamHandler = handler[0]
}
// Call the LLM Completion Stream
completionResponse, err = llm.Stream(ctx, completionMessages, completionOptions, handler)
completionResponse, err = llmInstance.Stream(ctx, completionMessages, completionOptions, streamHandler)
if err != nil {
return nil, err
}
@ -81,17 +102,328 @@ func (ast *Assistant) Stream(ctx *context.Context, inputMessages []context.Messa
return &context.Response{Create: createResponse, Done: doneResponse, Completion: completionResponse}, nil
}
// GetConnector get the connector from the context
func (ast *Assistant) GetConnector(ctx *context.Context) string {
// GetConnector get the connector object, capabilities, and error with priority: createResponse > ctx > ast
// Note: createResponse.Connector is already applied to ctx.Connector by applyContextAdjustments in create.go
// Returns: (connector, capabilities, error)
func (ast *Assistant) GetConnector(ctx *context.Context) (connector.Connector, *context.ModelCapabilities, error) {
// Determine connector ID with priority
connectorID := ast.Connector
if ctx.Connector != "" {
return ctx.Connector
connectorID = ctx.Connector
}
return ast.Connector
// If empty, return error
if connectorID == "" {
return nil, nil, fmt.Errorf("connector not specified")
}
// Load gou connector
conn, err := connector.Select(connectorID)
if err != nil {
return nil, nil, err
}
// Get connector capabilities from settings
capabilities := ast.getConnectorCapabilities(connectorID)
return conn, capabilities, nil
}
// BuildLLMRequest build the LLM request
func (ast *Assistant) BuildLLMRequest(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse) ([]context.Message, *llm.CompletionOptions, error) {
return messages, nil, nil
// getConnectorCapabilities get the capabilities of a connector from settings
func (ast *Assistant) getConnectorCapabilities(connectorID string) *context.ModelCapabilities {
// Get connector setting from global settings
setting, exists := connectorSettings[connectorID]
if !exists {
return nil
}
// Convert ConnectorSetting to ModelCapabilities
capabilities := &context.ModelCapabilities{}
if setting.Vision {
v := true
capabilities.Vision = &v
}
// Handle both Tools (deprecated) and ToolCalls
if setting.ToolCalls || setting.Tools {
v := true
capabilities.ToolCalls = &v
}
if setting.Audio {
v := true
capabilities.Audio = &v
}
if setting.Reasoning {
v := true
capabilities.Reasoning = &v
}
if setting.Streaming {
v := true
capabilities.Streaming = &v
}
if setting.JSON {
v := true
capabilities.JSON = &v
}
if setting.Multimodal {
v := true
capabilities.Multimodal = &v
}
return capabilities
}
// BuildRequest build the LLM request
func (ast *Assistant) BuildRequest(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse) ([]context.Message, *context.CompletionOptions, error) {
// Build final messages with proper priority
finalMessages, err := ast.buildMessages(ctx, messages, createResponse)
if err != nil {
return nil, nil, err
}
// Build completion options from createResponse and ctx
options := ast.buildCompletionOptions(ctx, createResponse)
return finalMessages, options, nil
}
// buildMessages builds the final message list with proper priority
// Priority: createResponse.Messages > input messages
// If createResponse is nil or has no messages, use input messages
func (ast *Assistant) buildMessages(ctx *context.Context, messages []context.Message, createResponse *context.HookCreateResponse) ([]context.Message, error) {
// If createResponse is nil or has no messages, return input messages as-is
if createResponse == nil || len(createResponse.Messages) == 0 {
return messages, nil
}
// createResponse.Messages takes highest priority
// Return them directly as they override everything
return createResponse.Messages, nil
}
// buildCompletionOptions builds completion options from multiple sources
// Priority (lowest to highest, later overrides earlier): ast > ctx > createResponse
// The priority means: if createResponse has a value, use it; else use ctx; else use ast
func (ast *Assistant) buildCompletionOptions(ctx *context.Context, createResponse *context.HookCreateResponse) *context.CompletionOptions {
options := &context.CompletionOptions{}
// Layer 1 (base): Apply ast - Assistant configuration
ast.applyAssistantOptions(options)
// Layer 2 (middle): Apply ctx - Context configuration (overrides ast)
ast.applyContextOptions(options, ctx)
// Layer 3 (highest): Apply createResponse - Hook configuration (overrides all)
if createResponse != nil {
ast.applyCreateResponseOptions(options, createResponse)
}
return options
}
// applyAssistantOptions applies options from ast.Options to CompletionOptions
// ast.Options can contain any OpenAI API parameters (temperature, top_p, stop, etc.)
func (ast *Assistant) applyAssistantOptions(options *context.CompletionOptions) {
if ast.Options == nil {
return
}
// Temperature
if v, ok := ast.Options["temperature"].(float64); ok {
options.Temperature = &v
}
// MaxTokens
if v, ok := ast.Options["max_tokens"].(float64); ok {
intVal := int(v)
options.MaxTokens = &intVal
} else if v, ok := ast.Options["max_tokens"].(int); ok {
options.MaxTokens = &v
}
// MaxCompletionTokens
if v, ok := ast.Options["max_completion_tokens"].(float64); ok {
intVal := int(v)
options.MaxCompletionTokens = &intVal
} else if v, ok := ast.Options["max_completion_tokens"].(int); ok {
options.MaxCompletionTokens = &v
}
// TopP
if v, ok := ast.Options["top_p"].(float64); ok {
options.TopP = &v
}
// N (number of choices)
if v, ok := ast.Options["n"].(float64); ok {
intVal := int(v)
options.N = &intVal
} else if v, ok := ast.Options["n"].(int); ok {
options.N = &v
}
// Stop sequences (can be string or []string)
if v, ok := ast.Options["stop"]; ok {
options.Stop = v
}
// PresencePenalty
if v, ok := ast.Options["presence_penalty"].(float64); ok {
options.PresencePenalty = &v
}
// FrequencyPenalty
if v, ok := ast.Options["frequency_penalty"].(float64); ok {
options.FrequencyPenalty = &v
}
// LogitBias
if v, ok := ast.Options["logit_bias"].(map[string]interface{}); ok {
logitBias := make(map[string]float64)
for key, val := range v {
if fval, ok := val.(float64); ok {
logitBias[key] = fval
}
}
if len(logitBias) > 0 {
options.LogitBias = logitBias
}
}
// User
if v, ok := ast.Options["user"].(string); ok {
options.User = v
}
// ResponseFormat
if v, ok := ast.Options["response_format"].(map[string]interface{}); ok {
options.ResponseFormat = v
}
// Seed
if v, ok := ast.Options["seed"].(float64); ok {
intVal := int(v)
options.Seed = &intVal
} else if v, ok := ast.Options["seed"].(int); ok {
options.Seed = &v
}
// Tools
if v, ok := ast.Options["tools"].([]interface{}); ok {
tools := make([]map[string]interface{}, 0, len(v))
for _, tool := range v {
if toolMap, ok := tool.(map[string]interface{}); ok {
tools = append(tools, toolMap)
}
}
if len(tools) > 0 {
options.Tools = tools
}
}
// ToolChoice
if v, ok := ast.Options["tool_choice"]; ok {
options.ToolChoice = v
}
// Stream
if v, ok := ast.Options["stream"].(bool); ok {
options.Stream = &v
}
}
// applyContextOptions applies options from ctx to CompletionOptions
// ctx provides Route and Metadata for CUI context
func (ast *Assistant) applyContextOptions(options *context.CompletionOptions, ctx *context.Context) {
// Set Route and Metadata from ctx
options.Route = ctx.Route
options.Metadata = ctx.Metadata
// Set wrapper configurations (assistant.Uses has priority over global settings)
// These can be overridden by createResponse
if visionWrapper := ast.getVisionWrapper(); visionWrapper != "" {
options.VisionWrapper = visionWrapper
}
if audioWrapper := ast.getAudioWrapper(); audioWrapper != "" {
options.AudioWrapper = audioWrapper
}
}
// applyCreateResponseOptions applies options from createResponse to CompletionOptions
// createResponse takes highest priority and overrides any previous settings
func (ast *Assistant) applyCreateResponseOptions(options *context.CompletionOptions, createResponse *context.HookCreateResponse) {
// Audio configuration
if createResponse.Audio != nil {
options.Audio = createResponse.Audio
}
// Temperature
if createResponse.Temperature != nil {
options.Temperature = createResponse.Temperature
}
// MaxTokens
if createResponse.MaxTokens != nil {
options.MaxTokens = createResponse.MaxTokens
}
// MaxCompletionTokens
if createResponse.MaxCompletionTokens != nil {
options.MaxCompletionTokens = createResponse.MaxCompletionTokens
}
// Route
if createResponse.Route != "" {
options.Route = createResponse.Route
}
// Metadata (merge with existing)
if createResponse.Metadata != nil {
if options.Metadata == nil {
options.Metadata = createResponse.Metadata
} else {
// Merge: createResponse.Metadata overrides existing
for key, value := range createResponse.Metadata {
options.Metadata[key] = value
}
}
}
}
// getVisionWrapper get the vision wrapper with priority: assistant.Uses > global settings
func (ast *Assistant) getVisionWrapper() string {
// Priority 1: Assistant-specific Uses configuration
if ast.Uses != nil && ast.Uses.Vision != "" {
return ast.Uses.Vision
}
// Priority 2: Global settings from globalUses
if globalUses != nil && globalUses.Vision != "" {
return globalUses.Vision
}
return ""
}
// getAudioWrapper get the audio wrapper with priority: assistant.Uses > global settings
func (ast *Assistant) getAudioWrapper() string {
// Priority 1: Assistant-specific Uses configuration
if ast.Uses != nil && ast.Uses.Audio != "" {
return ast.Uses.Audio
}
// Priority 2: Global settings from globalUses
if globalUses != nil && globalUses.Audio != "" {
return globalUses.Audio
}
return ""
}
// WithHistory with the history messages

262
agent/assistant/agent_test.go Normal file
View file

@ -0,0 +1,262 @@
package assistant_test
import (
stdContext "context"
"testing"
"github.com/yaoapp/gou/plan"
"github.com/yaoapp/yao/agent/assistant"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/testutils"
"github.com/yaoapp/yao/openapi/oauth/types"
)
// newTestContext creates a Context for testing with commonly used fields pre-populated
func newTestContext(chatID, assistantID string) *context.Context {
return &context.Context{
Context: stdContext.Background(),
Space: plan.NewMemorySharedSpace(),
ChatID: chatID,
AssistantID: assistantID,
Connector: "",
Locale: "en-us",
Theme: "light",
Client: context.Client{
Type: "web",
UserAgent: "TestAgent/1.0",
IP: "127.0.0.1",
},
Referer: context.RefererAPI,
Accept: context.AcceptWebCUI,
Route: "/test/route",
Metadata: map[string]interface{}{
"test": "context_metadata",
},
Authorized: &types.AuthorizedInfo{
Subject: "test-user",
ClientID: "test-client-id",
UserID: "test-user-123",
TeamID: "test-team-456",
TenantID: "test-tenant-789",
SessionID: "test-session-id",
},
}
}
// TestBuildRequest tests the BuildRequest function
func TestBuildRequest(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
agent, err := assistant.Get("tests.buildrequest")
if err != nil {
t.Fatalf("Failed to get tests.buildrequest assistant: %s", err.Error())
}
if agent.Script == nil {
t.Fatalf("The tests.buildrequest assistant has no script")
}
ctx := newTestContext("chat-test-buildrequest", "tests.buildrequest")
// Test 1: No override from hook - should use ast.Options and ctx values
t.Run("NoOverride", func(t *testing.T) {
inputMessages := []context.Message{{Role: "user", Content: "no_override"}}
// Call Create hook
createResponse, err := agent.Script.Create(ctx, inputMessages)
if err != nil {
t.Fatalf("Failed to call Create hook: %s", err.Error())
}
// Build LLM request
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
if err != nil {
t.Fatalf("Failed to build LLM request: %s", err.Error())
}
// Verify options - should use ast.Options values
if options.Temperature == nil {
t.Error("Expected temperature from ast.Options, got nil")
} else if *options.Temperature != 0.5 {
t.Errorf("Expected temperature 0.5 from ast.Options, got: %f", *options.Temperature)
}
if options.MaxTokens == nil {
t.Error("Expected max_tokens from ast.Options, got nil")
} else if *options.MaxTokens != 1000 {
t.Errorf("Expected max_tokens 1000 from ast.Options, got: %d", *options.MaxTokens)
}
if options.TopP == nil {
t.Error("Expected top_p from ast.Options, got nil")
} else if *options.TopP != 0.9 {
t.Errorf("Expected top_p 0.9 from ast.Options, got: %f", *options.TopP)
}
// Verify ctx values
if options.Route != "/test/route" {
t.Errorf("Expected route '/test/route' from ctx, got: %s", options.Route)
}
if options.Metadata == nil {
t.Error("Expected metadata from ctx, got nil")
} else if options.Metadata["test"] != "context_metadata" {
t.Errorf("Expected metadata from ctx, got: %v", options.Metadata)
}
t.Log("✓ No override: ast.Options and ctx values used correctly")
})
// Test 2: Override temperature - hook value should take priority
t.Run("OverrideTemperature", func(t *testing.T) {
inputMessages := []context.Message{{Role: "user", Content: "override_temperature"}}
createResponse, err := agent.Script.Create(ctx, inputMessages)
if err != nil {
t.Fatalf("Failed to call Create hook: %s", err.Error())
}
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
if err != nil {
t.Fatalf("Failed to build LLM request: %s", err.Error())
}
// Verify temperature override
if options.Temperature == nil {
t.Error("Expected temperature, got nil")
} else if *options.Temperature != 0.9 {
t.Errorf("Expected temperature 0.9 from hook, got: %f", *options.Temperature)
}
// Other values should still come from ast.Options
if options.MaxTokens == nil {
t.Error("Expected max_tokens from ast.Options, got nil")
} else if *options.MaxTokens != 1000 {
t.Errorf("Expected max_tokens 1000 from ast.Options, got: %d", *options.MaxTokens)
}
t.Log("✓ Temperature override: hook value takes priority over ast.Options")
})
// Test 3: Override all - all hook values should take priority
t.Run("OverrideAll", func(t *testing.T) {
inputMessages := []context.Message{{Role: "user", Content: "override_all"}}
createResponse, err := agent.Script.Create(ctx, inputMessages)
if err != nil {
t.Fatalf("Failed to call Create hook: %s", err.Error())
}
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
if err != nil {
t.Fatalf("Failed to build LLM request: %s", err.Error())
}
// Verify all overrides
if options.Temperature == nil || *options.Temperature != 0.8 {
t.Errorf("Expected temperature 0.8 from hook, got: %v", options.Temperature)
}
if options.MaxTokens == nil || *options.MaxTokens != 2000 {
t.Errorf("Expected max_tokens 2000 from hook, got: %v", options.MaxTokens)
}
if options.MaxCompletionTokens == nil || *options.MaxCompletionTokens != 1800 {
t.Errorf("Expected max_completion_tokens 1800 from hook, got: %v", options.MaxCompletionTokens)
}
if options.Audio == nil {
t.Error("Expected audio from hook, got nil")
} else {
if options.Audio.Voice != "alloy" {
t.Errorf("Expected voice 'alloy', got: %s", options.Audio.Voice)
}
if options.Audio.Format != "mp3" {
t.Errorf("Expected format 'mp3', got: %s", options.Audio.Format)
}
}
if options.Route != "/hook/route" {
t.Errorf("Expected route '/hook/route' from hook, got: %s", options.Route)
}
if options.Metadata == nil {
t.Error("Expected metadata from hook, got nil")
} else {
if options.Metadata["source"] != "hook" {
t.Errorf("Expected metadata['source'] = 'hook', got: %v", options.Metadata["source"])
}
}
t.Log("✓ Override all: all hook values take priority")
})
// Test 4: Override route and metadata - tests CUI context priority
t.Run("OverrideRouteMetadata", func(t *testing.T) {
inputMessages := []context.Message{{Role: "user", Content: "override_route_metadata"}}
createResponse, err := agent.Script.Create(ctx, inputMessages)
if err != nil {
t.Fatalf("Failed to call Create hook: %s", err.Error())
}
_, options, err := agent.BuildRequest(ctx, inputMessages, createResponse)
if err != nil {
t.Fatalf("Failed to build LLM request: %s", err.Error())
}
// Verify route override
if options.Route != "/custom/route" {
t.Errorf("Expected route '/custom/route' from hook, got: %s", options.Route)
}
// Verify metadata merge (ctx metadata should be merged with hook metadata)
if options.Metadata == nil {
t.Error("Expected metadata, got nil")
} else {
// Hook metadata should be present
if options.Metadata["custom"] != true {
t.Errorf("Expected metadata['custom'] = true from hook, got: %v", options.Metadata["custom"])
}
if options.Metadata["hook_data"] != "test" {
t.Errorf("Expected metadata['hook_data'] = 'test' from hook, got: %v", options.Metadata["hook_data"])
}
// Original ctx metadata should still be there (merged)
if options.Metadata["test"] != "context_metadata" {
t.Errorf("Expected original ctx metadata to be preserved, got: %v", options.Metadata)
}
}
// Other values should still come from ast.Options
if options.Temperature == nil || *options.Temperature != 0.5 {
t.Errorf("Expected temperature 0.5 from ast.Options, got: %v", options.Temperature)
}
t.Log("✓ Route and metadata override: hook values take priority, metadata merged")
})
// Test 5: Nil createResponse - should use ast.Options and ctx values
t.Run("NilCreateResponse", func(t *testing.T) {
// Create a fresh context for this test
freshCtx := newTestContext("chat-test-nil", "tests.buildrequest")
inputMessages := []context.Message{{Role: "user", Content: "test message"}}
_, options, err := agent.BuildRequest(freshCtx, inputMessages, nil)
if err != nil {
t.Fatalf("Failed to build LLM request: %s", err.Error())
}
// Should use ast.Options values
if options.Temperature == nil || *options.Temperature != 0.5 {
t.Errorf("Expected temperature 0.5 from ast.Options, got: %v", options.Temperature)
}
// Should use ctx values
if options.Route != "/test/route" {
t.Errorf("Expected route '/test/route' from ctx, got: %s", options.Route)
}
t.Log("✓ Nil createResponse: ast.Options and ctx values used")
})
}

2
agent/assistant/hook/done.go
View file

@ -5,6 +5,6 @@ import (
)
// Done done hook
func (s *Script) Done(ctx *context.Context, inputMessages []context.Message, completionResponse *context.ResponseCompletion, mcpResponse *context.ResponseHookMCP) (*context.ResponseHookDone, error) {
func (s *Script) Done(ctx *context.Context, inputMessages []context.Message, completionResponse *context.CompletionResponse, mcpResponse *context.ResponseHookMCP) (*context.ResponseHookDone, error) {
return &context.ResponseHookDone{}, nil
}

6
agent/assistant/hook/failback.go
View file

@ -1,8 +1,10 @@
package hook
import "github.com/yaoapp/yao/agent/context"
import (
"github.com/yaoapp/yao/agent/context"
)
// Failback failback hook
func (s *Script) Failback(ctx *context.Context, inputMessages []context.Message, completionResponse *context.ResponseCompletion) (*context.ResponseHookFailback, error) {
func (s *Script) Failback(ctx *context.Context, inputMessages []context.Message, completionResponse *context.CompletionResponse) (*context.ResponseHookFailback, error) {
return &context.ResponseHookFailback{}, nil
}

6
agent/assistant/load.go
View file

@ -29,6 +29,7 @@ var search interface{} = nil
var connectorSettings map[string]ConnectorSetting = map[string]ConnectorSetting{}
var vision *agentvision.Vision = nil
var defaultConnector string = "" // default connector
var globalUses *store.Uses = nil // global uses configuration from agent.yml
// LoadBuiltIn load the built-in assistants
func LoadBuiltIn() error {
@ -145,6 +146,11 @@ func SetConnector(c string) {
defaultConnector = c
}
// SetGlobalUses set the global uses configuration
func SetGlobalUses(uses *store.Uses) {
globalUses = uses
}
// SetCache set the cache
func SetCache(capacity int) {
ClearCache()

11
agent/assistant/types.go
View file

@ -109,9 +109,16 @@ type Assistant struct {
}
// ConnectorSetting the connector setting
// Defines the capabilities of a connector/model
type ConnectorSetting struct {
Vision bool `json:"vision,omitempty" yaml:"vision,omitempty"`
Tools bool `json:"tools,omitempty" yaml:"tools,omitempty"`
Vision bool `json:"vision,omitempty" yaml:"vision,omitempty"` // Supports vision/image input
Tools bool `json:"tools,omitempty" yaml:"tools,omitempty"` // Supports tool/function calling (deprecated, use ToolCalls)
ToolCalls bool `json:"tool_calls,omitempty" yaml:"tool_calls,omitempty"` // Supports tool/function calling
Audio bool `json:"audio,omitempty" yaml:"audio,omitempty"` // Supports audio input/output
Reasoning bool `json:"reasoning,omitempty" yaml:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1)
Streaming bool `json:"streaming,omitempty" yaml:"streaming,omitempty"` // Supports streaming responses
JSON bool `json:"json,omitempty" yaml:"json,omitempty"` // Supports JSON mode
Multimodal bool `json:"multimodal,omitempty" yaml:"multimodal,omitempty"` // Supports multimodal input
}
// VisionCapableModels list of LLM models that support vision capabilities

5
agent/context/types.go
View file

@ -188,7 +188,7 @@ type Response struct {
MCP *ResponseHookMCP `json:"mcp,omitempty"`
Done *ResponseHookDone `json:"done,omitempty"`
Failback *ResponseHookFailback `json:"failback,omitempty"`
Completion *ResponseCompletion `json:"completion,omitempty"`
Completion *CompletionResponse `json:"completion,omitempty"`
}
// HookCreateResponse the response of the create hook
@ -223,9 +223,6 @@ type ResponseHookMCP struct{}
// ResponseHookFailback the response of the failback hook
type ResponseHookFailback struct{}
// ResponseCompletion the response of the completion
type ResponseCompletion struct{}
// Message Structure ( OpenAI Chat Completion Input Message Structure, https://platform.openai.com/docs/api-reference/chat/create#chat/create-messages )
// ===============================

146
agent/context/types_llm.go Normal file
View file

@ -0,0 +1,146 @@
package context
// ModelCapabilities defines the capabilities of a language model
// Used by LLM to select appropriate provider and validate requests
type ModelCapabilities struct {
Vision *bool `json:"vision,omitempty"` // Supports vision/image input
ToolCalls *bool `json:"tool_calls,omitempty"` // Supports tool/function calling
Audio *bool `json:"audio,omitempty"` // Supports audio input/output
Reasoning *bool `json:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1)
Streaming *bool `json:"streaming,omitempty"` // Supports streaming responses
JSON *bool `json:"json,omitempty"` // Supports JSON mode
Multimodal *bool `json:"multimodal,omitempty"` // Supports multimodal input (text + images + audio)
}
// CompletionOptions the completion request options
// These options are extracted from HookCreateResponse and Context, then passed to the LLM connector
// Compatible with OpenAI Chat Completion API: https://platform.openai.com/docs/api-reference/chat/create
type CompletionOptions struct {
// Model capabilities (used by LLM to select appropriate provider)
// nil means capabilities are not specified/checked
Capabilities *ModelCapabilities `json:"capabilities,omitempty"`
// Wrapper configurations for vision and audio processing
// Format: "agent" (default) or "mcp:mcp_server_id"
VisionWrapper string `json:"vision_wrapper,omitempty"` // Vision processing wrapper (for image/video description)
AudioWrapper string `json:"audio_wrapper,omitempty"` // Audio processing wrapper (for speech-to-text/text-to-speech)
// Audio configuration (for models that support audio output)
Audio *AudioConfig `json:"audio,omitempty"`
// Generation parameters
Temperature *float64 `json:"temperature,omitempty"` // Sampling temperature (0-2), defaults to 1
MaxTokens *int `json:"max_tokens,omitempty"` // Maximum tokens to generate (deprecated, use MaxCompletionTokens)
MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` // Maximum tokens in completion
TopP *float64 `json:"top_p,omitempty"` // Nucleus sampling parameter (0-1), alternative to temperature
N *int `json:"n,omitempty"` // Number of chat completion choices to generate
// Control parameters
Stop interface{} `json:"stop,omitempty"` // Up to 4 sequences where the API will stop generating (string or []string)
PresencePenalty *float64 `json:"presence_penalty,omitempty"` // Presence penalty (-2.0 to 2.0)
FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` // Frequency penalty (-2.0 to 2.0)
LogitBias map[string]float64 `json:"logit_bias,omitempty"` // Modify likelihood of specified tokens appearing
// User and response format
User string `json:"user,omitempty"` // Unique identifier representing end-user
ResponseFormat map[string]interface{} `json:"response_format,omitempty"` // Format of the response (e.g., {"type": "json_object"})
Seed *int `json:"seed,omitempty"` // Seed for deterministic sampling
// Tool calling
Tools []map[string]interface{} `json:"tools,omitempty"` // List of tools the model may call
ToolChoice interface{} `json:"tool_choice,omitempty"` // Controls which tool is called ("none", "auto", "required", or specific tool)
// Streaming configuration
Stream *bool `json:"stream,omitempty"` // If true, stream partial message deltas
StreamOptions *StreamOptions `json:"stream_options,omitempty"` // Options for streaming response
// CUI Context information (from Context)
Route string `json:"route,omitempty"` // Route of the request for CUI context
Metadata map[string]interface{} `json:"metadata,omitempty"` // Metadata to pass to the page for CUI context
}
// CompletionResponse represents the unified completion response
// Compatible with OpenAI chat completion response format
type CompletionResponse struct {
// Response metadata
ID string `json:"id"` // Unique identifier for the completion
Object string `json:"object"` // Object type (e.g., "chat.completion")
Created int64 `json:"created"` // Unix timestamp of creation
Model string `json:"model"` // Model used for completion
// Completion content (these fields can coexist)
Content string `json:"content"` // Text content (regular response text)
ReasoningContent string `json:"reasoning_content,omitempty"` // Reasoning/thinking content (for o1, DeepSeek R1, etc.)
ToolCalls []ToolCallResult `json:"tool_calls,omitempty"` // Tool calls made by the model
Refusal string `json:"refusal,omitempty"` // Refusal message if model refused to answer
ContentTypes []ContentType `json:"content_types"` // Types of content present (can have multiple simultaneously)
// Raw response data
Raw interface{} `json:"raw,omitempty"` // Original raw response from the LLM provider (for debugging and special cases)
// Completion metadata
FinishReason string `json:"finish_reason"` // Reason for completion (stop, length, tool_calls, content_filter, etc.)
// Usage statistics
Usage *UsageInfo `json:"usage,omitempty"` // Token usage statistics
// Additional metadata
SystemFingerprint string `json:"system_fingerprint,omitempty"` // System fingerprint for reproducibility
Metadata map[string]interface{} `json:"metadata,omitempty"` // Additional metadata
}
// ContentType represents the type of content in the response
// A response can contain multiple content types simultaneously
type ContentType string
// Content type constants - a response can have multiple types simultaneously
// For example: text + reasoning, or text + tool_call, or all three
const (
ContentTypeText ContentType = "text" // Regular text content
ContentTypeReasoning ContentType = "reasoning" // Reasoning/thinking content (o1, DeepSeek R1, etc.)
ContentTypeToolCall ContentType = "tool_call" // Tool/function call
ContentTypeRefusal ContentType = "refusal" // Model refused to answer
ContentTypeEmpty ContentType = "empty" // Empty response (no content)
)
// UsageInfo represents token usage statistics
type UsageInfo struct {
PromptTokens int `json:"prompt_tokens"` // Tokens in the prompt
CompletionTokens int `json:"completion_tokens"` // Tokens in the completion
TotalTokens int `json:"total_tokens"` // Total tokens used
// Detailed token breakdown (for models with reasoning)
PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"` // Detailed prompt token breakdown
CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"` // Detailed completion token breakdown
}
// TokenDetails provides detailed token usage breakdown
type TokenDetails struct {
CachedTokens int `json:"cached_tokens,omitempty"` // Tokens from cache
ReasoningTokens int `json:"reasoning_tokens,omitempty"` // Tokens used for reasoning/thinking
AudioTokens int `json:"audio_tokens,omitempty"` // Tokens used for audio
TextTokens int `json:"text_tokens,omitempty"` // Tokens used for text
}
// ToolCallResult represents a tool call result in the completion
type ToolCallResult struct {
ID string `json:"id"` // Tool call ID
Type string `json:"type"` // Tool call type (usually "function")
Function FunctionCallResult `json:"function"` // Function call details
}
// FunctionCallResult represents a function call result
type FunctionCallResult struct {
Name string `json:"name"` // Function name
Arguments string `json:"arguments"` // Function arguments as JSON string
}
// FinishReason constants
const (
FinishReasonStop = "stop" // Natural stop point
FinishReasonLength = "length" // Max tokens reached
FinishReasonToolCalls = "tool_calls" // Tool calls made
FinishReasonContentFilter = "content_filter" // Content filtered
FinishReasonFunctionCall = "function_call" // Function call (deprecated)
FinishReasonError = "error" // Error occurred
)

49
agent/context/types_wrapper.go Normal file
View file

@ -0,0 +1,49 @@
package context
import "strings"
// WrapperType represents the type of wrapper for processing
type WrapperType string
const (
WrapperTypeAgent WrapperType = "agent" // Use agent for processing
WrapperTypeMCP WrapperType = "mcp" // Use MCP server for processing
)
// ParseWrapper parses a wrapper string and returns the type and ID
// Format: "agent" or "mcp:mcp_server_id"
func ParseWrapper(wrapper string) (WrapperType, string) {
if wrapper == "" || wrapper == "agent" {
return WrapperTypeAgent, ""
}
if strings.HasPrefix(wrapper, "mcp:") {
mcpID := strings.TrimPrefix(wrapper, "mcp:")
return WrapperTypeMCP, mcpID
}
// Default to agent if format is unknown
return WrapperTypeAgent, ""
}
// IsAgentWrapper checks if the wrapper is an agent wrapper
func IsAgentWrapper(wrapper string) bool {
wrapperType, _ := ParseWrapper(wrapper)
return wrapperType == WrapperTypeAgent
}
// IsMCPWrapper checks if the wrapper is an MCP wrapper
func IsMCPWrapper(wrapper string) bool {
wrapperType, _ := ParseWrapper(wrapper)
return wrapperType == WrapperTypeMCP
}
// GetMCPServerID extracts the MCP server ID from wrapper string
// Returns empty string if not an MCP wrapper
func GetMCPServerID(wrapper string) string {
wrapperType, id := ParseWrapper(wrapper)
if wrapperType == WrapperTypeMCP {
return id
}
return ""
}

51
agent/llm/handlers/handlers.go Normal file
View file

@ -0,0 +1,51 @@
package handlers
import (
"github.com/yaoapp/yao/agent/context"
)
// Handler interface for stream handlers
type Handler interface {
OnChunk(chunk *StreamChunk) error
OnComplete() error
OnError(err error) error
}
// NewDefaultHandler creates a default handler that sends chunks via context
func NewDefaultHandler(ctx *context.Context) Handler {
return &DefaultHandler{
ctx: ctx,
}
}
// DefaultHandler default stream handler implementation
type DefaultHandler struct {
ctx *context.Context
}
// OnChunk handles a streaming chunk
func (h *DefaultHandler) OnChunk(chunk *StreamChunk) error {
// TODO: Implement chunk handling
// - Send chunk via ctx
// - Handle different chunk types
// - Aggregate content for final response
return SendStreamChunk(h.ctx, chunk)
}
// OnComplete handles stream completion
func (h *DefaultHandler) OnComplete() error {
// TODO: Implement completion handling
// - Send final message
// - Close stream
// - Return aggregated response
return nil
}
// OnError handles stream errors
func (h *DefaultHandler) OnError(err error) error {
// TODO: Implement error handling
// - Send error message to client
// - Log error
// - Clean up resources
return err
}

86
agent/llm/handlers/stream.go Normal file
View file

@ -0,0 +1,86 @@
package handlers
import (
"github.com/yaoapp/yao/agent/context"
)
// DefaultStreamHandler creates a default stream handler that sends messages via context
// This handler is used when no custom handler is provided
func DefaultStreamHandler(ctx *context.Context) context.StreamFunc {
return func(data []byte) int {
// TODO: Implement default stream handling
// - Parse streaming chunk data
// - Extract content from chunk
// - Send message via ctx (SSE, WebSocket, etc.)
// - Handle different chunk types (content, tool_calls, reasoning)
// - Return 1 to continue streaming, 0 to stop
return 1
}
}
// SendStreamChunk sends a stream chunk via context
// Used internally by DefaultStreamHandler
func SendStreamChunk(ctx *context.Context, chunk *StreamChunk) error {
// TODO: Implement sending stream chunk
// - Format chunk for transport (SSE, WebSocket)
// - Send via ctx's connection
// - Handle errors and retries
return nil
}
// StreamChunk represents a parsed streaming chunk
type StreamChunk struct {
Type ChunkType `json:"type"` // Type of chunk (content, reasoning, tool_call, etc.)
Content string `json:"content,omitempty"` // Text content
// For reasoning chunks
ReasoningContent string `json:"reasoning_content,omitempty"`
// For tool call chunks
ToolCallID string `json:"tool_call_id,omitempty"`
ToolCallFunction string `json:"tool_call_function,omitempty"`
ToolCallArgs string `json:"tool_call_args,omitempty"`
// Metadata
Done bool `json:"done"` // Whether this is the final chunk
FinishReason string `json:"finish_reason,omitempty"` // Reason for completion (if done)
}
// ChunkType represents the type of streaming chunk
type ChunkType string
const (
ChunkTypeContent ChunkType = "content" // Regular text content
ChunkTypeReasoning ChunkType = "reasoning" // Reasoning/thinking content
ChunkTypeToolCall ChunkType = "tool_call" // Tool call chunk
ChunkTypeDone ChunkType = "done" // Final chunk (completion)
ChunkTypeError ChunkType = "error" // Error chunk
)
// ParseStreamChunk parses raw streaming data into StreamChunk
func ParseStreamChunk(data []byte) (*StreamChunk, error) {
// TODO: Implement stream chunk parsing
// - Parse SSE format (data: {...})
// - Handle different provider formats (OpenAI, DeepSeek, etc.)
// - Extract content, reasoning, tool calls
// - Detect completion (done: true)
return nil, nil
}
// FormatSSE formats a StreamChunk as Server-Sent Events format
func FormatSSE(chunk *StreamChunk) string {
// TODO: Implement SSE formatting
// - Format as "data: {...}\n\n"
// - Handle special cases (done, error)
// - Ensure proper JSON encoding
return ""
}
// FormatWebSocket formats a StreamChunk as WebSocket message
func FormatWebSocket(chunk *StreamChunk) []byte {
// TODO: Implement WebSocket formatting
// - Format as JSON message
// - Add message type/metadata
// - Handle binary vs text frames
return nil
}

4
agent/llm/interfaces.go
View file

@ -4,6 +4,6 @@ import "github.com/yaoapp/yao/agent/context"
// LLM the LLM interface
type LLM interface {
Stream(ctx *context.Context, messages []context.Message, options *CompletionOptions, handler context.StreamFunc) (*context.ResponseCompletion, error)
Post(ctx *context.Context, messages []context.Message, options *CompletionOptions) (*context.ResponseCompletion, error)
Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error)
Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error)
}

13
agent/llm/llm.go
View file

@ -1,6 +1,15 @@
package llm
import (
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm/providers"
)
// New create a new LLM instance
func New(connector string) (LLM, error) {
return nil, nil
// conn: connector object from connector.Select()
// options: completion options containing capabilities and other settings
func New(conn connector.Connector, options *context.CompletionOptions) (LLM, error) {
// Select appropriate provider based on capabilities
return providers.SelectProvider(conn, options)
}

319
agent/llm/providers/README.md Normal file
View file

@ -0,0 +1,319 @@
# LLM Providers Architecture
## Overview
This directory contains different LLM provider implementations, each optimized for specific model capabilities.
## Provider Selection Strategy
The `factory.SelectProvider()` function automatically selects the appropriate provider based on model capabilities:
```go
Priority 1: Reasoning models → reasoning.Provider
Priority 2: Native tool support → openai.Provider
Priority 3: Legacy models → legacy.Provider
```
## Provider Types
### 1. Base Provider (`base/`)
**Purpose**: Common functionality shared across all providers
**Features**:
- Message preprocessing
- Request body building
- Response parsing
**Usage**: Embedded in all other providers
---
### 2. OpenAI Provider (`openai/`)
**Purpose**: OpenAI-compatible models with full feature support
**Capabilities**:
- ✅ Vision (image input)
- ✅ Native tool calls
- ✅ Streaming
- ✅ JSON mode
**Models**:
- GPT-4, GPT-4o, GPT-4-turbo
- GPT-3.5-turbo
- Claude (via OpenAI-compatible API)
---
### 3. Reasoning Provider (`reasoning/`)
**Purpose**: Reasoning models with special response format
**Capabilities**:
- ✅ Reasoning content (`reasoning_content` field)
- ✅ Thinking + Answer phases
- ⚠️ Tool calls support varies by model
**Models**:
- **OpenAI o1** (supports native tool calls)
- **DeepSeek R1** (no native tool calls, uses prompt engineering)
**Special Handling**:
```go
// DeepSeek R1 scenario
if !supportsNativeTools && hasTools {
// Inject tool instructions into prompt
messages = injectToolInstructions(messages, tools)
// Extract tool calls from text response
toolCalls = extractToolCallsFromText(response.Content)
}
```
**Response Format**:
```json
{
"content": "The answer is 42",
"reasoning_content": "Let me think... first we need to...",
"content_types": ["text", "reasoning"]
}
```
---
### 4. Legacy Provider (`legacy/`)
**Purpose**: Older models without native tool calling
**Capabilities**:
- ✅ Text generation
- ⚠️ Tool calls via prompt engineering
- ❌ No native vision support
- ❌ No native tool API
**Models**:
- GPT-3 (davinci, curie)
- Older open-source models
- Custom models without tool API
**Tool Call Flow**:
1. Inject tool schemas into system prompt
2. Model returns tool call in text format (JSON)
3. Extract and parse tool calls from text
4. Execute tools
5. Continue conversation
---
### 5. Vision Utils (`vision/`)
**Purpose**: Vision-related preprocessing utilities
**Functions**:
- `PreprocessVisionMessages()` - Handle image content
- `ConvertImageToText()` - Convert images to descriptions (for non-vision models)
- `ValidateImageURL()` - Validate image URLs
- `ExtractImagesFromMessages()` - Extract all images from messages
**Usage**:
```go
// When model doesn't support vision
if !supportsVision {
messages = vision.PreprocessVisionMessages(messages, false)
// Images converted to text descriptions
}
```
---
### 6. Audio Utils (`audio/`)
**Purpose**: Audio-related preprocessing utilities
**Functions**:
- `PreprocessAudioMessages()` - Handle audio content
- `ConvertAudioToText()` - Convert audio to text transcription (for non-audio models)
- `ValidateAudioFormat()` - Validate audio format and encoding
- `ExtractAudioFromMessages()` - Extract all audio data from messages
- `RemoveAudioConfig()` - Remove audio configuration from options
**Usage**:
```go
// When model doesn't support audio
if !supportsAudio {
messages = audio.PreprocessAudioMessages(messages, false)
options = audio.RemoveAudioConfig(options)
// Audio converted to text transcriptions
}
```
---
## Special Scenarios
### Scenario 1: DeepSeek R1 (Reasoning + No Tool Support)
**Provider**: `reasoning.Provider`
**Handling**:
```go
// Check if reasoning model supports tools
if !p.supportsNativeTools && len(options.Tools) > 0 {
// Use prompt engineering approach
messages = p.injectToolInstructions(messages, tools)
options = p.removeToolsFromOptions(options)
}
// After getting response
if !p.supportsNativeTools {
toolCalls = p.extractToolCallsFromText(response.Content)
}
```
**Why reasoning provider?**
- Primary characteristic is reasoning (special response format)
- Tool handling is secondary concern
- Reuses tool injection logic from legacy approach
---
### Scenario 2: Legacy Model + Vision/Audio Request
**Provider**: `legacy.Provider`
**Handling**:
```go
import (
"github.com/yaoapp/yao/agent/llm/providers/vision"
"github.com/yaoapp/yao/agent/llm/providers/audio"
)
// Preprocess to remove/convert vision content
if !supportsVision {
messages = vision.PreprocessVisionMessages(messages, false)
// Images converted to text: "[Image: description]"
}
// Preprocess to remove/convert audio content
if !supportsAudio {
messages = audio.PreprocessAudioMessages(messages, false)
options = audio.RemoveAudioConfig(options)
// Audio converted to text: "[Audio transcription: ...]"
}
```
---
### Scenario 3: OpenAI o1 (Reasoning + Tool Support)
**Provider**: `reasoning.Provider`
**Handling**:
```go
// o1 supports native tools, no special handling needed
if p.supportsNativeTools {
// Use standard OpenAI tool calling API
}
```
---
## Configuration Example
In `connectors.yml`:
```yaml
# GPT-4o with all features
gpt-4o:
vision: true
tool_calls: true
audio: true
streaming: true
json: true
multimodal: true
# OpenAI o1 - reasoning with tool support
o1-preview:
reasoning: true
tool_calls: true
streaming: true
# DeepSeek R1 - reasoning without tool support
deepseek-reasoner:
reasoning: true
tool_calls: false # Will use prompt engineering
streaming: true
# GPT-3 - legacy model
gpt-3.5-turbo-instruct:
tool_calls: false # Will use prompt engineering
vision: false # Will convert images to text
audio: false # Will convert audio to text
streaming: false
# GPT-4 Vision only
gpt-4-vision:
vision: true
tool_calls: true
audio: false # No audio support
streaming: true
```
---
## Adding a New Provider
1. Create new directory: `providers/newprovider/`
2. Implement `LLM` interface:
```go
type Provider struct {
*base.Provider
}
func (p *Provider) Stream(...) (*CompletionResponse, error)
func (p *Provider) Post(...) (*CompletionResponse, error)
```
3. Update `factory.SelectProvider()` selection logic
4. Add capability flags to `ConnectorSetting`
---
## Testing
Each provider should have tests for:
- Standard completion
- Streaming completion
- Tool calling (if supported)
- Vision input (if supported)
- Error handling
- Response parsing
---
## Performance Considerations
- **Caching**: Consider caching connector instances
- **Pooling**: HTTP connection pooling for high throughput
- **Timeouts**: Configurable timeouts per provider
- **Retries**: Exponential backoff for transient errors

59
agent/llm/providers/audio/audio.go Normal file
View file

@ -0,0 +1,59 @@
package audio
import (
"github.com/yaoapp/yao/agent/context"
)
// PreprocessAudioMessages preprocess messages to handle audio content
// Removes or converts audio content for models that don't support it
func PreprocessAudioMessages(messages []context.Message, supportsAudio bool) []context.Message {
// TODO: Implement audio message preprocessing
// If supportsAudio is false:
// - Remove input_audio content parts
// - Convert to text-only messages
// - Optionally add audio transcriptions
// If supportsAudio is true:
// - Validate audio format
// - Ensure proper encoding
return messages
}
// ConvertAudioToText convert audio content to text transcription
// Used when model doesn't support audio input
func ConvertAudioToText(audioData string) (string, error) {
// TODO: Implement audio to text conversion
// - Call speech-to-text API (Whisper, etc.)
// - Generate transcription
// - Return as text content
return "", nil
}
// ValidateAudioFormat validate audio format and encoding
func ValidateAudioFormat(audioConfig *context.AudioConfig) error {
// TODO: Implement audio format validation
// - Check format (wav, mp3, etc.)
// - Validate encoding
// - Check sample rate
return nil
}
// ExtractAudioFromMessages extract all audio data from messages
func ExtractAudioFromMessages(messages []context.Message) []string {
// TODO: Implement audio extraction
// - Iterate through messages
// - Find ContentPart with type="input_audio"
// - Collect all audio data
return nil
}
// RemoveAudioConfig remove audio configuration from options
// Used when model doesn't support audio output
func RemoveAudioConfig(options *context.CompletionOptions) *context.CompletionOptions {
// TODO: Remove audio config from options
if options == nil {
return options
}
newOptions := *options
newOptions.Audio = nil
return &newOptions
}

65
agent/llm/providers/base/base.go Normal file
View file

@ -0,0 +1,65 @@
package base
import (
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
)
// Provider base provider implementation
// Provides common functionality for all LLM providers
type Provider struct {
Connector connector.Connector
Capabilities *context.ModelCapabilities
}
// NewProvider create a new base provider
func NewProvider(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
return &Provider{
Connector: conn,
Capabilities: capabilities,
}
}
// PreprocessMessages preprocess messages before sending to LLM
// Handles vision messages, audio messages, tool messages, etc.
func (p *Provider) PreprocessMessages(messages []context.Message) ([]context.Message, error) {
// TODO: Implement message preprocessing
// - Remove vision content if not supported
// - Remove audio content if not supported
// - Convert tool messages if needed
// - Validate message format
return messages, nil
}
// SupportsVision check if this provider supports vision
func (p *Provider) SupportsVision() bool {
return p.Capabilities != nil && p.Capabilities.Vision != nil && *p.Capabilities.Vision
}
// SupportsAudio check if this provider supports audio
func (p *Provider) SupportsAudio() bool {
return p.Capabilities != nil && p.Capabilities.Audio != nil && *p.Capabilities.Audio
}
// SupportsTools check if this provider supports tool calls
func (p *Provider) SupportsTools() bool {
return p.Capabilities != nil && p.Capabilities.ToolCalls != nil && *p.Capabilities.ToolCalls
}
// BuildRequestBody build the request body for the LLM API
func (p *Provider) BuildRequestBody(messages []context.Message, options *context.CompletionOptions) (map[string]interface{}, error) {
// TODO: Implement request body building
// - Convert messages to API format
// - Apply options (temperature, max_tokens, etc.)
// - Add model-specific parameters
return nil, nil
}
// ParseResponse parse the response from LLM API
func (p *Provider) ParseResponse(data []byte, isStreaming bool) (*context.CompletionResponse, error) {
// TODO: Implement response parsing
// - Parse JSON response
// - Extract content, tool calls, reasoning, etc.
// - Handle streaming chunks
return nil, nil
}

56
agent/llm/providers/factory.go Normal file
View file

@ -0,0 +1,56 @@
package providers
import (
"fmt"
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm/providers/legacy"
"github.com/yaoapp/yao/agent/llm/providers/openai"
"github.com/yaoapp/yao/agent/llm/providers/reasoning"
)
// LLM interface (copied to avoid import cycle)
type LLM interface {
Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error)
Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error)
}
// SelectProvider select the appropriate provider based on connector and capabilities
func SelectProvider(conn connector.Connector, options *context.CompletionOptions) (LLM, error) {
if options == nil || options.Capabilities == nil {
return nil, fmt.Errorf("options and capabilities are required")
}
capabilities := options.Capabilities
// Priority 1: Reasoning models (special response format)
if capabilities.Reasoning != nil && *capabilities.Reasoning {
return reasoning.New(conn, capabilities), nil
}
// Priority 2: Check if model supports native tool calls
if capabilities.ToolCalls != nil && *capabilities.ToolCalls {
// Use OpenAI-compatible provider (supports tools, vision, streaming)
return openai.New(conn, capabilities), nil
}
// Priority 3: Legacy models (no native tool support)
// Will use prompt engineering for tool calls
return legacy.New(conn, capabilities), nil
}
// DetectProvider detect provider type from connector
func DetectProvider(conn connector.Connector) string {
// TODO: Implement provider detection
// - Check connector type (Is(connector.OPENAI))
// - Check connector settings
// - Determine provider type (openai, claude, deepseek, etc.)
if conn.Is(connector.OPENAI) {
return "openai"
}
// Default to OpenAI-compatible
return "openai"
}

64
agent/llm/providers/legacy/legacy.go Normal file
View file

@ -0,0 +1,64 @@
package legacy
import (
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm/providers/base"
)
// Provider legacy LLM provider (no native tool calling support)
// Implements tool calling via prompt engineering
type Provider struct {
*base.Provider
}
// New create a new legacy provider
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
return &Provider{
Provider: base.NewProvider(conn, capabilities),
}
}
// Stream stream completion from legacy model
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
// TODO: Implement legacy model streaming
// - Preprocess messages (remove tool-specific fields, vision, audio)
// - Remove vision content (convert to text description)
// - Remove audio content (convert to text transcription)
// - Remove tool messages
// - Add tool calling instructions to system prompt if tools provided
// - Build request body without native tool parameters
// - Make streaming HTTP request
// - Parse response and detect tool calls from text
// - Extract tool calls using regex/JSON parsing
// - Call handler for each chunk
return nil, nil
}
// Post post completion request to legacy model
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
// TODO: Implement legacy model non-streaming completion
// - Preprocess messages
// - Add tool instructions to prompt
// - Make HTTP POST request
// - Parse response and extract tool calls from text
return nil, nil
}
// InjectToolInstructions inject tool calling instructions into system prompt
func (p *Provider) InjectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message {
// TODO: Implement tool instruction injection
// - Generate tool description prompt
// - Add to system message or create new system message
// - Include tool schemas and usage instructions
return messages
}
// ExtractToolCallsFromText extract tool calls from model's text response
func (p *Provider) ExtractToolCallsFromText(text string) []context.ToolCallResult {
// TODO: Implement tool call extraction
// - Look for JSON blocks or specific patterns
// - Parse tool name and arguments
// - Return structured tool calls
return nil
}

50
agent/llm/providers/openai/openai.go Normal file
View file

@ -0,0 +1,50 @@
package openai
import (
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm/providers/base"
)
// Provider OpenAI-compatible provider
// Supports: vision, tool calls, streaming, JSON mode
type Provider struct {
*base.Provider
}
// New create a new OpenAI provider
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
return &Provider{
Provider: base.NewProvider(conn, capabilities),
}
}
// Stream stream completion from OpenAI API
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
// TODO: Implement OpenAI streaming
// - Preprocess messages (vision, audio, tools)
// - Remove vision content if not supported
// - Remove audio content if not supported
// - Convert to text where needed
// - Build request body
// - Make streaming HTTP request
// - Parse SSE chunks
// - Call handler for each chunk
// - Aggregate final response
return nil, nil
}
// Post post completion request to OpenAI API
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
// TODO: Implement OpenAI non-streaming completion
// - Preprocess messages
// - Build request body
// - Make HTTP POST request
// - Parse response
return nil, nil
}
// SupportsAudio check if this provider supports audio
func (p *Provider) SupportsAudio() bool {
return p.Capabilities != nil && p.Capabilities.Audio != nil && *p.Capabilities.Audio
}

115
agent/llm/providers/reasoning/reasoning.go Normal file
View file

@ -0,0 +1,115 @@
package reasoning
import (
"github.com/yaoapp/gou/connector"
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm/providers/base"
)
// Provider reasoning model provider (o1, DeepSeek R1, etc.)
// Handles special response format with reasoning_content
// Note: Some reasoning models (e.g. DeepSeek R1) don't support native tool calls
type Provider struct {
*base.Provider
supportsNativeTools bool // Whether this reasoning model supports native tool calling
}
// New create a new reasoning provider
func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider {
// Check if this reasoning model supports native tool calls
supportsTools := false
if capabilities != nil && capabilities.ToolCalls != nil && *capabilities.ToolCalls {
supportsTools = true
}
return &Provider{
Provider: base.NewProvider(conn, capabilities),
supportsNativeTools: supportsTools,
}
}
// Stream stream completion from reasoning model
func (p *Provider) Stream(ctx *context.Context, messages []context.Message, options *context.CompletionOptions, handler context.StreamFunc) (*context.CompletionResponse, error) {
// TODO: Implement reasoning model streaming
// - Preprocess messages (reasoning models have restrictions)
// Handle tool calls based on model support
if !p.supportsNativeTools && options != nil && len(options.Tools) > 0 {
// Model doesn't support native tool calls (e.g. DeepSeek R1)
// Inject tool instructions into messages
messages = p.injectToolInstructions(messages, options.Tools)
// Remove tools from options to avoid API error
options = p.removeToolsFromOptions(options)
}
// - Build request body (special parameters for reasoning)
// - Make streaming HTTP request
// - Parse SSE chunks with reasoning_content
// - Handle both thinking and answer phases
// - Call handler for each chunk
// If tools were injected, extract tool calls from text response
// if !p.supportsNativeTools && hasTools {
// toolCalls = p.extractToolCallsFromText(response.Content)
// }
// - Aggregate final response with reasoning content
return nil, nil
}
// Post post completion request to reasoning model
func (p *Provider) Post(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) {
// TODO: Implement reasoning model non-streaming completion
// - Preprocess messages
// - Build request body
// - Make HTTP POST request
// - Parse response with reasoning_content field
// - Separate thinking from final answer
return nil, nil
}
// ParseReasoningResponse parse response with reasoning content
// Handles both OpenAI o1 format and DeepSeek R1 format
func (p *Provider) ParseReasoningResponse(data []byte) (*context.CompletionResponse, error) {
// TODO: Implement reasoning response parsing
// - Detect format (OpenAI vs DeepSeek)
// - Extract reasoning_content
// - Extract final content
// - Set ContentTypes correctly (text + reasoning)
return nil, nil
}
// injectToolInstructions inject tool calling instructions into messages
// Used for reasoning models that don't support native tool calls (e.g. DeepSeek R1)
func (p *Provider) injectToolInstructions(messages []context.Message, tools []map[string]interface{}) []context.Message {
// TODO: Implement tool instruction injection for reasoning models
// - Generate tool description prompt (optimized for reasoning models)
// - Add to system message or create new system message
// - Include tool schemas and usage instructions
// - Format should encourage reasoning about tool usage
return messages
}
// extractToolCallsFromText extract tool calls from reasoning model's text response
// Used when model doesn't support native tool calls
func (p *Provider) extractToolCallsFromText(text string) []context.ToolCallResult {
// TODO: Implement tool call extraction from text
// - Look for JSON blocks or specific patterns
// - Parse tool name and arguments
// - Return structured tool calls
// - Handle reasoning model's specific output format
return nil
}
// removeToolsFromOptions remove tool-related parameters from options
// Used when sending request to models that don't support native tool calls
func (p *Provider) removeToolsFromOptions(options *context.CompletionOptions) *context.CompletionOptions {
// TODO: Create a copy of options without tool parameters
// - Remove Tools field
// - Remove ToolChoice field
// - Keep other options intact
newOptions := *options
newOptions.Tools = nil
newOptions.ToolChoice = nil
return &newOptions
}

47
agent/llm/providers/vision/vision.go Normal file
View file

@ -0,0 +1,47 @@
package vision
import (
"github.com/yaoapp/yao/agent/context"
)
// PreprocessVisionMessages preprocess messages to handle vision content
// Removes or converts vision content for models that don't support it
func PreprocessVisionMessages(messages []context.Message, supportsVision bool) []context.Message {
// TODO: Implement vision message preprocessing
// If supportsVision is false:
// - Remove image_url content parts
// - Convert to text-only messages
// - Optionally add image descriptions from vision API
// If supportsVision is true:
// - Validate image URLs
// - Ensure proper format
return messages
}
// ConvertImageToText convert image content to text description
// Used when model doesn't support vision
func ConvertImageToText(imageURL string) (string, error) {
// TODO: Implement image to text conversion
// - Call vision API (if configured)
// - Generate description
// - Return as text content
return "", nil
}
// ValidateImageURL validate image URL format
func ValidateImageURL(imageURL string) error {
// TODO: Implement image URL validation
// - Check URL format
// - Validate image type
// - Check accessibility
return nil
}
// ExtractImagesFromMessages extract all image URLs from messages
func ExtractImagesFromMessages(messages []context.Message) []string {
// TODO: Implement image extraction
// - Iterate through messages
// - Find ContentPart with type="image_url"
// - Collect all image URLs
return nil
}

12
agent/llm/stream.go Normal file
View file

@ -0,0 +1,12 @@
package llm
import (
"github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/llm/handlers"
)
// DefaultStreamHandler creates a default stream handler
// This is a convenience function that wraps handlers.DefaultStreamHandler
func DefaultStreamHandler(ctx *context.Context) context.StreamFunc {
return handlers.DefaultStreamHandler(ctx)
}

4
agent/llm/types.go
View file

@ -1,4 +0,0 @@
package llm
// CompletionOptions the completion request
type CompletionOptions struct{}

11
agent/load.go
View file

@ -171,6 +171,17 @@ func initAssistant() error {
assistant.SetVision(api.Agent.DSL.Vision)
}
// Set global Uses configuration
if api.Agent.DSL.Use != nil {
globalUses := &store.Uses{
Vision: api.Agent.DSL.Use.Vision,
Audio: api.Agent.DSL.Use.Audio,
Search: api.Agent.DSL.Use.Search,
Fetch: api.Agent.DSL.Use.Fetch,
}
assistant.SetGlobalUses(globalUses)
}
if api.Agent.DSL.Connectors != nil {
assistant.SetConnectorSettings(api.Agent.DSL.Connectors)
}

9
agent/store/types/convert.go
View file

@ -173,6 +173,15 @@ func ToMySQLTime(v interface{}) string {
}
}
// Uses the wrapper configurations for assistant
// Used to specify which assistant or MCP server to use for vision, audio, etc.
type Uses struct {
Vision string `json:"vision,omitempty"` // Vision processing wrapper. Format: "agent" or "mcp:mcp_server_id"
Audio string `json:"audio,omitempty"` // Audio processing wrapper. Format: "agent" or "mcp:mcp_server_id"
Search string `json:"search,omitempty"` // Search wrapper. Format: "agent" or "mcp:mcp_server_id"
Fetch string `json:"fetch,omitempty"` // Fetch wrapper. Format: "agent" or "mcp:mcp_server_id"
}
// ToAssistantModel converts various types to AssistantModel
func ToAssistantModel(v interface{}) (*AssistantModel, error) {
if v == nil {

1
agent/store/types/types.go
View file

@ -155,6 +155,7 @@ type AssistantModel struct {
Workflow *Workflow `json:"workflow,omitempty"` // Workflow configuration
Placeholder *Placeholder `json:"placeholder,omitempty"` // Assistant Placeholder
Locales i18n.Map `json:"locales,omitempty"` // Assistant Locales
Uses *Uses `json:"uses,omitempty"` // Assistant-specific wrapper configurations for vision, audio, etc. If not set, use global settings
CreatedAt int64 `json:"created_at"` // Creation timestamp
UpdatedAt int64 `json:"updated_at"` // Last update timestamp

17
agent/store/xun/assistant.go
View file

@ -158,6 +158,7 @@ func (conv *Xun) SaveAssistant(assistant *types.AssistantModel) (string, error)
"tools": assistant.Tools,
"placeholder": assistant.Placeholder,
"locales": assistant.Locales,
"uses": assistant.Uses,
}
for field, value := range jsonFields {
@ -216,7 +217,7 @@ func (conv *Xun) UpdateAssistant(assistantID string, updates map[string]interfac
data := make(map[string]interface{})
// List of fields that need JSON marshaling
jsonFields := []string{"options", "tags", "prompts", "kb", "mcp", "workflow", "tools", "placeholder", "locales"}
jsonFields := []string{"options", "tags", "prompts", "kb", "mcp", "workflow", "tools", "placeholder", "locales", "uses"}
jsonFieldSet := make(map[string]bool)
for _, field := range jsonFields {
jsonFieldSet[field] = true
@ -416,7 +417,7 @@ func (conv *Xun) GetAssistants(filter types.AssistantFilter, locale ...string) (
// Convert rows to types.AssistantModel slice
assistants := make([]*types.AssistantModel, 0, len(rows))
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales"}
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales", "uses"}
for _, row := range rows {
data := row.ToMap()
@ -473,7 +474,7 @@ func (conv *Xun) GetAssistant(assistantID string, locale ...string) (*types.Assi
}
// Parse JSON fields
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales"}
jsonFields := []string{"tags", "options", "prompts", "workflow", "kb", "mcp", "tools", "placeholder", "locales", "uses"}
conv.parseJSONFields(data, jsonFields)
// Convert map to types.AssistantModel
@ -578,6 +579,16 @@ func (conv *Xun) GetAssistant(assistantID string, locale ...string) (*types.Assi
}
}
if uses, has := data["uses"]; has && uses != nil {
raw, err := jsoniter.Marshal(uses)
if err == nil {
var u types.Uses
if err := jsoniter.Unmarshal(raw, &u); err == nil {
model.Uses = &u
}
}
}
// Apply i18n translation if locale is provided
if len(locale) > 0 && locale[0] != "" {
conv.translate(model, assistantID, locale[0])

218
agent/store/xun/assistant_test.go
View file

@ -197,6 +197,125 @@ func TestSaveAssistant(t *testing.T) {
t.Errorf("Expected 3 tags, got %d", len(retrieved.Tags))
}
})
t.Run("UsesConfiguration", func(t *testing.T) {
// Test assistant with Uses configuration
assistant := &types.AssistantModel{
Name: "Uses Test Assistant",
Type: "assistant",
Connector: "openai",
Share: "private",
Uses: &types.Uses{
Vision: "mcp:vision-server",
Audio: "agent",
Search: "mcp:search-server",
Fetch: "agent",
},
}
id, err := store.SaveAssistant(assistant)
if err != nil {
t.Fatalf("Failed to save assistant with uses: %v", err)
}
// Retrieve and verify uses configuration
retrieved, err := store.GetAssistant(id)
if err != nil {
t.Fatalf("Failed to retrieve assistant: %v", err)
}
if retrieved.Uses == nil {
t.Fatal("Expected uses to be set")
}
if retrieved.Uses.Vision != "mcp:vision-server" {
t.Errorf("Expected vision 'mcp:vision-server', got '%s'", retrieved.Uses.Vision)
}
if retrieved.Uses.Audio != "agent" {
t.Errorf("Expected audio 'agent', got '%s'", retrieved.Uses.Audio)
}
if retrieved.Uses.Search != "mcp:search-server" {
t.Errorf("Expected search 'mcp:search-server', got '%s'", retrieved.Uses.Search)
}
if retrieved.Uses.Fetch != "agent" {
t.Errorf("Expected fetch 'agent', got '%s'", retrieved.Uses.Fetch)
}
t.Logf("Successfully saved and retrieved assistant with uses configuration")
})
t.Run("NilUses", func(t *testing.T) {
// Test assistant without Uses configuration
assistant := &types.AssistantModel{
Name: "No Uses Assistant",
Type: "assistant",
Connector: "openai",
Share: "private",
}
id, err := store.SaveAssistant(assistant)
if err != nil {
t.Fatalf("Failed to save assistant without uses: %v", err)
}
// Retrieve and verify uses is nil
retrieved, err := store.GetAssistant(id)
if err != nil {
t.Fatalf("Failed to retrieve assistant: %v", err)
}
if retrieved.Uses != nil {
t.Errorf("Expected uses to be nil, got %+v", retrieved.Uses)
}
})
t.Run("PartialUsesConfiguration", func(t *testing.T) {
// Test assistant with partial Uses configuration
assistant := &types.AssistantModel{
Name: "Partial Uses Assistant",
Type: "assistant",
Connector: "openai",
Share: "private",
Uses: &types.Uses{
Vision: "mcp:vision-only",
// Audio, Search, Fetch not set
},
}
id, err := store.SaveAssistant(assistant)
if err != nil {
t.Fatalf("Failed to save assistant with partial uses: %v", err)
}
// Retrieve and verify
retrieved, err := store.GetAssistant(id)
if err != nil {
t.Fatalf("Failed to retrieve assistant: %v", err)
}
if retrieved.Uses == nil {
t.Fatal("Expected uses to be set")
}
if retrieved.Uses.Vision != "mcp:vision-only" {
t.Errorf("Expected vision 'mcp:vision-only', got '%s'", retrieved.Uses.Vision)
}
if retrieved.Uses.Audio != "" {
t.Errorf("Expected audio to be empty, got '%s'", retrieved.Uses.Audio)
}
if retrieved.Uses.Search != "" {
t.Errorf("Expected search to be empty, got '%s'", retrieved.Uses.Search)
}
if retrieved.Uses.Fetch != "" {
t.Errorf("Expected fetch to be empty, got '%s'", retrieved.Uses.Fetch)
}
})
}
// TestDeleteAssistant tests deleting a single assistant
@ -1953,6 +2072,105 @@ func TestUpdateAssistant(t *testing.T) {
}
})
t.Run("UpdateUses", func(t *testing.T) {
// Create assistant without uses
assistant := &types.AssistantModel{
Name: "Uses Update Test",
Type: "assistant",
Connector: "openai",
Share: "private",
}
id, err := store.SaveAssistant(assistant)
if err != nil {
t.Fatalf("Failed to create assistant: %v", err)
}
// Update with uses configuration
updates := map[string]interface{}{
"uses": &types.Uses{
Vision: "mcp:new-vision",
Audio: "mcp:new-audio",
Search: "agent",
Fetch: "mcp:fetch-server",
},
}
err = store.UpdateAssistant(id, updates)
if err != nil {
t.Fatalf("Failed to update uses: %v", err)
}
// Verify updates
retrieved, err := store.GetAssistant(id)
if err != nil {
t.Fatalf("Failed to retrieve assistant: %v", err)
}
if retrieved.Uses == nil {
t.Fatal("Expected uses to be set")
}
if retrieved.Uses.Vision != "mcp:new-vision" {
t.Errorf("Expected vision 'mcp:new-vision', got '%s'", retrieved.Uses.Vision)
}
if retrieved.Uses.Audio != "mcp:new-audio" {
t.Errorf("Expected audio 'mcp:new-audio', got '%s'", retrieved.Uses.Audio)
}
if retrieved.Uses.Search != "agent" {
t.Errorf("Expected search 'agent', got '%s'", retrieved.Uses.Search)
}
if retrieved.Uses.Fetch != "mcp:fetch-server" {
t.Errorf("Expected fetch 'mcp:fetch-server', got '%s'", retrieved.Uses.Fetch)
}
// Update to change uses
updates2 := map[string]interface{}{
"uses": &types.Uses{
Vision: "agent",
Audio: "agent",
},
}
err = store.UpdateAssistant(id, updates2)
if err != nil {
t.Fatalf("Failed to update uses again: %v", err)
}
// Verify second update
retrieved2, err := store.GetAssistant(id)
if err != nil {
t.Fatalf("Failed to retrieve assistant: %v", err)
}
if retrieved2.Uses.Vision != "agent" {
t.Errorf("Expected vision 'agent', got '%s'", retrieved2.Uses.Vision)
}
if retrieved2.Uses.Audio != "agent" {
t.Errorf("Expected audio 'agent', got '%s'", retrieved2.Uses.Audio)
}
// Update to remove uses (set to nil)
updates3 := map[string]interface{}{
"uses": nil,
}
err = store.UpdateAssistant(id, updates3)
if err != nil {
t.Fatalf("Failed to set uses to nil: %v", err)
}
// Verify uses is nil
retrieved3, err := store.GetAssistant(id)
if err != nil {
t.Fatalf("Failed to retrieve assistant: %v", err)
}
if retrieved3.Uses != nil {
t.Errorf("Expected uses to be nil, got %+v", retrieved3.Uses)
}
})
t.Run("UpdatePermissionFields", func(t *testing.T) {
// Create assistant with permission fields
assistant := &types.AssistantModel{

3
agent/types/types.go
View file

@ -44,7 +44,8 @@ type Use struct {
Default string `json:"default,omitempty" yaml:"default,omitempty"` // The default assistant to use
Title string `json:"title,omitempty" yaml:"title,omitempty"` // The assistant for generating the topic title.
Prompt string `json:"prompt,omitempty" yaml:"prompt,omitempty"` // The assistant for generating the prompt.
Vision string `json:"vision,omitempty" yaml:"vision,omitempty"` // The assistant for generating the image/video description, if the assistant enable the vision and model not support vision, use the vision model to describe the image/video, and return the messages with the image/video's description.
Vision string `json:"vision,omitempty" yaml:"vision,omitempty"` // The assistant for generating the image/video description, if the assistant enable the vision and model not support vision, use the vision model to describe the image/video, and return the messages with the image/video's description. Format: "agent" or "mcp:mcp_server_id"
Audio string `json:"audio,omitempty" yaml:"audio,omitempty"` // The assistant for processing audio (speech-to-text, text-to-speech). If the model doesn't support audio, use this to convert audio to text. Format: "agent" or "mcp:mcp_server_id"
Search string `json:"search,omitempty" yaml:"search,omitempty"` // The assistant for searching the knowledge, global web search. If not set, and the assistant enable the knowledge, it will search the result from the knowledge automatically.
Fetch string `json:"fetch,omitempty" yaml:"fetch,omitempty"` // The assistant for fetching the http/https/ftp/sftp/etc. file, and return the file's content. if not set, use the http process to fetch the file.
}

284
data/bindata.go
View file

File diff suppressed because it is too large Load diff

7
yao/models/agent/assistant.mod.yao
View file

@ -184,6 +184,13 @@
"comment": "Assistant i18n locales",
"nullable": true
},
{
"name": "uses",
"type": "json",
"label": "Uses",
"comment": "Assistant-specific wrapper configurations for vision, audio, etc. If not set, use global settings",
"nullable": true
},
{
"name": "automated",
"type": "boolean",