diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index c4d3f53b..5bf37a28 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -49,6 +49,13 @@ env: DEEPSEEK_MODELS_V3: ${{ secrets.DEEPSEEK_MODELS_V3 }} DEEPSEEK_MODELS_V3_1: ${{ secrets.DEEPSEEK_MODELS_V3_1 }} + # Claude API Configuration + CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }} + CLAUDE_PROXY: ${{ secrets.CLAUDE_PROXY }} + CLAUDE_API_HOST: ${{ secrets.CLAUDE_API_HOST }} + CLAUDE_SONNET_4: ${{ secrets.CLAUDE_SONNET_4 }} + CLAUDE_SONNET_4_THINKING: ${{ secrets.CLAUDE_SONNET_4_THINKING }} + TAB_NAME: "::PET ADMIN" PAGE_SIZE: "20" PAGE_LINK: "https://yaoapps.com" diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 37fb2b10..e82d6772 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -53,6 +53,13 @@ env: DEEPSEEK_MODELS_V3: ${{ secrets.DEEPSEEK_MODELS_V3 }} DEEPSEEK_MODELS_V3_1: ${{ secrets.DEEPSEEK_MODELS_V3_1 }} + # Claude API Configuration + CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }} + CLAUDE_PROXY: ${{ secrets.CLAUDE_PROXY }} + CLAUDE_API_HOST: ${{ secrets.CLAUDE_API_HOST }} + CLAUDE_SONNET_4: ${{ secrets.CLAUDE_SONNET_4 }} + CLAUDE_SONNET_4_THINKING: ${{ secrets.CLAUDE_SONNET_4_THINKING }} + TAB_NAME: "::PET ADMIN" PAGE_SIZE: "20" PAGE_LINK: "https://yaoapps.com" diff --git a/agent/context/types_llm.go b/agent/context/types_llm.go index 354b31f8..211d4267 100644 --- a/agent/context/types_llm.go +++ b/agent/context/types_llm.go @@ -9,17 +9,61 @@ type Uses struct { Fetch string `json:"fetch,omitempty"` // Fetch/retrieval tool. Format: "agent" or "mcp:server_id" } +// VisionFormat specifies the vision input format +type VisionFormat string + +// Vision format constants define how image inputs are processed +const ( + // VisionFormatNone indicates no vision support + VisionFormatNone VisionFormat = "" + // VisionFormatOpenAI indicates OpenAI format (image_url with URL) + VisionFormatOpenAI VisionFormat = "openai" + // VisionFormatClaude indicates Claude/Anthropic format (image with base64) + VisionFormatClaude VisionFormat = "claude" + // VisionFormatBase64 forces base64 conversion (alias for claude) + VisionFormatBase64 VisionFormat = "base64" + // VisionFormatDefault enables auto-detection of format + VisionFormatDefault VisionFormat = "default" +) + // ModelCapabilities defines the capabilities of a language model // Used by LLM to select appropriate provider and validate requests type ModelCapabilities struct { - Vision *bool `json:"vision,omitempty"` // Supports vision/image input - ToolCalls *bool `json:"tool_calls,omitempty"` // Supports tool/function calling - Audio *bool `json:"audio,omitempty"` // Supports audio input/output - Reasoning *bool `json:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1) - Streaming *bool `json:"streaming,omitempty"` // Supports streaming responses - JSON *bool `json:"json,omitempty"` // Supports JSON mode - Multimodal *bool `json:"multimodal,omitempty"` // Supports multimodal input (text + images + audio) - TemperatureAdjustable *bool `json:"temperature_adjustable,omitempty"` // Supports temperature adjustment (reasoning models typically don't) + Vision interface{} `json:"vision,omitempty"` // Supports vision/image input: bool or VisionFormat string ("openai", "claude"/"base64", "default") + ToolCalls *bool `json:"tool_calls,omitempty"` // Supports tool/function calling + Audio *bool `json:"audio,omitempty"` // Supports audio input/output + Reasoning *bool `json:"reasoning,omitempty"` // Supports reasoning/thinking mode (o1, DeepSeek R1) + Streaming *bool `json:"streaming,omitempty"` // Supports streaming responses + JSON *bool `json:"json,omitempty"` // Supports JSON mode + Multimodal *bool `json:"multimodal,omitempty"` // Supports multimodal input (text + images + audio) + TemperatureAdjustable *bool `json:"temperature_adjustable,omitempty"` // Supports temperature adjustment (reasoning models typically don't) +} + +// GetVisionSupport returns whether vision is supported and the format +func (m *ModelCapabilities) GetVisionSupport() (bool, VisionFormat) { + if m == nil || m.Vision == nil { + return false, VisionFormatNone + } + + switch v := m.Vision.(type) { + case bool: + // Legacy bool format + return v, VisionFormatDefault + case string: + // String format + if v == "" || v == string(VisionFormatNone) { + return false, VisionFormatNone + } + return true, VisionFormat(v) + case VisionFormat: + // Direct VisionFormat type + if v == VisionFormatNone || v == "" { + return false, VisionFormatNone + } + return true, v + default: + return false, VisionFormatNone + } } // CompletionOptions the completion request options diff --git a/agent/llm/adapters/vision.go b/agent/llm/adapters/vision.go index 0e746a65..b0a5ff0b 100644 --- a/agent/llm/adapters/vision.go +++ b/agent/llm/adapters/vision.go @@ -1,6 +1,13 @@ package adapters import ( + "encoding/base64" + "fmt" + "io" + "net/http" + "strings" + "time" + "github.com/yaoapp/yao/agent/context" ) @@ -9,47 +16,64 @@ import ( type VisionAdapter struct { *BaseAdapter nativeSupport bool + format context.VisionFormat } // NewVisionAdapter creates a new vision adapter -func NewVisionAdapter(nativeSupport bool) *VisionAdapter { +func NewVisionAdapter(nativeSupport bool, format context.VisionFormat) *VisionAdapter { return &VisionAdapter{ BaseAdapter: NewBaseAdapter("VisionAdapter"), nativeSupport: nativeSupport, + format: format, } } // PreprocessMessages removes or converts image content if not supported func (a *VisionAdapter) PreprocessMessages(messages []context.Message) ([]context.Message, error) { - if a.nativeSupport { - // Native support, no preprocessing needed + if !a.nativeSupport { + // No vision support, remove image content + return a.removeImageContent(messages), nil + } + + // Check if we need to convert format + needsConversion := a.format == context.VisionFormatClaude || a.format == context.VisionFormatBase64 + + if !needsConversion { + // Native support with OpenAI format or default, no preprocessing needed return messages, nil } - // Process messages to remove image content + // Convert image_url format to Claude base64 format + return a.convertToBase64Format(messages) +} + +// removeImageContent removes image content from messages +func (a *VisionAdapter) removeImageContent(messages []context.Message) []context.Message { processed := make([]context.Message, 0, len(messages)) for _, msg := range messages { processedMsg := msg - // Handle multimodal content (array of ContentPart) - if contentParts, ok := msg.Content.([]context.ContentPart); ok { - filteredParts := make([]context.ContentPart, 0) + // Handle multimodal content (array of map) + if contentParts, ok := msg.Content.([]map[string]interface{}); ok { + filteredParts := make([]map[string]interface{}, 0) for _, part := range contentParts { - // Skip image content if not supported - if part.Type == context.ContentImageURL { - // TODO: Optionally convert to text description - continue + partType, _ := part["type"].(string) + // Skip image content + if partType != "image_url" && partType != "image" { + filteredParts = append(filteredParts, part) } - filteredParts = append(filteredParts, part) } // If all parts were filtered out, add placeholder text if len(filteredParts) == 0 { processedMsg.Content = "[Image content not supported by this model]" - } else if len(filteredParts) == 1 && filteredParts[0].Type == context.ContentText { - // Single text part, convert to string - processedMsg.Content = filteredParts[0].Text + } else if len(filteredParts) == 1 { + if textVal, ok := filteredParts[0]["text"].(string); ok { + processedMsg.Content = textVal + } else { + processedMsg.Content = filteredParts + } } else { processedMsg.Content = filteredParts } @@ -58,5 +82,154 @@ func (a *VisionAdapter) PreprocessMessages(messages []context.Message) ([]contex processed = append(processed, processedMsg) } + return processed +} + +// convertToBase64Format converts image_url format to Claude base64 format +func (a *VisionAdapter) convertToBase64Format(messages []context.Message) ([]context.Message, error) { + processed := make([]context.Message, 0, len(messages)) + + for _, msg := range messages { + processedMsg := msg + + // Handle multimodal content + if contentParts, ok := msg.Content.([]map[string]interface{}); ok { + convertedParts := make([]map[string]interface{}, 0) + + for _, part := range contentParts { + partType, _ := part["type"].(string) + + if partType == "image_url" { + // Convert to base64 format + convertedPart, err := a.convertImageURLToBase64(part) + if err != nil { + // If conversion fails, skip this image + continue + } + convertedParts = append(convertedParts, convertedPart) + } else { + // Keep non-image parts as-is + convertedParts = append(convertedParts, part) + } + } + + processedMsg.Content = convertedParts + } + + processed = append(processed, processedMsg) + } + return processed, nil } + +// convertImageURLToBase64 converts OpenAI image_url format to Claude base64 format +func (a *VisionAdapter) convertImageURLToBase64(part map[string]interface{}) (map[string]interface{}, error) { + // Extract URL from image_url object + imageURLObj, ok := part["image_url"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid image_url format") + } + + url, ok := imageURLObj["url"].(string) + if !ok || url == "" { + return nil, fmt.Errorf("missing or invalid URL in image_url") + } + + // Check if already base64 data URL + if strings.HasPrefix(url, "data:") { + // Extract media type and base64 data from data URL + // Format: data:image/jpeg;base64, + parts := strings.SplitN(url, ",", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid data URL format") + } + + // Extract media type from first part + mediaParts := strings.Split(parts[0], ";") + mediaType := strings.TrimPrefix(mediaParts[0], "data:") + base64Data := parts[1] + + return map[string]interface{}{ + "type": "image", + "source": map[string]interface{}{ + "type": "base64", + "media_type": mediaType, + "data": base64Data, + }, + }, nil + } + + // Download image from URL and convert to base64 + base64Data, mediaType, err := a.downloadAndEncodeImage(url) + if err != nil { + return nil, fmt.Errorf("failed to download image: %w", err) + } + + // Return Claude/Anthropic format + return map[string]interface{}{ + "type": "image", + "source": map[string]interface{}{ + "type": "base64", + "media_type": mediaType, + "data": base64Data, + }, + }, nil +} + +// downloadAndEncodeImage downloads an image from URL and returns base64 encoded data +func (a *VisionAdapter) downloadAndEncodeImage(url string) (string, string, error) { + // Create HTTP client with timeout + client := &http.Client{ + Timeout: 30 * time.Second, + } + + // Download image + resp, err := client.Get(url) + if err != nil { + return "", "", fmt.Errorf("failed to download image: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return "", "", fmt.Errorf("failed to download image: HTTP %d", resp.StatusCode) + } + + // Read image data + imageData, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", fmt.Errorf("failed to read image data: %w", err) + } + + // Detect media type from Content-Type header + mediaType := resp.Header.Get("Content-Type") + + // Normalize media type (remove charset and other parameters) + if mediaType != "" { + // Split by semicolon to remove parameters like "; charset=utf-8" + if idx := strings.Index(mediaType, ";"); idx != -1 { + mediaType = strings.TrimSpace(mediaType[:idx]) + } + } + + if mediaType == "" { + // Fallback to detecting from URL extension or default to jpeg + urlLower := strings.ToLower(url) + if strings.HasSuffix(urlLower, ".png") { + mediaType = "image/png" + } else if strings.HasSuffix(urlLower, ".gif") { + mediaType = "image/gif" + } else if strings.HasSuffix(urlLower, ".webp") { + mediaType = "image/webp" + } else if strings.Contains(urlLower, ".jpg") || strings.Contains(urlLower, ".jpeg") { + mediaType = "image/jpeg" + } else { + // Default to jpeg + mediaType = "image/jpeg" + } + } + + // Encode to base64 + base64Data := base64.StdEncoding.EncodeToString(imageData) + + return base64Data, mediaType, nil +} diff --git a/agent/llm/providers/base/base.go b/agent/llm/providers/base/base.go index dee1f889..1f281888 100644 --- a/agent/llm/providers/base/base.go +++ b/agent/llm/providers/base/base.go @@ -71,7 +71,11 @@ func (p *Provider) PreprocessMessages(messages []context.Message) ([]context.Mes // SupportsVision check if this provider supports vision func (p *Provider) SupportsVision() bool { - return p.Capabilities != nil && p.Capabilities.Vision != nil && *p.Capabilities.Vision + if p.Capabilities == nil { + return false + } + supported, _ := p.Capabilities.GetVisionSupport() + return supported } // SupportsAudio check if this provider supports audio diff --git a/agent/llm/providers/openai/claude_test.go b/agent/llm/providers/openai/claude_test.go new file mode 100644 index 00000000..63ad1819 --- /dev/null +++ b/agent/llm/providers/openai/claude_test.go @@ -0,0 +1,593 @@ +package openai_test + +import ( + gocontext "context" + "testing" + + "github.com/yaoapp/gou/connector" + "github.com/yaoapp/gou/plan" + "github.com/yaoapp/yao/agent/context" + "github.com/yaoapp/yao/agent/llm" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/openapi/oauth/types" + "github.com/yaoapp/yao/test" +) + +// newClaudeTestContext creates a real Context for testing Claude provider +func newClaudeTestContext(chatID, connectorID string) *context.Context { + return &context.Context{ + Context: gocontext.Background(), + Space: plan.NewMemorySharedSpace(), + ChatID: chatID, + AssistantID: "test-assistant", + Connector: connectorID, + Locale: "en-us", + Theme: "light", + Client: context.Client{ + Type: "web", + UserAgent: "ClaudeProviderTest/1.0", + IP: "127.0.0.1", + }, + Referer: context.RefererAPI, + Accept: context.AcceptStandard, + Route: "/api/test", + Metadata: make(map[string]interface{}), + Authorized: &types.AuthorizedInfo{ + Subject: "test-user", + ClientID: "test-client", + UserID: "test-user-123", + TeamID: "test-team-456", + TenantID: "test-tenant-789", + SessionID: "test-session-id", + Constraints: types.DataConstraints{ + TeamOnly: true, + Extra: map[string]interface{}{ + "test": "claude-provider", + }, + }, + }, + } +} + +// TestClaudeSonnet4StreamBasic tests basic streaming completion with Claude Sonnet 4 +func TestClaudeSonnet4StreamBasic(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + conn, err := connector.Select("claude.sonnet-4_0") + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &trueVal, + Reasoning: &falseVal, // Claude Sonnet 4 (non-thinking) doesn't expose reasoning + ToolCalls: &trueVal, + Vision: &trueVal, + Multimodal: &trueVal, + }, + } + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: "What is 3+3? Reply with just the number.", + }, + } + + maxTokens := 100 + options.MaxTokens = &maxTokens + + ctx := newClaudeTestContext("test-claude-sonnet4-basic", "claude.sonnet-4_0") + + var chunks []string + handler := func(chunkType context.StreamChunkType, data []byte) int { + chunks = append(chunks, string(data)) + t.Logf("Stream chunk [%s]: %s", chunkType, string(data)) + return 0 + } + + response, err := llmInstance.Stream(ctx, messages, options, handler) + if err != nil { + t.Fatalf("Stream failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + // Basic validation + if response.ID == "" { + t.Error("Response ID is empty") + } + if response.Model == "" { + t.Error("Response Model is empty") + } + + // Validate content + contentStr, ok := response.Content.(string) + if !ok { + t.Errorf("Content is not a string: %T", response.Content) + } + if len(contentStr) == 0 { + t.Error("Content is empty") + } + + t.Logf("Response content: %v", response.Content) + t.Logf("Usage: prompt=%d, completion=%d, total=%d", + response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens) + + t.Logf("Final response: %+v", response) + t.Logf("Total chunks received: %d", len(chunks)) +} + +// TestClaudeSonnet4PostBasic tests non-streaming completion +func TestClaudeSonnet4PostBasic(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + conn, err := connector.Select("claude.sonnet-4_0") + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &falseVal, + Reasoning: &falseVal, + ToolCalls: &trueVal, + Vision: &trueVal, + Multimodal: &trueVal, + }, + } + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: "What is 4+4? Reply with just the number.", + }, + } + + maxTokens := 100 + options.MaxTokens = &maxTokens + + ctx := newClaudeTestContext("test-claude-sonnet4-post", "claude.sonnet-4_0") + + response, err := llmInstance.Post(ctx, messages, options) + if err != nil { + t.Fatalf("Post failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + // Validate content + contentStr, ok := response.Content.(string) + if !ok { + t.Fatalf("Content is not a string: %T", response.Content) + } + + t.Logf("Response content: %s", contentStr) + t.Logf("Usage: prompt=%d, completion=%d, total=%d", + response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens) + + // Basic content validation + if len(contentStr) == 0 { + t.Error("Content is empty") + } + + t.Logf("Response: %+v", response) +} + +// TestClaudeSonnet4WithToolCalls tests tool calling capability +func TestClaudeSonnet4WithToolCalls(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + conn, err := connector.Select("claude.sonnet-4_0") + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &falseVal, + Reasoning: &falseVal, + ToolCalls: &trueVal, + Vision: &trueVal, + Multimodal: &trueVal, + }, + } + + // Define a simple tool with minimal parameters + simpleTool := map[string]interface{}{ + "type": "function", + "function": map[string]interface{}{ + "name": "get_info", + "description": "Get information", + "parameters": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "query": map[string]interface{}{ + "type": "string", + "description": "Query string (single letter)", + }, + "count": map[string]interface{}{ + "type": "number", + "description": "Count (single digit)", + }, + }, + "required": []string{"query", "count"}, + }, + }, + } + + options.Tools = []map[string]interface{}{simpleTool} + options.ToolChoice = "auto" + + // Set lower max_tokens for faster response + maxTokens := 50 + options.MaxTokens = &maxTokens + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: "Call get_info with query='A' and count=1", + }, + } + + ctx := newClaudeTestContext("test-claude-sonnet4-tools", "claude.sonnet-4_0") + + response, err := llmInstance.Post(ctx, messages, options) + if err != nil { + t.Fatalf("Post failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + // Validate tool calls + if len(response.ToolCalls) == 0 { + t.Error("No tool calls in response") + } else { + tc := response.ToolCalls[0] + t.Logf("✓ Tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments) + + if tc.Function.Name != "get_info" { + t.Errorf("Expected tool name 'get_info', got '%s'", tc.Function.Name) + } + } + + t.Logf("Usage: prompt=%d, completion=%d, total=%d", + response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens) + + t.Logf("Response: %+v", response) +} + +// TestClaudeSonnet4Vision tests vision capability with image input +func TestClaudeSonnet4Vision(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + conn, err := connector.Select("claude.sonnet-4_0") + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &falseVal, + Reasoning: &falseVal, + ToolCalls: &trueVal, + Vision: &trueVal, + Multimodal: &trueVal, + }, + } + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + // Use a test image URL + imageURL := "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: []map[string]interface{}{ + { + "type": "text", + "text": "Describe this image in one sentence.", + }, + { + "type": "image_url", + "image_url": map[string]string{ + "url": imageURL, + }, + }, + }, + }, + } + + maxTokens := 150 + options.MaxTokens = &maxTokens + + ctx := newClaudeTestContext("test-claude-sonnet4-vision", "claude.sonnet-4_0") + + response, err := llmInstance.Post(ctx, messages, options) + if err != nil { + t.Fatalf("Post failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + // Validate content + contentStr, ok := response.Content.(string) + if !ok { + t.Fatalf("Content is not a string: %T", response.Content) + } + + if len(contentStr) == 0 { + t.Error("Image description is empty") + } + + t.Logf("Image description: %s", contentStr) + t.Logf("Usage: prompt=%d, completion=%d, total=%d", + response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens) +} + +// TestClaudeSonnet4ThinkingStream tests Claude Sonnet 4 Thinking with streaming +func TestClaudeSonnet4ThinkingStream(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + conn, err := connector.Select("claude.sonnet-4_0-thinking") + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &trueVal, + Reasoning: &trueVal, // Claude Thinking mode exposes reasoning + ToolCalls: &falseVal, + Vision: &trueVal, + Multimodal: &trueVal, + }, + } + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: "If Sally has 3 apples and gives 2 to John, how many does she have left? Think through this step by step.", + }, + } + + maxTokens := 500 + options.MaxTokens = &maxTokens + + ctx := newClaudeTestContext("test-claude-thinking-stream", "claude.sonnet-4_0-thinking") + + var thinkingChunks []string + var textChunks []string + handler := func(chunkType context.StreamChunkType, data []byte) int { + t.Logf("Stream chunk [%s]: %s", chunkType, string(data)) + if chunkType == context.ChunkThinking { + thinkingChunks = append(thinkingChunks, string(data)) + } else if chunkType == context.ChunkText { + textChunks = append(textChunks, string(data)) + } + return 0 + } + + response, err := llmInstance.Stream(ctx, messages, options, handler) + if err != nil { + t.Fatalf("Stream failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + // Validate response + contentStr, ok := response.Content.(string) + if !ok { + t.Errorf("Content is not a string: %T", response.Content) + } + + t.Logf("Reasoning/Thinking content length: %d characters", len(response.ReasoningContent)) + t.Logf("Response content: %v", contentStr) + t.Logf("Usage: prompt=%d, completion=%d, total=%d", + response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens) + + if response.Usage != nil && response.Usage.CompletionTokensDetails != nil { + t.Logf("Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens) + } + + t.Logf("Received %d thinking chunks", len(thinkingChunks)) + t.Logf("Received %d text chunks", len(textChunks)) + t.Logf("Final response: %+v", response) +} + +// TestClaudeSonnet4ThinkingPost tests Claude Sonnet 4 Thinking in non-streaming mode +func TestClaudeSonnet4ThinkingPost(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + conn, err := connector.Select("claude.sonnet-4_0-thinking") + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &falseVal, + Reasoning: &trueVal, + ToolCalls: &falseVal, + Vision: &trueVal, + Multimodal: &trueVal, + }, + } + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: "Is 7 greater than 5? Explain your reasoning.", + }, + } + + maxTokens := 500 + options.MaxTokens = &maxTokens + + ctx := newClaudeTestContext("test-claude-thinking-post", "claude.sonnet-4_0-thinking") + + response, err := llmInstance.Post(ctx, messages, options) + if err != nil { + t.Fatalf("Post failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + // Validate content + contentStr, ok := response.Content.(string) + if !ok { + t.Fatalf("Content is not a string: %T", response.Content) + } + + t.Logf("Reasoning content: %s", response.ReasoningContent) + t.Logf("Final answer: %s", contentStr) + + // Check for reasoning content + if len(response.ReasoningContent) > 0 { + t.Logf("✓ Reasoning content present: %d characters", len(response.ReasoningContent)) + } + + if response.Usage != nil && response.Usage.CompletionTokensDetails != nil && response.Usage.CompletionTokensDetails.ReasoningTokens > 0 { + t.Logf("✓ Reasoning tokens: %d", response.Usage.CompletionTokensDetails.ReasoningTokens) + } + + t.Logf("Usage: prompt=%d, completion=%d, total=%d", + response.Usage.PromptTokens, response.Usage.CompletionTokens, response.Usage.TotalTokens) + + t.Logf("Response: %+v", response) +} + +// TestClaudeTemperatureHandling tests that Claude models handle temperature parameter correctly +func TestClaudeTemperatureHandling(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + tests := []struct { + name string + connector string + temperature float64 + reasoning bool + }{ + { + name: "Sonnet 4 with temperature 0.7", + connector: "claude.sonnet-4_0", + temperature: 0.7, + reasoning: false, + }, + { + name: "Sonnet 4 Thinking with temperature 0.5", + connector: "claude.sonnet-4_0-thinking", + temperature: 0.5, + reasoning: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := connector.Select(tt.connector) + if err != nil { + t.Fatalf("Failed to select connector: %v", err) + } + + trueVal := true + falseVal := false + options := &context.CompletionOptions{ + Capabilities: &context.ModelCapabilities{ + Streaming: &falseVal, + Reasoning: &tt.reasoning, + ToolCalls: &trueVal, + Vision: &trueVal, + }, + } + + llmInstance, err := llm.New(conn, options) + if err != nil { + t.Fatalf("Failed to create LLM instance: %v", err) + } + + messages := []context.Message{ + { + Role: context.RoleUser, + Content: "Say 'hello'.", + }, + } + + maxTokens := 50 + options.MaxTokens = &maxTokens + options.Temperature = &tt.temperature + + ctx := newClaudeTestContext("test-claude-temp-"+tt.connector, tt.connector) + + response, err := llmInstance.Post(ctx, messages, options) + if err != nil { + t.Fatalf("Post failed: %v", err) + } + + if response == nil { + t.Fatal("Response is nil") + } + + t.Logf("✓ %s completed successfully with temperature=%.1f", tt.name, tt.temperature) + }) + } +} diff --git a/agent/llm/providers/openai/deepseek_v3_test.go b/agent/llm/providers/openai/deepseek_v3_test.go index 9ee3b27e..793e9fba 100644 --- a/agent/llm/providers/openai/deepseek_v3_test.go +++ b/agent/llm/providers/openai/deepseek_v3_test.go @@ -234,32 +234,36 @@ func TestDeepSeekV3WithToolCalls(t *testing.T) { }, } - // Define a weather tool - weatherTool := map[string]interface{}{ + // Define a simple tool with minimal parameters + simpleTool := map[string]interface{}{ "type": "function", "function": map[string]interface{}{ - "name": "get_weather", - "description": "Get current weather for a location", + "name": "get_info", + "description": "Get information", "parameters": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ - "location": map[string]interface{}{ + "query": map[string]interface{}{ "type": "string", - "description": "City name", + "description": "Query string (single letter)", }, - "unit": map[string]interface{}{ - "type": "string", - "enum": []string{"celsius", "fahrenheit"}, + "count": map[string]interface{}{ + "type": "number", + "description": "Count (single digit)", }, }, - "required": []string{"location"}, + "required": []string{"query", "count"}, }, }, } - options.Tools = []map[string]interface{}{weatherTool} + options.Tools = []map[string]interface{}{simpleTool} options.ToolChoice = "auto" + // Set lower max_tokens for faster response + maxTokens := 50 + options.MaxTokens = &maxTokens + llmInstance, err := llm.New(conn, options) if err != nil { t.Fatalf("Failed to create LLM instance: %v", err) @@ -268,7 +272,7 @@ func TestDeepSeekV3WithToolCalls(t *testing.T) { messages := []context.Message{ { Role: context.RoleUser, - Content: "What's the weather in Beijing?", + Content: "Call get_info with query='A' and count=1", }, } @@ -290,8 +294,8 @@ func TestDeepSeekV3WithToolCalls(t *testing.T) { tc := response.ToolCalls[0] t.Logf("✓ Tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments) - if tc.Function.Name != "get_weather" { - t.Errorf("Expected tool name 'get_weather', got '%s'", tc.Function.Name) + if tc.Function.Name != "get_info" { + t.Errorf("Expected tool name 'get_info', got '%s'", tc.Function.Name) } } @@ -332,11 +336,11 @@ func TestDeepSeekV3NoReasoningEffort(t *testing.T) { messages := []context.Message{ { Role: context.RoleUser, - Content: "Say 'OK'", + Content: "Reply with just: OK", }, } - maxTokens := 10 + maxTokens := 20 options.MaxTokens = &maxTokens ctx := newDeepSeekV3TestContext("test-deepseek-v3-no-reasoning", "deepseek.v3") diff --git a/agent/llm/providers/openai/openai.go b/agent/llm/providers/openai/openai.go index 12ad6899..11f3c9f4 100644 --- a/agent/llm/providers/openai/openai.go +++ b/agent/llm/providers/openai/openai.go @@ -110,6 +110,19 @@ type Provider struct { adapters []adapters.CapabilityAdapter } +// buildAPIURL builds the complete API URL from host and endpoint +// If host ends with /, it's used as-is (user has specified full path) +// Otherwise, /v1 prefix is added automatically (standard for OpenAI-compatible APIs) +func buildAPIURL(host, endpoint string) string { + // If host ends with /, use it as-is (user has specified full path like /v1/ or /api/) + // Otherwise, add /v1 prefix (standard for OpenAI-compatible APIs) + if !strings.HasSuffix(host, "/") { + endpoint = "/v1" + endpoint + } + host = strings.TrimSuffix(host, "/") + return host + endpoint +} + // New create a new OpenAI provider with capability adapters func New(conn connector.Connector, capabilities *context.ModelCapabilities) *Provider { return &Provider{ @@ -132,8 +145,12 @@ func buildAdapters(cap *context.ModelCapabilities) []adapters.CapabilityAdapter } // Vision adapter - if cap.Vision != nil { - result = append(result, adapters.NewVisionAdapter(*cap.Vision)) + visionSupport, visionFormat := cap.GetVisionSupport() + if visionSupport { + result = append(result, adapters.NewVisionAdapter(true, visionFormat)) + } else if cap.Vision != nil { + // Vision explicitly disabled, add adapter to remove image content + result = append(result, adapters.NewVisionAdapter(false, context.VisionFormatNone)) } // Audio adapter @@ -288,22 +305,34 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess } } - // Preprocess options through adapters + // Preprocess messages and options through adapters + processedMessages := messages processedOptions := options for _, adapter := range p.adapters { + // Preprocess messages + newMessages, err := adapter.PreprocessMessages(processedMessages) + if err != nil { + if handler != nil { + handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s message preprocessing failed: %v", adapter.Name(), err))) + } + return nil, fmt.Errorf("adapter %s message preprocessing failed: %w", adapter.Name(), err) + } + processedMessages = newMessages + + // Preprocess options newOpts, err := adapter.PreprocessOptions(processedOptions) if err != nil { // Send error to handler if handler != nil { - handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s preprocessing failed: %v", adapter.Name(), err))) + handler(context.ChunkError, []byte(fmt.Sprintf("adapter %s option preprocessing failed: %v", adapter.Name(), err))) } - return nil, fmt.Errorf("adapter %s preprocessing failed: %w", adapter.Name(), err) + return nil, fmt.Errorf("adapter %s option preprocessing failed: %w", adapter.Name(), err) } processedOptions = newOpts } // Build request body - requestBody, err := p.buildRequestBody(messages, processedOptions, true) + requestBody, err := p.buildRequestBody(processedMessages, processedOptions, true) if err != nil { // Send stream_end with error if handler != nil { @@ -334,12 +363,7 @@ func (p *Provider) streamWithRetry(ctx *context.Context, messages []context.Mess } // Build URL - endpoint := "/chat/completions" - if host == "https://api.openai.com" && !strings.HasPrefix(endpoint, "/v1") { - endpoint = "/v1" + endpoint - } - host = strings.TrimSuffix(host, "/") - url := host + endpoint + url := buildAPIURL(host, "/chat/completions") // Create HTTP request with proxy support req := http.New(url). @@ -836,18 +860,27 @@ func (p *Provider) Post(ctx *context.Context, messages []context.Message, option // postWithRetry performs a single POST request attempt func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Message, options *context.CompletionOptions) (*context.CompletionResponse, error) { - // Preprocess options through adapters + // Preprocess messages and options through adapters + processedMessages := messages processedOptions := options for _, adapter := range p.adapters { + // Preprocess messages + newMessages, err := adapter.PreprocessMessages(processedMessages) + if err != nil { + return nil, fmt.Errorf("adapter %s message preprocessing failed: %w", adapter.Name(), err) + } + processedMessages = newMessages + + // Preprocess options newOpts, err := adapter.PreprocessOptions(processedOptions) if err != nil { - return nil, fmt.Errorf("adapter %s preprocessing failed: %w", adapter.Name(), err) + return nil, fmt.Errorf("adapter %s option preprocessing failed: %w", adapter.Name(), err) } processedOptions = newOpts } // Build request body - requestBody, err := p.buildRequestBody(messages, processedOptions, false) + requestBody, err := p.buildRequestBody(processedMessages, processedOptions, false) if err != nil { return nil, fmt.Errorf("failed to build request body: %w", err) } @@ -865,12 +898,7 @@ func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Messag } // Build URL - endpoint := "/chat/completions" - if host == "https://api.openai.com" && !strings.HasPrefix(endpoint, "/v1") { - endpoint = "/v1" + endpoint - } - host = strings.TrimSuffix(host, "/") - url := host + endpoint + url := buildAPIURL(host, "/chat/completions") // Create HTTP request with proxy support req := http.New(url). @@ -880,7 +908,24 @@ func (p *Provider) postWithRetry(ctx *context.Context, messages []context.Messag // Make request resp := req.Post(requestBody) if resp.Code != 200 { - return nil, fmt.Errorf("HTTP %d: %s", resp.Code, resp.Message) + // Try to get detailed error message from response + errorMsg := resp.Message + if resp.Data != nil { + if errorData, ok := resp.Data.(map[string]interface{}); ok { + if errObj, ok := errorData["error"]; ok { + if errMap, ok := errObj.(map[string]interface{}); ok { + if msg, ok := errMap["message"].(string); ok { + errorMsg = msg + } + } + } + } + // Log full response data for debugging + if respJSON, err := jsoniter.Marshal(resp.Data); err == nil { + log.Error("OpenAI API error response: %s", string(respJSON)) + } + } + return nil, fmt.Errorf("HTTP %d: %s", resp.Code, errorMsg) } // Parse response