opencode/ocgo
1
0
Fork 1
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

Fix image input support

Browse source
This commit is contained in:
Emanuel Casco 2026-05-08 21:04:23 +02:00
parent 1ecb921068
commit ba84a9c522
2 changed files with 382 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

269
cmd/ocgo/main.go
View file

@ -89,12 +89,23 @@ type ResponseTool struct {
type OAIMessage struct {
Role string `json:"role"`
Content string `json:"content,omitempty"`
Content any `json:"content,omitempty"`
ToolCalls []OAIToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
ReasoningContent string `json:"reasoning_content,omitempty"`
}
type OAIContentPart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *OAIImageURL `json:"image_url,omitempty"`
}
type OAIImageURL struct {
URL string `json:"url"`
Detail string `json:"detail,omitempty"`
}
type OAITool struct {
Type string `json:"type"`
Function OAIFunction `json:"function"`
@ -171,6 +182,22 @@ func knownModelIDs() []string {
return []string{"glm-5.1", "glm-5", "kimi-k2.6", "kimi-k2.5", "mimo-v2.5-pro", "mimo-v2.5", "mimo-v2-pro", "mimo-v2-omni", "minimax-m2.7", "minimax-m2.5", "deepseek-v4-pro", "deepseek-v4-flash", "qwen3.6-plus", "qwen3.5-plus"}
}
func modelSupportsImages(model string) bool {
switch model {
case "kimi-k2.6", "kimi-k2.5", "mimo-v2-omni":
return true
default:
return false
}
}
func modelInputModalities(model string) []string {
if modelSupportsImages(model) {
return []string{"text", "image"}
}
return []string{"text"}
}
func launchCmd() *cobra.Command {
var model string
var yes bool
@ -334,6 +361,10 @@ func proxyMessages(w http.ResponseWriter, r *http.Request, cfg Config) {
return
}
or := convertRequest(ar)
if err := validateImageSupport(or); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
body, _ := json.Marshal(or)
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, openAIURL, bytes.NewReader(body))
if err != nil {
@ -370,6 +401,11 @@ func proxyChatCompletions(w http.ResponseWriter, r *http.Request, cfg Config) {
http.Error(w, "invalid request", http.StatusBadRequest)
return
}
body, err = prepareChatBody(body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, openAIURL, bytes.NewReader(body))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -399,6 +435,10 @@ func proxyResponses(w http.ResponseWriter, r *http.Request, cfg Config) {
return
}
or := responsesToChat(rr)
if err := validateImageSupport(or); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
body, _ := json.Marshal(or)
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, openAIURL, bytes.NewReader(body))
if err != nil {
@ -433,6 +473,88 @@ func copyHeaders(dst, src http.Header) {
}
}
func prepareChatBody(body []byte) ([]byte, error) {
var req map[string]any
if json.Unmarshal(body, &req) != nil {
return body, nil
}
model, _ := req["model"].(string)
if !rawChatBodyHasImages(req) {
return body, nil
}
if !modelSupportsImages(model) {
return nil, unsupportedImageModelError(model)
}
changed := stripRawChatImageDetails(req)
if !changed {
return body, nil
}
out, err := json.Marshal(req)
if err != nil {
return body, nil
}
return out, nil
}
func rawChatBodyHasImages(req map[string]any) bool {
messages, _ := req["messages"].([]any)
for _, item := range messages {
msg, ok := item.(map[string]any)
if !ok {
continue
}
if contentHasImage(msg["content"]) {
return true
}
}
return false
}
func validateImageSupport(or OAIRequest) error {
if requestHasImages(or) && !modelSupportsImages(or.Model) {
return unsupportedImageModelError(or.Model)
}
return nil
}
func unsupportedImageModelError(model string) error {
if model == "" {
model = "unknown"
}
return fmt.Errorf("model %s does not support image inputs", model)
}
func stripRawChatImageDetails(req map[string]any) bool {
changed := false
messages, _ := req["messages"].([]any)
for _, item := range messages {
msg, ok := item.(map[string]any)
if !ok {
continue
}
parts, _ := msg["content"].([]any)
for _, part := range parts {
p, ok := part.(map[string]any)
if !ok {
continue
}
if _, ok := p["detail"]; ok {
delete(p, "detail")
changed = true
}
image, ok := p["image_url"].(map[string]any)
if !ok {
continue
}
if _, ok := image["detail"]; ok {
delete(image, "detail")
changed = true
}
}
}
return changed
}
func convertRequest(ar AnthropicRequest) OAIRequest {
model := ar.Model
if model == "" || strings.HasPrefix(model, "claude-") {
@ -469,6 +591,37 @@ func responsesToChat(rr ResponsesRequest) OAIRequest {
return out
}
func requestHasImages(or OAIRequest) bool {
for _, m := range or.Messages {
if contentHasImage(m.Content) {
return true
}
}
return false
}
func contentHasImage(content any) bool {
switch v := content.(type) {
case []OAIContentPart:
for _, part := range v {
if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" {
return true
}
}
case []any:
for _, item := range v {
m, ok := item.(map[string]any)
if !ok {
continue
}
if typ, _ := m["type"].(string); typ == "image_url" || typ == "input_image" {
return true
}
}
}
return false
}
func responsesInputToMessages(raw json.RawMessage) []OAIMessage {
if len(raw) == 0 {
return nil
@ -495,7 +648,7 @@ func responsesInputToMessages(raw json.RawMessage) []OAIMessage {
if role == "" {
role = "user"
}
out = append(out, OAIMessage{Role: role, Content: responsesContentText(item["content"])})
out = append(out, OAIMessage{Role: role, Content: responsesContent(item["content"])})
case "function_call":
var id, callID, name, args string
_ = json.Unmarshal(item["id"], &id)
@ -557,6 +710,66 @@ func cacheReasoningContent(calls []OAIToolCall, reasoning string) {
}
}
func responsesContent(raw json.RawMessage) any {
if len(raw) == 0 {
return ""
}
var s string
if json.Unmarshal(raw, &s) == nil {
return s
}
var parts []map[string]json.RawMessage
if json.Unmarshal(raw, &parts) != nil {
return string(raw)
}
var text strings.Builder
var out []OAIContentPart
hasImage := false
for _, p := range parts {
var typ string
_ = json.Unmarshal(p["type"], &typ)
switch typ {
case "input_text", "output_text", "text":
for _, key := range []string{"text", "output_text"} {
var v string
if json.Unmarshal(p[key], &v) == nil {
text.WriteString(v)
out = append(out, OAIContentPart{Type: "text", Text: v})
break
}
}
case "input_image", "image_url":
if image := responsesImageURL(p); image != nil {
hasImage = true
out = append(out, OAIContentPart{Type: "image_url", ImageURL: image})
}
}
}
if hasImage {
return out
}
return text.String()
}
func responsesImageURL(p map[string]json.RawMessage) *OAIImageURL {
var url string
if json.Unmarshal(p["image_url"], &url) != nil {
var obj struct {
URL string `json:"url"`
}
if json.Unmarshal(p["image_url"], &obj) == nil {
url = obj.URL
}
}
if url == "" {
_ = json.Unmarshal(p["url"], &url)
}
if url == "" {
return nil
}
return &OAIImageURL{URL: url}
}
func responsesContentText(raw json.RawMessage) string {
if len(raw) == 0 {
return ""
@ -591,6 +804,8 @@ func contentToOpenAI(m AMessage) []OAIMessage {
return []OAIMessage{{Role: m.Role, Content: string(m.Content)}}
}
var text strings.Builder
var parts []OAIContentPart
hasImage := false
var calls []OAIToolCall
var toolMsgs []OAIMessage
for _, b := range blocks {
@ -601,6 +816,14 @@ func contentToOpenAI(m AMessage) []OAIMessage {
var v string
_ = json.Unmarshal(b["text"], &v)
text.WriteString(v)
if v != "" {
parts = append(parts, OAIContentPart{Type: "text", Text: v})
}
case "image":
if image := anthropicImageURL(b); image != nil {
hasImage = true
parts = append(parts, OAIContentPart{Type: "image_url", ImageURL: image})
}
case "tool_use":
var id, name string
_ = json.Unmarshal(b["id"], &id)
@ -618,7 +841,7 @@ func contentToOpenAI(m AMessage) []OAIMessage {
}
if len(calls) > 0 {
msg := assistantToolCallsMessage(calls)
msg.Content = text.String()
msg.Content = openAIContentValue(text.String(), parts, hasImage)
return []OAIMessage{msg}
}
if len(toolMsgs) > 0 {
@ -631,7 +854,43 @@ func contentToOpenAI(m AMessage) []OAIMessage {
}
return out
}
return []OAIMessage{{Role: m.Role, Content: text.String()}}
return []OAIMessage{{Role: m.Role, Content: openAIContentValue(text.String(), parts, hasImage)}}
}
func openAIContentValue(text string, parts []OAIContentPart, hasImage bool) any {
if hasImage {
return parts
}
return text
}
func anthropicImageURL(b map[string]json.RawMessage) *OAIImageURL {
var source struct {
Type string `json:"type"`
MediaType string `json:"media_type"`
Data string `json:"data"`
URL string `json:"url"`
}
if json.Unmarshal(b["source"], &source) != nil {
return nil
}
if source.URL != "" || source.Type == "url" {
if source.URL == "" {
return nil
}
return &OAIImageURL{URL: source.URL}
}
if source.Data == "" {
return nil
}
if strings.HasPrefix(source.Data, "data:") {
return &OAIImageURL{URL: source.Data}
}
mediaType := source.MediaType
if mediaType == "" {
mediaType = "image/png"
}
return &OAIImageURL{URL: "data:" + mediaType + ";base64," + source.Data}
}
func systemText(raw json.RawMessage) string {
@ -1150,7 +1409,7 @@ func writeCodexModelCatalog(path string) error {
"auto_compact_token_limit": nil,
"effective_context_window_percent": 95,
"experimental_supported_tools": []any{},
"input_modalities": []string{"text"},
"input_modalities": modelInputModalities(id),
"supports_search_tool": false,
})
}

120
cmd/ocgo/main_test.go
View file

@ -69,13 +69,34 @@ func TestWriteCodexModelCatalog(t *testing.T) {
t.Fatal(err)
}
content := string(b)
for _, want := range []string{`"models"`, `"slug": "deepseek-v4-pro"`, `"context_window": 128000`, `"truncation_policy"`} {
for _, want := range []string{`"models"`, `"slug": "deepseek-v4-pro"`, `"context_window": 128000`, `"truncation_policy"`, `"supports_image_detail_original": false`, `"image"`} {
if !strings.Contains(content, want) {
t.Fatalf("missing %q in:\n%s", want, content)
}
}
}
func TestCodexModelCatalogAllowsImagesForKnownVisionModels(t *testing.T) {
if !modelSupportsImages("kimi-k2.6") {
t.Fatal("kimi-k2.6 should support image inputs")
}
if modelSupportsImages("deepseek-v4-pro") {
t.Fatal("deepseek-v4-pro should not support image inputs")
}
for _, tc := range []struct {
model string
want []string
}{
{model: "kimi-k2.6", want: []string{"text", "image"}},
{model: "deepseek-v4-pro", want: []string{"text"}},
} {
got := modelInputModalities(tc.model)
if strings.Join(got, ",") != strings.Join(tc.want, ",") {
t.Fatalf("%s modalities = %+v, want %+v", tc.model, got, tc.want)
}
}
}
func TestCompareVersions(t *testing.T) {
if compareVersions("0.80.9", "0.81.0") >= 0 {
t.Fatal("0.80.9 should be older")
@ -141,11 +162,106 @@ func TestAnthropicToolResultPreservesFollowingUserText(t *testing.T) {
if messages[0].Role != "tool" || messages[0].ToolCallID != "call_123" || messages[0].Content != "09:33:16" {
t.Fatalf("bad tool result conversion: %+v", messages[0])
}
if messages[1].Role != "user" || !strings.Contains(messages[1].Content, "figma.example") {
if messages[1].Role != "user" || !strings.Contains(contentString(messages[1].Content), "figma.example") {
t.Fatalf("following user text was not preserved: %+v", messages[1])
}
}
func TestResponsesInputPreservesImages(t *testing.T) {
messages := responsesInputToMessages([]byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe this"},{"type":"input_image","image_url":"data:image/png;base64,abc","detail":"high"}]}]`))
if len(messages) != 1 {
t.Fatalf("got %d messages", len(messages))
}
parts, ok := messages[0].Content.([]OAIContentPart)
if !ok {
t.Fatalf("content should be multimodal parts: %+v", messages[0].Content)
}
if len(parts) != 2 || parts[0].Type != "text" || parts[0].Text != "describe this" {
t.Fatalf("bad text part: %+v", parts)
}
if parts[1].Type != "image_url" || parts[1].ImageURL == nil || parts[1].ImageURL.URL != "data:image/png;base64,abc" || parts[1].ImageURL.Detail != "" {
t.Fatalf("bad image part: %+v", parts[1])
}
}
func TestResponsesImageKeepsKimiModel(t *testing.T) {
req := ResponsesRequest{Model: "kimi-k2.6", Input: []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe this"},{"type":"input_image","image_url":"data:image/png;base64,abc"}]}]`)}
out := responsesToChat(req)
if out.Model != "kimi-k2.6" {
t.Fatalf("image request should keep Kimi model, got %q", out.Model)
}
if err := validateImageSupport(out); err != nil {
t.Fatalf("Kimi image request should validate: %v", err)
}
}
func TestResponsesImageRejectsUnsupportedModel(t *testing.T) {
req := ResponsesRequest{Model: "deepseek-v4-pro", Input: []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":"describe this"},{"type":"input_image","image_url":"data:image/png;base64,abc"}]}]`)}
out := responsesToChat(req)
if err := validateImageSupport(out); err == nil || !strings.Contains(err.Error(), "deepseek-v4-pro") {
t.Fatalf("DeepSeek image request should be rejected, got %v", err)
}
}
func TestRawChatImageKeepsKimiAndStripsDetail(t *testing.T) {
body, err := prepareChatBody([]byte(`{"model":"kimi-k2.6","messages":[{"role":"user","content":[{"type":"text","text":"describe this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,abc","detail":"high"}}]}]}`))
if err != nil {
t.Fatalf("Kimi image request should validate: %v", err)
}
if !strings.Contains(string(body), `"model":"kimi-k2.6"`) {
t.Fatalf("image chat body should keep Kimi model: %s", string(body))
}
if strings.Contains(string(body), `"detail"`) {
t.Fatalf("image detail should be stripped for compatibility: %s", string(body))
}
}
func TestRawChatImageRejectsUnsupportedModel(t *testing.T) {
_, err := prepareChatBody([]byte(`{"model":"deepseek-v4-pro","messages":[{"role":"user","content":[{"type":"text","text":"describe this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}]}]}`))
if err == nil || !strings.Contains(err.Error(), "deepseek-v4-pro") {
t.Fatalf("DeepSeek image request should be rejected, got %v", err)
}
}
func TestAnthropicContentPreservesImages(t *testing.T) {
messages := contentToOpenAI(AMessage{Role: "user", Content: []byte(`[{"type":"text","text":"what is this?"},{"type":"image","source":{"type":"base64","media_type":"image/jpeg","data":"abc"}}]`)})
if len(messages) != 1 {
t.Fatalf("got %d messages", len(messages))
}
parts, ok := messages[0].Content.([]OAIContentPart)
if !ok {
t.Fatalf("content should be multimodal parts: %+v", messages[0].Content)
}
if len(parts) != 2 || parts[0].Text != "what is this?" {
t.Fatalf("bad text part: %+v", parts)
}
if parts[1].ImageURL == nil || parts[1].ImageURL.URL != "data:image/jpeg;base64,abc" {
t.Fatalf("bad image part: %+v", parts[1])
}
}
func TestAnthropicImageKeepsKimiModel(t *testing.T) {
out := convertRequest(AnthropicRequest{Model: "kimi-k2.6", Messages: []AMessage{{Role: "user", Content: []byte(`[{"type":"text","text":"what is this?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc"}}]`)}}})
if out.Model != "kimi-k2.6" {
t.Fatalf("image request should keep Kimi model, got %q", out.Model)
}
if err := validateImageSupport(out); err != nil {
t.Fatalf("Kimi image request should validate: %v", err)
}
}
func TestAnthropicImageRejectsUnsupportedModel(t *testing.T) {
out := convertRequest(AnthropicRequest{Model: "deepseek-v4-pro", Messages: []AMessage{{Role: "user", Content: []byte(`[{"type":"text","text":"what is this?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc"}}]`)}}})
if err := validateImageSupport(out); err == nil || !strings.Contains(err.Error(), "deepseek-v4-pro") {
t.Fatalf("DeepSeek image request should be rejected, got %v", err)
}
}
func contentString(v any) string {
s, _ := v.(string)
return s
}
func TestStreamAnthropicForwardsToolCalls(t *testing.T) {
reasoningContentCache.Lock()
reasoningContentCache.byCallID = map[string]string{}