From 5c96dc9005972fbc5d11cbc81108bb107519c9d7 Mon Sep 17 00:00:00 2001 From: Anton Bogdanovich <27antonb@gmail.com> Date: Sun, 3 May 2026 17:48:25 -0700 Subject: [PATCH 1/3] fix openai oauth codex and transcription --- pkg/audio/asr/asr.go | 8 ++- pkg/audio/asr/whisper_transcriber.go | 38 +++++++++++--- pkg/audio/asr/whisper_transcriber_test.go | 60 +++++++++++++++++++++++ pkg/auth/openai.go | 32 ++++++++++++ pkg/providers/oauth/codex_provider.go | 39 ++++++--------- 5 files changed, 144 insertions(+), 33 deletions(-) create mode 100644 pkg/auth/openai.go diff --git a/pkg/audio/asr/asr.go b/pkg/audio/asr/asr.go index 1482f40bb..7c9ee9f8c 100644 --- a/pkg/audio/asr/asr.go +++ b/pkg/audio/asr/asr.go @@ -57,7 +57,7 @@ func supportsWhisperTranscription(modelCfg *config.ModelConfig) bool { } func whisperModelID(modelCfg *config.ModelConfig) string { - if modelCfg == nil || modelCfg.APIKey() == "" { + if modelCfg == nil { return "" } @@ -66,7 +66,11 @@ func whisperModelID(modelCfg *config.ModelConfig) string { } _, modelID := providers.ExtractProtocol(modelCfg) - if strings.Contains(strings.ToLower(modelID), "whisper") { + normalized := strings.ToLower(modelID) + if strings.Contains(normalized, "whisper") || strings.Contains(normalized, "transcribe") { + if modelCfg.APIKey() == "" && modelCfg.AuthMethod != "oauth" { + return "" + } return modelID } return "" diff --git a/pkg/audio/asr/whisper_transcriber.go b/pkg/audio/asr/whisper_transcriber.go index fc1101e1c..5367c31bd 100644 --- a/pkg/audio/asr/whisper_transcriber.go +++ b/pkg/audio/asr/whisper_transcriber.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/sipeed/picoclaw/pkg/auth" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" @@ -24,6 +25,7 @@ type WhisperTranscriber struct { apiBase string modelID string providerName string + tokenSource func() (string, error) httpClient *http.Client } @@ -46,12 +48,17 @@ func NewWhisperTranscriber(modelCfg *config.ModelConfig) *WhisperTranscriber { if tr == nil { return nil } + if modelCfg.AuthMethod == "oauth" && protocol == "openai" { + tr.tokenSource = createOpenAITranscriptionTokenSource() + } logger.DebugCF("voice", "Creating whisper transcriber", map[string]any{ - "api_base": tr.apiBase, - "has_key": tr.apiKey != "", - "model": tr.modelID, - "provider": tr.providerName, + "api_base": tr.apiBase, + "auth_method": modelCfg.AuthMethod, + "has_key": tr.apiKey != "", + "has_oauth": tr.tokenSource != nil, + "model": tr.modelID, + "provider": tr.providerName, }) return tr } @@ -86,6 +93,13 @@ func (t *WhisperTranscriber) transcriptionURL() string { return base + "/audio/transcriptions" } +func (t *WhisperTranscriber) authorizationToken() (string, error) { + if t.tokenSource != nil { + return t.tokenSource() + } + return t.apiKey, nil +} + func (t *WhisperTranscriber) TranscribeData( ctx context.Context, data []byte, @@ -189,8 +203,13 @@ func (t *WhisperTranscriber) doRequest( } req.Header.Set("Content-Type", contentType) - if t.apiKey != "" { - req.Header.Set("Authorization", "Bearer "+t.apiKey) + token, err := t.authorizationToken() + if err != nil { + logger.ErrorCF("voice", "Failed to load transcription auth token", map[string]any{"error": err}) + return nil, fmt.Errorf("failed to load transcription auth token: %w", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) } logger.DebugCF("voice", "Sending whisper transcription request", map[string]any{ @@ -243,3 +262,10 @@ func (t *WhisperTranscriber) doRequest( func (t *WhisperTranscriber) Name() string { return "whisper" } + +func createOpenAITranscriptionTokenSource() func() (string, error) { + return func() (string, error) { + token, _, err := auth.GetOpenAIToken() + return token, err + } +} diff --git a/pkg/audio/asr/whisper_transcriber_test.go b/pkg/audio/asr/whisper_transcriber_test.go index a2a5178d1..ff506f974 100644 --- a/pkg/audio/asr/whisper_transcriber_test.go +++ b/pkg/audio/asr/whisper_transcriber_test.go @@ -100,3 +100,63 @@ func TestWhisperTranscriberUsesEndpointAPIBaseWithoutDoubleAppend(t *testing.T) t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions") } } + +func TestWhisperTranscriberUsesOAuthTokenSource(t *testing.T) { + var gotAuth string + var gotModel string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + + reader, err := r.MultipartReader() + if err != nil { + t.Fatalf("MultipartReader() error: %v", err) + } + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart() error: %v", err) + } + data, err := io.ReadAll(part) + if err != nil { + t.Fatalf("ReadAll() error: %v", err) + } + if part.FormName() == "model" { + gotModel = string(data) + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "oauth transcription"}); err != nil { + t.Fatalf("Encode() error: %v", err) + } + })) + defer server.Close() + + tr := NewWhisperTranscriber(&config.ModelConfig{ + Model: "openai/gpt-4o-transcribe", + APIBase: server.URL, + AuthMethod: "oauth", + }) + tr.httpClient = server.Client() + tr.tokenSource = func() (string, error) { + return "oauth-token", nil + } + + resp, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.mp3") + if err != nil { + t.Fatalf("TranscribeData() error: %v", err) + } + if resp.Text != "oauth transcription" { + t.Errorf("Text = %q, want %q", resp.Text, "oauth transcription") + } + if gotAuth != "Bearer oauth-token" { + t.Errorf("Authorization = %q, want %q", gotAuth, "Bearer oauth-token") + } + if gotModel != "gpt-4o-transcribe" { + t.Errorf("model field = %q, want %q", gotModel, "gpt-4o-transcribe") + } +} diff --git a/pkg/auth/openai.go b/pkg/auth/openai.go new file mode 100644 index 000000000..2cad4537d --- /dev/null +++ b/pkg/auth/openai.go @@ -0,0 +1,32 @@ +package auth + +import "fmt" + +// GetOpenAIToken returns the current OpenAI credential, refreshing OAuth +// credentials when they are close to expiry. The account ID is returned for +// Codex backend calls that require the Chatgpt-Account-Id header. +func GetOpenAIToken() (accessToken, accountID string, err error) { + cred, err := GetCredential("openai") + if err != nil { + return "", "", fmt.Errorf("loading auth credentials: %w", err) + } + if cred == nil { + return "", "", fmt.Errorf("no credentials for openai. Run: picoclaw auth login --provider openai") + } + + if cred.AuthMethod == "oauth" && cred.NeedsRefresh() && cred.RefreshToken != "" { + refreshed, err := RefreshAccessToken(cred, OpenAIOAuthConfig()) + if err != nil { + return "", "", fmt.Errorf("refreshing token: %w", err) + } + if refreshed.AccountID == "" { + refreshed.AccountID = cred.AccountID + } + if err := SetCredential("openai", refreshed); err != nil { + return "", "", fmt.Errorf("saving refreshed token: %w", err) + } + return refreshed.AccessToken, refreshed.AccountID, nil + } + + return cred.AccessToken, cred.AccountID, nil +} diff --git a/pkg/providers/oauth/codex_provider.go b/pkg/providers/oauth/codex_provider.go index 0b125997b..88da42af7 100644 --- a/pkg/providers/oauth/codex_provider.go +++ b/pkg/providers/oauth/codex_provider.go @@ -104,8 +104,16 @@ func (p *CodexProvider) Chat( defer stream.Close() var resp *responses.Response + var streamText strings.Builder for stream.Next() { evt := stream.Current() + if evt.Type == "response.output_text.done" { + textDone := evt.AsResponseOutputTextDone() + if textDone.Text != "" { + streamText.Reset() + streamText.WriteString(textDone.Text) + } + } if evt.Type == "response.completed" || evt.Type == "response.failed" || evt.Type == "response.incomplete" { evtResp := evt.Response if evtResp.ID != "" { @@ -153,7 +161,11 @@ func (p *CodexProvider) Chat( return nil, fmt.Errorf("codex API call: stream ended without completed response") } - return orc.ParseResponseFromStruct(resp), nil + parsed := orc.ParseResponseFromStruct(resp) + if parsed.Content == "" && len(parsed.ToolCalls) == 0 && streamText.Len() > 0 { + parsed.Content = streamText.String() + } + return parsed, nil } func (p *CodexProvider) GetDefaultModel() string { @@ -242,29 +254,6 @@ func buildCodexParams( func CreateCodexTokenSource() func() (string, string, error) { return func() (string, string, error) { - cred, err := auth.GetCredential("openai") - if err != nil { - return "", "", fmt.Errorf("loading auth credentials: %w", err) - } - if cred == nil { - return "", "", fmt.Errorf("no credentials for openai. Run: picoclaw auth login --provider openai") - } - - if cred.AuthMethod == "oauth" && cred.NeedsRefresh() && cred.RefreshToken != "" { - oauthCfg := auth.OpenAIOAuthConfig() - refreshed, err := auth.RefreshAccessToken(cred, oauthCfg) - if err != nil { - return "", "", fmt.Errorf("refreshing token: %w", err) - } - if refreshed.AccountID == "" { - refreshed.AccountID = cred.AccountID - } - if err := auth.SetCredential("openai", refreshed); err != nil { - return "", "", fmt.Errorf("saving refreshed token: %w", err) - } - return refreshed.AccessToken, refreshed.AccountID, nil - } - - return cred.AccessToken, cred.AccountID, nil + return auth.GetOpenAIToken() } } From 1627fd02404b16aeee054d7b6c7777cce6907611 Mon Sep 17 00:00:00 2001 From: Anton Bogdanovich <27antonb@gmail.com> Date: Sun, 3 May 2026 19:55:37 -0700 Subject: [PATCH 2/3] fix codex oauth streamed tool calls --- pkg/providers/oauth/codex_provider.go | 47 +++++++++++++ pkg/providers/oauth/codex_provider_test.go | 77 ++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/pkg/providers/oauth/codex_provider.go b/pkg/providers/oauth/codex_provider.go index 88da42af7..32ebcdb11 100644 --- a/pkg/providers/oauth/codex_provider.go +++ b/pkg/providers/oauth/codex_provider.go @@ -2,6 +2,7 @@ package oauthprovider import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -9,6 +10,7 @@ import ( "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" "github.com/openai/openai-go/v3/responses" + "github.com/openai/openai-go/v3/shared" "github.com/sipeed/picoclaw/pkg/auth" "github.com/sipeed/picoclaw/pkg/logger" @@ -105,6 +107,7 @@ func (p *CodexProvider) Chat( var resp *responses.Response var streamText strings.Builder + var streamToolCalls []ToolCall for stream.Next() { evt := stream.Current() if evt.Type == "response.output_text.done" { @@ -114,6 +117,12 @@ func (p *CodexProvider) Chat( streamText.WriteString(textDone.Text) } } + if evt.Type == "response.output_item.done" { + done := evt.AsResponseOutputItemDone() + if tc, ok := codexToolCallFromOutputItem(done.Item); ok { + streamToolCalls = append(streamToolCalls, tc) + } + } if evt.Type == "response.completed" || evt.Type == "response.failed" || evt.Type == "response.incomplete" { evtResp := evt.Response if evtResp.ID != "" { @@ -165,9 +174,44 @@ func (p *CodexProvider) Chat( if parsed.Content == "" && len(parsed.ToolCalls) == 0 && streamText.Len() > 0 { parsed.Content = streamText.String() } + if len(parsed.ToolCalls) == 0 && len(streamToolCalls) > 0 { + parsed.ToolCalls = streamToolCalls + parsed.FinishReason = "tool_calls" + } return parsed, nil } +func codexToolCallFromOutputItem(item responses.ResponseOutputItemUnion) (ToolCall, bool) { + if item.Type != "function_call" { + return ToolCall{}, false + } + + call := item.AsFunctionCall() + if call.Name == "" { + return ToolCall{}, false + } + + var args map[string]any + if err := json.Unmarshal([]byte(call.Arguments), &args); err != nil { + args = map[string]any{"raw": call.Arguments} + } + + id := call.CallID + if id == "" { + id = call.ID + } + + return ToolCall{ + ID: id, + Name: call.Name, + Arguments: args, + Function: &FunctionCall{ + Name: call.Name, + Arguments: call.Arguments, + }, + }, true +} + func (p *CodexProvider) GetDefaultModel() string { return codexDefaultModel } @@ -229,6 +273,9 @@ func buildCodexParams( OfInputItemList: inputItems, }, Store: openai.Opt(false), + Reasoning: shared.ReasoningParam{ + Effort: shared.ReasoningEffortNone, + }, } if instructions != "" { diff --git a/pkg/providers/oauth/codex_provider_test.go b/pkg/providers/oauth/codex_provider_test.go index aeeb18360..7b13f0365 100644 --- a/pkg/providers/oauth/codex_provider_test.go +++ b/pkg/providers/oauth/codex_provider_test.go @@ -374,6 +374,83 @@ func TestCodexProvider_ChatRoundTrip(t *testing.T) { } } +func TestCodexProvider_ChatRoundTrip_ToolCallFromStreamItem(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/responses" { + http.Error(w, "not found: "+r.URL.Path, http.StatusNotFound) + return + } + + var reqBody map[string]any + if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil { + http.Error(w, "invalid json", http.StatusBadRequest) + return + } + if reqBody["stream"] != true { + http.Error(w, "stream must be true", http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "text/event-stream") + itemDone := map[string]any{ + "type": "response.output_item.done", + "sequence_number": 1, + "output_index": 0, + "item": map[string]any{ + "id": "fc_1", + "type": "function_call", + "call_id": "call_abc", + "name": "nutritiondb__list_weight_entries", + "arguments": `{"limit":5}`, + "status": "completed", + }, + } + b, _ := json.Marshal(itemDone) + fmt.Fprintf(w, "event: response.output_item.done\n") + fmt.Fprintf(w, "data: %s\n\n", string(b)) + + resp := map[string]any{ + "id": "resp_test", + "object": "response", + "status": "completed", + "output": []map[string]any{}, + "usage": map[string]any{ + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": map[string]any{"cached_tokens": 0}, + "output_tokens_details": map[string]any{"reasoning_tokens": 0}, + }, + } + writeCompletedSSE(w, resp) + })) + defer server.Close() + + provider := NewCodexProvider("test-token", "acc-123") + provider.client = createOpenAITestClient(server.URL, "test-token", "acc-123") + + resp, err := provider.Chat(t.Context(), []Message{{Role: "user", Content: "latest weights"}}, nil, "gpt-5.4", nil) + if err != nil { + t.Fatalf("Chat() error: %v", err) + } + if len(resp.ToolCalls) != 1 { + t.Fatalf("len(ToolCalls) = %d, want 1", len(resp.ToolCalls)) + } + tc := resp.ToolCalls[0] + if tc.ID != "call_abc" { + t.Errorf("ToolCall.ID = %q, want call_abc", tc.ID) + } + if tc.Name != "nutritiondb__list_weight_entries" { + t.Errorf("ToolCall.Name = %q, want nutritiondb__list_weight_entries", tc.Name) + } + if tc.Arguments["limit"] != float64(5) { + t.Errorf("ToolCall.Arguments[limit] = %v, want 5", tc.Arguments["limit"]) + } + if resp.FinishReason != "tool_calls" { + t.Errorf("FinishReason = %q, want tool_calls", resp.FinishReason) + } +} + func TestCodexProvider_ChatRoundTrip_WebSearchDisabled(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/responses" { From 647e0db124b14f0a1d586a50f9ed674e1fc963a3 Mon Sep 17 00:00:00 2001 From: Anton Bogdanovich <27antonb@gmail.com> Date: Sun, 3 May 2026 20:04:08 -0700 Subject: [PATCH 3/3] support codex oauth thinking level --- pkg/providers/oauth/codex_provider.go | 22 +++++++++++- pkg/providers/oauth/codex_provider_test.go | 42 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/pkg/providers/oauth/codex_provider.go b/pkg/providers/oauth/codex_provider.go index 32ebcdb11..50205c70e 100644 --- a/pkg/providers/oauth/codex_provider.go +++ b/pkg/providers/oauth/codex_provider.go @@ -220,6 +220,10 @@ func (p *CodexProvider) SupportsNativeSearch() bool { return p.enableWebSearch } +func (p *CodexProvider) SupportsThinking() bool { + return true +} + func resolveCodexModel(model string) (string, string) { m := strings.ToLower(strings.TrimSpace(model)) if m == "" { @@ -274,7 +278,7 @@ func buildCodexParams( }, Store: openai.Opt(false), Reasoning: shared.ReasoningParam{ - Effort: shared.ReasoningEffortNone, + Effort: codexReasoningEffort(options["thinking_level"]), }, } @@ -299,6 +303,22 @@ func buildCodexParams( return params } +func codexReasoningEffort(raw any) shared.ReasoningEffort { + level, _ := raw.(string) + switch strings.ToLower(strings.TrimSpace(level)) { + case "low": + return shared.ReasoningEffortLow + case "medium", "adaptive": + return shared.ReasoningEffortMedium + case "high": + return shared.ReasoningEffortHigh + case "xhigh", "max": + return shared.ReasoningEffortXhigh + default: + return shared.ReasoningEffortNone + } +} + func CreateCodexTokenSource() func() (string, string, error) { return func() (string, string, error) { return auth.GetOpenAIToken() diff --git a/pkg/providers/oauth/codex_provider_test.go b/pkg/providers/oauth/codex_provider_test.go index 7b13f0365..2ed7aa997 100644 --- a/pkg/providers/oauth/codex_provider_test.go +++ b/pkg/providers/oauth/codex_provider_test.go @@ -10,6 +10,7 @@ import ( "github.com/openai/openai-go/v3" openaiopt "github.com/openai/openai-go/v3/option" "github.com/openai/openai-go/v3/responses" + "github.com/openai/openai-go/v3/shared" orc "github.com/sipeed/picoclaw/pkg/providers/openai_responses_common" ) @@ -34,6 +35,9 @@ func TestBuildCodexParams_BasicMessage(t *testing.T) { if params.MaxOutputTokens.Valid() { t.Fatalf("MaxOutputTokens should not be set for Codex backend") } + if params.Reasoning.Effort != shared.ReasoningEffortNone { + t.Fatalf("Reasoning.Effort = %q, want none", params.Reasoning.Effort) + } } func TestBuildCodexParams_SystemAsInstructions(t *testing.T) { @@ -50,6 +54,44 @@ func TestBuildCodexParams_SystemAsInstructions(t *testing.T) { } } +func TestBuildCodexParams_ThinkingLevel(t *testing.T) { + tests := []struct { + name string + level any + want shared.ReasoningEffort + }{ + {name: "default", level: nil, want: shared.ReasoningEffortNone}, + {name: "off", level: "off", want: shared.ReasoningEffortNone}, + {name: "low", level: "low", want: shared.ReasoningEffortLow}, + {name: "medium", level: "medium", want: shared.ReasoningEffortMedium}, + {name: "adaptive", level: "adaptive", want: shared.ReasoningEffortMedium}, + {name: "high", level: "high", want: shared.ReasoningEffortHigh}, + {name: "xhigh", level: "xhigh", want: shared.ReasoningEffortXhigh}, + {name: "max", level: "max", want: shared.ReasoningEffortXhigh}, + {name: "unknown", level: "banana", want: shared.ReasoningEffortNone}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := map[string]any{} + if tt.level != nil { + opts["thinking_level"] = tt.level + } + params := buildCodexParams([]Message{{Role: "user", Content: "Hi"}}, nil, "gpt-5.4", opts, false) + if params.Reasoning.Effort != tt.want { + t.Fatalf("Reasoning.Effort = %q, want %q", params.Reasoning.Effort, tt.want) + } + }) + } +} + +func TestCodexProvider_SupportsThinking(t *testing.T) { + provider := NewCodexProvider("test-token", "acc-123") + if !provider.SupportsThinking() { + t.Fatal("CodexProvider should support thinking_level") + } +} + func TestBuildCodexParams_ToolCallConversation(t *testing.T) { messages := []Message{ {Role: "user", Content: "What's the weather?"},