Merge branch 'main' into refactor-inbound-context-routing-session

# Conflicts:
#	pkg/agent/eventbus_test.go
#	pkg/agent/loop.go
#	pkg/bus/bus.go
#	pkg/bus/types.go
#	pkg/channels/pico/pico.go
#	pkg/channels/telegram/telegram.go
#	pkg/config/config.go
#	web/backend/api/session.go
#	web/backend/api/session_test.go
This commit is contained in:
Hoshina 2026-04-07 21:41:02 +08:00
commit e32a209683
282 changed files with 33064 additions and 3251 deletions

62
.github/workflows/create_dmg.yml vendored Normal file
View file

@ -0,0 +1,62 @@
name: Create macOS DMG
on:
workflow_dispatch:
jobs:
build:
name: Build ${{ matrix.arch }}
runs-on: macos-latest
strategy:
matrix:
# This creates two parallel jobs
arch: [arm64, amd64]
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
ref: main
# 1. 安装指定版本的 Go (可选,但推荐)
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
# 2. 安装 pnpm
- name: Install pnpm
run: brew install pnpm
# 3. 运行你的 Makefile 编译二进制文件
- name: Build with Make
run: make build ARCH=${{ matrix.arch }} && make build-macos-app ARCH=${{ matrix.arch }}
# 4. 签名
- name: Ad-hoc Sign
run: codesign --force --deep --sign - "build/PicoClaw Launcher.app"
# 5. 安装打包工具
- name: Install create-dmg
run: brew install create-dmg
# 6. 执行打包命令
- name: Create DMG
run: |
mkdir -p dist
create-dmg \
--volname "PicoClaw Installer" \
--window-pos 200 120 \
--window-size 800 400 \
--icon-size 100 \
--icon "PicoClaw Launcher.app" 200 190 \
--hide-extension "PicoClaw Launcher.app" \
--app-drop-link 600 185 \
"dist/picoclaw-${{ matrix.arch }}.dmg" \
"build/PicoClaw Launcher.app"
# 7. 上传文件到 GitHub Artifacts (供你下载)
- name: Upload DMG
uses: actions/upload-artifact@v7
with:
name: macos-dmg-${{ matrix.arch }}
path: dist/*.dmg

2
.gitignore vendored
View file

@ -67,3 +67,5 @@ web/backend/dist/*
.claude/
docker/data
.omc/

View file

@ -12,6 +12,7 @@ linters:
- exhaustruct
- funcorder
- gochecknoglobals
- gosmopolitan # Project legitimately uses CJK text in tests (FTS5, token counting)
- godot
- intrange
- ireturn
@ -61,6 +62,9 @@ linters:
- usestdlibvars
- usetesting
settings:
gomoddirectives:
replace-allow-list:
- github.com/bwmarrin/discordgo
errcheck:
check-type-assertions: true
check-blank: true

View file

@ -5,6 +5,7 @@ BINARY_NAME=picoclaw
BUILD_DIR=build
CMD_DIR=cmd/$(BINARY_NAME)
MAIN_GO=$(CMD_DIR)/main.go
EXT=
# Version
VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
@ -69,9 +70,11 @@ WORKSPACE_DIR?=$(PICOCLAW_HOME)/workspace
WORKSPACE_SKILLS_DIR=$(WORKSPACE_DIR)/skills
BUILTIN_SKILLS_DIR=$(CURDIR)/skills
LNCMD=ln -sf
# OS detection
UNAME_S:=$(shell uname -s)
UNAME_M:=$(shell uname -m)
UNAME_S?=$(shell uname -s)
UNAME_M?=$(shell uname -m)
# Platform-specific settings
ifeq ($(UNAME_S),Linux)
@ -93,17 +96,30 @@ ifeq ($(UNAME_S),Linux)
endif
else ifeq ($(UNAME_S),Darwin)
PLATFORM=darwin
WEB_GO=CGO_ENABLED=1 go
WEB_GO=CGO_LDFLAGS="-mmacosx-version-min=10.11" CGO_CFLAGS="-mmacosx-version-min=10.11" CGO_ENABLED=1 go
ifeq ($(UNAME_M),x86_64)
ARCH=amd64
ARCH?=amd64
else ifeq ($(UNAME_M),arm64)
ARCH=arm64
ARCH?=arm64
else
ARCH=$(UNAME_M)
ARCH?=$(UNAME_M)
endif
else
PLATFORM=$(UNAME_S)
ARCH=$(UNAME_M)
ifeq ($(UNAME_M),x86_64)
ARCH?=amd64
else
ARCH?=$(UNAME_M)
endif
# Detect Windows (Git Bash / MSYS2)
IS_WINDOWS:=$(if $(findstring MINGW,$(UNAME_S)),yes,$(if $(findstring MSYS,$(UNAME_S)),yes,$(if $(findstring CYGWIN,$(UNAME_S)),yes,no)))
ifeq ($(IS_WINDOWS),yes)
EXT=.exe
LNCMD=cp
else ifeq ($(UNAME_S),windows) # failsafe for force windows build in other OS using UNAME_S=windows
EXT=.exe
endif
endif
BINARY_PATH=$(BUILD_DIR)/$(BINARY_NAME)-$(PLATFORM)-$(ARCH)
@ -120,23 +136,23 @@ generate:
## build: Build the picoclaw binary for current platform
build: generate
@echo "Building $(BINARY_NAME) for $(PLATFORM)/$(ARCH)..."
@echo "Building $(BINARY_NAME)$(EXT) for $(PLATFORM)/$(ARCH)..."
@mkdir -p $(BUILD_DIR)
@$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./$(CMD_DIR)
@echo "Build complete: $(BINARY_PATH)"
@ln -sf $(BINARY_NAME)-$(PLATFORM)-$(ARCH) $(BUILD_DIR)/$(BINARY_NAME)
@GOARCH=${ARCH} $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH)$(EXT) ./$(CMD_DIR)
@echo "Build complete: $(BINARY_PATH)$(EXT)"
@$(LNCMD) $(BINARY_NAME)-$(PLATFORM)-$(ARCH)$(EXT) $(BUILD_DIR)/$(BINARY_NAME)$(EXT)
## build-launcher: Build the picoclaw-launcher (web console) binary
build-launcher:
@echo "Building picoclaw-launcher for $(PLATFORM)/$(ARCH)..."
@mkdir -p $(BUILD_DIR)
@$(MAKE) -C web build \
OUTPUT="$(CURDIR)/$(BUILD_DIR)/picoclaw-launcher-$(PLATFORM)-$(ARCH)" \
@GOARCH=${ARCH} $(MAKE) -C web build \
OUTPUT="$(CURDIR)/$(BUILD_DIR)/picoclaw-launcher-$(PLATFORM)-$(ARCH)$(EXT)" \
WEB_GO='$(WEB_GO)' \
GO_BUILD_TAGS='$(GO_BUILD_TAGS)' \
LDFLAGS='$(LDFLAGS)'
@ln -sf picoclaw-launcher-$(PLATFORM)-$(ARCH) $(BUILD_DIR)/picoclaw-launcher
@echo "Build complete: $(BUILD_DIR)/picoclaw-launcher"
@$(LNCMD) picoclaw-launcher-$(PLATFORM)-$(ARCH)$(EXT) $(BUILD_DIR)/picoclaw-launcher$(EXT)
@echo "Build complete: $(BUILD_DIR)/picoclaw-launcher$(EXT)"
build-launcher-frontend:
@$(MAKE) -C web build-frontend
@ -324,16 +340,34 @@ docker-clean:
## build-macos-app: Build PicoClaw macOS .app bundle (no terminal window)
build-macos-app:
build-macos-app:build-launcher
@echo "Building macOS .app bundle..."
@if [ "$(UNAME_S)" != "Darwin" ]; then \
echo "Error: This target is only available on macOS"; \
exit 1; \
fi
@cd web && $(MAKE) build && cd ..
@./scripts/build-macos-app.sh $(BINARY_NAME)-$(PLATFORM)-$(ARCH)
@./scripts/build-macos-app.sh $(PLATFORM)-$(ARCH)
@echo "macOS .app bundle created: $(BUILD_DIR)/PicoClaw.app"
## mem: Build membench, download LOCOMO data (if needed), run benchmark, and show results
mem:
@echo "Building membench..."
@mkdir -p $(BUILD_DIR)
@$(GO) build -o $(BUILD_DIR)/membench ./cmd/membench
@echo "Build complete: $(BUILD_DIR)/membench"
@if [ ! -f $(BUILD_DIR)/memdata/locomo10.json ]; then \
echo "Downloading LOCOMO dataset..."; \
mkdir -p $(BUILD_DIR)/memdata; \
curl -sfL "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json" \
-o $(BUILD_DIR)/memdata/locomo10.json && [ -s $(BUILD_DIR)/memdata/locomo10.json ] || { echo "Error: LOCOMO download failed"; exit 1; }; \
echo "Download complete"; \
else \
echo "LOCOMO dataset already exists, skipping download"; \
fi
@echo "Running benchmark..."
@rm -rf $(BUILD_DIR)/memout
@$(BUILD_DIR)/membench run --data $(BUILD_DIR)/memdata --out $(BUILD_DIR)/memout --budget 4000
## help: Show this help message
help:
@echo "picoclaw Makefile"

View file

@ -57,6 +57,8 @@
## 📢 Actualités
2026-03-31 📱 **Support Android !** PicoClaw fonctionne maintenant sur Android ! Téléchargez l'APK sur [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 publiée !** Refonte de l'architecture Agent (SubTurn, Hooks, Steering, EventBus), intégration WeChat/WeCom, renforcement de la sécurité (.security.yml, filtrage des données sensibles), nouveaux providers (AWS Bedrock, Azure, Xiaomi MiMo), et 35 corrections de bugs. PicoClaw a atteint **26K Stars** !
2026-03-17 🚀 **v0.2.3 publiée !** Interface system tray (Windows & Linux), requête de statut des sous-agents (`spawn_status`), rechargement à chaud expérimental du Gateway, sécurisation Cron, et 2 correctifs de sécurité. PicoClaw a atteint **25K Stars** !
@ -304,7 +306,25 @@ Pour la documentation détaillée du TUI, voir [docs.picoclaw.io](https://docs.p
Donnez une seconde vie à votre téléphone vieux de dix ans ! Transformez-le en assistant IA intelligent avec PicoClaw.
**Option 1 : Termux (disponible maintenant)**
**Option 1 : Installation APK**
Aperçu :
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Téléchargez l'APK depuis [picoclaw.io](https://picoclaw.io/download/) et installez-le directement. Pas besoin de Termux !
**Option 2 : Termux**
<details>
<summary><b>Terminal Launcher (pour les environnements à ressources limitées)</b></summary>
1. Installez [Termux](https://github.com/termux/termux-app) (téléchargez depuis [GitHub Releases](https://github.com/termux/termux-app/releases), ou cherchez dans F-Droid / Google Play)
2. Exécutez les commandes suivantes :
@ -321,13 +341,6 @@ Suivez ensuite la section Terminal Launcher ci-dessous pour terminer la configur
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**Option 2 : Installation APK (bientôt disponible)**
Un APK Android autonome avec WebUI intégré est en développement. Restez à l'écoute !
<details>
<summary><b>Terminal Launcher (pour les environnements à ressources limitées)</b></summary>
Pour les environnements minimaux où seul le binaire principal `picoclaw` est disponible (sans Launcher UI), vous pouvez tout configurer via la ligne de commande et un fichier de configuration JSON.
**1. Initialiser**

View file

@ -56,6 +56,8 @@
## 📢 Berita
2026-03-31 📱 **Dukungan Android!** PicoClaw sekarang berjalan di Android! Unduh APK di [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 Dirilis!** Perombakan arsitektur Agent (SubTurn, Hooks, Steering, EventBus), integrasi WeChat/WeCom, penguatan keamanan (.security.yml, penyaringan data sensitif), provider baru (AWS Bedrock, Azure, Xiaomi MiMo), dan 35 perbaikan bug. PicoClaw telah mencapai **26K Stars**!
2026-03-17 🚀 **v0.2.3 Dirilis!** UI system tray (Windows & Linux), pelacakan status sub-agent (`spawn_status`), eksperimental Gateway hot-reload, gerbang keamanan Cron, dan 2 perbaikan keamanan. PicoClaw telah mencapai **25K Stars**!
@ -301,7 +303,25 @@ Untuk dokumentasi TUI lengkap, lihat [docs.picoclaw.io](https://docs.picoclaw.io
Berikan kehidupan kedua untuk ponsel lama Anda! Ubah menjadi Asisten AI pintar dengan PicoClaw.
**Opsi 1: Termux (tersedia sekarang)**
**Opsi 1: Instal APK**
Pratinjau:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Unduh APK dari [picoclaw.io](https://picoclaw.io/download/) dan instal langsung. Tanpa Termux!
**Opsi 2: Termux**
<details>
<summary><b>Terminal Launcher (untuk lingkungan dengan sumber daya terbatas)</b></summary>
1. Instal [Termux](https://github.com/termux/termux-app) (unduh dari [GitHub Releases](https://github.com/termux/termux-app/releases), atau cari di F-Droid / Google Play)
2. Jalankan perintah berikut:
@ -318,13 +338,6 @@ Kemudian ikuti bagian Terminal Launcher di bawah untuk menyelesaikan konfigurasi
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**Opsi 2: Instal APK (segera hadir)**
APK Android mandiri dengan WebUI bawaan sedang dalam pengembangan. Pantau terus!
<details>
<summary><b>Terminal Launcher (untuk lingkungan dengan sumber daya terbatas)</b></summary>
Untuk lingkungan minimal di mana hanya binary inti `picoclaw` yang tersedia (tanpa Launcher UI), Anda dapat mengonfigurasi semuanya melalui command line dan file konfigurasi JSON.
**1. Inisialisasi**

View file

@ -56,6 +56,8 @@
## 📢 Novità
2026-03-31 📱 **Supporto Android!** PicoClaw ora funziona su Android! Scarica l'APK su [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 rilasciata!** Revisione dell'architettura Agent (SubTurn, Hooks, Steering, EventBus), integrazione WeChat/WeCom, rafforzamento della sicurezza (.security.yml, filtraggio dati sensibili), nuovi provider (AWS Bedrock, Azure, Xiaomi MiMo) e 35 correzioni di bug. PicoClaw raggiunge **26K Stars**!
2026-03-17 🚀 **v0.2.3 rilasciata!** Interfaccia system tray (Windows & Linux), query sullo stato dei sub-agent (`spawn_status`), hot-reload sperimentale del Gateway, gate di sicurezza per Cron e 2 correzioni di sicurezza. PicoClaw raggiunge **25K Stars**!
@ -301,7 +303,25 @@ Per la documentazione dettagliata del TUI, vedi [docs.picoclaw.io](https://docs.
Dai una seconda vita al tuo telefono di dieci anni fa! Trasformalo in un assistente IA intelligente con PicoClaw.
**Opzione 1: Termux (disponibile ora)**
**Opzione 1: Installazione APK**
Anteprima:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Scarica l'APK da [picoclaw.io](https://picoclaw.io/download/) e installa direttamente. Senza Termux!
**Opzione 2: Termux**
<details>
<summary><b>Terminal Launcher (per ambienti con risorse limitate)</b></summary>
1. Installa [Termux](https://github.com/termux/termux-app) (scarica da [GitHub Releases](https://github.com/termux/termux-app/releases), o cerca su F-Droid / Google Play)
2. Esegui i seguenti comandi:
@ -318,13 +338,6 @@ Poi segui la sezione Terminal Launcher qui sotto per completare la configurazion
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**Opzione 2: APK Install (prossimamente)**
Un APK Android standalone con WebUI integrato è in sviluppo. Resta sintonizzato!
<details>
<summary><b>Terminal Launcher (per ambienti con risorse limitate)</b></summary>
Per ambienti minimali dove è disponibile solo il binario core `picoclaw` (senza Launcher UI), puoi configurare tutto tramite riga di comando e un file di configurazione JSON.
**1. Inizializza**

View file

@ -56,6 +56,8 @@
## 📢 ニュース
2026-03-31 📱 **Android サポート!** PicoClawがAndroidで動作APKは[picoclaw.io](https://picoclaw.io/download)からダウンロード
2026-03-25 🚀 **v0.2.4 リリース!** Agent アーキテクチャ全面刷新SubTurn、Hooks、Steering、EventBus、WeChat/WeCom 統合、セキュリティ強化(.security.yml、機密データフィルタリング、新プロバイダーAWS Bedrock、Azure、Xiaomi MiMo、35 件のバグ修正。PicoClaw **26K ⭐** 達成!
2026-03-17 🚀 **v0.2.3 リリース!** システムトレイ UIWindows & Linux、サブエージェントステータス追跡`spawn_status`)、実験的 Gateway ホットリロード、cron セキュリティゲート、セキュリティ修正 2 件。PicoClaw **25K ⭐** 達成!
@ -301,7 +303,25 @@ TUI の詳細なドキュメントは [docs.picoclaw.io](https://docs.picoclaw.i
10 年前のスマホに第二の人生をPicoClaw でスマート AI アシスタントに変身させましょう。
**オプション 1: Termux現在利用可能**
**オプション 1: APK インストール**
プレビュー:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
[picoclaw.io](https://picoclaw.io/download/) から APK をダウンロードして直接インストール。Termux 不要!
**オプション 2: Termux**
<details>
<summary><b>Terminal Launcherリソース制約環境向け</b></summary>
1. [Termux](https://github.com/termux/termux-app) をインストール([GitHub Releases](https://github.com/termux/termux-app/releases) からダウンロード、または F-Droid / Google Play で検索)
2. 以下のコマンドを実行:
@ -318,13 +338,6 @@ termux-chroot ./picoclaw onboard # chroot で標準的な Linux ファイル
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**オプション 2: APK インストール(近日公開)**
内蔵 WebUI を備えたスタンドアロン Android APK を開発中です。お楽しみに!
<details>
<summary><b>Terminal Launcherリソース制約環境向け</b></summary>
`picoclaw` コアバイナリのみが利用可能な最小環境Launcher UI なし)では、コマンドラインと JSON 設定ファイルですべてを設定できます。
**1. 初期化**

View file

@ -56,6 +56,8 @@
## 📢 News
2026-03-31 📱 **Android Support!** PicoClaw now runs on Android! Download the APK at [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 Released!** Agent architecture overhaul (SubTurn, Hooks, Steering, EventBus), WeChat/WeCom integration, security hardening (.security.yml, sensitive data filtering), new providers (AWS Bedrock, Azure, Xiaomi MiMo), and 35 bug fixes. PicoClaw has reached **26K Stars**!
2026-03-17 🚀 **v0.2.3 Released!** System tray UI (Windows & Linux), sub-agent status query (`spawn_status`), experimental Gateway hot-reload, Cron security gating, and 2 security fixes. PicoClaw has reached **25K Stars**!
@ -301,7 +303,25 @@ For detailed TUI documentation, see [docs.picoclaw.io](https://docs.picoclaw.io)
Give your decade-old phone a second life! Turn it into a smart AI Assistant with PicoClaw.
**Option 1: Termux (available now)**
**Option 1: APK Install**
Preview:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Download the APK from [picoclaw.io](https://picoclaw.io/download/) and install directly. No Termux required!
**Option 2: Termux**
<details>
<summary><b>Terminal Launcher (for resource-constrained environments)</b></summary>
1. Install [Termux](https://github.com/termux/termux-app) (download from [GitHub Releases](https://github.com/termux/termux-app/releases), or search in F-Droid / Google Play)
2. Run the following commands:
@ -318,13 +338,6 @@ Then follow the Terminal Launcher section below to complete configuration.
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**Option 2: APK Install (coming soon)**
A standalone Android APK with built-in WebUI is in development. Stay tuned!
<details>
<summary><b>Terminal Launcher (for resource-constrained environments)</b></summary>
For minimal environments where only the `picoclaw` core binary is available (no Launcher UI), you can configure everything via the command line and a JSON config file.
**1. Initialize**
@ -441,7 +454,7 @@ For full provider configuration details, see [Providers & Models](docs/providers
## 💬 Channels (Chat Apps)
Talk to your PicoClaw through 17+ messaging platforms:
Talk to your PicoClaw through 18+ messaging platforms:
| Channel | Setup | Protocol | Docs |
|---------|-------|----------|------|
@ -456,6 +469,7 @@ Talk to your PicoClaw through 17+ messaging platforms:
| **Feishu / Lark** | Medium (App ID + Secret) | WebSocket/SDK | [Guide](docs/channels/feishu/README.md) |
| **LINE** | Medium (credentials + webhook) | Webhook | [Guide](docs/channels/line/README.md) |
| **WeCom** | Easy (QR login or manual) | WebSocket | [Guide](docs/channels/wecom/README.md) |
| **VK** | Easy (group token) | Long Poll | [Guide](docs/channels/vk/README.md) |
| **IRC** | Medium (server + nick) | IRC protocol | [Guide](docs/chat-apps.md#irc) |
| **OneBot** | Medium (WebSocket URL) | OneBot v11 | [Guide](docs/channels/onebot/README.md) |
| **MaixCam** | Easy (enable) | TCP socket | [Guide](docs/channels/maixcam/README.md) |

View file

@ -56,6 +56,8 @@
## 📢 Berita
2026-03-31 📱 **Sokongan Android!** PicoClaw sekarang berjalan di Android! Muat turun APK di [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 Dikeluarkan!** Penstrukturan semula seni bina Agent (SubTurn, Hooks, Steering, EventBus), integrasi WeChat/WeCom, penguatan keselamatan (.security.yml, penapisan data sensitif), penyedia baharu (AWS Bedrock, Azure, Xiaomi MiMo), dan 35 pembetulan pepijat. PicoClaw mencapai **26K Stars**!
2026-03-17 🚀 **v0.2.3 Dikeluarkan!** UI dulang sistem (Windows & Linux), pertanyaan status sub-agent (`spawn_status`), muat semula panas Gateway eksperimental, kawalan keselamatan Cron, dan 2 pembetulan keselamatan. PicoClaw mencapai **25K Stars**!
@ -298,7 +300,25 @@ Untuk dokumentasi TUI terperinci, lihat [docs.picoclaw.io](https://docs.picoclaw
Berikan telefon lama anda kehidupan baru! Jadikannya Pembantu AI pintar dengan PicoClaw.
**Pilihan 1: Termux (tersedia sekarang)**
**Pilihan 1: Pasang APK**
Pratonton:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Muat turun APK dari [picoclaw.io](https://picoclaw.io/download/) dan pasang secara langsung. Tiada Termux diperlukan!
**Pilihan 2: Termux**
<details>
<summary><b>Pelancar Terminal (untuk persekitaran terhad sumber)</b></summary>
1. Pasang [Termux](https://github.com/termux/termux-app) (muat turun dari [GitHub Releases](https://github.com/termux/termux-app/releases), atau cari di F-Droid / Google Play)
2. Jalankan arahan berikut:
@ -315,13 +335,6 @@ Kemudian ikuti bahagian Pelancar Terminal di bawah untuk melengkapkan konfiguras
<img src="assets/termux.jpg" alt="PicoClaw pada Termux" width="512">
**Pilihan 2: APK (akan datang)**
APK Android bebas dengan WebUI terbina dalam sedang dalam pembangunan. Nantikan!
<details>
<summary><b>Pelancar Terminal (untuk persekitaran terhad sumber)</b></summary>
Untuk persekitaran minimal di mana hanya binari teras `picoclaw` tersedia (tiada UI Pelancar), anda boleh mengkonfigurasi semua melalui baris arahan dan fail konfigurasi JSON.
**1. Mulakan**

View file

@ -56,6 +56,8 @@
## 📢 Novidades
2026-03-31 📱 **Suporte Android!** PicoClaw agora roda no Android! Baixe o APK em [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 Lançada!** Reformulação da arquitetura Agent (SubTurn, Hooks, Steering, EventBus), integração WeChat/WeCom, fortalecimento de segurança (.security.yml, filtragem de dados sensíveis), novos providers (AWS Bedrock, Azure, Xiaomi MiMo) e 35 correções de bugs. O PicoClaw atingiu **26K Stars**!
2026-03-17 🚀 **v0.2.3 Lançada!** UI na bandeja do sistema (Windows e Linux), consulta de status de sub-agent (`spawn_status`), hot-reload experimental do Gateway, controle de segurança do Cron e 2 correções de segurança. O PicoClaw atingiu **25K Stars**!
@ -301,7 +303,25 @@ Para documentação detalhada do TUI, veja [docs.picoclaw.io](https://docs.picoc
Dê uma segunda vida ao seu celular de uma década! Transforme-o em um Assistente de IA inteligente com o PicoClaw.
**Opção 1: Termux (disponível agora)**
**Opção 1: Instalação via APK**
Pré-visualização:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Baixe o APK de [picoclaw.io](https://picoclaw.io/download/) e instale diretamente. Sem necessidade de Termux!
**Opção 2: Termux**
<details>
<summary><b>Terminal Launcher (para ambientes com recursos limitados)</b></summary>
1. Instale o [Termux](https://github.com/termux/termux-app) (baixe nas [GitHub Releases](https://github.com/termux/termux-app/releases), ou pesquise no F-Droid / Google Play)
2. Execute os seguintes comandos:
@ -318,13 +338,6 @@ Em seguida, siga a seção Terminal Launcher abaixo para concluir a configuraç
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**Opção 2: Instalação via APK (em breve)**
Um APK Android independente com WebUI integrado está em desenvolvimento. Fique ligado!
<details>
<summary><b>Terminal Launcher (para ambientes com recursos limitados)</b></summary>
Para ambientes mínimos onde apenas o binário principal `picoclaw` está disponível (sem Launcher UI), você pode configurar tudo via linha de comando e um arquivo de configuração JSON.
**1. Inicializar**

View file

@ -56,6 +56,8 @@
## 📢 Tin tức
2026-03-31 📱 **Hỗ trợ Android!** PicoClaw giờ chạy trên Android! Tải APK tại [picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 đã phát hành!** Tái cấu trúc kiến trúc Agent (SubTurn, Hooks, Steering, EventBus), tích hợp WeChat/WeCom, tăng cường bảo mật (.security.yml, lọc dữ liệu nhạy cảm), provider mới (AWS Bedrock, Azure, Xiaomi MiMo) và 35 bản vá lỗi. PicoClaw đã đạt **26K Stars**!
2026-03-17 🚀 **v0.2.3 đã phát hành!** Giao diện system tray (Windows & Linux), truy vấn trạng thái sub-agent (`spawn_status`), thử nghiệm Gateway hot-reload, bảo mật Cron, và 2 bản vá bảo mật. PicoClaw đã đạt **25K Stars**!
@ -301,7 +303,25 @@ Sử dụng menu TUI để: **1)** Cấu hình Provider -> **2)** Cấu hình Ch
Hãy cho chiếc điện thoại cũ của bạn một cuộc sống mới! Biến nó thành Trợ lý AI thông minh với PicoClaw.
**Tùy chọn 1: Termux (có sẵn ngay)**
**Tùy chọn 1: Cài đặt APK**
Xem trước:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
Tải APK từ [picoclaw.io](https://picoclaw.io/download/) và cài đặt trực tiếp. Không cần Termux!
**Tùy chọn 2: Termux**
<details>
<summary><b>Terminal Launcher (cho môi trường hạn chế tài nguyên)</b></summary>
1. Cài đặt [Termux](https://github.com/termux/termux-app) (tải từ [GitHub Releases](https://github.com/termux/termux-app/releases), hoặc tìm kiếm trong F-Droid / Google Play)
2. Chạy các lệnh sau:
@ -318,13 +338,6 @@ Sau đó làm theo phần Terminal Launcher bên dưới để hoàn tất cấu
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**Tùy chọn 2: Cài đặt APK (sắp ra mắt)**
Một APK Android độc lập với WebUI tích hợp đang được phát triển. Hãy đón chờ!
<details>
<summary><b>Terminal Launcher (cho môi trường hạn chế tài nguyên)</b></summary>
Đối với các môi trường tối giản chỉ có binary lõi `picoclaw` (không có Launcher UI), bạn có thể cấu hình mọi thứ qua dòng lệnh và tệp cấu hình JSON.
**1. Khởi tạo**

View file

@ -56,6 +56,8 @@
## 📢 新闻
2026-03-31 📱 **Android 支持!** PicoClaw 现可在 Android 上运行APK 下载地址:[picoclaw.io](https://picoclaw.io/download)
2026-03-25 🚀 **v0.2.4 发布!** Agent 架构全面重构SubTurn、Hook、Steering、EventBus、微信/企业微信深度集成、安全体系升级(.security.yml、敏感数据过滤、新增 ProviderAWS Bedrock、Azure、小米 MiMo以及 35 项 Bug 修复。PicoClaw 已达 **26K ⭐**
2026-03-17 🚀 **v0.2.3 发布!** 系统托盘 UIWindows & Linux、子 Agent 状态查询 (`spawn_status`)、实验性 Gateway 热重载、Cron 安全门控,以及 2 项安全修复。PicoClaw 已达 **25K ⭐**
@ -301,7 +303,25 @@ picoclaw-launcher-tui
让你十年前的旧手机焕发新生!将它变成你的 AI 助手。
**方式一Termux现已可用**
**方式一APK 安装**
预览:
<table>
<tr>
<td><img src="assets/fui_main_page.jpg" width="200"></td>
<td><img src="assets/fui_web_page.jpg" width="200"></td>
<td><img src="assets/fui_log_page.jpg" width="200"></td>
<td><img src="assets/fui_setting_page.jpg" width="200"></td>
</tr>
</table>
从 [picoclaw.io](https://picoclaw.io/download/) 下载 APK 并直接安装,无需 Termux
**方式二Termux**
<details>
<summary><b>Terminal Launcher适用于资源受限环境</b></summary>
1. 安装 [Termux](https://github.com/termux/termux-app)(可从 [GitHub Releases](https://github.com/termux/termux-app/releases) 下载,或在 F-Droid / Google Play 中搜索)
2. 执行以下命令:
@ -318,13 +338,6 @@ termux-chroot ./picoclaw onboard # chroot 提供标准 Linux 文件系统布
<img src="assets/termux.jpg" alt="PicoClaw on Termux" width="512">
**方式二APK 安装(即将推出)**
内置 WebUI 的独立 Android APK 正在开发中,敬请期待!
<details>
<summary><b>Terminal Launcher适用于资源受限环境</b></summary>
对于只有 `picoclaw` 核心二进制文件的极简环境(无 Launcher UI可通过命令行和 JSON 配置文件完成所有配置。
**1. 初始化**
@ -435,7 +448,7 @@ PicoClaw 通过 `model_list` 配置支持 30+ LLM Provider使用 `协议/模
## 💬 Channels聊天应用
通过 17+ 消息平台与你的 PicoClaw 对话:
通过 18+ 消息平台与你的 PicoClaw 对话:
| Channel | 配置难度 | 协议 | 文档 |
|---------|----------|------|------|
@ -450,6 +463,7 @@ PicoClaw 通过 `model_list` 配置支持 30+ LLM Provider使用 `协议/模
| **飞书 / Lark** | 中等App ID + Secret | WebSocket/SDK | [指南](docs/channels/feishu/README.zh.md) |
| **LINE** | 中等credentials + webhook | Webhook | [指南](docs/channels/line/README.zh.md) |
| **企业微信** | 简单(扫码登录或手动配置) | WebSocket | [指南](docs/channels/wecom/README.zh.md) |
| **VK** | 简单(群组 token | Long Poll | [指南](docs/channels/vk/README.md) |
| **IRC** | 中等server + nick | IRC 协议 | [指南](docs/zh/chat-apps.md#irc) |
| **OneBot** | 中等WebSocket URL | OneBot v11 | [指南](docs/channels/onebot/README.zh.md) |
| **MaixCam** | 简单(启用即可) | TCP socket | [指南](docs/channels/maixcam/README.zh.md) |

BIN
assets/fui_log_page.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

BIN
assets/fui_main_page.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 34 KiB

BIN
assets/fui_setting_page.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

BIN
assets/fui_web_page.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 365 KiB

After

Width:  |  Height:  |  Size: 362 KiB

366
cmd/membench/eval.go Normal file
View file

@ -0,0 +1,366 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/sipeed/picoclaw/pkg/seahorse"
)
// EvalResult holds per-sample evaluation results for one mode.
type EvalResult struct {
Mode string `json:"mode"`
SampleID string `json:"sampleId"`
QAResults []QAResult `json:"qaResults"`
Agg AggMetrics `json:"aggregated"`
}
// QAResult holds metrics for a single QA pair.
type QAResult struct {
Question string `json:"question"`
Category int `json:"category"`
GoldAnswer string `json:"goldAnswer"`
TokenF1 float64 `json:"tokenF1"`
HitRate float64 `json:"hitRate"`
}
// AggMetrics holds aggregated evaluation metrics.
type AggMetrics struct {
OverallF1 float64 `json:"overallF1"`
OverallHitRate float64 `json:"overallHitRate"`
ByCategory map[int]*CatMetrics `json:"byCategory"`
TotalQuestions int `json:"totalQuestions"`
}
// CatMetrics holds metrics for a single category.
type CatMetrics struct {
F1 float64 `json:"f1"`
HitRate float64 `json:"hitRate"`
QuestionCount int `json:"questionCount"`
}
// EvalLegacy evaluates using legacy session store (raw history + budget truncation).
func EvalLegacy(
ctx context.Context,
samples []LocomoSample,
legacy *LegacyStore,
budgetTokens int,
) []EvalResult {
results := make([]EvalResult, 0, len(samples))
for si := range samples {
sample := &samples[si]
history := legacy.GetHistory(sample.SampleID)
// Convert messages to content strings
allContent := make([]string, 0, len(history))
for _, msg := range history {
allContent = append(allContent, msg.Content)
}
qaResults := make([]QAResult, 0, len(sample.QA))
for qi := range sample.QA {
qa := &sample.QA[qi]
// Budget truncate the full history
truncated, _ := BudgetTruncate(allContent, budgetTokens)
context := StringListToContent(truncated)
f1 := TokenOverlapF1(context, qa.AnswerString())
hitRate := RecallHitRate(qa.Evidence, sample, context)
qaResults = append(qaResults, QAResult{
Question: qa.Question,
Category: qa.Category,
GoldAnswer: qa.AnswerString(),
TokenF1: f1,
HitRate: hitRate,
})
}
results = append(results, EvalResult{
Mode: "legacy",
SampleID: sample.SampleID,
QAResults: qaResults,
Agg: aggregateMetrics(qaResults),
})
}
return results
}
// EvalSeahorse evaluates using seahorse short memory (per-keyword search + expand).
func EvalSeahorse(
ctx context.Context,
samples []LocomoSample,
ir *SeahorseIngestResult,
budgetTokens int,
) []EvalResult {
store := ir.Engine.GetRetrieval().Store()
retrieval := ir.Engine.GetRetrieval()
results := make([]EvalResult, 0, len(samples))
for si := range samples {
sample := &samples[si]
convID, ok := ir.ConvMap[sample.SampleID]
if !ok {
log.Printf("WARN: no conversation ID for sample %s", sample.SampleID)
continue
}
qaResults := make([]QAResult, 0, len(sample.QA))
for qi := range sample.QA {
qa := &sample.QA[qi]
keywords := ExtractKeywords(qa.Question)
// Search each keyword individually and union results,
// tracking best BM25 rank per message for relevance sorting.
bestRank := map[int64]float64{}
for _, kw := range keywords {
searchResults, err := store.SearchMessages(ctx, seahorse.SearchInput{
Pattern: kw,
ConversationID: convID,
Limit: 20,
})
if err != nil {
log.Printf("WARN: search failed for keyword %q: %v", kw, err)
continue
}
for _, sr := range searchResults {
if sr.MessageID > 0 {
if prev, ok := bestRank[sr.MessageID]; !ok || sr.Rank < prev {
bestRank[sr.MessageID] = sr.Rank
}
}
}
}
// Sort messageIDs by rank ascending (best/most-negative first).
// BudgetTruncate walks from the front, keeping best-ranked messages.
// Note: SQLite FTS5 bm25() returns negative values where more
// negative = better match.
messageIDs := make([]int64, 0, len(bestRank))
for id := range bestRank {
messageIDs = append(messageIDs, id)
}
sort.Slice(messageIDs, func(i, j int) bool {
return bestRank[messageIDs[i]] < bestRank[messageIDs[j]]
})
// Expand messages to get full content
var contentParts []string
if len(messageIDs) > 0 {
expandResult, err := retrieval.ExpandMessages(ctx, messageIDs)
if err != nil {
log.Printf("WARN: expand failed for sample %s: %v", sample.SampleID, err)
} else {
for _, msg := range expandResult.Messages {
contentParts = append(contentParts, msg.Content)
}
}
}
if len(contentParts) == 0 {
qaResults = append(qaResults, QAResult{
Question: qa.Question,
Category: qa.Category,
GoldAnswer: qa.AnswerString(),
TokenF1: 0.0,
HitRate: 0.0,
})
continue
}
// Budget truncate (drop worst-ranked)
truncated, _ := BudgetTruncate(contentParts, budgetTokens)
context := StringListToContent(truncated)
f1 := TokenOverlapF1(context, qa.AnswerString())
hitRate := RecallHitRate(qa.Evidence, sample, context)
qaResults = append(qaResults, QAResult{
Question: qa.Question,
Category: qa.Category,
GoldAnswer: qa.AnswerString(),
TokenF1: f1,
HitRate: hitRate,
})
}
results = append(results, EvalResult{
Mode: "seahorse",
SampleID: sample.SampleID,
QAResults: qaResults,
Agg: aggregateMetrics(qaResults),
})
}
return results
}
// aggregateMetrics computes overall and per-category metrics.
func aggregateMetrics(qaResults []QAResult) AggMetrics {
byCat := map[int]*CatMetrics{}
totalF1 := 0.0
totalHitRate := 0.0
for _, qr := range qaResults {
totalF1 += qr.TokenF1
totalHitRate += qr.HitRate
cat, ok := byCat[qr.Category]
if !ok {
cat = &CatMetrics{}
byCat[qr.Category] = cat
}
cat.F1 += qr.TokenF1
cat.HitRate += qr.HitRate
cat.QuestionCount++
}
n := len(qaResults)
if n == 0 {
n = 1
}
agg := AggMetrics{
OverallF1: totalF1 / float64(n),
OverallHitRate: totalHitRate / float64(n),
ByCategory: byCat,
TotalQuestions: len(qaResults),
}
for _, cat := range agg.ByCategory {
if cat.QuestionCount > 0 {
cat.F1 /= float64(cat.QuestionCount)
cat.HitRate /= float64(cat.QuestionCount)
}
}
return agg
}
// SaveResults writes per-sample eval results to JSON files.
func SaveResults(results []EvalResult, outDir string) error {
if err := os.MkdirAll(outDir, 0o755); err != nil {
return fmt.Errorf("create output dir: %w", err)
}
for _, r := range results {
path := filepath.Join(outDir, fmt.Sprintf("eval_%s_%s.json", r.Mode, r.SampleID))
data, err := json.MarshalIndent(r, "", " ")
if err != nil {
return fmt.Errorf("marshal result: %w", err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return fmt.Errorf("write result: %w", err)
}
}
return nil
}
// SaveAggregated writes a combined results.json with all modes.
func SaveAggregated(results []EvalResult, outDir string) error {
byMode := map[string][]EvalResult{}
for _, r := range results {
byMode[r.Mode] = append(byMode[r.Mode], r)
}
aggMap := map[string]AggMetrics{}
for mode, modeResults := range byMode {
aggMap[mode] = computeModeAgg(modeResults)
}
data, err := json.MarshalIndent(aggMap, "", " ")
if err != nil {
return err
}
return os.WriteFile(filepath.Join(outDir, "results.json"), data, 0o644)
}
// computeModeAgg aggregates results for a single mode using weighted averaging
// (weighted by question count per sample). All modes must have the same Mode field.
func computeModeAgg(results []EvalResult) AggMetrics {
agg := AggMetrics{ByCategory: map[int]*CatMetrics{}}
for _, r := range results {
agg.OverallF1 += r.Agg.OverallF1 * float64(r.Agg.TotalQuestions)
agg.OverallHitRate += r.Agg.OverallHitRate * float64(r.Agg.TotalQuestions)
agg.TotalQuestions += r.Agg.TotalQuestions
for cat, cm := range r.Agg.ByCategory {
existing, ok := agg.ByCategory[cat]
if !ok {
existing = &CatMetrics{}
agg.ByCategory[cat] = existing
}
existing.F1 += cm.F1 * float64(cm.QuestionCount)
existing.HitRate += cm.HitRate * float64(cm.QuestionCount)
existing.QuestionCount += cm.QuestionCount
}
}
if agg.TotalQuestions > 0 {
agg.OverallF1 /= float64(agg.TotalQuestions)
agg.OverallHitRate /= float64(agg.TotalQuestions)
}
for _, cat := range agg.ByCategory {
if cat.QuestionCount > 0 {
cat.F1 /= float64(cat.QuestionCount)
cat.HitRate /= float64(cat.QuestionCount)
}
}
return agg
}
// printSection prints a single comparison table section.
func printSection(title string, results []EvalResult) {
fmt.Printf("\n--- %s ---\n", title)
byMode := map[string][]EvalResult{}
for _, r := range results {
byMode[r.Mode] = append(byMode[r.Mode], r)
}
modes := map[string]AggMetrics{}
for mode, modeResults := range byMode {
modes[mode] = computeModeAgg(modeResults)
}
modeKeys := make([]string, 0, len(modes))
for k := range modes {
modeKeys = append(modeKeys, k)
}
sort.Strings(modeKeys)
// Collect all category keys across modes
catSet := map[int]bool{}
for _, agg := range modes {
for cat := range agg.ByCategory {
catSet[cat] = true
}
}
cats := make([]int, 0, len(catSet))
for cat := range catSet {
cats = append(cats, cat)
}
sort.Ints(cats)
fmt.Printf("%-10s %-8s %-8s", "Mode", "HitRate", "F1")
for _, cat := range cats {
fmt.Printf(" %-7s", fmt.Sprintf("C%d", cat))
}
fmt.Println()
fmt.Println(strings.Repeat("-", 10+8+8+7*len(cats)+8))
for _, mode := range modeKeys {
agg := modes[mode]
fmt.Printf("%-10s %-8.4f %-8.4f", mode, agg.OverallHitRate, agg.OverallF1)
for _, cat := range cats {
if cm, ok := agg.ByCategory[cat]; ok {
fmt.Printf(" %-7.4f", cm.HitRate)
} else {
fmt.Printf(" %-7s", "N/A")
}
}
fmt.Println()
}
}
// PrintComparison outputs a human-readable comparison table to stdout.
func PrintComparison(results []EvalResult, llmResults []EvalResult) {
printSection("No LLM generation", results)
if len(llmResults) > 0 {
printSection("With LLM", llmResults)
}
}

104
cmd/membench/eval_test.go Normal file
View file

@ -0,0 +1,104 @@
package main
import (
"math"
"testing"
)
func TestComputeModeAggAllCategories(t *testing.T) {
results := []EvalResult{
{
Mode: "test",
SampleID: "s1",
QAResults: []QAResult{
{Category: 1, TokenF1: 0.5, HitRate: 0.8},
{Category: 2, TokenF1: 0.3, HitRate: 0.6},
{Category: 3, TokenF1: 0.1, HitRate: 0.4},
{Category: 4, TokenF1: 0.7, HitRate: 0.9},
{Category: 5, TokenF1: 0.2, HitRate: 0.1},
},
},
}
for i := range results {
results[i].Agg = aggregateMetrics(results[i].QAResults)
}
got := computeModeAgg(results)
// Should have all 5 categories
for cat := 1; cat <= 5; cat++ {
cm, ok := got.ByCategory[cat]
if !ok {
t.Errorf("ByCategory missing category %d", cat)
continue
}
if cm.QuestionCount != 1 {
t.Errorf("ByCategory[%d].QuestionCount = %d, want 1", cat, cm.QuestionCount)
}
}
// Verify specific F1 values per category
wantF1 := map[int]float64{1: 0.5, 2: 0.3, 3: 0.1, 4: 0.7, 5: 0.2}
for cat, want := range wantF1 {
if cm, ok := got.ByCategory[cat]; ok {
if math.Abs(cm.F1-want) > 1e-9 {
t.Errorf("ByCategory[%d].F1 = %.4f, want %.4f", cat, cm.F1, want)
}
}
}
}
func TestComputeModeAgg(t *testing.T) {
// Two samples with different question counts:
// sample-a: 2 questions, F1 = [0.4, 0.6] → avg 0.5
// sample-b: 8 questions, F1 = [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1] → avg 0.1
//
// Unweighted (PrintComparison bug): (0.5 + 0.1) / 2 = 0.3
// Weighted (correct): (0.4+0.6 + 0.1*8) / 10 = 1.8 / 10 = 0.18
results := []EvalResult{
{
Mode: "test",
SampleID: "sample-a",
QAResults: []QAResult{
{TokenF1: 0.4, HitRate: 0.5},
{TokenF1: 0.6, HitRate: 0.7},
},
},
{
Mode: "test",
SampleID: "sample-b",
QAResults: []QAResult{
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
{TokenF1: 0.1, HitRate: 0.2},
},
},
}
// Compute per-sample aggregates
for i := range results {
results[i].Agg = aggregateMetrics(results[i].QAResults)
}
got := computeModeAgg(results)
// Weighted: (0.4+0.6+0.1*8) / 10 = 1.8/10 = 0.18
wantF1 := 0.18
if math.Abs(got.OverallF1-wantF1) > 1e-9 {
t.Errorf("OverallF1 = %.6f, want %.6f (weighted average)", got.OverallF1, wantF1)
}
// Weighted: (0.5+0.7+0.2*8) / 10 = 2.8/10 = 0.28
wantRecall := 0.28
if math.Abs(got.OverallHitRate-wantRecall) > 1e-9 {
t.Errorf("OverallHitRate = %.6f, want %.6f (weighted average)", got.OverallHitRate, wantRecall)
}
if got.TotalQuestions != 10 {
t.Errorf("TotalQuestions = %d, want 10", got.TotalQuestions)
}
}

85
cmd/membench/ingest.go Normal file
View file

@ -0,0 +1,85 @@
package main
import (
"context"
"fmt"
"log"
"github.com/sipeed/picoclaw/pkg/seahorse"
)
// ConvMap stores the mapping from sampleID to seahorse ConversationID.
type ConvMap map[string]int64
// SeahorseIngestResult holds the results of ingesting into seahorse.
type SeahorseIngestResult struct {
Engine *seahorse.Engine
ConvMap ConvMap // sampleID → conversationID
}
// IngestSeahorse loads all LOCOMO samples into a seahorse Engine.
// Returns the engine and a mapping from sampleID to conversationID for scoped retrieval.
func IngestSeahorse(ctx context.Context, samples []LocomoSample, dbPath string) (*SeahorseIngestResult, error) {
noopFn := func(ctx context.Context, prompt string, opts seahorse.CompleteOptions) (string, error) {
return "", nil
}
engine, err := seahorse.NewEngine(seahorse.Config{
DBPath: dbPath,
}, noopFn)
if err != nil {
return nil, fmt.Errorf("create seahorse engine: %w", err)
}
store := engine.GetRetrieval().Store()
convMap := make(ConvMap)
for si := range samples {
sample := &samples[si]
sessionKey := "locomo-" + sample.SampleID
// Check if conversation already exists (idempotent)
existing, _ := store.GetConversationBySessionKey(ctx, sessionKey)
if existing != nil {
convMap[sample.SampleID] = existing.ConversationID
log.Printf("Skipping existing sample %s: convID=%d", sample.SampleID, existing.ConversationID)
continue
}
turns := GetTurns(sample)
// Convert turns to seahorse messages
msgs := make([]seahorse.Message, 0, len(turns))
for _, turn := range turns {
content := turn.Speaker + ": " + turn.Text
msgs = append(msgs, seahorse.Message{
Role: "user",
Content: content,
TokenCount: len(turn.Text) / 4,
})
}
// Ingest all turns for this sample
_, err := engine.Ingest(ctx, sessionKey, msgs)
if err != nil {
return nil, fmt.Errorf("ingest sample %s: %w", sample.SampleID, err)
}
// Get the conversation ID for scoped retrieval
conv, err := store.GetConversationBySessionKey(ctx, sessionKey)
if err != nil {
return nil, fmt.Errorf("get conversation for %s: %w", sample.SampleID, err)
}
if conv == nil {
return nil, fmt.Errorf("conversation not found for %s after ingest", sample.SampleID)
}
convMap[sample.SampleID] = conv.ConversationID
log.Printf("Ingested sample %s: %d turns, convID=%d", sample.SampleID, len(turns), conv.ConversationID)
}
log.Printf("Seahorse ingestion complete: %d samples, %d conversations", len(samples), len(convMap))
return &SeahorseIngestResult{
Engine: engine,
ConvMap: convMap,
}, nil
}

View file

@ -0,0 +1,79 @@
package main
import (
"context"
"encoding/json"
"path/filepath"
"testing"
"github.com/sipeed/picoclaw/pkg/seahorse"
)
func TestIngestSeahorseIdempotent(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "test.db")
// Minimal test data
samples := []LocomoSample{
{
SampleID: "test-1",
Conversation: map[string]json.RawMessage{
"session_1": json.RawMessage(`[
{"speaker":"A","dia_id":"D1:1","text":"hello world this is a test message"},
{"speaker":"B","dia_id":"D1:2","text":"another message for testing purposes"}
]`),
},
},
}
// First ingestion
result1, err := IngestSeahorse(ctx, samples, dbPath)
if err != nil {
t.Fatalf("first ingest failed: %v", err)
}
convCount1 := len(result1.ConvMap)
result1.Engine.Close()
// Second ingestion on same DB — should reuse existing data
result2, err := IngestSeahorse(ctx, samples, dbPath)
if err != nil {
t.Fatalf("second ingest failed: %v", err)
}
defer result2.Engine.Close()
// ConvMap should have same number of entries (no duplicates)
if len(result2.ConvMap) != convCount1 {
t.Errorf("second ingest convMap has %d entries, want %d (same as first)",
len(result2.ConvMap), convCount1)
}
// Verify conversation IDs are the same (reused, not new ones)
for id, cid1 := range result1.ConvMap {
cid2, ok := result2.ConvMap[id]
if !ok {
t.Errorf("sample %s missing from second ConvMap", id)
continue
}
if cid2 != cid1 {
t.Errorf("sample %s: second ingest got convID %d, want %d (reused)", id, cid2, cid1)
}
}
// Verify no duplicate messages by counting
store := result2.Engine.GetRetrieval().Store()
for _, convID := range result2.ConvMap {
msgs, err := store.SearchMessages(ctx, seahorse.SearchInput{
Pattern: "test",
ConversationID: convID,
Limit: 100,
})
if err != nil {
t.Fatalf("search failed: %v", err)
}
// Should find exactly 1 message containing "test" (the first turn)
if len(msgs) > 2 {
t.Errorf("found %d messages for 'test' in conv %d, expected ≤2 (no duplicates)", len(msgs), convID)
}
}
}

View file

@ -0,0 +1,34 @@
package main
import (
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/session"
)
// LegacyStore wraps session.SessionManager for legacy baseline.
type LegacyStore struct {
sm *session.SessionManager
}
// NewLegacyStore creates a new in-memory session manager.
func NewLegacyStore() *LegacyStore {
return &LegacyStore{
sm: session.NewSessionManager(""),
}
}
// IngestSample loads all turns from a LOCOMO sample into the legacy session store.
func (ls *LegacyStore) IngestSample(sample *LocomoSample) {
sessionKey := "locomo-" + sample.SampleID
turns := GetTurns(sample)
for _, turn := range turns {
content := turn.Speaker + ": " + turn.Text
ls.sm.AddMessage(sessionKey, "user", content)
}
}
// GetHistory returns all messages for a sample's session.
func (ls *LegacyStore) GetHistory(sampleID string) []providers.Message {
sessionKey := "locomo-" + sampleID
return ls.sm.GetHistory(sessionKey)
}

142
cmd/membench/locomo.go Normal file
View file

@ -0,0 +1,142 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// LocomoSample represents one conversation sample from the LOCOMO dataset.
type LocomoSample struct {
SampleID string `json:"sample_id"`
Conversation map[string]json.RawMessage `json:"conversation"`
QA []LocomoQA `json:"qa"`
}
// LocomoTurn represents a single turn in a conversation.
type LocomoTurn struct {
Speaker string `json:"speaker"`
DiaID string `json:"dia_id"`
Text string `json:"text"`
}
// LocomoQA represents a question-answer pair with evidence.
type LocomoQA struct {
Question string `json:"question"`
Answer json.RawMessage `json:"answer"` // can be string or int (category 1-4)
AdversarialAnswer string `json:"adversarial_answer"` // category 5 only
Evidence []string `json:"evidence"`
Category int `json:"category"` // 1=single-hop, 2=multi-hop, 3=open-ended, 5=adversarial
}
// AnswerString returns the answer as a string, handling both string and int types.
func (qa *LocomoQA) AnswerString() string {
// Prefer answer field (category 1-4)
if len(qa.Answer) > 0 {
var s string
if err := json.Unmarshal(qa.Answer, &s); err == nil {
return s
}
var n json.Number
if err := json.Unmarshal(qa.Answer, &n); err == nil {
return n.String()
}
return strings.Trim(string(qa.Answer), `"`)
}
// Fallback to adversarial_answer (category 5)
return qa.AdversarialAnswer
}
// LoadDataset reads all JSON files from dataDir and returns parsed samples.
func LoadDataset(dataDir string) ([]LocomoSample, error) {
entries, err := os.ReadDir(dataDir)
if err != nil {
return nil, fmt.Errorf("read data dir %s: %w", dataDir, err)
}
var samples []LocomoSample
for _, entry := range entries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".json") {
path := filepath.Join(dataDir, entry.Name())
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read file %s: %w", path, err)
}
var batch []LocomoSample
if err := json.Unmarshal(data, &batch); err != nil {
return nil, fmt.Errorf("parse file %s: %w", path, err)
}
samples = append(samples, batch...)
}
}
return samples, nil
}
// GetSessionNames returns sorted session keys (session_1, session_2, ...) from conversation.
func GetSessionNames(conv map[string]json.RawMessage) []string {
var names []string
for k := range conv {
if strings.HasPrefix(k, "session_") && !strings.Contains(k, "_date_time") {
names = append(names, k)
}
}
sort.Slice(names, func(i, j int) bool {
ni := sessionNum(names[i])
nj := sessionNum(names[j])
return ni < nj
})
return names
}
func sessionNum(key string) int {
// "session_1" → 1, "session_10" → 10
parts := strings.SplitN(key, "_", 2)
if len(parts) < 2 {
return 0
}
n, _ := strconv.Atoi(parts[1])
return n
}
// GetTurns flattens all sessions' turns in chronological order.
func GetTurns(sample *LocomoSample) []LocomoTurn {
names := GetSessionNames(sample.Conversation)
var all []LocomoTurn
for _, name := range names {
raw, ok := sample.Conversation[name]
if !ok {
continue
}
var turns []LocomoTurn
if err := json.Unmarshal(raw, &turns); err != nil {
log.Printf("WARNING: unmarshal failed for session %q in sample %s: %v", name, sample.SampleID, err)
continue
}
all = append(all, turns...)
}
return all
}
// GetTurnByDiaID finds a specific turn by dia_id (e.g. "D1:3").
func GetTurnByDiaID(sample *LocomoSample, diaID string) *LocomoTurn {
turns := GetTurns(sample)
for i := range turns {
if turns[i].DiaID == diaID {
return &turns[i]
}
}
return nil
}
// GetSpeakers returns the two speaker names from conversation metadata.
func GetSpeakers(conv map[string]json.RawMessage) (string, string) {
var a, b string
json.Unmarshal(conv["speaker_a"], &a)
json.Unmarshal(conv["speaker_b"], &b)
return a, b
}

View file

@ -0,0 +1,67 @@
package main
import (
"encoding/json"
"testing"
)
func TestAnswerString(t *testing.T) {
tests := []struct {
name string
json string
want string
}{
{
"string answer",
`{"question":"Q","answer":"Paris","evidence":[],"category":1}`,
"Paris",
},
{
"int answer",
`{"question":"Q","answer":42,"evidence":[],"category":1}`,
"42",
},
{
"adversarial answer (category 5)",
`{"question":"Q","evidence":[],"category":5,"adversarial_answer":"self-care is important"}`,
"self-care is important",
},
{
"both answer and adversarial_answer present",
`{"question":"Q","answer":"normal","evidence":[],"category":5,"adversarial_answer":"adversarial"}`,
"normal",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var qa LocomoQA
if err := json.Unmarshal([]byte(tt.json), &qa); err != nil {
t.Fatalf("unmarshal: %v", err)
}
got := qa.AnswerString()
if got != tt.want {
t.Errorf("AnswerString() = %q, want %q", got, tt.want)
}
})
}
}
func TestGetSessionNames(t *testing.T) {
conv := map[string]json.RawMessage{
"session_2": {},
"session_1": {},
"session_10": {},
"session_1_date_time": {},
"speaker_a": {},
}
names := GetSessionNames(conv)
want := []string{"session_1", "session_2", "session_10"}
if len(names) != len(want) {
t.Fatalf("got %v, want %v", names, want)
}
for i, n := range names {
if n != want[i] {
t.Errorf("names[%d] = %q, want %q", i, n, want[i])
}
}
}

208
cmd/membench/main.go Normal file
View file

@ -0,0 +1,208 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"github.com/sipeed/picoclaw/pkg/logger"
)
var (
flagData string
flagOut string
flagMode string
flagBudget int
)
func main() {
// Suppress seahorse INFO logs during benchmark
logger.SetLevel(logger.WARN)
rootCmd := &cobra.Command{
Use: "membench",
Short: "Memory benchmark tool for picoclaw",
}
ingestCmd := &cobra.Command{
Use: "ingest",
Short: "Load LOCOMO data into storage backends",
RunE: runIngest,
}
ingestCmd.Flags().StringVar(&flagData, "data", "", "LOCOMO dataset directory (required)")
ingestCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
ingestCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to ingest: legacy, seahorse, or all")
evalCmd := &cobra.Command{
Use: "eval",
Short: "Run QA evaluation against ingested data",
RunE: runEval,
}
evalCmd.Flags().StringVar(&flagData, "data", "", "LOCOMO dataset directory (required)")
evalCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
evalCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to evaluate: legacy, seahorse, or all")
evalCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval")
reportCmd := &cobra.Command{
Use: "report",
Short: "Output comparison results from evaluation",
RunE: runReport,
}
reportCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
runCmd := &cobra.Command{
Use: "run",
Short: "Convenience: eval + report (ingestion is done inline)",
RunE: runAll,
}
runCmd.Flags().StringVar(&flagData, "data", "", "LOCOMO dataset directory (required)")
runCmd.Flags().StringVar(&flagOut, "out", "./bench-out", "output working directory")
runCmd.Flags().StringVar(&flagMode, "mode", "all", "modes to run: legacy, seahorse, or all")
runCmd.Flags().IntVar(&flagBudget, "budget", 4000, "token budget for retrieval")
rootCmd.AddCommand(ingestCmd, evalCmd, reportCmd, runCmd)
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
}
func modesFromFlag() []string {
switch strings.ToLower(flagMode) {
case "all":
return []string{"legacy", "seahorse"}
default:
return []string{strings.ToLower(flagMode)}
}
}
func runIngest(cmd *cobra.Command, args []string) error {
if flagData == "" {
return fmt.Errorf("--data is required")
}
modes := modesFromFlag()
if len(modes) == 0 {
return nil
}
ctx := context.Background()
samples, err := LoadDataset(flagData)
if err != nil {
return fmt.Errorf("load dataset: %w", err)
}
log.Printf("Loaded %d samples from %s", len(samples), flagData)
for _, mode := range modes {
switch mode {
case "legacy":
legacy := NewLegacyStore()
for i := range samples {
legacy.IngestSample(&samples[i])
}
log.Printf("legacy: ingested %d samples", len(samples))
case "seahorse":
dbPath := filepath.Join(flagOut, "seahorse.db")
if err := os.MkdirAll(flagOut, 0o755); err != nil {
return fmt.Errorf("create out dir: %w", err)
}
_, err := IngestSeahorse(ctx, samples, dbPath)
if err != nil {
return fmt.Errorf("ingest seahorse: %w", err)
}
}
}
return nil
}
func runEval(cmd *cobra.Command, args []string) error {
if flagData == "" {
return fmt.Errorf("--data is required")
}
modes := modesFromFlag()
if len(modes) == 0 {
return nil
}
ctx := context.Background()
samples, err := LoadDataset(flagData)
if err != nil {
return fmt.Errorf("load dataset: %w", err)
}
log.Printf("Loaded %d samples", len(samples))
var allResults []EvalResult
for _, mode := range modes {
switch mode {
case "legacy":
legacy := NewLegacyStore()
for i := range samples {
legacy.IngestSample(&samples[i])
}
results := EvalLegacy(ctx, samples, legacy, flagBudget)
allResults = append(allResults, results...)
log.Printf("legacy: evaluated %d samples", len(results))
case "seahorse":
dbPath := filepath.Join(flagOut, "seahorse.db")
ir, err := IngestSeahorse(ctx, samples, dbPath)
if err != nil {
return fmt.Errorf("ingest seahorse: %w", err)
}
results := EvalSeahorse(ctx, samples, ir, flagBudget)
allResults = append(allResults, results...)
log.Printf("seahorse: evaluated %d samples", len(results))
}
}
if err := SaveResults(allResults, flagOut); err != nil {
return fmt.Errorf("save results: %w", err)
}
if err := SaveAggregated(allResults, flagOut); err != nil {
return fmt.Errorf("save aggregated: %w", err)
}
PrintComparison(allResults, nil)
return nil
}
func runReport(cmd *cobra.Command, args []string) error {
entries, err := os.ReadDir(flagOut)
if err != nil {
return fmt.Errorf("read out dir: %w", err)
}
var allResults []EvalResult
for _, entry := range entries {
if !entry.IsDir() && strings.HasPrefix(entry.Name(), "eval_") && strings.HasSuffix(entry.Name(), ".json") {
path := filepath.Join(flagOut, entry.Name())
var r EvalResult
data, err := os.ReadFile(path)
if err != nil {
log.Printf("WARN: read %s: %v", path, err)
continue
}
if err := json.Unmarshal(data, &r); err != nil {
log.Printf("WARN: parse %s: %v", path, err)
continue
}
allResults = append(allResults, r)
}
}
if len(allResults) == 0 {
return fmt.Errorf("no eval results found in %s", flagOut)
}
PrintComparison(allResults, nil)
return nil
}
func runAll(cmd *cobra.Command, args []string) error {
return runEval(cmd, args)
}

227
cmd/membench/metrics.go Normal file
View file

@ -0,0 +1,227 @@
package main
import (
"fmt"
"log"
"regexp"
"strconv"
"strings"
"unicode"
)
// diaIDRe matches valid dia_id patterns like "D1:3", "D30:5".
var diaIDRe = regexp.MustCompile(`^D(\d+):(\d+)$`)
// SplitEvidenceIDs splits an evidence string that may contain multiple
// semicolon-separated or space-separated dia_ids. Only returns valid IDs.
// Example: "D8:6; D9:17" → ["D8:6", "D9:17"]
// Example: "D9:1 D4:4 D4:6" → ["D9:1", "D4:4", "D4:6"]
func SplitEvidenceIDs(evidence string) []string {
if evidence == "" {
return nil
}
// Split on semicolons first, then spaces
parts := strings.Split(evidence, ";")
var ids []string
for _, part := range parts {
for _, token := range strings.Fields(strings.TrimSpace(part)) {
token = strings.TrimSpace(token)
if diaIDRe.MatchString(token) {
ids = append(ids, NormalizeDiaID(token))
}
}
}
if len(ids) == 0 {
return nil
}
return ids
}
// NormalizeDiaID strips leading zeros from the number parts of a dia_id.
// "D30:05" → "D30:5", "D10:003" → "D10:3"
func NormalizeDiaID(id string) string {
m := diaIDRe.FindStringSubmatch(id)
if m == nil {
return id
}
session, _ := strconv.Atoi(m[1])
turn, _ := strconv.Atoi(m[2])
return fmt.Sprintf("D%d:%d", session, turn)
}
// stopwords is a fixed English stopword list for deterministic keyword extraction.
var stopwords = map[string]struct{}{
"a": {}, "an": {}, "the": {},
"is": {}, "are": {}, "was": {}, "were": {},
"did": {}, "does": {}, "do": {},
"when": {}, "where": {}, "what": {}, "who": {},
"how": {}, "why": {},
"to": {}, "of": {}, "in": {}, "on": {}, "at": {},
"for": {}, "and": {}, "or": {}, "but": {}, "not": {},
"it": {}, "this": {}, "that": {}, "with": {},
"from": {}, "by": {}, "as": {},
"if": {}, "then": {}, "than": {}, "so": {},
"no": {}, "yes": {},
"all": {}, "any": {}, "each": {}, "every": {},
"some": {}, "such": {},
"about": {}, "into": {}, "over": {},
"after": {}, "before": {}, "between": {},
"through": {}, "during": {}, "until": {},
"would": {}, "could": {}, "should": {},
"may": {}, "might": {}, "can": {},
"will": {}, "shall": {}, "must": {},
"have": {}, "has": {}, "had": {},
"been": {}, "being": {}, "be": {},
"go": {}, "went": {}, "gone": {},
"i": {}, "you": {}, "me": {}, "my": {}, "your": {},
"we": {}, "they": {}, "them": {}, "our": {},
"its": {}, "their": {}, "he": {}, "she": {},
"his": {}, "her": {},
}
// ExtractKeywords removes stopwords and punctuation, returns individual keywords.
// Deterministic: uses fixed stopword list, no LLM.
func ExtractKeywords(question string) []string {
// Lowercase and split on whitespace/punctuation
lower := strings.ToLower(question)
words := strings.FieldsFunc(lower, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
var keywords []string
for _, w := range words {
if w == "" || len(w) < 2 {
continue
}
if _, ok := stopwords[w]; ok {
continue
}
keywords = append(keywords, w)
if len(keywords) >= 6 {
break
}
}
return keywords
}
// TokenOverlapF1 computes token-level F1 between prediction and reference.
// Both strings are lowercased and split on whitespace.
// NOTE: This metric underestimates quality for multi-hop (cat 2) and
// open-ended (cat 3) questions where the gold answer uses different phrasing
// than the source text. LLM-Judge scoring is a v2 follow-up.
func TokenOverlapF1(prediction, reference string) float64 {
predTokens := tokenize(prediction)
refTokens := tokenize(reference)
if len(predTokens) == 0 && len(refTokens) == 0 {
return 1.0
}
if len(predTokens) == 0 || len(refTokens) == 0 {
return 0.0
}
// Count matches
refCount := map[string]int{}
for _, t := range refTokens {
refCount[t]++
}
predCount := map[string]int{}
for _, t := range predTokens {
predCount[t]++
}
var matches float64
for token, pc := range predCount {
if rc, ok := refCount[token]; ok {
matches += float64(min(pc, rc))
}
}
precision := matches / float64(len(predTokens))
recall := matches / float64(len(refTokens))
if precision+recall == 0 {
return 0.0
}
return 2 * precision * recall / (precision + recall)
}
func tokenize(s string) []string {
lower := strings.ToLower(s)
return strings.Fields(lower)
}
// RecallHitRate computes fraction of evidence IDs found in retrieved content.
// For each evidence dia_id, looks up the turn text and checks substring match.
// Logs a warning for turns with text < 20 chars (higher false-positive risk).
func RecallHitRate(evidenceIDs []string, sample *LocomoSample, retrievedContent string) float64 {
if len(evidenceIDs) == 0 {
return 1.0 // no evidence required = perfect
}
// Expand any multi-ID evidence entries (e.g. "D8:6; D9:17" or "D9:1 D4:4")
var expanded []string
for _, id := range evidenceIDs {
split := SplitEvidenceIDs(id)
if split != nil {
expanded = append(expanded, split...)
}
}
if len(expanded) == 0 {
log.Printf("WARNING: no valid dia_ids after expanding evidence %v", evidenceIDs)
return float64(0) / float64(len(evidenceIDs))
}
// Build turn index once (avoids re-parsing JSON per ID)
turns := GetTurns(sample)
turnMap := make(map[string]*LocomoTurn, len(turns))
for i := range turns {
turnMap[turns[i].DiaID] = &turns[i]
}
lowerRetrieved := strings.ToLower(retrievedContent)
found := 0
resolvable := 0
for _, diaID := range expanded {
turn, ok := turnMap[diaID]
if !ok {
log.Printf("WARNING: dia_id %q not found in sample %s", diaID, sample.SampleID)
continue
}
resolvable++
if len(turn.Text) < 20 {
log.Printf("WARNING: short turn text (%d chars) for dia_id %s: %q",
len(turn.Text), diaID, turn.Text)
}
if strings.Contains(lowerRetrieved, strings.ToLower(turn.Text)) {
found++
}
}
if resolvable == 0 {
return 0.0 // no resolvable evidence = can't evaluate
}
return float64(found) / float64(resolvable)
}
// BudgetTruncate truncates messages to fit within a token budget.
// Returns the truncated messages and total token count.
func BudgetTruncate(messages []string, budgetTokens int) ([]string, int) {
var result []string
total := 0
// Walk from the front (best first) and keep until budget exhausted.
for i := 0; i < len(messages); i++ {
tokens := len(messages[i]) / 4
if total+tokens > budgetTokens && len(result) > 0 {
break
}
result = append(result, messages[i])
total += tokens
}
return result, total
}
// StringListToContent joins a list of strings into a single content string.
func StringListToContent(parts []string) string {
return strings.Join(parts, "\n")
}

View file

@ -0,0 +1,239 @@
package main
import (
"encoding/json"
"math"
"testing"
)
func TestSplitEvidenceIDs(t *testing.T) {
tests := []struct {
input string
want []string
}{
{"D1:3", []string{"D1:3"}},
{"D8:6; D9:17", []string{"D8:6", "D9:17"}},
{"D9:1 D4:4 D4:6", []string{"D9:1", "D4:4", "D4:6"}},
{"D22:1 D22:2 D9:10 D9:11", []string{"D22:1", "D22:2", "D9:10", "D9:11"}},
{"D21:18 D21:22 D11:15 D11:19", []string{"D21:18", "D21:22", "D11:15", "D11:19"}},
{"D30:05", []string{"D30:5"}},
{"D", nil},
{"D:", nil},
{"", nil},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
got := SplitEvidenceIDs(tt.input)
if len(got) != len(tt.want) {
t.Fatalf("SplitEvidenceIDs(%q) = %v, want %v", tt.input, got, tt.want)
}
for i := range got {
if got[i] != tt.want[i] {
t.Errorf("[%d] = %q, want %q", i, got[i], tt.want[i])
}
}
})
}
}
func TestNormalizeDiaID(t *testing.T) {
tests := []struct {
input string
want string
}{
{"D1:3", "D1:3"},
{"D30:05", "D30:5"},
{"D10:003", "D10:3"},
{"D1:0", "D1:0"},
}
for _, tt := range tests {
got := NormalizeDiaID(tt.input)
if got != tt.want {
t.Errorf("NormalizeDiaID(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestTokenOverlapF1(t *testing.T) {
tests := []struct {
name string
prediction string
reference string
want float64
}{
{"exact match", "hello world", "hello world", 1.0},
{"no overlap", "foo bar", "baz qux", 0.0},
{"empty both", "", "", 1.0},
{"empty prediction", "", "hello", 0.0},
{"empty reference", "hello", "", 0.0},
{"partial overlap", "the cat sat on the mat", "the cat on the floor", 8.0 / 11.0},
{"case insensitive", "Hello World", "hello world", 1.0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := TokenOverlapF1(tt.prediction, tt.reference)
if math.Abs(got-tt.want) > 1e-9 {
t.Errorf("TokenOverlapF1(%q, %q) = %.4f, want %.4f",
tt.prediction, tt.reference, got, tt.want)
}
})
}
}
func TestBudgetTruncate(t *testing.T) {
t.Run("within budget returns all", func(t *testing.T) {
msgs := []string{"short", "message", "here"}
result, total := BudgetTruncate(msgs, 1000)
if len(result) != 3 {
t.Errorf("expected 3 messages, got %d", len(result))
}
if total == 0 {
t.Error("expected non-zero token count")
}
})
t.Run("over budget keeps best first", func(t *testing.T) {
msgs := []string{
"best message that is quite long and takes up tokens",
"good message also fairly long content",
"worst short",
}
result, _ := BudgetTruncate(msgs, 5) // very small budget
if len(result) == 0 {
t.Fatal("expected at least one message")
}
// Best-ranked (first) should be kept
if result[0] != "best message that is quite long and takes up tokens" {
t.Errorf("expected best message kept first, got %q", result[0])
}
})
t.Run("over budget keeps best ranked first", func(t *testing.T) {
// Messages are sorted by bm25 rank ascending (best/most-negative first).
// When budget is insufficient, BudgetTruncate must keep the front
// (best-ranked) messages, not the tail (worst-ranked).
msgs := []string{
"best ranked message with some content here",
"second best message also has content",
"third message here too",
"worst ranked short",
}
// Budget only fits ~1 message (~10 tokens per message, budget=12)
result, _ := BudgetTruncate(msgs, 12)
if len(result) == 0 {
t.Fatal("expected at least one message")
}
if result[0] != "best ranked message with some content here" {
t.Errorf("expected best-ranked (first) message kept, got %q", result[0])
}
// Worst-ranked (last) must NOT appear
for _, m := range result {
if m == "worst ranked short" {
t.Error("worst-ranked message should have been truncated")
}
}
})
t.Run("preserves original order", func(t *testing.T) {
msgs := []string{"alpha", "beta", "gamma"}
result, _ := BudgetTruncate(msgs, 100)
for i, got := range result {
if got != msgs[i] {
t.Errorf("result[%d] = %q, want %q", i, got, msgs[i])
}
}
})
t.Run("empty input", func(t *testing.T) {
result, total := BudgetTruncate(nil, 100)
if len(result) != 0 {
t.Errorf("expected 0 messages, got %d", len(result))
}
if total != 0 {
t.Errorf("expected 0 tokens, got %d", total)
}
})
}
func TestRecallHitRate(t *testing.T) {
// Build a sample with known turns
sample := &LocomoSample{
SampleID: "test-sample",
Conversation: map[string]json.RawMessage{
"session_1": json.RawMessage(`[
{"speaker":"A","dia_id":"D1:1","text":"hello world this is a test message with enough length"},
{"speaker":"B","dia_id":"D1:2","text":"another message for testing recall computation purposes here"},
{"speaker":"A","dia_id":"D1:3","text":"third turn with some more content to test"}
]`),
},
}
t.Run("all evidence found", func(t *testing.T) {
retrieved := "hello world this is a test message with enough length another message for testing recall computation purposes here"
got := RecallHitRate([]string{"D1:1", "D1:2"}, sample, retrieved)
if math.Abs(got-1.0) > 1e-9 {
t.Errorf("RecallHitRate all found = %.4f, want 1.0", got)
}
})
t.Run("partial evidence found", func(t *testing.T) {
retrieved := "hello world this is a test message with enough length"
got := RecallHitRate([]string{"D1:1", "D1:2"}, sample, retrieved)
if math.Abs(got-0.5) > 1e-9 {
t.Errorf("RecallHitRate partial = %.4f, want 0.5", got)
}
})
t.Run("no evidence required", func(t *testing.T) {
got := RecallHitRate(nil, sample, "anything")
if got != 1.0 {
t.Errorf("RecallHitRate no evidence = %.4f, want 1.0", got)
}
})
t.Run("missing turn excluded from denominator", func(t *testing.T) {
// D1:1 is found, D99:1 does not exist in sample
// Should only count resolvable turns in denominator
retrieved := "hello world this is a test message with enough length"
got := RecallHitRate([]string{"D1:1", "D99:1"}, sample, retrieved)
if math.Abs(got-1.0) > 1e-9 {
t.Errorf("RecallHitRate missing turn = %.4f, want 1.0 (unresolvable excluded)", got)
}
})
}
func TestExtractKeywords(t *testing.T) {
tests := []struct {
name string
input string
want []string
}{
{"simple", "What is the capital of France", []string{"capital", "france"}},
{
"stops removed",
"Who is the president of the United States",
[]string{"president", "united", "states"},
},
{
"max 6 keywords",
"one two three four five six seven eight nine ten",
[]string{"one", "two", "three", "four", "five", "six"},
},
{"short words filtered", "I am a go to the store", []string{"am", "store"}},
{"empty", "", nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ExtractKeywords(tt.input)
if len(got) != len(tt.want) {
t.Fatalf("ExtractKeywords(%q) = %v (len %d), want %v (len %d)",
tt.input, got, len(got), tt.want, len(tt.want))
}
for i := range got {
if got[i] != tt.want[i] {
t.Errorf("[%d] = %q, want %q", i, got[i], tt.want[i])
}
}
})
}
}

View file

@ -9,6 +9,7 @@ package main
import (
"fmt"
"os"
"time"
"github.com/spf13/cobra"
@ -24,10 +25,11 @@ import (
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/status"
"github.com/sipeed/picoclaw/cmd/picoclaw/internal/version"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/updater"
)
func NewPicoclawCommand() *cobra.Command {
short := fmt.Sprintf("%s picoclaw - Personal AI Assistant v%s\n\n", internal.Logo, config.GetVersion())
short := fmt.Sprintf("%s picoclaw - Personal AI Assistant %s\n\n", internal.Logo, config.GetVersion())
cmd := &cobra.Command{
Use: "picoclaw",
@ -45,6 +47,7 @@ func NewPicoclawCommand() *cobra.Command {
migrate.NewMigrateCommand(),
skills.NewSkillsCommand(),
model.NewModelCommand(),
updater.NewUpdateCommand("picoclaw"),
version.NewVersionCommand(),
)
@ -66,6 +69,21 @@ const (
func main() {
fmt.Printf("%s", banner)
tz_env := os.Getenv("TZ")
if tz_env != "" {
fmt.Println("TZ environment:", tz_env)
zoneinfo_env := os.Getenv("ZONEINFO")
fmt.Println("ZONEINFO environment:", zoneinfo_env)
loc, err := time.LoadLocation(tz_env)
if err != nil {
fmt.Println("Error loading time zone:", err)
} else {
fmt.Println("Time zone loaded successfully:", loc)
time.Local = loc //nolint:gosmopolitan // We intentionally set local timezone from TZ env
}
}
cmd := NewPicoclawCommand()
if err := cmd.Execute(); err != nil {
os.Exit(1)

View file

@ -17,7 +17,7 @@ func TestNewPicoclawCommand(t *testing.T) {
require.NotNil(t, cmd)
short := fmt.Sprintf("%s picoclaw - Personal AI Assistant v%s\n\n", internal.Logo, config.GetVersion())
short := fmt.Sprintf("%s picoclaw - Personal AI Assistant %s\n\n", internal.Logo, config.GetVersion())
assert.Equal(t, "picoclaw", cmd.Use)
assert.Equal(t, short, cmd.Short)
@ -43,6 +43,7 @@ func TestNewPicoclawCommand(t *testing.T) {
"onboard",
"skills",
"status",
"update",
"version",
}

View file

@ -48,6 +48,11 @@
"model": "deepseek/deepseek-chat",
"api_key": "sk-your-deepseek-key"
},
{
"model_name": "venice-uncensored",
"model": "venice/venice-uncensored",
"api_key": "your-venice-api-key"
},
{
"model_name": "lmstudio-local",
"model": "lmstudio/openai/gpt-oss-20b"
@ -416,7 +421,11 @@
"enabled": true
},
"read_file": {
"enabled": true
"enabled": true,
"mode": "bytes"
},
"send_tts": {
"enabled": false
},
"spawn": {
"enabled": true

View file

@ -9,4 +9,4 @@ COPY $TARGETPLATFORM/picoclaw-launcher /usr/local/bin/picoclaw-launcher
COPY $TARGETPLATFORM/picoclaw-launcher-tui /usr/local/bin/picoclaw-launcher-tui
ENTRYPOINT ["picoclaw-launcher"]
CMD ["-public", "-no-browser"]
CMD ["-console", "-public", "-no-browser"]

View file

@ -45,8 +45,11 @@ services:
- launcher
environment:
- PICOCLAW_GATEWAY_HOST=0.0.0.0
# Set a fixed dashboard token instead of a random one each restart.
# If not set, a random token is generated and printed to the console on startup.
#- PICOCLAW_LAUNCHER_TOKEN=your-secret-token-here
ports:
- "127.0.0.1:18800:18800"
- "127.0.0.1:18790:18790"
- "18800:18800"
- "18790:18790"
volumes:
- ./data:/root/.picoclaw

View file

@ -13,18 +13,20 @@ Le canal Telegram utilise le long polling via l'API Bot Telegram pour une commun
"enabled": true,
"token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
"allow_from": ["123456789"],
"proxy": ""
"proxy": "",
"use_markdown_v2": false
}
}
}
```
| Champ | Type | Requis | Description |
| ---------- | ------ | ------ | ------------------------------------------------------------------------ |
| enabled | bool | Oui | Activer ou non le canal Telegram |
| token | string | Oui | Token de l'API Bot Telegram |
| allow_from | array | Non | Liste blanche d'identifiants utilisateur ; vide signifie tous les utilisateurs |
| proxy | string | Non | URL du proxy pour se connecter à l'API Telegram (ex. http://127.0.0.1:7890) |
| Champ | Type | Requis | Description |
| --------------- | ------ | ------ | ------------------------------------------------------------------------ |
| enabled | bool | Oui | Activer ou non le canal Telegram |
| token | string | Oui | Token de l'API Bot Telegram |
| allow_from | array | Non | Liste blanche d'identifiants utilisateur ; vide signifie tous les utilisateurs |
| proxy | string | Non | URL du proxy pour se connecter à l'API Telegram (ex. http://127.0.0.1:7890) |
| use_markdown_v2 | bool | Non | Activer le formatage Telegram MarkdownV2 |
## Configuration initiale
@ -33,3 +35,20 @@ Le canal Telegram utilise le long polling via l'API Bot Telegram pour une commun
3. Obtenir le Token de l'API HTTP
4. Renseigner le Token dans le fichier de configuration
5. (Optionnel) Configurer `allow_from` pour restreindre les identifiants utilisateur autorisés à interagir (les IDs peuvent être obtenus via `@userinfobot`)
## Formatage avancées
Vous pouvez définir `use_markdown_v2: true` pour activer les options de formatage améliorées. Cela permet au bot d'utiliser toutes les fonctionnalités de Telegram MarkdownV2, y compris les styles imbriqués, les spoilers et les blocs de largeur fixe personnalisés.
```json
{
"channels": {
"telegram": {
"enabled": true,
"token": "YOUR_BOT_TOKEN",
"allow_from": ["YOUR_USER_ID"],
"use_markdown_v2": true
}
}
}
```

View file

@ -13,18 +13,20 @@ Telegram チャンネルは、Telegram Bot API を使用したロングポーリ
"enabled": true,
"token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
"allow_from": ["123456789"],
"proxy": ""
"proxy": "",
"use_markdown_v2": false
}
}
}
```
| フィールド | 型 | 必須 | 説明 |
| ---------- | ------ | ---- | ----------------------------------------------------------------- |
| enabled | bool | はい | Telegram チャンネルを有効にするかどうか |
| token | string | はい | Telegram Bot API トークン |
| allow_from | array | いいえ | 許可するユーザーIDのリスト。空の場合はすべてのユーザーを許可 |
| proxy | string | いいえ | Telegram API への接続に使用するプロキシ URL (例: http://127.0.0.1:7890) |
| フィールド | 型 | 必須 | 説明 |
| --------------- | ------ | ---- | ----------------------------------------------------------------- |
| enabled | bool | はい | Telegram チャンネルを有効にするかどうか |
| token | string | はい | Telegram Bot API トークン |
| allow_from | array | いいえ | 許可するユーザーIDのリスト。空の場合はすべてのユーザーを許可 |
| proxy | string | いいえ | Telegram API への接続に使用するプロキシ URL (例: http://127.0.0.1:7890) |
| use_markdown_v2 | bool | いいえ | Telegram MarkdownV2 フォーマットを有効にする |
## セットアップ手順
@ -33,3 +35,20 @@ Telegram チャンネルは、Telegram Bot API を使用したロングポーリ
3. HTTP API トークンを取得する
4. 設定ファイルにトークンを入力する
5. (任意) `allow_from` を設定して、対話を許可するユーザー ID を制限するID は `@userinfobot` で取得可能)
## 高度なフォーマット
`use_markdown_v2: true` を設定することで、增强されたフォーマットオプションを有効にできます。これにより、ボットは Telegram MarkdownV2 の全機能(ネストされたスタイル、スポイラー、カスタム固定幅ブロックなど)を利用できます。
```json
{
"channels": {
"telegram": {
"enabled": true,
"token": "YOUR_BOT_TOKEN",
"allow_from": ["YOUR_USER_ID"],
"use_markdown_v2": true
}
}
}
```

View file

@ -13,18 +13,20 @@ The Telegram channel uses long polling via the Telegram Bot API for bot-based co
"enabled": true,
"token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
"allow_from": ["123456789"],
"proxy": ""
"proxy": "",
"use_markdown_v2": false
}
}
}
```
| Field | Type | Required | Description |
| ---------- | ------ | -------- | ------------------------------------------------------------------ |
| enabled | bool | Yes | Whether to enable the Telegram channel |
| token | string | Yes | Telegram Bot API Token |
| allow_from | array | No | Allowlist of user IDs; empty means all users are allowed |
| proxy | string | No | Proxy URL for connecting to the Telegram API (e.g. http://127.0.0.1:7890) |
| Field | Type | Required | Description |
| ---------------- | ------ | -------- | ------------------------------------------------------------------ |
| enabled | bool | Yes | Whether to enable the Telegram channel |
| token | string | Yes | Telegram Bot API Token |
| allow_from | array | No | Allowlist of user IDs; empty means all users are allowed |
| proxy | string | No | Proxy URL for connecting to the Telegram API (e.g. http://127.0.0.1:7890) |
| use_markdown_v2 | bool | No | Enable Telegram MarkdownV2 formatting |
## Setup
@ -53,3 +55,20 @@ Examples:
/use git
explain how to squash the last 3 commits
```
## Advanced Formatting
You can set `use_markdown_v2: true` to enable enhanced formatting options. This allows the bot to utilize the full range of Telegram MarkdownV2 features, including nested styles, spoilers, and custom fixed-width blocks.
```json
{
"channels": {
"telegram": {
"enabled": true,
"token": "YOUR_BOT_TOKEN",
"allow_from": ["YOUR_USER_ID"],
"use_markdown_v2": true
}
}
}
```

View file

@ -13,18 +13,20 @@ O canal Telegram utiliza long polling via a API de Bot do Telegram para comunica
"enabled": true,
"token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
"allow_from": ["123456789"],
"proxy": ""
"proxy": "",
"use_markdown_v2": false
}
}
}
```
| Campo | Tipo | Obrigatório | Descrição |
| ---------- | ------ | ----------- | -------------------------------------------------------------------------- |
| enabled | bool | Sim | Se o canal Telegram deve ser habilitado |
| token | string | Sim | Token da API de Bot do Telegram |
| allow_from | array | Não | Lista de IDs de usuários permitidos; vazio significa todos os usuários |
| proxy | string | Não | URL do proxy para conexão com a API do Telegram (ex. http://127.0.0.1:7890) |
| Campo | Tipo | Obrigatório | Descrição |
| --------------- | ------ | ----------- | -------------------------------------------------------------------------- |
| enabled | bool | Sim | Se o canal Telegram deve ser habilitado |
| token | string | Sim | Token da API de Bot do Telegram |
| allow_from | array | Não | Lista de IDs de usuários permitidos; vazio significa todos os usuários |
| proxy | string | Não | URL do proxy para conexão com a API do Telegram (ex. http://127.0.0.1:7890) |
| use_markdown_v2 | bool | Não | Habilitar formatação Telegram MarkdownV2 |
## Configuração inicial
@ -33,3 +35,20 @@ O canal Telegram utiliza long polling via a API de Bot do Telegram para comunica
3. Obtenha o Token da API HTTP
4. Preencha o Token no arquivo de configuração
5. (Opcional) Configure `allow_from` para restringir quais IDs de usuário podem interagir (os IDs podem ser obtidos via `@userinfobot`)
## Formatação Avançada
Você pode definir `use_markdown_v2: true` para habilitar opções de formatação aprimoradas. Isso permite que o bot utilize todos os recursos do Telegram MarkdownV2, incluindo estilos aninhados, spoilers e blocos de largura fixa personalizados.
```json
{
"channels": {
"telegram": {
"enabled": true,
"token": "YOUR_BOT_TOKEN",
"allow_from": ["YOUR_USER_ID"],
"use_markdown_v2": true
}
}
}
```

View file

@ -13,18 +13,20 @@ Kênh Telegram sử dụng long polling qua Telegram Bot API để giao tiếp d
"enabled": true,
"token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
"allow_from": ["123456789"],
"proxy": ""
"proxy": "",
"use_markdown_v2": false
}
}
}
```
| Trường | Kiểu | Bắt buộc | Mô tả |
| ---------- | ------ | -------- | ------------------------------------------------------------------------ |
| enabled | bool | Có | Có bật kênh Telegram hay không |
| token | string | Có | Token API Bot Telegram |
| allow_from | array | Không | Danh sách trắng ID người dùng; để trống nghĩa là cho phép tất cả |
| proxy | string | Không | URL proxy để kết nối với Telegram API (ví dụ: http://127.0.0.1:7890) |
| Trường | Kiểu | Bắt buộc | Mô tả |
| -------------- | ------ | -------- | ------------------------------------------------------------------------ |
| enabled | bool | Có | Có bật kênh Telegram hay không |
| token | string | Có | Token API Bot Telegram |
| allow_from | array | Không | Danh sách trắng ID người dùng; để trống nghĩa là cho phép tất cả |
| proxy | string | Không | URL proxy để kết nối với Telegram API (ví dụ: http://127.0.0.1:7890) |
| use_markdown_v2 | bool | Không | Bật định dạng Telegram MarkdownV2 |
## Hướng dẫn thiết lập
@ -33,3 +35,20 @@ Kênh Telegram sử dụng long polling qua Telegram Bot API để giao tiếp d
3. Lấy Token API HTTP
4. Điền Token vào file cấu hình
5. (Tùy chọn) Cấu hình `allow_from` để giới hạn ID người dùng được phép tương tác (có thể lấy ID qua `@userinfobot`)
## Định dạng nâng cao
Bạn có thể đặt `use_markdown_v2: true` để bật các tùy chọn định dạng nâng cao. Điều này cho phép bot sử dụng toàn bộ các tính năng của Telegram MarkdownV2, bao gồm các kiểu lồng nhau, spoiler và các khối chiều rộng cố định tùy chỉnh.
```json
{
"channels": {
"telegram": {
"enabled": true,
"token": "YOUR_BOT_TOKEN",
"allow_from": ["YOUR_USER_ID"],
"use_markdown_v2": true
}
}
}
```

View file

@ -13,18 +13,20 @@ Telegram Channel 通过 Telegram 机器人 API 使用长轮询实现基于机器
"enabled": true,
"token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
"allow_from": ["123456789"],
"proxy": ""
"proxy": "",
"use_markdown_v2": false
}
}
}
```
| 字段 | 类型 | 必填 | 描述 |
| ---------- | ------ | ---- | --------------------------------------------------------- |
| enabled | bool | 是 | 是否启用 Telegram 频道 |
| token | string | 是 | Telegram 机器人 API Token |
| allow_from | array | 否 | 用户ID白名单空表示允许所有用户 |
| proxy | string | 否 | 连接 Telegram API 的代理 URL (例如 http://127.0.0.1:7890) |
| 字段 | 类型 | 必填 | 描述 |
| ---------------- | ------ | ---- | --------------------------------------------------------- |
| enabled | bool | 是 | 是否启用 Telegram 频道 |
| token | string | 是 | Telegram 机器人 API Token |
| allow_from | array | 否 | 用户ID白名单空表示允许所有用户 |
| proxy | string | 否 | 连接 Telegram API 的代理 URL (例如 http://127.0.0.1:7890) |
| use_markdown_v2 | bool | 否 | 启用 Telegram MarkdownV2 格式化 |
## 设置流程
@ -50,6 +52,23 @@ Telegram 会在启动时自动注册 PicoClaw 的顶级 Bot 命令,包括 `/st
```text
/list skills
/use git explain how to squash the last 3 commits
/use italiapersonalfinance
dammi le ultime news
/use git
explain how to squash the last 3 commits
```
## 高级格式化
您可以设置 `use_markdown_v2: true` 来启用增强的格式化选项。这允许机器人使用 Telegram MarkdownV2 的全部功能,包括嵌套样式、剧透和自定义等宽代码块。
```json
{
"channels": {
"telegram": {
"enabled": true,
"token": "YOUR_BOT_TOKEN",
"allow_from": ["YOUR_USER_ID"],
"use_markdown_v2": true
}
}
}
```

194
docs/channels/vk/README.md Normal file
View file

@ -0,0 +1,194 @@
# VK (VKontakte)
The VK channel uses Bots Long Poll API for bot-based communication with VK social network. It supports text messages, media attachments (photos, videos, audio, documents, stickers), and group chat interactions.
## Configuration
```json
{
"channels": {
"vk": {
"enabled": true,
"token": "NOT_HERE",
"group_id": 123456789,
"allow_from": ["123456789"],
"group_trigger": {
"mention_only": false,
"prefixes": ["/bot", "!bot"]
}
}
}
}
```
| Field | Type | Required | Description |
| ---------------- | ------ | -------- | ------------------------------------------------------------------ |
| enabled | bool | Yes | Whether to enable the VK channel |
| token | string | Yes | Set to `NOT_HERE` - token is stored securely (see Token Storage) |
| group_id | int | Yes | VK Community ID (Group ID) |
| allow_from | array | No | Allowlist of user IDs; empty means all users are allowed |
| group_trigger | object | No | Configuration for group chat triggers |
### Token Storage
For security reasons, the VK access token should not be stored directly in the configuration file. Instead:
1. Set `token` to `"NOT_HERE"` in the configuration
2. Store the actual token using one of these methods:
- **Environment variable**: Set `PICOCLAW_CHANNELS_VK_TOKEN` environment variable
- **Secure storage**: Use PicoClaw's secure token storage mechanism
Example using environment variable:
```bash
export PICOCLAW_CHANNELS_VK_TOKEN="vk1.a.abc123..."
```
### Group Trigger Configuration
| Field | Type | Description |
| ------------ | -------- | ------------------------------------------------------------------ |
| mention_only | bool | Only respond when bot is mentioned in group chats |
| prefixes | []string | List of prefixes that trigger bot response in group chats |
## Setup
### 1. Create a VK Community
1. Go to [VK](https://vk.com) and log in
2. Create a new community or use an existing one
3. Note your Community ID (found in the community URL, e.g., `public123456789`)
### 2. Enable Messages
1. Go to your community page
2. Click "Manage" → "Messages" → "Community Messages"
3. Enable community messages
### 3. Create Access Token
1. Go to "Manage" → "API usage" → "Access tokens"
2. Click "Create token"
3. Select the following permissions:
- `messages` - Access to messages
- `photos` - Access to photos (optional)
- `docs` - Access to documents (optional)
4. Copy the generated access token
5. Store the token securely (see Token Storage section below)
### 4. Configure PicoClaw
1. Add the token to your PicoClaw configuration
2. Set the `group_id` to your community ID (numeric value)
3. (Optional) Configure `allow_from` to restrict which user IDs can interact
## Features
### Supported Message Types
- **Text messages**: Full support for text messages
- **Photos**: Photos are displayed as `[photo]` placeholder
- **Videos**: Videos are displayed as `[video]` placeholder
- **Audio**: Audio files are displayed as `[audio]` placeholder
- **Voice messages**: Voice messages are displayed as `[voice]` placeholder and support transcription
- **Documents**: Documents are displayed as `[document: filename]`
- **Stickers**: Stickers are displayed as `[sticker]` placeholder
### Voice Support
The VK channel supports both voice message reception and text-to-speech capabilities:
- **ASR (Automatic Speech Recognition)**: Voice messages can be transcribed to text using configured voice models
- **TTS (Text-to-Speech)**: Text responses can be converted to voice messages
To enable voice transcription, configure a voice model in your providers setup. See [Voice Transcription](../../providers.md#voice-transcription) for details.
### Group Chat Support
The VK channel supports group chats with configurable triggers:
- **Mention-only mode**: Bot only responds when mentioned
- **Prefix mode**: Bot responds to messages starting with specified prefixes
- **Permissive mode**: Bot responds to all messages (default)
### Message Length
VK has a maximum message length of 4000 characters. PicoClaw automatically splits longer messages into multiple parts.
## Example Configuration
### Basic Configuration
```json
{
"channels": {
"vk": {
"enabled": true,
"token": "NOT_HERE",
"group_id": 123456789
}
}
}
```
### With User Whitelist
```json
{
"channels": {
"vk": {
"enabled": true,
"token": "NOT_HERE",
"group_id": 123456789,
"allow_from": ["123456789", "987654321"]
}
}
}
```
### With Group Chat Triggers
```json
{
"channels": {
"vk": {
"enabled": true,
"token": "NOT_HERE",
"group_id": 123456789,
"group_trigger": {
"prefixes": ["/bot", "!bot"]
}
}
}
}
```
## Troubleshooting
### Bot Not Responding
1. Check that the access token is valid
2. Verify that the `group_id` is correct
3. Ensure the user ID is in `allow_from` if configured
4. Check PicoClaw logs for error messages
### Permission Errors
Make sure the access token has the necessary permissions:
- `messages` - Required for sending and receiving messages
- `photos` - Optional, for handling photo attachments
- `docs` - Optional, for handling document attachments
### Group Chat Issues
If the bot doesn't respond in group chats:
1. Check `group_trigger` configuration
2. Try using a prefix to trigger the bot
3. Check if the bot has permission to read group messages
## API Reference
The VK channel uses the [VK SDK for Go](https://github.com/SevereCloud/vksdk) library, which supports VK API version 5.199.
For more information about VK API, see:
- [VK API Documentation](https://dev.vk.com/en)
- [VK Bots Long Poll API](https://dev.vk.com/en/api/bots-long-poll/getting-started)

View file

@ -246,6 +246,66 @@ Even with `restrict_to_workspace: false`, the `exec` tool blocks these dangerous
| `tools.allow_read_paths` | string[] | `[]` | Additional paths allowed for reading outside workspace |
| `tools.allow_write_paths` | string[] | `[]` | Additional paths allowed for writing outside workspace |
### Read File Mode
`read_file` has two mutually exclusive implementations selected by config. PicoClaw registers exactly one of them at startup:
| Config Key | Type | Default | Description |
|------------|------|---------|-------------|
| `tools.read_file.enabled` | bool | `true` | Enables the `read_file` tool |
| `tools.read_file.mode` | string | `bytes` | Selects the `read_file` implementation: `bytes` or `lines` |
| `tools.read_file.max_read_file_size` | int | `65536` | Maximum bytes returned by `read_file` |
#### Mode: `bytes`
Optimized for arbitrary files and binary-safe pagination.
Parameters:
* `path` (required): File path
* `offset` (optional): Starting byte offset, default `0`
* `length` (optional): Maximum number of bytes to read, default `max_read_file_size`
Use `bytes` when:
* You may read binary files
* You want deterministic byte-range pagination
#### Mode: `lines`
Text-oriented behavior, optimized for source files, markdown, logs, and configs. The tool reads sequentially by line and stops when the configured byte budget is reached.
Parameters:
* `path` (required): File path
* `start_line` (optional): Starting line number, 1-indexed and inclusive, default `1`
* `max_lines` (optional): Maximum number of lines to read, default = all remaining lines until EOF or byte budget
Behavior notes:
* Binary-looking files are rejected with guidance to switch `read_file` to `mode = bytes`
* Extremely long single lines are truncated rather than skipped
Use `mode = lines` when:
* The agent mostly reads text files
* You want line-based pagination in prompts and tool calls
* You want cleaner chunks for code review, logs, and documentation
#### Example
```json
{
"tools": {
"read_file": {
"enabled": true,
"mode": "lines",
"max_read_file_size": 65536
}
}
}
```
### Exec Security
| Config Key | Type | Default | Description |

View file

@ -99,6 +99,24 @@ Cette conception permet également le **support multi-agents** avec une sélecti
}
```
#### Champs d'entrée `model_list`
| Champ | Type | Requis | Description |
|-------|------|--------|-------------|
| `model_name` | string | Oui | Nom unique pour référencer ce modèle dans la config agent |
| `model` | string | Oui | Identifiant fournisseur/modèle (ex : `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) |
| `api_keys` | string[] | Oui* | Clé(s) API pour l'authentification. Plusieurs clés permettent la rotation par requête. Non requis pour les fournisseurs locaux (Ollama, LM Studio, VLLM) |
| `api_base` | string | Non | Remplace l'URL de base API par défaut |
| `proxy` | string | Non | URL du proxy HTTP pour cette entrée de modèle |
| `user_agent` | string | Non | En-tête `User-Agent` personnalisé pour les requêtes API (supporté par les providers OpenAI-compatible, Anthropic et Azure) |
| `request_timeout` | int | Non | Délai d'expiration de la requête en secondes (la valeur par défaut varie selon le provider) |
| `max_tokens_field` | string | Non | Remplace le nom du champ max tokens dans le corps de la requête (ex : `max_completion_tokens` pour les modèles o1) |
| `thinking_level` | string | Non | Niveau de pensée étendue : `off`, `low`, `medium`, `high`, `xhigh` ou `adaptive` |
| `extra_body` | object | Non | Champs supplémentaires à injecter dans chaque corps de requête |
| `rpm` | int | Non | Limite de requêtes par minute |
| `fallbacks` | string[] | Non | Noms des modèles de secours pour le basculement automatique |
| `enabled` | bool | Non | Activer ou désactiver cette entrée de modèle (par défaut : `true`) |
#### Exemples par Vendor
**OpenAI**
@ -190,6 +208,7 @@ Pour l'accès direct à l'API Anthropic ou les endpoints personnalisés qui ne p
"model": "openai/custom-model",
"api_base": "https://my-proxy.com/v1",
"api_keys": ["sk-..."],
"user_agent": "MyApp/1.0",
"request_timeout": 300
}
```

View file

@ -1,219 +0,0 @@
# ⚙️ Guida alla Configurazione
> Torna al [README](../../README.md)
## ⚙️ Configurazione
File di configurazione: `~/.picoclaw/config.json`
### Variabili d'Ambiente
Puoi sovrascrivere i percorsi predefiniti usando variabili d'ambiente. Questo è utile per installazioni portatili, distribuzioni containerizzate, o per eseguire picoclaw come servizio di sistema. Queste variabili sono indipendenti e controllano percorsi diversi.
| Variabile | Descrizione | Percorso Predefinito |
|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------|---------------------------|
| `PICOCLAW_CONFIG` | Sovrascrive il percorso al file di configurazione. Indica direttamente a picoclaw quale `config.json` caricare, ignorando tutte le altre posizioni. | `~/.picoclaw/config.json` |
| `PICOCLAW_HOME` | Sovrascrive la directory radice per i dati di picoclaw. Modifica la posizione predefinita del `workspace` e delle altre directory dati. | `~/.picoclaw` |
**Esempi:**
```bash
# Esegui picoclaw usando un file di configurazione specifico
# Il percorso del workspace verrà letto da quel file di configurazione
PICOCLAW_CONFIG=/etc/picoclaw/production.json picoclaw gateway
# Esegui picoclaw con tutti i dati salvati in /opt/picoclaw
# La configurazione verrà caricata dal percorso predefinito ~/.picoclaw/config.json
# Il workspace verrà creato in /opt/picoclaw/workspace
PICOCLAW_HOME=/opt/picoclaw picoclaw agent
# Usa entrambi per un setup completamente personalizzato
PICOCLAW_HOME=/srv/picoclaw PICOCLAW_CONFIG=/srv/picoclaw/main.json picoclaw gateway
```
### Struttura del Workspace
PicoClaw salva i dati nel workspace configurato (predefinito: `~/.picoclaw/workspace`):
```
~/.picoclaw/workspace/
├── sessions/ # Sessioni di conversazione e cronologia
├── memory/ # Memoria a lungo termine (MEMORY.md)
├── state/ # Stato persistente (ultimo canale, ecc.)
├── cron/ # Database dei job pianificati
├── skills/ # Skill personalizzate
├── AGENTS.md # Guida al comportamento dell'agent
├── HEARTBEAT.md # Prompt per task periodici (controllato ogni 30 min)
├── IDENTITY.md # Identità dell'agent
├── SOUL.md # Anima dell'agent
└── USER.md # Preferenze dell'utente
```
> **Nota:** Le modifiche a `AGENTS.md`, `SOUL.md`, `USER.md`, `IDENTITY.md` e `memory/MEMORY.md` vengono rilevate automaticamente a runtime tramite il tracciamento della data di modifica (mtime). **Non è necessario riavviare il gateway** dopo aver modificato questi file — l'agent caricherà il nuovo contenuto alla prossima richiesta.
### Sorgenti delle Skill
Per impostazione predefinita, le skill vengono caricate da:
1. `~/.picoclaw/workspace/skills` (workspace)
2. `~/.picoclaw/skills` (globale)
3. `<current-working-directory>/skills` (builtin)
Per configurazioni avanzate/di test, puoi sovrascrivere la directory radice delle skill builtin con:
```bash
export PICOCLAW_BUILTIN_SKILLS=/path/to/skills
```
### Politica Unificata di Esecuzione dei Comandi
- I comandi slash generici vengono eseguiti tramite un unico percorso in `pkg/agent/loop.go` via `commands.Executor`.
- Gli adattatori dei canali non consumano più localmente i comandi generici; inoltrano il testo in entrata al percorso bus/agent. Telegram registra ancora automaticamente i comandi supportati all'avvio.
- Un comando slash sconosciuto (ad esempio `/foo`) viene passato all'elaborazione LLM come se fosse un messaggio dell'utente.
- Un comando registrato ma non supportato sul canale corrente (ad esempio `/show` su WhatsApp) restituisce un errore esplicito all'utente e interrompe l'elaborazione.
### 🔒 Sandbox di Sicurezza
PicoClaw esegue in un ambiente sandboxed per impostazione predefinita. L'agent può accedere solo ai file ed eseguire comandi all'interno del workspace configurato.
#### Configurazione Predefinita
```json
{
"agents": {
"defaults": {
"workspace": "~/.picoclaw/workspace",
"restrict_to_workspace": true
}
}
}
```
| Opzione | Predefinito | Descrizione |
| ----------------------- | ----------------------- | ---------------------------------------------------- |
| `workspace` | `~/.picoclaw/workspace` | Directory di lavoro dell'agent |
| `restrict_to_workspace` | `true` | Limita l'accesso a file/comandi al workspace |
#### Strumenti Protetti
Quando `restrict_to_workspace: true`, i seguenti strumenti sono in sandbox:
| Strumento | Funzione | Restrizione |
| ------------- | ------------------------- | ---------------------------------------------------- |
| `read_file` | Legge file | Solo file all'interno del workspace |
| `write_file` | Scrive file | Solo file all'interno del workspace |
| `list_dir` | Elenca directory | Solo directory all'interno del workspace |
| `edit_file` | Modifica file | Solo file all'interno del workspace |
| `append_file` | Aggiunge ai file | Solo file all'interno del workspace |
| `exec` | Esegue comandi | I percorsi dei comandi devono essere nel workspace |
#### Protezione Exec Aggiuntiva
Anche con `restrict_to_workspace: false`, lo strumento `exec` blocca questi comandi pericolosi:
* `rm -rf`, `del /f`, `rmdir /s` — Cancellazione di massa
* `format`, `mkfs`, `diskpart` — Formattazione del disco
* `dd if=` — Imaging del disco
* Scrittura su `/dev/sd[a-z]` — Scritture dirette su disco
* `shutdown`, `reboot`, `poweroff` — Spegnimento del sistema
* Fork bomb `:(){ :|:& };:`
### Controllo Accesso ai File
| Chiave di configurazione | Tipo | Predefinito | Descrizione |
|--------------------------|------|-------------|-------------|
| `tools.allow_read_paths` | string[] | `[]` | Percorsi aggiuntivi consentiti per la lettura al di fuori del workspace |
| `tools.allow_write_paths` | string[] | `[]` | Percorsi aggiuntivi consentiti per la scrittura al di fuori del workspace |
### Sicurezza Exec
| Chiave di configurazione | Tipo | Predefinito | Descrizione |
|--------------------------|------|-------------|-------------|
| `tools.exec.allow_remote` | bool | `false` | Consente lo strumento exec da canali remoti (Telegram/Discord ecc.) |
| `tools.exec.enable_deny_patterns` | bool | `true` | Abilita l'intercettazione dei comandi pericolosi |
| `tools.exec.custom_deny_patterns` | string[] | `[]` | Pattern regex personalizzati da bloccare |
| `tools.exec.custom_allow_patterns` | string[] | `[]` | Pattern regex personalizzati da consentire |
> **Nota di sicurezza:** La protezione dei symlink è abilitata per impostazione predefinita — tutti i percorsi file vengono risolti tramite `filepath.EvalSymlinks` prima del confronto con la whitelist, prevenendo attacchi di escape tramite symlink.
#### Limitazione Nota: Processi Figlio degli Strumenti di Build
Il controllo di sicurezza exec ispeziona solo la riga di comando avviata direttamente da PicoClaw. Non ispeziona ricorsivamente i processi figlio generati da strumenti di sviluppo consentiti come `make`, `go run`, `cargo`, `npm run` o script di build personalizzati.
Ciò significa che un comando di primo livello può comunque compilare o avviare altri binari dopo aver superato il controllo iniziale. In pratica, tratta gli script di build, i Makefile, gli script di pacchetti e i binari generati come codice eseguibile che richiede lo stesso livello di revisione di un comando shell diretto.
Per ambienti ad alto rischio:
* Esamina gli script di build prima dell'esecuzione.
* Preferisci l'approvazione/revisione manuale per i workflow di compilazione ed esecuzione.
* Esegui PicoClaw in un container o VM se hai bisogno di un isolamento più forte di quello fornito dal controllo integrato.
#### Esempi di Errore
```
[ERROR] tool: Tool execution failed
{tool=exec, error=Command blocked by safety guard (path outside working dir)}
```
```
[ERROR] tool: Tool execution failed
{tool=exec, error=Command blocked by safety guard (dangerous pattern detected)}
```
#### Disabilitare le Restrizioni (Rischio di Sicurezza)
Se hai bisogno che l'agent acceda a percorsi al di fuori del workspace:
**Metodo 1: File di configurazione**
```json
{
"agents": {
"defaults": {
"restrict_to_workspace": false
}
}
}
```
**Metodo 2: Variabile d'ambiente**
```bash
export PICOCLAW_AGENTS_DEFAULTS_RESTRICT_TO_WORKSPACE=false
```
> ⚠️ **Attenzione**: Disabilitare questa restrizione consente all'agent di accedere a qualsiasi percorso sul tuo sistema. Usare con cautela solo in ambienti controllati.
#### Coerenza dei Confini di Sicurezza
L'impostazione `restrict_to_workspace` si applica in modo coerente a tutti i percorsi di esecuzione:
| Percorso di esecuzione | Confine di sicurezza |
| ---------------------- | --------------------------------- |
| Main Agent | `restrict_to_workspace` ✅ |
| Subagent / Spawn | Eredita la stessa restrizione ✅ |
| Heartbeat tasks | Eredita la stessa restrizione ✅ |
Tutti i percorsi condividono la stessa restrizione del workspace — non è possibile aggirare il confine di sicurezza tramite subagent o task pianificati.
### Heartbeat (Task Periodici)
PicoClaw può eseguire task periodici automaticamente. Crea un file `HEARTBEAT.md` nel tuo workspace:
```markdown
# Periodic Tasks
- Check my email for important messages
- Review my calendar for upcoming events
- Check the weather forecast
```
L'agent leggerà questo file ogni 30 minuti (configurabile) ed eseguirà tutti i task usando gli strumenti disponibili.
#### Task Asincroni con Spawn
Per task di lunga durata (ricerca web, chiamate API), usa lo strumento `spawn` per creare un **subagent**:
```markdown
# Periodic Tasks
```

View file

@ -99,6 +99,24 @@
}
```
#### `model_list` エントリフィールド
| フィールド | 型 | 必須 | 説明 |
|-----------|------|------|------|
| `model_name` | string | はい | agent 設定でこのモデルを参照するための一意の名前 |
| `model` | string | はい | ベンダー/モデル識別子(例:`openai/gpt-5.4``azure/gpt-5.4``anthropic/claude-sonnet-4.6` |
| `api_keys` | string[] | はい* | 認証キー。複数キーでリクエストごとのローテーションが可能。ローカル providerOllama、LM Studio、VLLMには不要 |
| `api_base` | string | いいえ | デフォルトの API エンドポイント URL を上書き |
| `proxy` | string | いいえ | このモデルエントリの HTTP プロキシ URL |
| `user_agent` | string | いいえ | カスタム `User-Agent` リクエストヘッダーOpenAI 互換、Anthropic、Azure provider で対応) |
| `request_timeout` | int | いいえ | リクエストタイムアウト(秒)。デフォルト値は provider により異なる |
| `max_tokens_field` | string | いいえ | リクエストボディの max tokens フィールド名を上書きo1 モデルでは `max_completion_tokens` |
| `thinking_level` | string | いいえ | 拡張思考レベル:`off``low``medium``high``xhigh``adaptive` |
| `extra_body` | object | いいえ | 各リクエストボディに注入する追加フィールド |
| `rpm` | int | いいえ | 1 分あたりのリクエストレート制限 |
| `fallbacks` | string[] | いいえ | 自動フェイルオーバーのフォールバックモデル名 |
| `enabled` | bool | いいえ | このモデルエントリを有効にするかどうか(デフォルト:`true` |
#### ベンダー別設定例
**OpenAI**
@ -201,6 +219,7 @@ Anthropic API への直接アクセスや、Anthropic のネイティブメッ
"model": "openai/custom-model",
"api_base": "https://my-proxy.com/v1",
"api_keys": ["sk-..."],
"user_agent": "MyApp/1.0",
"request_timeout": 300
}
```

View file

@ -16,6 +16,7 @@
| `openrouter` | LLM (recommended, access to all models) | [openrouter.ai](https://openrouter.ai) |
| `anthropic` | LLM (Claude direct) | [console.anthropic.com](https://console.anthropic.com) |
| `openai` | LLM (GPT direct) | [platform.openai.com](https://platform.openai.com) |
| `venice` | LLM (Venice AI direct) | [venice.ai](https://venice.ai) |
| `deepseek` | LLM (DeepSeek direct) | [platform.deepseek.com](https://platform.deepseek.com) |
| `qwen` | LLM (Qwen direct) | [dashscope.console.aliyun.com](https://dashscope.console.aliyun.com) |
| `groq` | LLM + **Voice transcription** (Whisper) | [console.groq.com](https://console.groq.com) |
@ -46,6 +47,7 @@ This design also enables **multi-agent support** with flexible provider selectio
| Vendor | `model` Prefix | Default API Base | Protocol | API Key |
| ------------------- | ----------------- |-----------------------------------------------------| --------- | ---------------------------------------------------------------- |
| **OpenAI** | `openai/` | `https://api.openai.com/v1` | OpenAI | [Get Key](https://platform.openai.com) |
| **Venice AI** | `venice/` | `https://api.venice.ai/api/v1` | OpenAI | [Get Key](https://venice.ai) |
| **Anthropic** | `anthropic/` | `https://api.anthropic.com/v1` | Anthropic | [Get Key](https://console.anthropic.com) |
| **智谱 AI (GLM)** | `zhipu/` | `https://open.bigmodel.cn/api/paas/v4` | OpenAI | [Get Key](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) |
| **Z.AI Coding Plan** | `openai/` | `https://api.z.ai/api/coding/paas/v4` | OpenAI | [Get Key](https://z.ai/manage-apikey/apikey-list) |
@ -106,6 +108,25 @@ This design also enables **multi-agent support** with flexible provider selectio
}
```
#### `model_list` Entry Fields
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `model_name` | string | Yes | Unique name used to reference this model in agent config |
| `model` | string | Yes | Vendor/model identifier (e.g., `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) |
| `api_keys` | string[] | Yes* | API key(s) for authentication. Multiple keys enable per-request rotation. Not required for local providers (Ollama, LM Studio, VLLM) |
| `api_base` | string | No | Override the default API endpoint URL |
| `proxy` | string | No | HTTP proxy URL for this model entry |
| `user_agent` | string | No | Custom `User-Agent` header sent with API requests (supported by OpenAI-compatible, Anthropic, and Azure providers) |
| `request_timeout` | int | No | Request timeout in seconds (default varies by provider) |
| `max_tokens_field` | string | No | Override the max tokens field name in request body (e.g., `max_completion_tokens` for o1 models) |
| `thinking_level` | string | No | Extended thinking level: `off`, `low`, `medium`, `high`, `xhigh`, or `adaptive` |
| `extra_body` | object | No | Additional fields to inject into every request body |
| `custom_headers` | object | No | Additional HTTP headers to inject into every request (e.g., `{"X-Source":"coding-plan"}`). If a key matches a built-in header, the custom value overrides the built-in one (e.g., `Authorization`, `User-Agent`, `Content-Type`, `Accept`). |
| `rpm` | int | No | Per-minute request rate limit |
| `fallbacks` | string[] | No | Fallback model names for automatic failover |
| `enabled` | bool | No | Whether this model entry is active (default: `true`) |
#### Voice Transcription
You can configure a dedicated model for audio transcription with `voice.model_name`. This lets you reuse existing multimodal providers that support audio input instead of relying only on Groq.
@ -247,6 +268,7 @@ PicoClaw sends OpenAI-compatible requests to LM Studio, and strips the `lmstudio
"model": "openai/custom-model",
"api_base": "https://my-proxy.com/v1",
"api_keys": ["sk-..."],
"user_agent": "MyApp/1.0",
"request_timeout": 300
}
```

View file

@ -99,6 +99,24 @@ Este design também permite **suporte multi-agente** com seleção flexível de
}
```
#### Campos de entrada `model_list`
| Campo | Tipo | Obrigatório | Descrição |
|-------|------|-------------|-----------|
| `model_name` | string | Sim | Nome único para referenciar este modelo na config do agent |
| `model` | string | Sim | Identificador fornecedor/modelo (ex: `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) |
| `api_keys` | string[] | Sim* | Chave(s) API para autenticação. Múltiplas chaves permitem rotação por requisição. Não necessário para providers locais (Ollama, LM Studio, VLLM) |
| `api_base` | string | Não | Substitui a URL base da API padrão |
| `proxy` | string | Não | URL do proxy HTTP para esta entrada de modelo |
| `user_agent` | string | Não | Cabeçalho `User-Agent` personalizado enviado com requisições API (suportado por providers OpenAI-compatible, Anthropic e Azure) |
| `request_timeout` | int | Não | Timeout de requisição em segundos (o padrão varia por provider) |
| `max_tokens_field` | string | Não | Substitui o nome do campo max tokens no corpo da requisição (ex: `max_completion_tokens` para modelos o1) |
| `thinking_level` | string | Não | Nível de pensamento estendido: `off`, `low`, `medium`, `high`, `xhigh` ou `adaptive` |
| `extra_body` | object | Não | Campos adicionais para injetar em cada corpo de requisição |
| `rpm` | int | Não | Limite de requisições por minuto |
| `fallbacks` | string[] | Não | Nomes dos modelos de fallback para failover automático |
| `enabled` | bool | Não | Ativar ou desativar esta entrada de modelo (padrão: `true`) |
#### Exemplos por Vendor
**OpenAI**
@ -190,6 +208,7 @@ Para acesso direto à API Anthropic ou endpoints personalizados que suportam ape
"model": "openai/custom-model",
"api_base": "https://my-proxy.com/v1",
"api_keys": ["sk-..."],
"user_agent": "MyApp/1.0",
"request_timeout": 300
}
```

95
docs/rate-limiting.md Normal file
View file

@ -0,0 +1,95 @@
# Dynamic Rate Limiting
PicoClaw prevents 429 errors from LLM provider APIs by enforcing configurable per-model request-rate limits **before** sending each request. Unlike the reactive cooldown/fallback system (which activates *after* a 429 is received), rate limiting is **proactive**: it keeps outbound QPS within the provider's free-tier or plan limits.
## How it works
### Token-bucket algorithm
Each rate-limited model gets a token bucket:
- **Capacity** = `rpm` (burst size equals the per-minute limit)
- **Refill rate** = `rpm / 60` tokens per second
- Tokens are consumed one per LLM call; if the bucket is empty, the call blocks until a token refills or the request context is cancelled
### Call chain integration
```
AgentLoop.callLLM()
└─ FallbackChain.Execute() ← iterate candidates
├─ CooldownTracker.IsAvailable() ← skip if post-429 cooldown active
├─ RateLimiterRegistry.Wait() ← NEW: block until token available
└─ provider.Chat() ← actual LLM HTTP call
```
The rate limiter runs **after** the cooldown check and **before** the provider call, so:
- Candidates already in cooldown are skipped entirely (no token consumed)
- Candidates that are available get throttled to the configured RPM
The same check applies in `ExecuteImage`.
### Thread safety
`RateLimiterRegistry` is safe for concurrent use. The per-limiter token bucket uses a fine-grained mutex so concurrent goroutines each acquire their own token independently.
## Configuration
Set `rpm` on any model in `model_list`:
```yaml
model_list:
- model_name: gpt-4o-free
model: openai/gpt-4o
api_base: https://api.openai.com/v1
rpm: 3 # max 3 requests per minute
api_keys:
- sk-...
- model_name: claude-haiku
model: anthropic/claude-haiku-4-5
rpm: 60 # 60 rpm (Anthropic free tier)
api_keys:
- sk-ant-...
- model_name: local-llm
model: openai/llama3
api_base: http://localhost:11434/v1
# no rpm → unrestricted
```
| Field | Type | Default | Description |
|---|---|---|---|
| `rpm` | `int` | `0` | Requests per minute. `0` means no limit. |
### Interaction with fallbacks
When a model has fallbacks configured, each candidate is rate-limited **independently**:
```yaml
model_list:
- model_name: gpt4-with-fallback
model: openai/gpt-4o
rpm: 5
fallbacks:
- gpt-4o-mini # must also be in model_list; its own rpm applies
```
If the current candidate's bucket is empty and there are more candidates available, PicoClaw skips the locally saturated candidate and tries the next fallback immediately. Only the last remaining candidate waits for a token to refill. If the context deadline is hit while waiting on that last candidate, the wait error propagates.
For `model_list` aliases that resolve to the same underlying provider/model, rate limiting is keyed by the stable config identity (for example `model_name`) rather than the resolved runtime model string. This preserves distinct RPM settings for multi-key and alias-based configurations.
### Burst behaviour
The bucket starts **full** (burst = RPM). For `rpm: 3`, the first 3 requests fire instantly; subsequent requests are spaced ~20 s apart.
To reduce burstiness for strict APIs, set a lower `rpm` and rely on the steady-state refill.
## Files changed
| File | What |
|---|---|
| `pkg/providers/ratelimiter.go` | `RateLimiter` (token bucket) + `RateLimiterRegistry` |
| `pkg/providers/ratelimiter_test.go` | Unit tests for limiter and registry |
| `pkg/providers/fallback.go` | `FallbackCandidate.RPM` field; `FallbackChain.rl`; `Wait()` call in `Execute`/`ExecuteImage` |
| `pkg/agent/model_resolution.go` | Resolves candidates from `model_list`, preserving stable config identity and propagating `RPM` into `FallbackCandidate` |
| `pkg/agent/loop.go` | Build `RateLimiterRegistry`, register all agents' candidates, pass to `NewFallbackChain` |

View file

@ -528,6 +528,9 @@ For example:
- `PICOCLAW_TOOLS_EXEC_ENABLE_DENY_PATTERNS=false`
- `PICOCLAW_TOOLS_CRON_EXEC_TIMEOUT_MINUTES=10`
- `PICOCLAW_TOOLS_MCP_ENABLED=true`
- `PICOCLAW_TOOLS_MCP_MAX_INLINE_TEXT_CHARS=16384`
Note: Nested map-style config (for example `tools.mcp.servers.<name>.*`) is configured in `config.json` rather than
environment variables.
For MCP tools, `tools.mcp.max_inline_text_chars` controls how much text result is kept inline in model context. The threshold is counted in Unicode characters (Go runes), not bytes. For example, `16384` means up to 16,384 characters inline, which may occupy more than 16 KB for multibyte text such as CJK. Above this threshold, PicoClaw saves the MCP text result as a local artifact in the agent workspace and gives the model a short note plus a structured `[file:...]` artifact path instead of injecting the full payload into context.

View file

@ -99,6 +99,24 @@ Thiết kế này cũng cho phép **hỗ trợ đa agent** với lựa chọn pr
}
```
#### Các trường entry `model_list`
| Trường | Kiểu | Bắt buộc | Mô tả |
|--------|------|----------|------|
| `model_name` | string | Có | Tên duy nhất để tham chiếu model này trong cấu hình agent |
| `model` | string | Có | Định danh nhà cung cấp/model (ví dụ: `openai/gpt-5.4`, `azure/gpt-5.4`, `anthropic/claude-sonnet-4.6`) |
| `api_keys` | string[] | Có* | Khóa API xác thực. Nhiều khóa cho phép xoay vòng theo yêu cầu. Không cần thiết cho provider nội bộ (Ollama, LM Studio, VLLM) |
| `api_base` | string | Không | Ghi đè URL endpoint API mặc định |
| `proxy` | string | Không | URL proxy HTTP cho entry model này |
| `user_agent` | string | Không | Header `User-Agent` tùy chỉnh gửi với yêu cầu API (được hỗ trợ bởi provider OpenAI-compatible, Anthropic và Azure) |
| `request_timeout` | int | Không | Timeout yêu cầu tính bằng giây (mặc định khác nhau tùy provider) |
| `max_tokens_field` | string | Không | Ghi đè tên trường max tokens trong request body (ví dụ: `max_completion_tokens` cho model o1) |
| `thinking_level` | string | Không | Mức độ tư duy mở rộng: `off`, `low`, `medium`, `high`, `xhigh` hoặc `adaptive` |
| `extra_body` | object | Không | Các trường bổ sung để chèn vào mỗi request body |
| `rpm` | int | Không | Giới hạn tốc độ yêu cầu mỗi phút |
| `fallbacks` | string[] | Không | Tên model dự phòng cho failover tự động |
| `enabled` | bool | Không | Kích hoạt hay vô hiệu hóa entry model này (mặc định: `true`) |
#### Ví Dụ Theo Vendor
**OpenAI**
@ -190,6 +208,7 @@ Thiết kế này cũng cho phép **hỗ trợ đa agent** với lựa chọn pr
"model": "openai/custom-model",
"api_base": "https://my-proxy.com/v1",
"api_keys": ["sk-..."],
"user_agent": "MyApp/1.0",
"request_timeout": 300
}
```

View file

@ -15,6 +15,7 @@
| `openrouter` | LLM (推荐,可访问所有模型) | [openrouter.ai](https://openrouter.ai) |
| `anthropic` | LLM (Claude 直连) | [console.anthropic.com](https://console.anthropic.com) |
| `openai` | LLM (GPT 直连) | [platform.openai.com](https://platform.openai.com) |
| `venice` | LLM (Venice AI 直连) | [venice.ai](https://venice.ai) |
| `deepseek` | LLM (DeepSeek 直连) | [platform.deepseek.com](https://platform.deepseek.com) |
| `qwen` | LLM (通义千问) | [dashscope.console.aliyun.com](https://dashscope.console.aliyun.com) |
| `groq` | LLM + **语音转录** (Whisper) | [console.groq.com](https://console.groq.com) |
@ -44,6 +45,7 @@
| 厂商 | `model` 前缀 | 默认 API Base | 协议 | 获取 API Key |
| ------------------- | ----------------- | --------------------------------------------------- | --------- | ----------------------------------------------------------------- |
| **OpenAI** | `openai/` | `https://api.openai.com/v1` | OpenAI | [获取密钥](https://platform.openai.com) |
| **Venice AI** | `venice/` | `https://api.venice.ai/api/v1` | OpenAI | [获取密钥](https://venice.ai) |
| **Anthropic** | `anthropic/` | `https://api.anthropic.com/v1` | Anthropic | [获取密钥](https://console.anthropic.com) |
| **智谱 AI (GLM)** | `zhipu/` | `https://open.bigmodel.cn/api/paas/v4` | OpenAI | [获取密钥](https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys) |
| **DeepSeek** | `deepseek/` | `https://api.deepseek.com/v1` | OpenAI | [获取密钥](https://platform.deepseek.com) |
@ -102,6 +104,25 @@
}
```
#### `model_list` 条目字段
| 字段 | 类型 | 必填 | 说明 |
|------|------|------|------|
| `model_name` | string | 是 | 在 agent 配置中引用此模型的唯一名称 |
| `model` | string | 是 | 厂商/模型标识符(如 `openai/gpt-5.4``azure/gpt-5.4``anthropic/claude-sonnet-4.6` |
| `api_keys` | string[] | 是* | 认证密钥。多个密钥可按请求轮换。本地 providerOllama、LM Studio、VLLM不需要 |
| `api_base` | string | 否 | 覆盖默认的 API 端点 URL |
| `proxy` | string | 否 | 此模型条目的 HTTP 代理 URL |
| `user_agent` | string | 否 | 自定义 `User-Agent` 请求头(支持 OpenAI 兼容、Anthropic 和 Azure provider |
| `request_timeout` | int | 否 | 请求超时时间(秒),默认值因 provider 而异 |
| `max_tokens_field` | string | 否 | 覆盖请求体中 max tokens 的字段名(如 o1 模型使用 `max_completion_tokens` |
| `thinking_level` | string | 否 | 扩展思考级别:`off``low``medium``high``xhigh``adaptive` |
| `extra_body` | object | 否 | 注入到每个请求体中的额外字段 |
| `custom_headers` | object | 否 | 注入到每个请求中的额外 HTTP 请求头(例如 `{"X-Source":"coding-plan"}`)。若键名与内置请求头同名,会覆盖内置值(如 `Authorization``User-Agent``Content-Type``Accept`)。 |
| `rpm` | int | 否 | 每分钟请求速率限制 |
| `fallbacks` | string[] | 否 | 自动故障转移的备用模型名称 |
| `enabled` | bool | 否 | 是否启用此模型条目(默认:`true` |
#### 语音转录
你可以通过 `voice.model_name` 为语音转录指定一个专用模型。这样可以直接复用已经配置好的、支持音频输入的多模态 provider而不必只依赖 Groq。
@ -232,6 +253,7 @@ PicoClaw 向 LM Studio 的 OpenAI 兼容终结点发送请求,且将移除首
"model": "openai/custom-model",
"api_base": "https://my-proxy.com/v1",
"api_keys": ["sk-..."],
"user_agent": "MyApp/1.0",
"request_timeout": 300
}
```

18
go.mod
View file

@ -5,8 +5,10 @@ go 1.25.8
require (
fyne.io/systray v1.12.0
github.com/BurntSushi/toml v1.6.0
github.com/SevereCloud/vksdk/v3 v3.3.1
github.com/adhocore/gronx v1.19.6
github.com/anthropics/anthropic-sdk-go v1.26.0
github.com/atc0005/go-teams-notify/v2 v2.14.0
github.com/atotto/clipboard v0.1.4
github.com/aws/aws-sdk-go-v2 v1.41.5
github.com/aws/aws-sdk-go-v2/config v1.32.12
@ -23,12 +25,15 @@ require (
github.com/h2non/filetype v1.1.3
github.com/larksuite/oapi-sdk-go/v3 v3.5.3
github.com/mdp/qrterminal/v3 v3.2.1
github.com/minio/selfupdate v0.6.0
github.com/modelcontextprotocol/go-sdk v1.4.1
github.com/mymmrac/telego v1.7.0
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
github.com/openai/openai-go/v3 v3.22.0
github.com/pion/rtp v1.10.1
github.com/pion/webrtc/v3 v3.3.6
github.com/rivo/tview v0.42.0
github.com/rs/zerolog v1.34.0
github.com/rs/zerolog v1.35.0
github.com/slack-go/slack v0.17.3
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
@ -41,11 +46,12 @@ require (
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
maunium.net/go/mautrix v0.26.4
modernc.org/sqlite v1.47.0
modernc.org/sqlite v1.48.0
rsc.io/qr v0.2.0
)
require (
aead.dev/minisign v0.2.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
@ -61,6 +67,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
github.com/aws/smithy-go v1.24.2 // indirect
github.com/beeper/argo-go v1.1.2 // indirect
github.com/cloudflare/circl v1.6.3 // indirect
github.com/coder/websocket v1.8.14 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
@ -76,6 +83,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.34 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
@ -83,6 +91,8 @@ require (
github.com/segmentio/encoding v0.5.4 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/vektah/gqlparser/v2 v2.5.27 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.mau.fi/libsignal v0.2.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.35.0 // indirect
@ -120,6 +130,8 @@ require (
golang.org/x/arch v0.24.0 // indirect
golang.org/x/crypto v0.49.0
golang.org/x/net v0.52.0
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sync v0.20.0
golang.org/x/sys v0.42.0
)
replace github.com/bwmarrin/discordgo => github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532

48
go.sum
View file

@ -1,3 +1,5 @@
aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk=
aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
@ -7,6 +9,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/SevereCloud/vksdk/v3 v3.3.1 h1:O86zsp5LQnHE+O5acvuXM/s6S1LyxzVTkF6+Lup0Jyg=
github.com/SevereCloud/vksdk/v3 v3.3.1/go.mod h1:c6WaA5aocUYsXfkcUbg2qy45V9M1VDcqHHmHIN14NAw=
github.com/adhocore/gronx v1.19.6 h1:5KNVcoR9ACgL9HhEqCm5QXsab/gI4QDIybTAWcXDKDc=
github.com/adhocore/gronx v1.19.6/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg=
github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
@ -17,6 +21,8 @@ github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwTo
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAfT7CoSYSac11PY=
github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q=
github.com/atc0005/go-teams-notify/v2 v2.14.0 h1:7N+xw+COnYANLREaAveQ65rsNQ12nIZJED9nMLyscCo=
github.com/atc0005/go-teams-notify/v2 v2.14.0/go.mod h1:EECsWM2b0Hvoz7O+QdlsvyN2KCUOFQCGj8bUBXv3A3Q=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
@ -53,8 +59,6 @@ github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs=
github.com/beeper/argo-go v1.1.2/go.mod h1:M+LJAnyowKVQ6Rdj6XYGEn+qcVFkb3R/MUpqkGR0hM4=
github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno=
github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE=
@ -65,11 +69,12 @@ github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoG
github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g=
github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
@ -108,7 +113,6 @@ github.com/go-resty/resty/v2 v2.17.1/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2m
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@ -173,17 +177,16 @@ github.com/larksuite/oapi-sdk-go/v3 v3.5.3 h1:xvf8Dv29kBXC5/DNDCLhHkAFW8l/0LlQJi
github.com/larksuite/oapi-sdk-go/v3 v3.5.3/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mdp/qrterminal/v3 v3.2.1 h1:6+yQjiiOsSuXT5n9/m60E54vdgFsw0zhADHhHLrFet4=
github.com/mdp/qrterminal/v3 v3.2.1/go.mod h1:jOTmXvnBsMy5xqLniO0R++Jmjs2sTm9dFSuQ5kpz/SU=
github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
github.com/modelcontextprotocol/go-sdk v1.4.1 h1:M4x9GyIPj+HoIlHNGpK2hq5o3BFhC+78PkEaldQRphc=
github.com/modelcontextprotocol/go-sdk v1.4.1/go.mod h1:Bo/mS87hPQqHSRkMv4dQq1XCu6zv4INdXnFZabkNU6s=
github.com/mymmrac/telego v1.7.0 h1:yRO/l00tFGG4nY66ufUKb4ARqv7qx9+LsjQv/b0NEyo=
@ -204,8 +207,13 @@ github.com/openai/openai-go/v3 v3.22.0 h1:6MEoNoV8sbjOVmXdvhmuX3BjVbVdcExbVyGixi
github.com/openai/openai-go/v3 v3.22.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo=
github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6 h1:rh2lKw/P/EqHa724vYH2+VVQ1YnW4u6EOXl0PMAovZE=
github.com/petermattis/goid v0.0.0-20260226131333-17d1149c6ac6/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
github.com/pion/rtp v1.10.1 h1:xP1prZcCTUuhO2c83XtxyOHJteISg6o8iPsE2acaMtA=
github.com/pion/rtp v1.10.1/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
github.com/pion/webrtc/v3 v3.3.6 h1:7XAh4RPtlY1Vul6/GmZrv7z+NnxKA6If0KStXBI2ZLE=
github.com/pion/webrtc/v3 v3.3.6/go.mod h1:zyN7th4mZpV27eXybfR/cnUf3J2DRy8zw/mdjD9JTNM=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
@ -218,9 +226,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
github.com/rs/zerolog v1.35.0 h1:VD0ykx7HMiMJytqINBsKcbLS+BJ4WYjz+05us+LRTdI=
github.com/rs/zerolog v1.35.0/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
@ -271,8 +278,14 @@ github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADT
github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE=
github.com/vektah/gqlparser/v2 v2.5.27 h1:RHPD3JOplpk5mP5JGX8RKZkt2/Vwj/PZv0HxTdwFp0s=
github.com/vektah/gqlparser/v2 v2.5.27/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo=
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532 h1:gxFHYeUDGziRb0zXYEqBFohC+NJbIW9L0tddaXMWr2o=
github.com/yeongaori/discordgo-fork v0.0.0-20260319072544-e8e546f5d532/go.mod h1:A0FcMFJKJ9fRjgSuZ2o+pIQ6mPS81SVuiLN2vYTa7Ao=
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -300,8 +313,9 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
@ -322,6 +336,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
@ -344,24 +359,25 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@ -442,8 +458,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.47.0 h1:R1XyaNpoW4Et9yly+I2EeX7pBza/w+pmYee/0HJDyKk=
modernc.org/sqlite v1.47.0/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig=
modernc.org/sqlite v1.48.0 h1:ElZyLop3Q2mHYk5IFPPXADejZrlHu7APbpB0sF78bq4=
modernc.org/sqlite v1.48.0/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

View file

@ -602,14 +602,16 @@ func (cb *ContextBuilder) BuildMessages(
// Add conversation history
messages = append(messages, history...)
// Add current user message
if strings.TrimSpace(currentMessage) != "" {
// Add current user message. Media-only turns must still be preserved so
// multimodal providers receive the uploaded image even when the user sends
// no accompanying text.
if strings.TrimSpace(currentMessage) != "" || len(media) > 0 {
msg := providers.Message{
Role: "user",
Content: currentMessage,
}
if len(media) > 0 {
msg.Media = media
msg.Media = append([]string(nil), media...)
}
messages = append(messages, msg)
}

View file

@ -6,10 +6,8 @@
package agent
import (
"encoding/json"
"unicode/utf8"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/tokenizer"
)
// parseTurnBoundaries returns the starting index of each Turn in the history.
@ -86,88 +84,16 @@ func findSafeBoundary(history []providers.Message, targetIndex int) int {
return 0
}
// estimateMessageTokens estimates the token count for a single message,
// including Content, ReasoningContent, ToolCalls arguments, ToolCallID
// metadata, and Media items. Uses a heuristic of 2.5 characters per token.
func estimateMessageTokens(msg providers.Message) int {
contentChars := utf8.RuneCountInString(msg.Content)
// SystemParts are structured system blocks used for cache-aware adapters.
// They carry the same content as Content, but in multiple blocks.
// We estimate them as an alternative representation, not additive.
systemPartsChars := 0
if len(msg.SystemParts) > 0 {
for _, part := range msg.SystemParts {
systemPartsChars += utf8.RuneCountInString(part.Text)
}
// Per-part overhead for JSON structure (type, text, cache_control).
const perPartOverhead = 20
systemPartsChars += len(msg.SystemParts) * perPartOverhead
}
// Use the larger of the two representations to stay conservative.
chars := contentChars
if systemPartsChars > chars {
chars = systemPartsChars
}
chars += utf8.RuneCountInString(msg.ReasoningContent)
for _, tc := range msg.ToolCalls {
chars += len(tc.ID) + len(tc.Type)
if tc.Function != nil {
// Count function name + arguments (the wire format for most providers).
// tc.Name mirrors tc.Function.Name — count only once to avoid double-counting.
chars += len(tc.Function.Name) + len(tc.Function.Arguments)
} else {
// Fallback: some provider formats use top-level Name without Function.
chars += len(tc.Name)
}
}
if msg.ToolCallID != "" {
chars += len(msg.ToolCallID)
}
// Per-message overhead for role label, JSON structure, separators.
const messageOverhead = 12
chars += messageOverhead
tokens := chars * 2 / 5
// Media items (images, files) are serialized by provider adapters into
// multipart or image_url payloads. Add a fixed per-item token estimate
// directly (not through the chars heuristic) since actual cost depends
// on resolution and provider-specific image tokenization.
const mediaTokensPerItem = 256
tokens += len(msg.Media) * mediaTokensPerItem
return tokens
// EstimateMessageTokens estimates the token count for a single message.
// Delegates to the shared tokenizer package for consistency across agent and seahorse.
func EstimateMessageTokens(msg providers.Message) int {
return tokenizer.EstimateMessageTokens(msg)
}
// estimateToolDefsTokens estimates the total token cost of tool definitions
// as they appear in the LLM request. Each tool's name, description, and
// JSON schema parameters contribute to the context window budget.
func estimateToolDefsTokens(defs []providers.ToolDefinition) int {
if len(defs) == 0 {
return 0
}
totalChars := 0
for _, d := range defs {
totalChars += len(d.Function.Name) + len(d.Function.Description)
if d.Function.Parameters != nil {
if paramJSON, err := json.Marshal(d.Function.Parameters); err == nil {
totalChars += len(paramJSON)
}
}
// Per-tool overhead: type field, JSON structure, separators.
totalChars += 20
}
return totalChars * 2 / 5
// EstimateToolDefsTokens estimates the total token cost of tool definitions
// as they appear in the LLM request. Delegates to the shared tokenizer package.
func EstimateToolDefsTokens(defs []providers.ToolDefinition) int {
return tokenizer.EstimateToolDefsTokens(defs)
}
// isOverContextBudget checks whether the assembled messages plus tool definitions
@ -181,10 +107,10 @@ func isOverContextBudget(
) bool {
msgTokens := 0
for _, m := range messages {
msgTokens += estimateMessageTokens(m)
msgTokens += EstimateMessageTokens(m)
}
toolTokens := estimateToolDefsTokens(toolDefs)
toolTokens := EstimateToolDefsTokens(toolDefs)
total := msgTokens + toolTokens + maxTokens
return total > contextWindow

View file

@ -417,9 +417,9 @@ func TestEstimateMessageTokens(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := estimateMessageTokens(tt.msg)
got := EstimateMessageTokens(tt.msg)
if got < tt.want {
t.Errorf("estimateMessageTokens() = %d, want >= %d", got, tt.want)
t.Errorf("EstimateMessageTokens() = %d, want >= %d", got, tt.want)
}
})
}
@ -443,8 +443,8 @@ func TestEstimateMessageTokens_ToolCallsContribute(t *testing.T) {
},
}
plainTokens := estimateMessageTokens(plain)
withTCTokens := estimateMessageTokens(withTC)
plainTokens := EstimateMessageTokens(plain)
withTCTokens := EstimateMessageTokens(withTC)
if withTCTokens <= plainTokens {
t.Errorf("message with ToolCalls (%d tokens) should exceed plain message (%d tokens)",
@ -457,7 +457,7 @@ func TestEstimateMessageTokens_MultibyteContent(t *testing.T) {
// but may map to different token counts. The heuristic should still produce
// reasonable estimates via RuneCountInString.
msg := msgUser("caf\u00e9 na\u00efve r\u00e9sum\u00e9 \u00fcber stra\u00dfe")
tokens := estimateMessageTokens(msg)
tokens := EstimateMessageTokens(msg)
if tokens <= 0 {
t.Errorf("multibyte message should produce positive token count, got %d", tokens)
}
@ -481,7 +481,7 @@ func TestEstimateMessageTokens_LargeArguments(t *testing.T) {
},
}
tokens := estimateMessageTokens(msg)
tokens := EstimateMessageTokens(msg)
// 5000+ chars → at least 2000 tokens with the 2.5 char/token heuristic
if tokens < 2000 {
t.Errorf("large tool call arguments should produce significant token count, got %d", tokens)
@ -496,8 +496,8 @@ func TestEstimateMessageTokens_ReasoningContent(t *testing.T) {
ReasoningContent: strings.Repeat("thinking step ", 200),
}
plainTokens := estimateMessageTokens(plain)
reasoningTokens := estimateMessageTokens(withReasoning)
plainTokens := EstimateMessageTokens(plain)
reasoningTokens := EstimateMessageTokens(withReasoning)
if reasoningTokens <= plainTokens {
t.Errorf("message with ReasoningContent (%d tokens) should exceed plain message (%d tokens)",
@ -513,8 +513,8 @@ func TestEstimateMessageTokens_MediaItems(t *testing.T) {
Media: []string{"media://img1.png", "media://img2.png"},
}
plainTokens := estimateMessageTokens(plain)
mediaTokens := estimateMessageTokens(withMedia)
plainTokens := EstimateMessageTokens(plain)
mediaTokens := EstimateMessageTokens(withMedia)
if mediaTokens <= plainTokens {
t.Errorf("message with Media (%d tokens) should exceed plain message (%d tokens)",
@ -540,8 +540,8 @@ func TestEstimateMessageTokens_SystemParts(t *testing.T) {
},
}
plainTokens := estimateMessageTokens(plain)
partsTokens := estimateMessageTokens(withParts)
plainTokens := EstimateMessageTokens(plain)
partsTokens := EstimateMessageTokens(withParts)
if partsTokens <= plainTokens {
t.Errorf("system message with SystemParts (%d) should exceed plain message (%d)",
@ -549,7 +549,7 @@ func TestEstimateMessageTokens_SystemParts(t *testing.T) {
}
}
// --- estimateToolDefsTokens tests ---
// --- EstimateToolDefsTokens tests ---
func TestEstimateToolDefsTokens(t *testing.T) {
tests := []struct {
@ -599,9 +599,9 @@ func TestEstimateToolDefsTokens(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := estimateToolDefsTokens(tt.defs)
got := EstimateToolDefsTokens(tt.defs)
if got < tt.want {
t.Errorf("estimateToolDefsTokens() = %d, want >= %d", got, tt.want)
t.Errorf("EstimateToolDefsTokens() = %d, want >= %d", got, tt.want)
}
})
}
@ -624,8 +624,8 @@ func TestEstimateToolDefsTokens_ScalesWithCount(t *testing.T) {
}
}
one := estimateToolDefsTokens([]providers.ToolDefinition{makeTool("tool_a")})
three := estimateToolDefsTokens([]providers.ToolDefinition{
one := EstimateToolDefsTokens([]providers.ToolDefinition{makeTool("tool_a")})
three := EstimateToolDefsTokens([]providers.ToolDefinition{
makeTool("tool_a"), makeTool("tool_b"), makeTool("tool_c"),
})
@ -770,7 +770,7 @@ func TestEstimateMessageTokens_WithReasoningAndMedia(t *testing.T) {
},
}
tokens := estimateMessageTokens(msg)
tokens := EstimateMessageTokens(msg)
// ReasoningContent alone is ~1700 chars → ~680 tokens.
// Content + TC + overhead adds more. Should be well above 500.
@ -781,7 +781,7 @@ func TestEstimateMessageTokens_WithReasoningAndMedia(t *testing.T) {
// Compare without reasoning to ensure it's counted.
msgNoReasoning := msg
msgNoReasoning.ReasoningContent = ""
tokensNoReasoning := estimateMessageTokens(msgNoReasoning)
tokensNoReasoning := EstimateMessageTokens(msgNoReasoning)
if tokens <= tokensNoReasoning {
t.Errorf("reasoning content should add tokens: with=%d, without=%d", tokens, tokensNoReasoning)

View file

@ -707,6 +707,38 @@ func TestEmptyWorkspaceBaselineDetectsNewFiles(t *testing.T) {
}
}
func TestBuildMessages_IncludesMediaOnlyCurrentMessage(t *testing.T) {
tmpDir := setupWorkspace(t, nil)
defer os.RemoveAll(tmpDir)
cb := NewContextBuilder(tmpDir)
msgs := cb.BuildMessages(
nil,
"",
"",
[]string{"data:image/png;base64,abc123"},
"pico",
"chat-1",
"",
"",
)
if len(msgs) != 2 {
t.Fatalf("len(msgs) = %d, want 2", len(msgs))
}
userMsg := msgs[1]
if userMsg.Role != "user" {
t.Fatalf("userMsg.Role = %q, want %q", userMsg.Role, "user")
}
if userMsg.Content != "" {
t.Fatalf("userMsg.Content = %q, want empty string", userMsg.Content)
}
if len(userMsg.Media) != 1 || userMsg.Media[0] != "data:image/png;base64,abc123" {
t.Fatalf("userMsg.Media = %#v, want image payload", userMsg.Media)
}
}
// BenchmarkBuildMessagesWithCache measures caching performance.
func BenchmarkBuildMessagesWithCache(b *testing.B) {
tmpDir, _ := os.MkdirTemp("", "picoclaw-bench-*")

379
pkg/agent/context_legacy.go Normal file
View file

@ -0,0 +1,379 @@
package agent
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
)
// legacyContextManager wraps the existing summarization/compression logic
// as a ContextManager implementation. It is the default when no other
// ContextManager is configured.
type legacyContextManager struct {
al *AgentLoop
summarizing sync.Map // dedup for async Compact (post-turn)
}
func (m *legacyContextManager) Assemble(_ context.Context, req *AssembleRequest) (*AssembleResponse, error) {
// Legacy: read history from session, return as-is.
// Budget enforcement happens in BuildMessages caller via
// isOverContextBudget + forceCompression.
agent := m.al.registry.GetDefaultAgent()
if agent == nil {
return &AssembleResponse{}, nil
}
history := agent.Sessions.GetHistory(req.SessionKey)
summary := agent.Sessions.GetSummary(req.SessionKey)
return &AssembleResponse{
History: history,
Summary: summary,
}, nil
}
func (m *legacyContextManager) Compact(_ context.Context, req *CompactRequest) error {
switch req.Reason {
case ContextCompressReasonProactive, ContextCompressReasonRetry:
// Sync emergency compression — budget exceeded.
if result, ok := m.forceCompression(req.SessionKey); ok {
m.al.emitEvent(
EventKindContextCompress,
m.al.newTurnEventScope("", req.SessionKey, nil).meta(0, "forceCompression", "turn.context.compress"),
ContextCompressPayload{
Reason: req.Reason,
DroppedMessages: result.DroppedMessages,
RemainingMessages: result.RemainingMessages,
},
)
}
case ContextCompressReasonSummarize:
m.maybeSummarize(req.SessionKey)
}
return nil
}
func (m *legacyContextManager) Ingest(_ context.Context, _ *IngestRequest) error {
// Legacy: no-op. Messages are persisted by Sessions JSONL.
return nil
}
// maybeSummarize triggers summarization if the session history exceeds thresholds.
// It runs asynchronously in a goroutine.
func (m *legacyContextManager) maybeSummarize(sessionKey string) {
agent := m.al.registry.GetDefaultAgent()
if agent == nil {
return
}
newHistory := agent.Sessions.GetHistory(sessionKey)
tokenEstimate := m.estimateTokens(newHistory)
threshold := agent.ContextWindow * agent.SummarizeTokenPercent / 100
if len(newHistory) > agent.SummarizeMessageThreshold || tokenEstimate > threshold {
summarizeKey := agent.ID + ":" + sessionKey
if _, loading := m.summarizing.LoadOrStore(summarizeKey, true); !loading {
go func() {
defer m.summarizing.Delete(summarizeKey)
defer func() {
if r := recover(); r != nil {
logger.WarnCF("agent", "Summarization panic recovered", map[string]any{
"session_key": sessionKey,
"panic": r,
})
}
}()
logger.Debug("Memory threshold reached. Optimizing conversation history...")
m.summarizeSession(agent, sessionKey)
}()
}
}
}
type compressionResult struct {
DroppedMessages int
RemainingMessages int
}
// forceCompression aggressively reduces context when the limit is hit.
// It drops the oldest ~50% of Turns (a Turn is a complete user→LLM→response
// cycle, as defined in #1316), so tool-call sequences are never split.
func (m *legacyContextManager) forceCompression(sessionKey string) (compressionResult, bool) {
agent := m.al.registry.GetDefaultAgent()
if agent == nil {
return compressionResult{}, false
}
history := agent.Sessions.GetHistory(sessionKey)
if len(history) <= 2 {
return compressionResult{}, false
}
turns := parseTurnBoundaries(history)
var mid int
if len(turns) >= 2 {
mid = turns[len(turns)/2]
} else {
mid = findSafeBoundary(history, len(history)/2)
}
var keptHistory []providers.Message
if mid <= 0 {
for i := len(history) - 1; i >= 0; i-- {
if history[i].Role == "user" {
keptHistory = []providers.Message{history[i]}
break
}
}
} else {
keptHistory = history[mid:]
}
droppedCount := len(history) - len(keptHistory)
existingSummary := agent.Sessions.GetSummary(sessionKey)
compressionNote := fmt.Sprintf(
"[Emergency compression dropped %d oldest messages due to context limit]",
droppedCount,
)
if existingSummary != "" {
compressionNote = existingSummary + "\n\n" + compressionNote
}
agent.Sessions.SetSummary(sessionKey, compressionNote)
agent.Sessions.SetHistory(sessionKey, keptHistory)
agent.Sessions.Save(sessionKey)
logger.WarnCF("agent", "Forced compression executed", map[string]any{
"session_key": sessionKey,
"dropped_msgs": droppedCount,
"new_count": len(keptHistory),
})
return compressionResult{
DroppedMessages: droppedCount,
RemainingMessages: len(keptHistory),
}, true
}
func (m *legacyContextManager) summarizeSession(agent *AgentInstance, sessionKey string) {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
history := agent.Sessions.GetHistory(sessionKey)
summary := agent.Sessions.GetSummary(sessionKey)
if len(history) <= 4 {
return
}
safeCut := findSafeBoundary(history, len(history)-4)
if safeCut <= 0 {
return
}
keepCount := len(history) - safeCut
toSummarize := history[:safeCut]
maxMessageTokens := agent.ContextWindow / 2
validMessages := make([]providers.Message, 0)
omitted := false
for _, msg := range toSummarize {
if msg.Role != "user" && msg.Role != "assistant" {
continue
}
msgTokens := len(msg.Content) / 2
if msgTokens > maxMessageTokens {
omitted = true
continue
}
validMessages = append(validMessages, msg)
}
if len(validMessages) == 0 {
return
}
const (
maxSummarizationMessages = 10
llmMaxRetries = 3
)
var finalSummary string
if len(validMessages) > maxSummarizationMessages {
mid := len(validMessages) / 2
mid = m.findNearestUserMessage(validMessages, mid)
part1 := validMessages[:mid]
part2 := validMessages[mid:]
s1, _ := m.summarizeBatch(ctx, agent, part1, "")
s2, _ := m.summarizeBatch(ctx, agent, part2, "")
mergePrompt := fmt.Sprintf(
"Merge these two conversation summaries into one cohesive summary:\n\n1: %s\n\n2: %s",
s1, s2,
)
resp, err := m.retryLLMCall(ctx, agent, mergePrompt, llmMaxRetries)
if err == nil && resp.Content != "" {
finalSummary = resp.Content
} else {
finalSummary = s1 + " " + s2
}
} else {
finalSummary, _ = m.summarizeBatch(ctx, agent, validMessages, summary)
}
if omitted && finalSummary != "" {
finalSummary += "\n[Note: Some oversized messages were omitted from this summary for efficiency.]"
}
if finalSummary != "" {
agent.Sessions.SetSummary(sessionKey, finalSummary)
agent.Sessions.TruncateHistory(sessionKey, keepCount)
agent.Sessions.Save(sessionKey)
m.al.emitEvent(
EventKindSessionSummarize,
m.al.newTurnEventScope(agent.ID, sessionKey, nil).meta(0, "summarizeSession", "turn.session.summarize"),
SessionSummarizePayload{
SummarizedMessages: len(validMessages),
KeptMessages: keepCount,
SummaryLen: len(finalSummary),
OmittedOversized: omitted,
},
)
}
}
func (m *legacyContextManager) findNearestUserMessage(messages []providers.Message, mid int) int {
originalMid := mid
for mid > 0 && messages[mid].Role != "user" {
mid--
}
if messages[mid].Role == "user" {
return mid
}
mid = originalMid
for mid < len(messages) && messages[mid].Role != "user" {
mid++
}
if mid < len(messages) {
return mid
}
return originalMid
}
func (m *legacyContextManager) retryLLMCall(
ctx context.Context,
agent *AgentInstance,
prompt string,
maxRetries int,
) (*providers.LLMResponse, error) {
const llmTemperature = 0.3
var resp *providers.LLMResponse
var err error
for attempt := 0; attempt < maxRetries; attempt++ {
m.al.activeRequests.Add(1)
resp, err = func() (*providers.LLMResponse, error) {
defer m.al.activeRequests.Done()
return agent.Provider.Chat(
ctx,
[]providers.Message{{Role: "user", Content: prompt}},
nil,
agent.Model,
map[string]any{
"max_tokens": agent.MaxTokens,
"temperature": llmTemperature,
"prompt_cache_key": agent.ID,
},
)
}()
if err == nil && resp != nil && resp.Content != "" {
return resp, nil
}
if attempt < maxRetries-1 {
time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond)
}
}
return resp, err
}
func (m *legacyContextManager) summarizeBatch(
ctx context.Context,
agent *AgentInstance,
batch []providers.Message,
existingSummary string,
) (string, error) {
const (
llmMaxRetries = 3
fallbackMinContentLength = 200
fallbackMaxContentPercent = 10
)
var sb strings.Builder
sb.WriteString("Provide a concise summary of this conversation segment, preserving core context and key points.\n")
if existingSummary != "" {
sb.WriteString("Existing context: ")
sb.WriteString(existingSummary)
sb.WriteString("\n")
}
sb.WriteString("\nCONVERSATION:\n")
for _, msg := range batch {
fmt.Fprintf(&sb, "%s: %s\n", msg.Role, msg.Content)
}
prompt := sb.String()
response, err := m.retryLLMCall(ctx, agent, prompt, llmMaxRetries)
if err == nil && response.Content != "" {
return strings.TrimSpace(response.Content), nil
}
var fallback strings.Builder
fallback.WriteString("Conversation summary: ")
for i, msg := range batch {
if i > 0 {
fallback.WriteString(" | ")
}
content := strings.TrimSpace(msg.Content)
runes := []rune(content)
if len(runes) == 0 {
fallback.WriteString(fmt.Sprintf("%s: ", msg.Role))
continue
}
keepLength := len(runes) * fallbackMaxContentPercent / 100
if keepLength < fallbackMinContentLength {
keepLength = fallbackMinContentLength
}
if keepLength > len(runes) {
keepLength = len(runes)
}
content = string(runes[:keepLength])
if keepLength < len(runes) {
content += "..."
}
fallback.WriteString(fmt.Sprintf("%s: %s", msg.Role, content))
}
return fallback.String(), nil
}
func (m *legacyContextManager) estimateTokens(messages []providers.Message) int {
total := 0
for _, msg := range messages {
total += EstimateMessageTokens(msg)
}
return total
}

View file

@ -0,0 +1,90 @@
package agent
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/sipeed/picoclaw/pkg/providers"
)
// ContextManager manages conversation context via a pluggable strategy.
// Exactly ONE ContextManager is active per AgentLoop, selected by config.
// The default ("legacy") preserves current summarization behavior.
type ContextManager interface {
// Assemble builds budget-aware context from the ContextManager's own storage.
// Called before BuildMessages. Returns assembled messages ready for LLM.
Assemble(ctx context.Context, req *AssembleRequest) (*AssembleResponse, error)
// Compact compresses conversation history.
// Called after turn completes (may be async internally) and on context overflow (sync).
Compact(ctx context.Context, req *CompactRequest) error
// Ingest records a message into the ContextManager's own storage.
// Called after each message is persisted to session JSONL.
Ingest(ctx context.Context, req *IngestRequest) error
}
// AssembleRequest is the input to Assemble.
type AssembleRequest struct {
SessionKey string // session identifier
Budget int // context window in tokens
MaxTokens int // max response tokens
}
// AssembleResponse is the output of Assemble.
type AssembleResponse struct {
History []providers.Message // assembled conversation history for BuildMessages
Summary string // conversation summary embedded into system prompt by BuildMessages
}
// CompactRequest is the input to Compact.
type CompactRequest struct {
SessionKey string // session identifier
Reason ContextCompressReason // proactive_budget | llm_retry | summarize
Budget int // context window budget (used for retry aggressive compaction)
}
// IngestRequest is the input to Ingest.
type IngestRequest struct {
SessionKey string // session identifier
Message providers.Message // the message just persisted
}
// ContextManagerFactory constructs a ContextManager from config.
// al provides access to the AgentLoop's runtime resources (provider, model, workspace, etc.)
// cfg is the raw JSON configuration from config.json (may be nil).
type ContextManagerFactory func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error)
var (
cmRegistryMu sync.RWMutex
cmRegistry = map[string]ContextManagerFactory{}
)
// RegisterContextManager registers a named ContextManager factory.
func RegisterContextManager(name string, factory ContextManagerFactory) error {
if name == "" {
return fmt.Errorf("context manager name is required")
}
if factory == nil {
return fmt.Errorf("context manager %q factory is nil", name)
}
cmRegistryMu.Lock()
defer cmRegistryMu.Unlock()
if _, exists := cmRegistry[name]; exists {
return fmt.Errorf("context manager %q is already registered", name)
}
cmRegistry[name] = factory
return nil
}
func lookupContextManager(name string) (ContextManagerFactory, bool) {
cmRegistryMu.RLock()
defer cmRegistryMu.RUnlock()
f, ok := cmRegistry[name]
return f, ok
}

View file

@ -0,0 +1,764 @@
package agent
import (
"context"
"encoding/json"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/providers"
)
// ---------------------------------------------------------------------------
// Factory registry tests
// ---------------------------------------------------------------------------
func TestRegisterContextManager_Success(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return &noopContextManager{}, nil
}
if err := RegisterContextManager("test_cm", factory); err != nil {
t.Fatalf("unexpected error: %v", err)
}
f, ok := lookupContextManager("test_cm")
if !ok {
t.Fatal("expected factory to be registered")
}
if f == nil {
t.Fatal("expected non-nil factory")
}
}
func TestRegisterContextManager_EmptyName(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
err := RegisterContextManager("", func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return &noopContextManager{}, nil
})
if err == nil {
t.Fatal("expected error for empty name")
}
if !strings.Contains(err.Error(), "name is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestRegisterContextManager_NilFactory(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
err := RegisterContextManager("nil_factory", nil)
if err == nil {
t.Fatal("expected error for nil factory")
}
if !strings.Contains(err.Error(), "factory is nil") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestRegisterContextManager_Duplicate(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return &noopContextManager{}, nil
}
if err := RegisterContextManager("dup_cm", factory); err != nil {
t.Fatalf("first registration failed: %v", err)
}
err := RegisterContextManager("dup_cm", factory)
if err == nil {
t.Fatal("expected error for duplicate registration")
}
if !strings.Contains(err.Error(), "already registered") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestLookupContextManager_Unknown(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
_, ok := lookupContextManager("nonexistent")
if ok {
t.Fatal("expected lookup to fail for unknown name")
}
}
// ---------------------------------------------------------------------------
// resolveContextManager tests
// ---------------------------------------------------------------------------
func TestResolveContextManager_Default(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "", // default → legacy
},
},
}
al := newCMTestAgentLoop(cfg)
cm := al.contextManager
if cm == nil {
t.Fatal("expected non-nil context manager")
}
if _, ok := cm.(*legacyContextManager); !ok {
t.Fatalf("expected *legacyContextManager, got %T", cm)
}
}
func TestResolveContextManager_ExplicitLegacy(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "legacy",
},
},
}
al := newCMTestAgentLoop(cfg)
if _, ok := al.contextManager.(*legacyContextManager); !ok {
t.Fatalf("expected *legacyContextManager, got %T", al.contextManager)
}
}
func TestResolveContextManager_UnknownFallsBackToLegacy(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "unknown_cm",
},
},
}
al := newCMTestAgentLoop(cfg)
if _, ok := al.contextManager.(*legacyContextManager); !ok {
t.Fatalf("expected fallback to *legacyContextManager, got %T", al.contextManager)
}
}
func TestResolveContextManager_RegisteredFactory(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return &noopContextManager{}, nil
}
if err := RegisterContextManager("custom_cm", factory); err != nil {
t.Fatalf("register failed: %v", err)
}
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "custom_cm",
},
},
}
al := newCMTestAgentLoop(cfg)
if _, ok := al.contextManager.(*noopContextManager); !ok {
t.Fatalf("expected *noopContextManager, got %T", al.contextManager)
}
}
func TestResolveContextManager_FactoryError(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return nil, os.ErrPermission
}
if err := RegisterContextManager("broken_cm", factory); err != nil {
t.Fatalf("register failed: %v", err)
}
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "broken_cm",
},
},
}
al := newCMTestAgentLoop(cfg)
// Should fall back to legacy when factory returns error
if _, ok := al.contextManager.(*legacyContextManager); !ok {
t.Fatalf("expected fallback to *legacyContextManager on factory error, got %T", al.contextManager)
}
}
// ---------------------------------------------------------------------------
// Legacy Assemble tests
// ---------------------------------------------------------------------------
func TestLegacyAssemble_Passthrough(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
agent := al.registry.GetDefaultAgent()
if agent == nil {
t.Fatal("expected default agent")
}
history := []providers.Message{
{Role: "user", Content: "hello"},
{Role: "assistant", Content: "hi there"},
}
agent.Sessions.SetHistory("test-session", history)
resp, err := al.contextManager.Assemble(context.Background(), &AssembleRequest{
SessionKey: "test-session",
Budget: 8000,
MaxTokens: 4096,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(resp.History) != len(history) {
t.Fatalf("expected %d messages, got %d", len(history), len(resp.History))
}
for i, msg := range resp.History {
if msg.Content != history[i].Content || msg.Role != history[i].Role {
t.Fatalf("message %d mismatch: want %+v, got %+v", i, history[i], msg)
}
}
}
func TestLegacyAssemble_EmptyHistory(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
resp, err := al.contextManager.Assemble(context.Background(), &AssembleRequest{
SessionKey: "test-session",
Budget: 8000,
MaxTokens: 4096,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(resp.History) != 0 {
t.Fatalf("expected empty messages, got %d", len(resp.History))
}
}
// ---------------------------------------------------------------------------
// Legacy Compact overflow tests
// ---------------------------------------------------------------------------
func TestLegacyCompact_Overflow(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
history := []providers.Message{
{Role: "user", Content: "msg 1"},
{Role: "assistant", Content: "resp 1"},
{Role: "user", Content: "msg 2"},
{Role: "assistant", Content: "resp 2"},
{Role: "user", Content: "msg 3"},
}
defaultAgent.Sessions.SetHistory("session-overflow", history)
sub := al.SubscribeEvents(16)
defer al.UnsubscribeEvents(sub.ID)
err := al.contextManager.Compact(context.Background(), &CompactRequest{
SessionKey: "session-overflow",
Reason: ContextCompressReasonRetry,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// After overflow compression, history should be shorter
newHistory := defaultAgent.Sessions.GetHistory("session-overflow")
if len(newHistory) >= len(history) {
t.Fatalf("expected compressed history, got %d messages (was %d)", len(newHistory), len(history))
}
// Summary should contain compression note
summary := defaultAgent.Sessions.GetSummary("session-overflow")
if !strings.Contains(summary, "Emergency compression") {
t.Fatalf("expected compression note in summary, got %q", summary)
}
// Event should carry the proactive reason
events := collectEventStream(sub.C)
compressEvt, ok := findEvent(events, EventKindContextCompress)
if !ok {
t.Fatal("expected context compress event")
}
payload, ok := compressEvt.Payload.(ContextCompressPayload)
if !ok {
t.Fatalf("expected ContextCompressPayload, got %T", compressEvt.Payload)
}
if payload.Reason != ContextCompressReasonRetry {
t.Fatalf("expected retry reason, got %q", payload.Reason)
}
}
func TestLegacyCompact_Overflow_ProactiveReason(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
history := []providers.Message{
{Role: "user", Content: "msg 1"},
{Role: "assistant", Content: "resp 1"},
{Role: "user", Content: "msg 2"},
{Role: "assistant", Content: "resp 2"},
{Role: "user", Content: "msg 3"},
}
defaultAgent.Sessions.SetHistory("session-proactive", history)
sub := al.SubscribeEvents(16)
defer al.UnsubscribeEvents(sub.ID)
err := al.contextManager.Compact(context.Background(), &CompactRequest{
SessionKey: "session-proactive",
Reason: ContextCompressReasonProactive,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
events := collectEventStream(sub.C)
compressEvt, ok := findEvent(events, EventKindContextCompress)
if !ok {
t.Fatal("expected context compress event")
}
payload, ok := compressEvt.Payload.(ContextCompressPayload)
if !ok {
t.Fatalf("expected ContextCompressPayload, got %T", compressEvt.Payload)
}
if payload.Reason != ContextCompressReasonProactive {
t.Fatalf("expected proactive reason, got %q", payload.Reason)
}
}
func TestLegacyCompact_Overflow_TooShortToCompress(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
history := []providers.Message{
{Role: "user", Content: "only one"},
}
defaultAgent.Sessions.SetHistory("session-tiny", history)
err := al.contextManager.Compact(context.Background(), &CompactRequest{
SessionKey: "session-tiny",
Reason: ContextCompressReasonRetry,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// History should be unchanged (too short to compress)
newHistory := defaultAgent.Sessions.GetHistory("session-tiny")
if len(newHistory) != len(history) {
t.Fatalf("expected history unchanged, got %d messages (was %d)", len(newHistory), len(history))
}
}
// ---------------------------------------------------------------------------
// Legacy Compact post-turn tests
// ---------------------------------------------------------------------------
func TestLegacyCompact_PostTurn_BelowThreshold(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
// Small history, below summarization thresholds
history := []providers.Message{
{Role: "user", Content: "hi"},
{Role: "assistant", Content: "hello"},
}
defaultAgent.Sessions.SetHistory("session-small", history)
err := al.contextManager.Compact(context.Background(), &CompactRequest{
SessionKey: "session-small",
Reason: ContextCompressReasonSummarize,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// History should remain unchanged
newHistory := defaultAgent.Sessions.GetHistory("session-small")
if len(newHistory) != len(history) {
t.Fatalf("expected unchanged history, got %d messages (was %d)", len(newHistory), len(history))
}
}
func TestLegacyCompact_PostTurn_ExceedsMessageThreshold(t *testing.T) {
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextWindow: 8000,
SummarizeMessageThreshold: 2,
SummarizeTokenPercent: 75,
},
},
}
msgBus := bus.NewMessageBus()
al := NewAgentLoop(cfg, msgBus, &simpleMockProvider{response: "summary"})
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
// 6 messages > threshold of 2
history := []providers.Message{
{Role: "user", Content: "q1"},
{Role: "assistant", Content: "a1"},
{Role: "user", Content: "q2"},
{Role: "assistant", Content: "a2"},
{Role: "user", Content: "q3"},
{Role: "assistant", Content: "a3"},
}
defaultAgent.Sessions.SetHistory("session-threshold", history)
err := al.contextManager.Compact(context.Background(), &CompactRequest{
SessionKey: "session-threshold",
Reason: ContextCompressReasonSummarize,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Wait for async summarization to complete via event
sub := al.SubscribeEvents(16)
defer al.UnsubscribeEvents(sub.ID)
waitForEvent(t, sub.C, 5*time.Second, func(evt Event) bool {
return evt.Kind == EventKindSessionSummarize
})
newHistory := defaultAgent.Sessions.GetHistory("session-threshold")
if len(newHistory) >= len(history) {
t.Fatalf("expected summarization to reduce history from %d messages, got %d", len(history), len(newHistory))
}
}
// ---------------------------------------------------------------------------
// Legacy Ingest tests
// ---------------------------------------------------------------------------
func TestLegacyIngest_NoOp(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
err := al.contextManager.Ingest(context.Background(), &IngestRequest{
SessionKey: "session-ingest",
Message: providers.Message{Role: "user", Content: "test"},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
// ---------------------------------------------------------------------------
// Mock ContextManager — verifies dispatch through AgentLoop
// ---------------------------------------------------------------------------
func TestAgentLoop_UsesCustomContextManager(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
mock := &trackingContextManager{}
factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return mock, nil
}
if err := RegisterContextManager("tracking_cm", factory); err != nil {
t.Fatalf("register failed: %v", err)
}
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "tracking_cm",
},
},
}
al := newCMTestAgentLoop(cfg)
// Verify the mock was installed
if al.contextManager != mock {
t.Fatalf("expected mock context manager, got %T", al.contextManager)
}
// Direct method calls
_, err := mock.Assemble(context.Background(), &AssembleRequest{
SessionKey: "s1",
Budget: 8000,
MaxTokens: 4096,
})
if err != nil {
t.Fatalf("Assemble error: %v", err)
}
if mock.assembleCalls.Load() != 1 {
t.Fatalf("expected 1 assemble call, got %d", mock.assembleCalls.Load())
}
err = mock.Compact(context.Background(), &CompactRequest{
SessionKey: "s1",
Reason: ContextCompressReasonRetry,
})
if err != nil {
t.Fatalf("Compact error: %v", err)
}
if mock.compactCalls.Load() != 1 {
t.Fatalf("expected 1 compact call, got %d", mock.compactCalls.Load())
}
err = mock.Ingest(context.Background(), &IngestRequest{
SessionKey: "s1",
Message: providers.Message{Role: "user", Content: "test"},
})
if err != nil {
t.Fatalf("Ingest error: %v", err)
}
if mock.ingestCalls.Load() != 1 {
t.Fatalf("expected 1 ingest call, got %d", mock.ingestCalls.Load())
}
}
func TestIngestCalledDuringTurn(t *testing.T) {
cleanup := resetCMRegistry()
defer cleanup()
mock := &trackingContextManager{}
factory := func(cfg json.RawMessage, al *AgentLoop) (ContextManager, error) {
return mock, nil
}
if err := RegisterContextManager("ingest_track_cm", factory); err != nil {
t.Fatalf("register failed: %v", err)
}
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
ContextManager: "ingest_track_cm",
},
},
}
msgBus := bus.NewMessageBus()
al := NewAgentLoop(cfg, msgBus, &simpleMockProvider{response: "done"})
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
// Run a turn — ingestMessage is called for user message and final assistant message
_, err := al.runAgentLoop(context.Background(), defaultAgent, processOptions{
SessionKey: "session-ingest-turn",
Channel: "cli",
ChatID: "direct",
UserMessage: "test ingest",
DefaultResponse: defaultResponse,
EnableSummary: false,
SendResponse: false,
})
if err != nil {
t.Fatalf("runAgentLoop failed: %v", err)
}
// Should have at least 2 ingest calls: user message + final assistant message
if mock.ingestCalls.Load() < 2 {
t.Fatalf("expected >= 2 ingest calls during turn, got %d", mock.ingestCalls.Load())
}
}
// ---------------------------------------------------------------------------
// forceCompression edge cases (via legacy Compact)
// ---------------------------------------------------------------------------
func TestLegacyCompact_Overflow_SingleTurnKeepsLastUserMessage(t *testing.T) {
cfg := testConfig(t)
al := newCMTestAgentLoop(cfg)
defaultAgent := al.registry.GetDefaultAgent()
if defaultAgent == nil {
t.Fatal("expected default agent")
}
// History with only 2 messages — forceCompression should still handle it
history := []providers.Message{
{Role: "user", Content: "first question"},
{Role: "assistant", Content: "first answer"},
}
defaultAgent.Sessions.SetHistory("session-2msg", history)
err := al.contextManager.Compact(context.Background(), &CompactRequest{
SessionKey: "session-2msg",
Reason: ContextCompressReasonRetry,
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
newHistory := defaultAgent.Sessions.GetHistory("session-2msg")
// With 2 messages, forceCompression returns false (len <= 2), so no compression
if len(newHistory) != len(history) {
t.Fatalf("expected no compression for 2-message history, got %d", len(newHistory))
}
}
// ---------------------------------------------------------------------------
// Test helpers
// ---------------------------------------------------------------------------
// noopContextManager is a minimal ContextManager that does nothing.
type noopContextManager struct{}
func (m *noopContextManager) Assemble(_ context.Context, req *AssembleRequest) (*AssembleResponse, error) {
return &AssembleResponse{}, nil
}
func (m *noopContextManager) Compact(_ context.Context, _ *CompactRequest) error { return nil }
func (m *noopContextManager) Ingest(_ context.Context, _ *IngestRequest) error { return nil }
// trackingContextManager tracks call counts for each method.
type trackingContextManager struct {
assembleCalls atomic.Int64
compactCalls atomic.Int64
ingestCalls atomic.Int64
mu sync.Mutex
lastAssemble *AssembleRequest
lastCompact *CompactRequest
lastIngest *IngestRequest
}
func (m *trackingContextManager) Assemble(_ context.Context, req *AssembleRequest) (*AssembleResponse, error) {
m.assembleCalls.Add(1)
m.mu.Lock()
m.lastAssemble = req
m.mu.Unlock()
return &AssembleResponse{}, nil
}
func (m *trackingContextManager) Compact(_ context.Context, req *CompactRequest) error {
m.compactCalls.Add(1)
m.mu.Lock()
m.lastCompact = req
m.mu.Unlock()
return nil
}
func (m *trackingContextManager) Ingest(_ context.Context, req *IngestRequest) error {
m.ingestCalls.Add(1)
m.mu.Lock()
m.lastIngest = req
m.mu.Unlock()
return nil
}
// resetCMRegistry clears the global factory registry and returns a cleanup
// function that restores the original state after the test.
func resetCMRegistry() func() {
cmRegistryMu.Lock()
original := make(map[string]ContextManagerFactory, len(cmRegistry))
for k, v := range cmRegistry {
original[k] = v
}
cmRegistry = make(map[string]ContextManagerFactory)
cmRegistryMu.Unlock()
return func() {
cmRegistryMu.Lock()
cmRegistry = original
cmRegistryMu.Unlock()
}
}
func testConfig(t *testing.T) *config.Config {
t.Helper()
return &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: t.TempDir(),
ModelName: "test-model",
MaxTokens: 4096,
MaxToolIterations: 10,
},
},
}
}
func newCMTestAgentLoop(cfg *config.Config) *AgentLoop {
msgBus := bus.NewMessageBus()
return NewAgentLoop(cfg, msgBus, &simpleMockProvider{response: "test"})
}

View file

@ -0,0 +1,269 @@
//go:build !mipsle && !netbsd
package agent
import (
"context"
"encoding/json"
"fmt"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/providers/protocoltypes"
"github.com/sipeed/picoclaw/pkg/seahorse"
"github.com/sipeed/picoclaw/pkg/session"
"github.com/sipeed/picoclaw/pkg/tokenizer"
)
// seahorseContextManager adapts seahorse.Engine to agent.ContextManager.
type seahorseContextManager struct {
engine *seahorse.Engine
sessions session.SessionStore // for startup bootstrap
}
// newSeahorseContextManager creates a seahorse-backed ContextManager.
func newSeahorseContextManager(_ json.RawMessage, al *AgentLoop) (ContextManager, error) {
if al == nil {
return nil, fmt.Errorf("seahorse: AgentLoop is required")
}
// Resolve workspace for DB path
// DB stores session data, so it goes in sessions/ directory
agent := al.registry.GetDefaultAgent()
dbPath := agent.Workspace + "/sessions/seahorse.db"
// Create CompleteFn from provider
completeFn := providerToCompleteFn(agent.Provider, agent.Model)
// Create engine
engine, err := seahorse.NewEngine(seahorse.Config{
DBPath: dbPath,
}, completeFn)
if err != nil {
return nil, fmt.Errorf("seahorse: create engine: %w", err)
}
mgr := &seahorseContextManager{
engine: engine,
sessions: agent.Sessions,
}
// Register seahorse tools with the agent's tool registry
retrieval := mgr.engine.GetRetrieval()
al.RegisterTool(seahorse.NewGrepTool(retrieval))
al.RegisterTool(seahorse.NewExpandTool(retrieval))
// Bootstrap all existing sessions at startup
if agent.Sessions != nil {
ctx := context.Background()
for _, sessionKey := range agent.Sessions.ListSessions() {
mgr.bootstrapSession(ctx, sessionKey)
}
}
return mgr, nil
}
// providerToCompleteFn wraps providers.LLMProvider as a seahorse.CompleteFn.
func providerToCompleteFn(provider providers.LLMProvider, model string) seahorse.CompleteFn {
return func(ctx context.Context, prompt string, opts seahorse.CompleteOptions) (string, error) {
resp, err := provider.Chat(
ctx,
[]providers.Message{{Role: "user", Content: prompt}},
nil, // no tools for summarization
model,
map[string]any{
"max_tokens": opts.MaxTokens,
"temperature": opts.Temperature,
"prompt_cache_key": "seahorse",
},
)
if err != nil {
return "", err
}
return resp.Content, nil
}
}
// Assemble builds budget-aware context from seahorse SQLite.
func (m *seahorseContextManager) Assemble(ctx context.Context, req *AssembleRequest) (*AssembleResponse, error) {
if req == nil {
return nil, fmt.Errorf("seahorse assemble: nil request")
}
budget := req.Budget
if budget <= 0 {
budget = 100000
}
// Reserve space for model response (spec lines 1400-1410)
effectiveBudget := budget - req.MaxTokens
if effectiveBudget <= 0 {
// MaxTokens >= budget is a configuration problem
// Use 50% as minimum to avoid guaranteed overflow
logger.WarnCF("agent", "MaxTokens >= budget, using 50% fallback",
map[string]any{"budget": budget, "max_tokens": req.MaxTokens})
effectiveBudget = budget / 2
}
result, err := m.engine.Assemble(ctx, req.SessionKey, seahorse.AssembleInput{
Budget: effectiveBudget,
})
if err != nil {
return nil, fmt.Errorf("seahorse assemble: %w", err)
}
history := seahorseToProviderMessages(result)
// Summary is already formatted as XML with system prompt addition by assembler
return &AssembleResponse{
History: history,
Summary: result.Summary,
}, nil
}
// Compact compresses conversation history via seahorse summarization.
func (m *seahorseContextManager) Compact(ctx context.Context, req *CompactRequest) error {
if req == nil {
return nil
}
// For retry (LLM overflow), use aggressive CompactUntilUnder to guarantee
// context shrinks below budget (spec lines ~1410).
if req.Reason == ContextCompressReasonRetry && req.Budget > 0 {
_, err := m.engine.CompactUntilUnder(ctx, req.SessionKey, req.Budget)
return err
}
_, err := m.engine.Compact(ctx, req.SessionKey, seahorse.CompactInput{
Force: req.Reason == ContextCompressReasonRetry,
Budget: &req.Budget,
})
return err
}
// Ingest records a message into seahorse SQLite.
// All existing sessions are bootstrapped at startup, so this only ingests new messages.
func (m *seahorseContextManager) Ingest(ctx context.Context, req *IngestRequest) error {
if req == nil {
return nil
}
msg := providerToSeahorseMessage(req.Message)
_, err := m.engine.Ingest(ctx, req.SessionKey, []seahorse.Message{msg})
return err
}
// bootstrapSession reconciles JSONL session history into seahorse SQLite.
func (m *seahorseContextManager) bootstrapSession(ctx context.Context, sessionKey string) {
if m.sessions == nil {
return
}
history := m.sessions.GetHistory(sessionKey)
if len(history) == 0 {
return
}
// Convert provider messages to seahorse messages
msgs := make([]seahorse.Message, len(history))
for i, h := range history {
msgs[i] = providerToSeahorseMessage(h)
}
if err := m.engine.Bootstrap(ctx, sessionKey, msgs); err != nil {
logger.WarnCF("seahorse", "bootstrap", map[string]any{
"session": sessionKey,
"error": err.Error(),
})
}
}
// providerToSeahorseMessage converts a providers.Message to a seahorse.Message.
func providerToSeahorseMessage(msg protocoltypes.Message) seahorse.Message {
result := seahorse.Message{
Role: msg.Role,
Content: msg.Content,
ReasoningContent: msg.ReasoningContent,
TokenCount: tokenizer.EstimateMessageTokens(msg),
}
// Convert ToolCalls → MessageParts
for _, tc := range msg.ToolCalls {
part := seahorse.MessagePart{
Type: "tool_use",
Name: tc.Function.Name,
Arguments: tc.Function.Arguments,
ToolCallID: tc.ID,
}
result.Parts = append(result.Parts, part)
}
// Convert tool result
if msg.ToolCallID != "" {
part := seahorse.MessagePart{
Type: "tool_result",
ToolCallID: msg.ToolCallID,
Text: msg.Content,
}
result.Parts = append(result.Parts, part)
}
// Convert media attachments
for _, mediaURI := range msg.Media {
part := seahorse.MessagePart{
Type: "media",
MediaURI: mediaURI,
}
result.Parts = append(result.Parts, part)
}
return result
}
// seahorseToProviderMessages converts a seahorse.AssembleResult to []providers.Message.
func seahorseToProviderMessages(result *seahorse.AssembleResult) []protocoltypes.Message {
messages := make([]protocoltypes.Message, 0, len(result.Messages))
// Convert assembled messages (which already include summary XML messages)
for _, msg := range result.Messages {
pm := protocoltypes.Message{
Role: msg.Role,
Content: msg.Content,
ReasoningContent: msg.ReasoningContent,
}
// Reconstruct ToolCalls from parts
for _, part := range msg.Parts {
if part.Type == "tool_use" {
pm.ToolCalls = append(pm.ToolCalls, protocoltypes.ToolCall{
ID: part.ToolCallID,
Type: "function", // Required by OpenAI-compatible APIs (GLM, etc.)
Function: &protocoltypes.FunctionCall{
Name: part.Name,
Arguments: part.Arguments,
},
})
}
if part.Type == "tool_result" {
pm.ToolCallID = part.ToolCallID
if pm.Content == "" && part.Text != "" {
pm.Content = part.Text
}
}
if part.Type == "media" && part.MediaURI != "" {
pm.Media = append(pm.Media, part.MediaURI)
}
}
messages = append(messages, pm)
}
return messages
}
func init() {
if err := RegisterContextManager("seahorse", newSeahorseContextManager); err != nil {
panic(fmt.Sprintf("register seahorse context manager: %v", err))
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,20 @@
//go:build mipsle || netbsd
package agent
import (
"encoding/json"
"fmt"
)
// newSeahorseContextManager is unavailable on platforms where modernc sqlite/libc
// currently has no stable build path for this project.
func newSeahorseContextManager(_ json.RawMessage, _ *AgentLoop) (ContextManager, error) {
return nil, fmt.Errorf("seahorse context manager is unavailable on this platform")
}
func init() {
if err := RegisterContextManager("seahorse", newSeahorseContextManager); err != nil {
panic(fmt.Sprintf("register seahorse context manager: %v", err))
}
}

View file

@ -511,8 +511,8 @@ func TestAgentLoop_EmitsSessionSummarizeEvent(t *testing.T) {
sub := al.SubscribeEvents(16)
defer al.UnsubscribeEvents(sub.ID)
turnScope := al.newTurnEventScope(defaultAgent.ID, "session-1", nil)
al.summarizeSession(defaultAgent, "session-1", turnScope)
lcm := &legacyContextManager{al: al}
lcm.summarizeSession(defaultAgent, "session-1")
events := collectEventStream(sub.C)
summaryEvt, ok := findEvent(events, EventKindSessionSummarize)

View file

@ -167,6 +167,8 @@ const (
ContextCompressReasonProactive ContextCompressReason = "proactive_budget"
// ContextCompressReasonRetry indicates compression during context-error retry handling.
ContextCompressReasonRetry ContextCompressReason = "llm_retry"
// ContextCompressReasonSummarize indicates post-turn async summarization.
ContextCompressReasonSummarize ContextCompressReason = "summarize"
)
// ContextCompressPayload describes a forced history compression.

View file

@ -51,6 +51,10 @@ type AgentInstance struct {
// LightProvider is the concrete provider instance for the configured light model.
// It is only used when routing selects the light tier for a turn.
LightProvider providers.LLMProvider
// CandidateProviders maps "provider/model" keys to per-candidate LLMProvider
// instances. This allows each fallback model to use its own api_base and api_key
// from model_list, instead of inheriting the primary model's provider config.
CandidateProviders map[string]providers.LLMProvider
}
// NewAgentInstance creates an agent instance from config.
@ -77,7 +81,12 @@ func NewAgentInstance(
if cfg.Tools.IsToolEnabled("read_file") {
maxReadFileSize := cfg.Tools.ReadFile.MaxReadFileSize
toolsRegistry.Register(tools.NewReadFileTool(workspace, readRestrict, maxReadFileSize, allowReadPaths))
switch cfg.Tools.ReadFile.EffectiveMode() {
case config.ReadFileModeLines:
toolsRegistry.Register(tools.NewReadFileLinesTool(workspace, readRestrict, maxReadFileSize, allowReadPaths))
default:
toolsRegistry.Register(tools.NewReadFileBytesTool(workspace, readRestrict, maxReadFileSize, allowReadPaths))
}
}
if cfg.Tools.IsToolEnabled("write_file") {
toolsRegistry.Register(tools.NewWriteFileTool(workspace, restrict, allowWritePaths))
@ -170,6 +179,9 @@ func NewAgentInstance(
// Resolve fallback candidates
candidates := resolveModelCandidates(cfg, defaults.Provider, model, fallbacks)
candidateProviders := make(map[string]providers.LLMProvider)
populateCandidateProvidersFromNames(cfg, workspace, fallbacks, candidateProviders)
// Model routing setup: pre-resolve light model candidates at creation time
// to avoid repeated model_list lookups on every incoming message.
var router *routing.Router
@ -194,6 +206,7 @@ func NewAgentInstance(
})
lightCandidates = resolved
lightProvider = lp
populateCandidateProvidersFromNames(cfg, workspace, []string{rc.LightModel}, candidateProviders)
}
}
} else {
@ -225,6 +238,43 @@ func NewAgentInstance(
Router: router,
LightCandidates: lightCandidates,
LightProvider: lightProvider,
CandidateProviders: candidateProviders,
}
}
// populateCandidateProvidersFromNames resolves each model name (alias or
// "provider/model") via resolvedModelConfig and creates a dedicated LLMProvider
// for it. This reuses the canonical config resolution path (GetModelConfig) so
// alias handling and load-balancing stay consistent with the rest of the codebase.
func populateCandidateProvidersFromNames(
cfg *config.Config,
workspace string,
names []string,
out map[string]providers.LLMProvider,
) {
if cfg == nil || len(names) == 0 {
return
}
for _, name := range names {
mc, err := resolvedModelConfig(cfg, strings.TrimSpace(name), workspace)
if err != nil {
logger.WarnCF("agent",
"fallback provider: no model_list entry found; will inherit primary provider credentials",
map[string]any{"name": name, "error": err.Error()})
continue
}
protocol, modelID := providers.ExtractProtocol(strings.TrimSpace(mc.Model))
key := providers.ModelKey(providers.NormalizeProvider(protocol), modelID)
if _, exists := out[key]; exists {
continue
}
p, _, err := providers.CreateProviderFromConfig(mc)
if err != nil {
logger.WarnCF("agent", "fallback provider: failed to create provider",
map[string]any{"model": mc.Model, "error": err.Error()})
continue
}
out[key] = p
}
}

View file

@ -9,6 +9,7 @@ import (
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/media"
"github.com/sipeed/picoclaw/pkg/providers"
)
func TestNewAgentInstance_UsesDefaultsTemperatureAndMaxTokens(t *testing.T) {
@ -165,6 +166,58 @@ func TestNewAgentInstance_ResolveCandidatesFromModelListAlias(t *testing.T) {
}
}
func TestNewAgentInstance_PreservesDistinctLimiterIdentityForSharedResolvedModel(t *testing.T) {
tmpDir := t.TempDir()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: tmpDir,
ModelName: "glm-4.7",
ModelFallbacks: []string{"glm-4.7__key_1"},
},
},
ModelList: []*config.ModelConfig{
{
ModelName: "glm-4.7",
Model: "zhipu/glm-4.7",
RPM: 1,
},
{
ModelName: "glm-4.7__key_1",
Model: "zhipu/glm-4.7",
RPM: 3,
},
},
}
agent := NewAgentInstance(nil, &cfg.Agents.Defaults, cfg, &mockProvider{})
if len(agent.Candidates) != 2 {
t.Fatalf("len(Candidates) = %d, want 2", len(agent.Candidates))
}
first := agent.Candidates[0]
second := agent.Candidates[1]
if first.Provider != "zhipu" || first.Model != "glm-4.7" {
t.Fatalf("first candidate = %s/%s, want zhipu/glm-4.7", first.Provider, first.Model)
}
if second.Provider != "zhipu" || second.Model != "glm-4.7" {
t.Fatalf("second candidate = %s/%s, want zhipu/glm-4.7", second.Provider, second.Model)
}
if first.IdentityKey != "model_name:glm-4.7" {
t.Fatalf("first identity key = %q, want %q", first.IdentityKey, "model_name:glm-4.7")
}
if second.IdentityKey != "model_name:glm-4.7__key_1" {
t.Fatalf("second identity key = %q, want %q", second.IdentityKey, "model_name:glm-4.7__key_1")
}
if first.RPM != 1 {
t.Fatalf("first RPM = %d, want 1", first.RPM)
}
if second.RPM != 3 {
t.Fatalf("second RPM = %d, want 3", second.RPM)
}
}
func TestNewAgentInstance_AllowsMediaTempDirForReadListAndExec(t *testing.T) {
workspace := t.TempDir()
mediaDir := media.TempDir()
@ -248,6 +301,240 @@ func TestNewAgentInstance_AllowsMediaTempDirForReadListAndExec(t *testing.T) {
}
}
// TestPopulateCandidateProviders_NilCfgIsNoop verifies that passing a nil
// config does not panic and leaves the output map empty.
func TestPopulateCandidateProviders_NilCfgIsNoop(t *testing.T) {
out := map[string]providers.LLMProvider{}
populateCandidateProvidersFromNames(nil, t.TempDir(), []string{"gpt-4o"}, out)
if len(out) != 0 {
t.Fatalf("expected empty map, got %d entries", len(out))
}
}
// TestPopulateCandidateProviders_SkipsExistingKeys verifies that a key already
// present in the output map is not overwritten.
func TestPopulateCandidateProviders_SkipsExistingKeys(t *testing.T) {
existing := &mockProvider{}
key := providers.ModelKey("openai", "gpt-4o")
out := map[string]providers.LLMProvider{key: existing}
cfg := &config.Config{
ModelList: []*config.ModelConfig{
{ModelName: "my-gpt", Model: "openai/gpt-4o", APIKeys: config.SimpleSecureStrings("test-key")},
},
}
populateCandidateProvidersFromNames(cfg, t.TempDir(), []string{"my-gpt"}, out)
if out[key] != existing {
t.Fatal("existing provider entry was overwritten; expected it to be preserved")
}
}
// TestPopulateCandidateProviders_ResolvesAlias verifies that a model_name
// alias (e.g. "my-gpt") is resolved via GetModelConfig and the provider
// is created using the underlying model's config.
func TestPopulateCandidateProviders_ResolvesAlias(t *testing.T) {
workspace := t.TempDir()
out := map[string]providers.LLMProvider{}
cfg := &config.Config{
ModelList: []*config.ModelConfig{
{ModelName: "my-gpt", Model: "openai/gpt-4o", APIBase: "https://api.openai.com/v1", Workspace: workspace},
},
}
populateCandidateProvidersFromNames(cfg, workspace, []string{"my-gpt"}, out)
key := providers.ModelKey("openai", "gpt-4o")
if out[key] == nil {
t.Fatalf("expected CandidateProviders[%q] to be populated for alias", key)
}
}
// TestPopulateCandidateProviders_ResolvesProtocolPrefix verifies that a
// model_list entry using full "provider/model" notation (e.g.
// "gemini/gemma-3-27b-it") is matched correctly when referenced by model_name.
func TestPopulateCandidateProviders_ResolvesProtocolPrefix(t *testing.T) {
workspace := t.TempDir()
out := map[string]providers.LLMProvider{}
cfg := &config.Config{
ModelList: []*config.ModelConfig{
{
ModelName: "gemma",
Model: "gemini/gemma-3-27b-it",
APIKeys: config.SimpleSecureStrings("gemini-test-key"),
Workspace: workspace,
},
},
}
populateCandidateProvidersFromNames(cfg, workspace, []string{"gemma"}, out)
key := providers.ModelKey("gemini", "gemma-3-27b-it")
if out[key] == nil {
t.Fatalf("expected CandidateProviders[%q] to be populated for protocol-prefixed model", key)
}
}
// TestPopulateCandidateProviders_EmptyNamesIsNoop verifies the early-exit
// path when the names slice is empty.
func TestPopulateCandidateProviders_EmptyNamesIsNoop(t *testing.T) {
out := map[string]providers.LLMProvider{}
cfg := &config.Config{
ModelList: []*config.ModelConfig{
{ModelName: "my-gpt", Model: "openai/gpt-4o", APIKeys: config.SimpleSecureStrings("key")},
},
}
populateCandidateProvidersFromNames(cfg, t.TempDir(), nil, out)
if len(out) != 0 {
t.Fatalf("expected empty map, got %d entries", len(out))
}
}
// TestPopulateCandidateProviders_EmptyModelListIsNoop verifies the early-exit
// path when model_list is empty — no provider can be created.
func TestPopulateCandidateProviders_EmptyModelListIsNoop(t *testing.T) {
out := map[string]providers.LLMProvider{}
cfg := &config.Config{}
populateCandidateProvidersFromNames(cfg, t.TempDir(), []string{"gpt-4o"}, out)
if len(out) != 0 {
t.Fatalf("expected empty map, got %d entries", len(out))
}
}
// TestPopulateCandidateProviders_UnmatchedNameIsSkipped verifies that a
// name with no matching model_list entry is skipped and does not
// cause a panic or leave a nil entry in the map.
func TestPopulateCandidateProviders_UnmatchedNameIsSkipped(t *testing.T) {
out := map[string]providers.LLMProvider{}
cfg := &config.Config{
ModelList: []*config.ModelConfig{
{ModelName: "my-gpt", Model: "openai/gpt-4o", APIKeys: config.SimpleSecureStrings("key")},
},
}
populateCandidateProvidersFromNames(cfg, t.TempDir(), []string{"nonexistent-model"}, out)
if len(out) != 0 {
t.Fatalf("expected empty map for unmatched name, got %d entries", len(out))
}
}
// TestNewAgentInstance_CandidateProvidersPopulatedForCrossProviderFallbacks
// mirrors the exact scenario from bug #2140: primary model on OpenRouter with
// Gemini fallbacks. Each entry must get its own provider instance so that
// fallback requests go to the correct API endpoint, not the primary's.
func TestNewAgentInstance_CandidateProvidersPopulatedForCrossProviderFallbacks(t *testing.T) {
workspace := t.TempDir()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: workspace,
ModelName: "mistral-small-3.1",
ModelFallbacks: []string{"gemma-3-27b", "gemini-images"},
},
},
ModelList: []*config.ModelConfig{
{
ModelName: "mistral-small-3.1",
Model: "openrouter/mistralai/mistral-small-3.1-24b-instruct:free",
APIBase: "https://openrouter.ai/api/v1",
APIKeys: config.SimpleSecureStrings("sk-or-test"),
Workspace: workspace,
},
{
ModelName: "gemma-3-27b",
Model: "gemini/gemma-3-27b-it",
APIKeys: config.SimpleSecureStrings("AIzaSy-test"),
Workspace: workspace,
},
{
ModelName: "gemini-images",
Model: "gemini/gemini-2.5-flash-lite",
APIKeys: config.SimpleSecureStrings("AIzaSy-test"),
Workspace: workspace,
},
},
}
primaryProvider := &mockProvider{}
agent := NewAgentInstance(nil, &cfg.Agents.Defaults, cfg, primaryProvider)
// Only fallback models need entries — the primary uses the injected provider directly.
wantKeys := []string{
providers.ModelKey("gemini", "gemma-3-27b-it"),
providers.ModelKey("gemini", "gemini-2.5-flash-lite"),
}
for _, key := range wantKeys {
p, ok := agent.CandidateProviders[key]
if !ok {
t.Errorf("CandidateProviders missing key %q", key)
continue
}
if p == nil {
t.Errorf("CandidateProviders[%q] is nil", key)
}
// Each fallback must use its own provider, not the injected primary.
if p == primaryProvider {
t.Errorf(
"CandidateProviders[%q] is the same instance as the primary provider; fallback would inherit primary credentials",
key,
)
}
}
if t.Failed() {
t.Logf("CandidateProviders keys present: %v", func() []string {
keys := make([]string, 0, len(agent.CandidateProviders))
for k := range agent.CandidateProviders {
keys = append(keys, k)
}
return keys
}())
}
}
func TestNewAgentInstance_ReadFileModeSelectsSchema(t *testing.T) {
workspace := t.TempDir()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: workspace,
ModelName: "test-model",
},
},
Tools: config.ToolsConfig{
ReadFile: config.ReadFileToolConfig{
Enabled: true,
Mode: config.ReadFileModeLines,
MaxReadFileSize: 4096,
},
},
}
agent := NewAgentInstance(nil, &cfg.Agents.Defaults, cfg, &mockProvider{})
readTool, ok := agent.Tools.Get("read_file")
if !ok {
t.Fatal("read_file tool not registered")
}
params := readTool.Parameters()
props, _ := params["properties"].(map[string]any)
if _, ok := props["start_line"]; !ok {
t.Fatalf("expected line-mode schema to expose start_line, got %#v", props)
}
if _, ok := props["max_lines"]; !ok {
t.Fatalf("expected line-mode schema to expose max_lines, got %#v", props)
}
if _, ok := props["offset"]; ok {
t.Fatalf("did not expect line-mode schema to expose offset, got %#v", props)
}
if _, ok := props["length"]; ok {
t.Fatalf("did not expect line-mode schema to expose length, got %#v", props)
}
}
func TestNewAgentInstance_InvalidExecConfigDoesNotExit(t *testing.T) {
workspace := t.TempDir()

View file

@ -18,6 +18,8 @@ import (
"sync/atomic"
"time"
"github.com/sipeed/picoclaw/pkg/audio/asr"
"github.com/sipeed/picoclaw/pkg/audio/tts"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/channels"
"github.com/sipeed/picoclaw/pkg/commands"
@ -32,7 +34,6 @@ import (
"github.com/sipeed/picoclaw/pkg/state"
"github.com/sipeed/picoclaw/pkg/tools"
"github.com/sipeed/picoclaw/pkg/utils"
"github.com/sipeed/picoclaw/pkg/voice"
)
type AgentLoop struct {
@ -48,11 +49,11 @@ type AgentLoop struct {
// Runtime state
running atomic.Bool
summarizing sync.Map
contextManager ContextManager
fallback *providers.FallbackChain
channelManager *channels.Manager
mediaStore media.MediaStore
transcriber voice.Transcriber
transcriber asr.Transcriber
cmdRegistry *commands.Registry
mcp mcpRuntime
hookRuntime hookRuntime
@ -116,9 +117,18 @@ func NewAgentLoop(
) *AgentLoop {
registry := NewAgentRegistry(cfg, provider)
// Set up shared fallback chain
// Set up shared fallback chain with rate limiting.
cooldown := providers.NewCooldownTracker()
fallbackChain := providers.NewFallbackChain(cooldown)
rl := providers.NewRateLimiterRegistry()
// Register rate limiters for all agents' candidates so that RPM limits
// configured in ModelConfig are enforced before each LLM call.
for _, agentID := range registry.ListAgentIDs() {
if agent, ok := registry.GetAgent(agentID); ok {
rl.RegisterCandidates(agent.Candidates)
rl.RegisterCandidates(agent.LightCandidates)
}
}
fallbackChain := providers.NewFallbackChain(cooldown, rl)
// Create state manager using default agent's workspace for channel recording
defaultAgent := registry.GetDefaultAgent()
@ -134,13 +144,13 @@ func NewAgentLoop(
registry: registry,
state: stateManager,
eventBus: eventBus,
summarizing: sync.Map{},
fallback: fallbackChain,
cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()),
steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)),
}
al.hooks = NewHookManager(eventBus)
configureHookManagerFromConfig(al.hooks, cfg)
al.contextManager = al.resolveContextManager()
// Register shared tools to all agents (now that al is created)
registerSharedTools(al, cfg, msgBus, registry, provider)
@ -157,6 +167,13 @@ func registerSharedTools(
provider providers.LLMProvider,
) {
allowReadPaths := buildAllowReadPatterns(cfg)
var ttsProvider tts.TTSProvider
if cfg.Tools.IsToolEnabled("send_tts") {
ttsProvider = tts.DetectTTS(cfg)
if ttsProvider == nil {
logger.WarnCF("voice-tts", "send_tts enabled but no TTS provider configured", nil)
}
}
for _, agentID := range registry.ListAgentIDs() {
agent, ok := registry.GetAgent(agentID)
@ -267,6 +284,21 @@ func registerSharedTools(
agent.Tools.Register(sendFileTool)
}
if ttsProvider != nil {
agent.Tools.Register(tools.NewSendTTSTool(ttsProvider, nil))
}
if cfg.Tools.IsToolEnabled("load_image") {
loadImageTool := tools.NewLoadImageTool(
agent.Workspace,
cfg.Agents.Defaults.RestrictToWorkspace,
cfg.Agents.Defaults.GetMaxMediaSize(),
nil,
allowReadPaths,
)
agent.Tools.Register(loadImageTool)
}
// Skill discovery and installation tools
skills_enabled := cfg.Tools.IsToolEnabled("skills")
find_skills_enable := cfg.Tools.IsToolEnabled("find_skills")
@ -309,6 +341,14 @@ func registerSharedTools(
subagentManager := tools.NewSubagentManager(provider, agent.Model, agent.Workspace)
subagentManager.SetLLMOptions(agent.MaxTokens, agent.Temperature)
// Inject a media resolver so the legacy RunToolLoop fallback path can
// resolve media:// refs in the same way the main AgentLoop does.
// This keeps subagent vision support working even when the optimized
// sub-turn spawner path is unavailable.
subagentManager.SetMediaResolver(func(msgs []providers.Message) []providers.Message {
return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize())
})
// Set the spawner that links into AgentLoop's turnState
subagentManager.SetSpawner(func(
ctx context.Context,
@ -1075,6 +1115,7 @@ func (al *AgentLoop) ReloadProviderAndConfig(
go func() {
defer func() {
if r := recover(); r != nil {
logger.RecoverPanicNoExit(r)
panicErr = fmt.Errorf("panic during registry creation: %v", r)
logger.ErrorCF("agent", "Panic during registry creation",
map[string]any{"panic": r})
@ -1115,8 +1156,15 @@ func (al *AgentLoop) ReloadProviderAndConfig(
al.cfg = cfg
al.registry = registry
// Also update fallback chain with new config
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker())
// Also update fallback chain with new config; rebuild rate limiter registry.
newRL := providers.NewRateLimiterRegistry()
for _, agentID := range registry.ListAgentIDs() {
if agent, ok := registry.GetAgent(agentID); ok {
newRL.RegisterCandidates(agent.Candidates)
newRL.RegisterCandidates(agent.LightCandidates)
}
}
al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL)
al.mu.Unlock()
@ -1174,10 +1222,15 @@ func (al *AgentLoop) SetMediaStore(s media.MediaStore) {
agent.Tools.SetMediaStore(s)
}
}
registry.ForEachTool("send_tts", func(t tools.Tool) {
if st, ok := t.(*tools.SendTTSTool); ok {
st.SetMediaStore(s)
}
})
}
// SetTranscriber injects a voice transcriber for agent-level audio transcription.
func (al *AgentLoop) SetTranscriber(t voice.Transcriber) {
func (al *AgentLoop) SetTranscriber(t asr.Transcriber) {
al.transcriber = t
}
@ -1198,19 +1251,23 @@ func (al *AgentLoop) transcribeAudioInMessage(ctx context.Context, msg bus.Inbou
// Transcribe each audio media ref in order.
var transcriptions []string
var keptMedia []string
for _, ref := range msg.Media {
path, meta, err := al.mediaStore.ResolveWithMeta(ref)
if err != nil {
logger.WarnCF("voice", "Failed to resolve media ref", map[string]any{"ref": ref, "error": err})
keptMedia = append(keptMedia, ref)
continue
}
if !utils.IsAudioFile(meta.Filename, meta.ContentType) {
keptMedia = append(keptMedia, ref)
continue
}
result, err := al.transcriber.Transcribe(ctx, path)
if err != nil {
logger.WarnCF("voice", "Transcription failed", map[string]any{"ref": ref, "error": err})
transcriptions = append(transcriptions, "")
keptMedia = append(keptMedia, ref)
continue
}
transcriptions = append(transcriptions, result.Text)
@ -1230,15 +1287,21 @@ func (al *AgentLoop) transcribeAudioInMessage(ctx context.Context, msg bus.Inbou
}
text := transcriptions[idx]
idx++
if text == "" {
return match
}
return "[voice: " + text + "]"
})
// Append any remaining transcriptions not matched by an annotation.
for ; idx < len(transcriptions); idx++ {
newContent += "\n[voice: " + transcriptions[idx] + "]"
if transcriptions[idx] != "" {
newContent += "\n[voice: " + transcriptions[idx] + "]"
}
}
msg.Content = newContent
msg.Media = keptMedia
return msg, true
}
@ -1825,8 +1888,15 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er
var history []providers.Message
var summary string
if !ts.opts.NoHistory {
history = ts.agent.Sessions.GetHistory(ts.sessionKey)
summary = ts.agent.Sessions.GetSummary(ts.sessionKey)
// ContextManager assembles budget-aware history and summary.
if resp, err := al.contextManager.Assemble(turnCtx, &AssembleRequest{
SessionKey: ts.sessionKey,
Budget: ts.agent.ContextWindow,
MaxTokens: ts.agent.MaxTokens,
}); err == nil && resp != nil {
history = resp.History
summary = resp.Summary
}
}
ts.captureRestorePoint(history, summary)
@ -1851,22 +1921,28 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er
if isOverContextBudget(ts.agent.ContextWindow, messages, toolDefs, ts.agent.MaxTokens) {
logger.WarnCF("agent", "Proactive compression: context budget exceeded before LLM call",
map[string]any{"session_key": ts.sessionKey})
if compression, ok := al.forceCompression(ts.agent, ts.sessionKey); ok {
al.emitEvent(
EventKindContextCompress,
ts.eventMeta("runTurn", "turn.context.compress"),
ContextCompressPayload{
Reason: ContextCompressReasonProactive,
DroppedMessages: compression.DroppedMessages,
RemainingMessages: compression.RemainingMessages,
},
)
ts.refreshRestorePointFromSession(ts.agent)
if err := al.contextManager.Compact(turnCtx, &CompactRequest{
SessionKey: ts.sessionKey,
Reason: ContextCompressReasonProactive,
Budget: ts.agent.ContextWindow,
}); err != nil {
logger.WarnCF("agent", "Proactive compact failed", map[string]any{
"session_key": ts.sessionKey,
"error": err.Error(),
})
}
ts.refreshRestorePointFromSession(ts.agent)
// Re-assemble from CM after compact.
if resp, err := al.contextManager.Assemble(turnCtx, &AssembleRequest{
SessionKey: ts.sessionKey,
Budget: ts.agent.ContextWindow,
MaxTokens: ts.agent.MaxTokens,
}); err == nil && resp != nil {
history = resp.History
summary = resp.Summary
}
newHistory := ts.agent.Sessions.GetHistory(ts.sessionKey)
newSummary := ts.agent.Sessions.GetSummary(ts.sessionKey)
messages = ts.agent.ContextBuilder.BuildMessages(
newHistory, newSummary, ts.userMessage,
history, summary, ts.userMessage,
ts.media, ts.channel, ts.chatID,
ts.opts.SenderID, ts.opts.SenderDisplayName,
activeSkillNames(ts.agent, ts.opts)...,
@ -1888,6 +1964,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState) (turnResult, er
ts.agent.Sessions.AddMessage(ts.sessionKey, rootMsg.Role, rootMsg.Content)
}
ts.recordPersistedMessage(rootMsg)
ts.ingestMessage(turnCtx, al, rootMsg)
}
activeCandidates, activeModel, usedLight := al.selectCandidates(ts.agent, ts.userMessage, messages)
@ -1963,6 +2040,7 @@ turnLoop:
if !ts.opts.NoHistory {
ts.agent.Sessions.AddFullMessage(ts.sessionKey, pm)
ts.recordPersistedMessage(pm)
ts.ingestMessage(turnCtx, al, pm)
}
logger.InfoCF("agent", "Injected steering message into context",
map[string]any{
@ -2016,6 +2094,14 @@ turnLoop:
providerToolDefs = filtered
}
// Resolve media:// refs produced by tool results (e.g. load_image).
// Skipped on iteration 1 because inbound user media is already resolved
// before entering the loop; only subsequent iterations can contain new
// tool-generated media refs that need base64 encoding.
if iteration > 1 {
messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize)
}
callMessages := messages
if gracefulTerminal {
callMessages = append(append([]providers.Message(nil), messages...), ts.interruptHintMessage())
@ -2115,7 +2201,11 @@ turnLoop:
providerCtx,
activeCandidates,
func(ctx context.Context, provider, model string) (*providers.LLMResponse, error) {
return activeProvider.Chat(ctx, messagesForCall, toolDefsForCall, model, llmOpts)
candidateProvider := activeProvider
if cp, ok := ts.agent.CandidateProviders[providers.ModelKey(provider, model)]; ok {
candidateProvider = cp
}
return candidateProvider.Chat(ctx, messagesForCall, toolDefsForCall, model, llmOpts)
},
)
if fbErr != nil {
@ -2221,23 +2311,28 @@ turnLoop:
))
}
if compression, ok := al.forceCompression(ts.agent, ts.sessionKey); ok {
al.emitEvent(
EventKindContextCompress,
ts.eventMeta("runTurn", "turn.context.compress"),
ContextCompressPayload{
Reason: ContextCompressReasonRetry,
DroppedMessages: compression.DroppedMessages,
RemainingMessages: compression.RemainingMessages,
},
)
ts.refreshRestorePointFromSession(ts.agent)
if compactErr := al.contextManager.Compact(turnCtx, &CompactRequest{
SessionKey: ts.sessionKey,
Reason: ContextCompressReasonRetry,
Budget: ts.agent.ContextWindow,
}); compactErr != nil {
logger.WarnCF("agent", "Context overflow compact failed", map[string]any{
"session_key": ts.sessionKey,
"error": compactErr.Error(),
})
}
ts.refreshRestorePointFromSession(ts.agent)
// Re-assemble from CM after compact.
if asmResp, asmErr := al.contextManager.Assemble(turnCtx, &AssembleRequest{
SessionKey: ts.sessionKey,
Budget: ts.agent.ContextWindow,
MaxTokens: ts.agent.MaxTokens,
}); asmErr == nil && asmResp != nil {
history = asmResp.History
summary = asmResp.Summary
}
newHistory := ts.agent.Sessions.GetHistory(ts.sessionKey)
newSummary := ts.agent.Sessions.GetSummary(ts.sessionKey)
messages = ts.agent.ContextBuilder.BuildMessages(
newHistory, newSummary, "",
history, summary, "",
nil, ts.channel, ts.chatID, ts.opts.SenderID, ts.opts.SenderDisplayName,
activeSkillNames(ts.agent, ts.opts)...,
)
@ -2409,6 +2504,7 @@ turnLoop:
if !ts.opts.NoHistory {
ts.agent.Sessions.AddFullMessage(ts.sessionKey, assistantMsg)
ts.recordPersistedMessage(assistantMsg)
ts.ingestMessage(turnCtx, al, assistantMsg)
}
ts.setPhase(TurnPhaseTools)
@ -2633,6 +2729,7 @@ turnLoop:
if toolResult == nil {
toolResult = tools.ErrorResult("hook returned nil tool result")
}
if len(toolResult.Media) > 0 && toolResult.ResponseHandled {
parts := make([]bus.MediaPart, 0, len(toolResult.Media))
for _, ref := range toolResult.Media {
@ -2675,6 +2772,13 @@ turnLoop:
}
if len(toolResult.Media) > 0 && !toolResult.ResponseHandled {
// For tools like load_image that produce media refs without sending them
// to the user channel (ResponseHandled == false), both Media and ArtifactTags
// coexist on the result:
// - Media: carries media:// refs that resolveMediaRefs will base64-encode
// into image_url parts in the next LLM iteration (enabling vision).
// - ArtifactTags: exposes the local file path as a structured [file:…] tag
// in the tool result text, so the LLM knows an artifact was produced.
toolResult.ArtifactTags = buildArtifactTags(al.mediaStore, toolResult.Media)
}
@ -2693,7 +2797,6 @@ turnLoop:
"content_len": len(toolResult.ForUser),
})
}
contentForLLM := toolResult.ContentForLLM()
// Filter sensitive data (API keys, tokens, secrets) before sending to LLM
@ -2706,6 +2809,9 @@ turnLoop:
Content: contentForLLM,
ToolCallID: toolCallID,
}
if len(toolResult.Media) > 0 && !toolResult.ResponseHandled {
toolResultMsg.Media = append(toolResultMsg.Media, toolResult.Media...)
}
al.emitEvent(
EventKindToolExecEnd,
ts.eventMeta("runTurn", "turn.tool.end"),
@ -2722,6 +2828,7 @@ turnLoop:
if !ts.opts.NoHistory {
ts.agent.Sessions.AddFullMessage(ts.sessionKey, toolResultMsg)
ts.recordPersistedMessage(toolResultMsg)
ts.ingestMessage(turnCtx, al, toolResultMsg)
}
if steerMsgs := al.dequeueSteeringMessagesForScope(ts.sessionKey); len(steerMsgs) > 0 {
@ -2821,6 +2928,7 @@ turnLoop:
if !ts.opts.NoHistory {
ts.agent.Sessions.AddMessage(ts.sessionKey, summaryMsg.Role, summaryMsg.Content)
ts.recordPersistedMessage(summaryMsg)
ts.ingestMessage(turnCtx, al, summaryMsg)
if err := ts.agent.Sessions.Save(ts.sessionKey); err != nil {
turnStatus = TurnEndStatusError
al.emitEvent(
@ -2835,7 +2943,7 @@ turnLoop:
}
}
if ts.opts.EnableSummary {
al.maybeSummarize(ts.agent, ts.sessionKey, ts.scope)
al.contextManager.Compact(turnCtx, &CompactRequest{SessionKey: ts.sessionKey, Reason: ContextCompressReasonSummarize, Budget: ts.agent.ContextWindow})
}
ts.setPhase(TurnPhaseCompleted)
@ -2890,6 +2998,7 @@ turnLoop:
finalMsg := providers.Message{Role: "assistant", Content: finalContent}
ts.agent.Sessions.AddMessage(ts.sessionKey, finalMsg.Role, finalMsg.Content)
ts.recordPersistedMessage(finalMsg)
ts.ingestMessage(turnCtx, al, finalMsg)
if err := ts.agent.Sessions.Save(ts.sessionKey); err != nil {
turnStatus = TurnEndStatusError
al.emitEvent(
@ -2905,7 +3014,14 @@ turnLoop:
}
if ts.opts.EnableSummary {
al.maybeSummarize(ts.agent, ts.sessionKey, ts.scope)
al.contextManager.Compact(
turnCtx,
&CompactRequest{
SessionKey: ts.sessionKey,
Reason: ContextCompressReasonSummarize,
Budget: ts.agent.ContextWindow,
},
)
}
ts.setPhase(TurnPhaseCompleted)
@ -2984,103 +3100,28 @@ func (al *AgentLoop) selectCandidates(
return agent.LightCandidates, resolvedCandidateModel(agent.LightCandidates, agent.Router.LightModel()), true
}
// maybeSummarize triggers summarization if the session history exceeds thresholds.
func (al *AgentLoop) maybeSummarize(agent *AgentInstance, sessionKey string, turnScope turnEventScope) {
newHistory := agent.Sessions.GetHistory(sessionKey)
tokenEstimate := al.estimateTokens(newHistory)
threshold := agent.ContextWindow * agent.SummarizeTokenPercent / 100
if len(newHistory) > agent.SummarizeMessageThreshold || tokenEstimate > threshold {
summarizeKey := agent.ID + ":" + sessionKey
if _, loading := al.summarizing.LoadOrStore(summarizeKey, true); !loading {
go func() {
defer al.summarizing.Delete(summarizeKey)
logger.Debug("Memory threshold reached. Optimizing conversation history...")
al.summarizeSession(agent, sessionKey, turnScope)
}()
}
// resolveContextManager selects the ContextManager implementation based on config.
func (al *AgentLoop) resolveContextManager() ContextManager {
name := al.cfg.Agents.Defaults.ContextManager
if name == "" || name == "legacy" {
return &legacyContextManager{al: al}
}
}
type compressionResult struct {
DroppedMessages int
RemainingMessages int
}
// forceCompression aggressively reduces context when the limit is hit.
// It drops the oldest ~50% of Turns (a Turn is a complete user→LLM→response
// cycle, as defined in #1316), so tool-call sequences are never split.
//
// If the history is a single Turn with no safe split point, the function
// falls back to keeping only the most recent user message. This breaks
// Turn atomicity as a last resort to avoid a context-exceeded loop.
//
// Session history contains only user/assistant/tool messages — the system
// prompt is built dynamically by BuildMessages and is NOT stored here.
// The compression note is recorded in the session summary so that
// BuildMessages can include it in the next system prompt.
func (al *AgentLoop) forceCompression(agent *AgentInstance, sessionKey string) (compressionResult, bool) {
history := agent.Sessions.GetHistory(sessionKey)
if len(history) <= 2 {
return compressionResult{}, false
factory, ok := lookupContextManager(name)
if !ok {
logger.WarnCF("agent", "Unknown context manager, falling back to legacy", map[string]any{
"name": name,
})
return &legacyContextManager{al: al}
}
// Split at a Turn boundary so no tool-call sequence is torn apart.
// parseTurnBoundaries gives us the start of each Turn; we drop the
// oldest half of Turns and keep the most recent ones.
turns := parseTurnBoundaries(history)
var mid int
if len(turns) >= 2 {
mid = turns[len(turns)/2]
} else {
// Fewer than 2 Turns — fall back to message-level midpoint
// aligned to the nearest Turn boundary.
mid = findSafeBoundary(history, len(history)/2)
cm, err := factory(al.cfg.Agents.Defaults.ContextManagerConfig, al)
if err != nil {
logger.WarnCF("agent", "Failed to create context manager, falling back to legacy", map[string]any{
"name": name,
"error": err.Error(),
})
return &legacyContextManager{al: al}
}
var keptHistory []providers.Message
if mid <= 0 {
// No safe Turn boundary — the entire history is a single Turn
// (e.g. one user message followed by a massive tool response).
// Keeping everything would leave the agent stuck in a context-
// exceeded loop, so fall back to keeping only the most recent
// user message. This breaks Turn atomicity as a last resort.
for i := len(history) - 1; i >= 0; i-- {
if history[i].Role == "user" {
keptHistory = []providers.Message{history[i]}
break
}
}
} else {
keptHistory = history[mid:]
}
droppedCount := len(history) - len(keptHistory)
// Record compression in the session summary so BuildMessages includes it
// in the system prompt. We do not modify history messages themselves.
existingSummary := agent.Sessions.GetSummary(sessionKey)
compressionNote := fmt.Sprintf(
"[Emergency compression dropped %d oldest messages due to context limit]",
droppedCount,
)
if existingSummary != "" {
compressionNote = existingSummary + "\n\n" + compressionNote
}
agent.Sessions.SetSummary(sessionKey, compressionNote)
agent.Sessions.SetHistory(sessionKey, keptHistory)
agent.Sessions.Save(sessionKey)
logger.WarnCF("agent", "Forced compression executed", map[string]any{
"session_key": sessionKey,
"dropped_msgs": droppedCount,
"new_count": len(keptHistory),
})
return compressionResult{
DroppedMessages: droppedCount,
RemainingMessages: len(keptHistory),
}, true
return cm
}
// GetStartupInfo returns information about loaded tools and skills for logging.
@ -3172,247 +3213,13 @@ func formatToolsForLog(toolDefs []providers.ToolDefinition) string {
}
// summarizeSession summarizes the conversation history for a session.
func (al *AgentLoop) summarizeSession(agent *AgentInstance, sessionKey string, turnScope turnEventScope) {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
history := agent.Sessions.GetHistory(sessionKey)
summary := agent.Sessions.GetSummary(sessionKey)
// Keep the most recent Turns for continuity, aligned to a Turn boundary
// so that no tool-call sequence is split.
if len(history) <= 4 {
return
}
safeCut := findSafeBoundary(history, len(history)-4)
if safeCut <= 0 {
return
}
keepCount := len(history) - safeCut
toSummarize := history[:safeCut]
// Oversized Message Guard
maxMessageTokens := agent.ContextWindow / 2
validMessages := make([]providers.Message, 0)
omitted := false
for _, m := range toSummarize {
if m.Role != "user" && m.Role != "assistant" {
continue
}
msgTokens := len(m.Content) / 2
if msgTokens > maxMessageTokens {
omitted = true
continue
}
validMessages = append(validMessages, m)
}
if len(validMessages) == 0 {
return
}
const (
maxSummarizationMessages = 10
llmMaxRetries = 3
llmTemperature = 0.3
fallbackMaxContentLength = 200
)
// Multi-Part Summarization
var finalSummary string
if len(validMessages) > maxSummarizationMessages {
mid := len(validMessages) / 2
mid = al.findNearestUserMessage(validMessages, mid)
part1 := validMessages[:mid]
part2 := validMessages[mid:]
s1, _ := al.summarizeBatch(ctx, agent, part1, "")
s2, _ := al.summarizeBatch(ctx, agent, part2, "")
mergePrompt := fmt.Sprintf(
"Merge these two conversation summaries into one cohesive summary:\n\n1: %s\n\n2: %s",
s1,
s2,
)
resp, err := al.retryLLMCall(ctx, agent, mergePrompt, llmMaxRetries)
if err == nil && resp.Content != "" {
finalSummary = resp.Content
} else {
finalSummary = s1 + " " + s2
}
} else {
finalSummary, _ = al.summarizeBatch(ctx, agent, validMessages, summary)
}
if omitted && finalSummary != "" {
finalSummary += "\n[Note: Some oversized messages were omitted from this summary for efficiency.]"
}
if finalSummary != "" {
agent.Sessions.SetSummary(sessionKey, finalSummary)
agent.Sessions.TruncateHistory(sessionKey, keepCount)
agent.Sessions.Save(sessionKey)
al.emitEvent(
EventKindSessionSummarize,
turnScope.meta(0, "summarizeSession", "turn.session.summarize"),
SessionSummarizePayload{
SummarizedMessages: len(validMessages),
KeptMessages: keepCount,
SummaryLen: len(finalSummary),
OmittedOversized: omitted,
},
)
}
}
// findNearestUserMessage finds the nearest user message to the given index.
// It searches backward first, then forward if no user message is found.
func (al *AgentLoop) findNearestUserMessage(messages []providers.Message, mid int) int {
originalMid := mid
for mid > 0 && messages[mid].Role != "user" {
mid--
}
if messages[mid].Role == "user" {
return mid
}
mid = originalMid
for mid < len(messages) && messages[mid].Role != "user" {
mid++
}
if mid < len(messages) {
return mid
}
return originalMid
}
// retryLLMCall calls the LLM with retry logic.
func (al *AgentLoop) retryLLMCall(
ctx context.Context,
agent *AgentInstance,
prompt string,
maxRetries int,
) (*providers.LLMResponse, error) {
const (
llmTemperature = 0.3
)
var resp *providers.LLMResponse
var err error
for attempt := 0; attempt < maxRetries; attempt++ {
al.activeRequests.Add(1)
resp, err = func() (*providers.LLMResponse, error) {
defer al.activeRequests.Done()
return agent.Provider.Chat(
ctx,
[]providers.Message{{Role: "user", Content: prompt}},
nil,
agent.Model,
map[string]any{
"max_tokens": agent.MaxTokens,
"temperature": llmTemperature,
"prompt_cache_key": agent.ID,
},
)
}()
if err == nil && resp != nil && resp.Content != "" {
return resp, nil
}
if attempt < maxRetries-1 {
time.Sleep(time.Duration(attempt+1) * 100 * time.Millisecond)
}
}
return resp, err
}
// summarizeBatch summarizes a batch of messages.
func (al *AgentLoop) summarizeBatch(
ctx context.Context,
agent *AgentInstance,
batch []providers.Message,
existingSummary string,
) (string, error) {
const (
llmMaxRetries = 3
llmTemperature = 0.3
fallbackMinContentLength = 200
fallbackMaxContentPercent = 10
)
var sb strings.Builder
sb.WriteString(
"Provide a concise summary of this conversation segment, preserving core context and key points.\n",
)
if existingSummary != "" {
sb.WriteString("Existing context: ")
sb.WriteString(existingSummary)
sb.WriteString("\n")
}
sb.WriteString("\nCONVERSATION:\n")
for _, m := range batch {
fmt.Fprintf(&sb, "%s: %s\n", m.Role, m.Content)
}
prompt := sb.String()
response, err := al.retryLLMCall(ctx, agent, prompt, llmMaxRetries)
if err == nil && response.Content != "" {
return strings.TrimSpace(response.Content), nil
}
var fallback strings.Builder
fallback.WriteString("Conversation summary: ")
for i, m := range batch {
if i > 0 {
fallback.WriteString(" | ")
}
content := strings.TrimSpace(m.Content)
runes := []rune(content)
if len(runes) == 0 {
fallback.WriteString(fmt.Sprintf("%s: ", m.Role))
continue
}
keepLength := len(runes) * fallbackMaxContentPercent / 100
if keepLength < fallbackMinContentLength {
keepLength = fallbackMinContentLength
}
if keepLength > len(runes) {
keepLength = len(runes)
}
content = string(runes[:keepLength])
if keepLength < len(runes) {
content += "..."
}
fallback.WriteString(fmt.Sprintf("%s: %s", m.Role, content))
}
return fallback.String(), nil
}
// estimateTokens estimates the number of tokens in a message list.
// Counts Content, ToolCalls arguments, and ToolCallID metadata so that
// tool-heavy conversations are not systematically undercounted.
func (al *AgentLoop) estimateTokens(messages []providers.Message) int {
total := 0
for _, m := range messages {
total += estimateMessageTokens(m)
}
return total
}
func (al *AgentLoop) handleCommand(
ctx context.Context,
msg bus.InboundMessage,
@ -3609,7 +3416,7 @@ func (al *AgentLoop) buildCommandsRuntime(agent *AgentInstance, opts *processOpt
return "", fmt.Errorf("failed to initialize model %q: %w", value, err)
}
nextCandidates := resolveModelCandidates(cfg, cfg.Agents.Defaults.Provider, modelCfg.Model, agent.Fallbacks)
nextCandidates := resolveModelCandidates(cfg, cfg.Agents.Defaults.Provider, value, agent.Fallbacks)
if len(nextCandidates) == 0 {
return "", fmt.Errorf("model %q did not resolve to any provider candidates", value)
}

View file

@ -126,6 +126,8 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error {
}
mcpTool := tools.NewMCPTool(mcpManager, serverName, tool)
mcpTool.SetWorkspace(agent.Workspace)
mcpTool.SetMaxInlineTextRunes(al.cfg.Tools.MCP.GetMaxInlineTextChars())
if registerAsHidden {
agent.Tools.RegisterHidden(mcpTool)

View file

@ -2132,6 +2132,162 @@ func TestProcessMessage_ModelRoutingUsesLightProvider(t *testing.T) {
}
}
// TestProcessMessage_FallbackUsesPerCandidateProvider is the loop-level test for
// bug #2140. It verifies that when the primary model returns a rate-limit error
// the fallback closure routes the retry to the fallback model's own provider
// (its own api_base), not back to the primary provider's endpoint.
func TestProcessMessage_FallbackUsesPerCandidateProvider(t *testing.T) {
workspace := t.TempDir()
primaryCalls := 0
primaryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
primaryCalls++
// Return 429 so FallbackChain classifies this as retriable and moves on.
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusTooManyRequests)
_ = json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{
"message": "rate limit exceeded",
"type": "rate_limit_error",
},
})
}))
defer primaryServer.Close()
fallbackCalls := 0
fallbackServer := newStrictChatCompletionTestServer(
t, "fallback", "gemma-3-27b-it", "fallback reply", &fallbackCalls,
)
defer fallbackServer.Close()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: workspace,
ModelName: "mistral-primary",
ModelFallbacks: []string{"gemma-fallback"},
MaxTokens: 4096,
MaxToolIterations: 3,
},
},
ModelList: []*config.ModelConfig{
{
ModelName: "mistral-primary",
Model: "openrouter/mistralai/mistral-small-3.1",
APIBase: primaryServer.URL,
APIKeys: config.SimpleSecureStrings("primary-key"),
Workspace: workspace,
},
{
ModelName: "gemma-fallback",
Model: "gemini/gemma-3-27b-it",
APIBase: fallbackServer.URL,
APIKeys: config.SimpleSecureStrings("fallback-key"),
Workspace: workspace,
},
},
}
provider, _, err := providers.CreateProvider(cfg)
if err != nil {
t.Fatalf("CreateProvider() error = %v", err)
}
msgBus := bus.NewMessageBus()
al := NewAgentLoop(cfg, msgBus, provider)
helper := testHelper{al: al}
resp := helper.executeAndGetResponse(t, context.Background(), bus.InboundMessage{
Channel: "telegram",
SenderID: "user1",
ChatID: "chat1",
Content: "hi",
})
if resp != "fallback reply" {
t.Fatalf("response = %q, want %q (fallback provider)", resp, "fallback reply")
}
if primaryCalls == 0 {
t.Fatal("primary server was never called; expected at least one attempt")
}
if fallbackCalls != 1 {
t.Fatalf("fallback server calls = %d, want 1", fallbackCalls)
}
}
// TestProcessMessage_FallbackUsesActiveProviderWhenCandidateNotRegistered verifies
// that when a candidate has no model_list entry it is absent from CandidateProviders
// and the fallback closure falls back to activeProvider instead of panicking.
func TestProcessMessage_FallbackUsesActiveProviderWhenCandidateNotRegistered(t *testing.T) {
workspace := t.TempDir()
// Primary server: returns 429 on first call, succeeds on second.
// Both the primary and the unregistered fallback share this server
// (same api_base) so activeProvider routes both calls here.
callCount := 0
primaryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
w.Header().Set("Content-Type", "application/json")
if callCount == 1 {
w.WriteHeader(http.StatusTooManyRequests)
_ = json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{"message": "rate limit", "type": "rate_limit_error"},
})
return
}
// Second call (fallback via activeProvider) succeeds.
_ = json.NewEncoder(w).Encode(map[string]any{
"choices": []map[string]any{
{"message": map[string]any{"content": "active provider reply"}, "finish_reason": "stop"},
},
})
}))
defer primaryServer.Close()
cfg := &config.Config{
Agents: config.AgentsConfig{
Defaults: config.AgentDefaults{
Workspace: workspace,
ModelName: "primary-model",
MaxTokens: 4096,
MaxToolIterations: 3,
// No model_list entry for this alias — absent from CandidateProviders.
ModelFallbacks: []string{"openrouter/fallback-model"},
},
},
ModelList: []*config.ModelConfig{
{
ModelName: "primary-model",
Model: "openrouter/primary-model",
APIBase: primaryServer.URL,
APIKeys: config.SimpleSecureStrings("primary-key"),
Workspace: workspace,
},
},
}
provider, _, err := providers.CreateProvider(cfg)
if err != nil {
t.Fatalf("CreateProvider() error = %v", err)
}
msgBus := bus.NewMessageBus()
al := NewAgentLoop(cfg, msgBus, provider)
helper := testHelper{al: al}
resp := helper.executeAndGetResponse(t, context.Background(), bus.InboundMessage{
Channel: "telegram",
SenderID: "user1",
ChatID: "chat1",
Content: "hi",
})
if resp != "active provider reply" {
t.Fatalf("response = %q, want %q", resp, "active provider reply")
}
if callCount < 2 {
t.Fatalf("primary server calls = %d, want >= 2 (one 429 + one success via activeProvider)", callCount)
}
}
// TestToolResult_SilentToolDoesNotSendUserMessage verifies silent tools don't trigger outbound
func TestToolResult_SilentToolDoesNotSendUserMessage(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "agent-test-*")

View file

@ -8,44 +8,102 @@ import (
"github.com/sipeed/picoclaw/pkg/providers"
)
func buildModelListResolver(cfg *config.Config) func(raw string) (string, bool) {
ensureProtocol := func(model string) string {
model = strings.TrimSpace(model)
if model == "" {
return ""
}
if strings.Contains(model, "/") {
return model
}
return "openai/" + model
func ensureProtocolModel(model string) string {
model = strings.TrimSpace(model)
if model == "" {
return ""
}
if strings.Contains(model, "/") {
return model
}
return "openai/" + model
}
func modelConfigIdentityKey(mc *config.ModelConfig) string {
if mc == nil {
return ""
}
if name := strings.TrimSpace(mc.ModelName); name != "" {
return "model_name:" + name
}
return ""
}
func candidateFromModelConfig(
defaultProvider string,
mc *config.ModelConfig,
) (providers.FallbackCandidate, bool) {
if mc == nil {
return providers.FallbackCandidate{}, false
}
return func(raw string) (string, bool) {
raw = strings.TrimSpace(raw)
if raw == "" || cfg == nil {
return "", false
}
if mc, err := cfg.GetModelConfig(raw); err == nil && mc != nil && strings.TrimSpace(mc.Model) != "" {
return ensureProtocol(mc.Model), true
}
for i := range cfg.ModelList {
fullModel := strings.TrimSpace(cfg.ModelList[i].Model)
if fullModel == "" {
continue
}
if fullModel == raw {
return ensureProtocol(fullModel), true
}
_, modelID := providers.ExtractProtocol(fullModel)
if modelID == raw {
return ensureProtocol(fullModel), true
}
}
return "", false
ref := providers.ParseModelRef(ensureProtocolModel(mc.Model), defaultProvider)
if ref == nil {
return providers.FallbackCandidate{}, false
}
return providers.FallbackCandidate{
Provider: ref.Provider,
Model: ref.Model,
RPM: mc.RPM,
IdentityKey: modelConfigIdentityKey(mc),
}, true
}
func lookupModelConfigByRef(cfg *config.Config, raw string) *config.ModelConfig {
raw = strings.TrimSpace(raw)
if raw == "" || cfg == nil {
return nil
}
if mc, err := cfg.GetModelConfig(raw); err == nil && mc != nil && strings.TrimSpace(mc.Model) != "" {
return mc
}
for i := range cfg.ModelList {
mc := cfg.ModelList[i]
if mc == nil {
continue
}
fullModel := strings.TrimSpace(mc.Model)
if fullModel == "" {
continue
}
if fullModel == raw {
return mc
}
_, modelID := providers.ExtractProtocol(fullModel)
if modelID == raw {
return mc
}
}
return nil
}
func resolveModelCandidate(
cfg *config.Config,
defaultProvider string,
raw string,
) (providers.FallbackCandidate, bool) {
raw = strings.TrimSpace(raw)
if raw == "" {
return providers.FallbackCandidate{}, false
}
if mc := lookupModelConfigByRef(cfg, raw); mc != nil {
return candidateFromModelConfig(defaultProvider, mc)
}
ref := providers.ParseModelRef(raw, defaultProvider)
if ref == nil {
return providers.FallbackCandidate{}, false
}
return providers.FallbackCandidate{
Provider: ref.Provider,
Model: ref.Model,
}, true
}
func resolveModelCandidates(
@ -54,14 +112,29 @@ func resolveModelCandidates(
primary string,
fallbacks []string,
) []providers.FallbackCandidate {
return providers.ResolveCandidatesWithLookup(
providers.ModelConfig{
Primary: primary,
Fallbacks: fallbacks,
},
defaultProvider,
buildModelListResolver(cfg),
)
seen := make(map[string]bool)
candidates := make([]providers.FallbackCandidate, 0, 1+len(fallbacks))
addCandidate := func(raw string) {
candidate, ok := resolveModelCandidate(cfg, defaultProvider, raw)
if !ok {
return
}
key := candidate.StableKey()
if seen[key] {
return
}
seen[key] = true
candidates = append(candidates, candidate)
}
addCandidate(primary)
for _, fallback := range fallbacks {
addCandidate(fallback)
}
return candidates
}
func resolvedCandidateModel(candidates []providers.FallbackCandidate, fallback string) string {

View file

@ -432,6 +432,7 @@ func spawnSubTurn(
// 7. Defer cleanup: deliver result (for async), emit End event, and recover from panics
defer func() {
if r := recover(); r != nil {
logger.RecoverPanicNoExit(r)
err = fmt.Errorf("subturn panicked: %v", r)
result = nil
logger.ErrorCF("subturn", "SubTurn panicked", map[string]any{
@ -515,6 +516,7 @@ func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, re
// We use defer/recover to catch any unlikely channel panics if it were ever closed.
defer func() {
if r := recover(); r != nil {
logger.RecoverPanicNoExit(r)
logger.WarnCF("subturn", "recovered panic sending to pendingResults", map[string]any{
"parent_id": parentTS.turnID,
"child_id": childID,
@ -607,6 +609,7 @@ type ephemeralSessionStoreIface interface {
SetHistory(key string, history []providers.Message)
TruncateHistory(key string, keepLast int)
Save(key string) error
ListSessions() []string
Close() error
}
@ -666,8 +669,9 @@ func (e *ephemeralSessionStore) TruncateHistory(_ string, keepLast int) {
e.history = e.history[len(e.history)-keepLast:]
}
func (e *ephemeralSessionStore) Save(_ string) error { return nil }
func (e *ephemeralSessionStore) Close() error { return nil }
func (e *ephemeralSessionStore) Save(_ string) error { return nil }
func (e *ephemeralSessionStore) Close() error { return nil }
func (e *ephemeralSessionStore) ListSessions() []string { return nil }
func (e *ephemeralSessionStore) truncateLocked() {
if len(e.history) > maxEphemeralHistorySize {

View file

@ -8,6 +8,7 @@ import (
"time"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/session"
"github.com/sipeed/picoclaw/pkg/tools"
@ -341,6 +342,23 @@ func (ts *turnState) refreshRestorePointFromSession(agent *AgentInstance) {
ts.captureRestorePoint(history, summary)
}
// ingestMessage calls the ContextManager's Ingest method for a persisted message.
// Errors are logged but never block the turn.
func (ts *turnState) ingestMessage(ctx context.Context, al *AgentLoop, msg providers.Message) {
if al.contextManager == nil {
return
}
if err := al.contextManager.Ingest(ctx, &IngestRequest{
SessionKey: ts.sessionKey,
Message: msg,
}); err != nil {
logger.WarnCF("agent", "Context manager ingest failed", map[string]any{
"session_key": ts.sessionKey,
"error": err.Error(),
})
}
}
func (ts *turnState) restoreSession(agent *AgentInstance) error {
ts.mu.RLock()
history := append([]providers.Message(nil), ts.restorePointHistory...)

166
pkg/audio/asr/README.md Normal file
View file

@ -0,0 +1,166 @@
# ASR (Automatic Speech Recognition)
This package handles speech-to-text for PicoClaw voice input.
If you are new to ASR setup, the simplest mental model is:
1. Add one or more ASR-capable entries to `model_list`.
2. Point `voice.model_name` at the one you want to use.
3. Put the API key in `.security.yml`.
## Quick Recommendation
For most new users, start with one of these:
| Provider | Example model | Why start here |
| --- | --- | --- |
| [Groq](https://console.groq.com/keys) | `groq/whisper-large-v3-turbo` | Fast Whisper-style transcription and a straightforward OpenAI-compatible API. Groq currently advertises a free tier plan for 2000 reqs/day. |
| [ElevenLabs](https://elevenlabs.io/pricing) | `elevenlabs/scribe_v1` | Easy setup and strong speech-to-text quality. ElevenLabs currently advertises a free plan that includes speech-to-text usage. |
Pricing and free-plan limits can change, so check the linked pricing pages before depending on them in production.
## How ASR Configuration Works
PicoClaw does not keep ASR API keys inside the `voice` section.
Instead:
- `voice.model_name` chooses a named entry from `model_list`.
- The matching `model_list` entry describes the actual provider and model.
- `.security.yml` stores the API key for that named model entry.
This is the recommended pattern because it is explicit, reusable, and consistent with the rest of PicoClaw's model configuration.
## Recommended Setup
### Option A: Groq Whisper
`config.json`
```json
{
"voice": {
"model_name": "groq-asr",
"echo_transcription": true
},
"model_list": [
{
"model_name": "groq-asr",
"model": "groq/whisper-large-v3-turbo"
}
]
}
```
`.security.yml`
```yaml
model_list:
groq-asr:
api_keys:
- "gsk_your_groq_key"
```
Notes:
- You can omit `api_base` and PicoClaw will use Groq's default API base automatically.
- If you set `api_base` manually for Groq Whisper, both of these forms work:
- `https://api.groq.com/openai/v1`
- `https://api.groq.com/openai/v1/audio/transcriptions`
- Any OpenAI-compatible Whisper model name containing `whisper` can use the Whisper transcription path, not only `whisper-large-v3-turbo`.
### Option B: ElevenLabs
`config.json`
```json
{
"voice": {
"model_name": "elevenlabs-asr",
"echo_transcription": true
},
"model_list": [
{
"model_name": "elevenlabs-asr",
"model": "elevenlabs/scribe_v1"
}
]
}
```
`.security.yml`
```yaml
model_list:
elevenlabs-asr:
api_keys:
- "sk-elevenlabs-your-key"
```
### Option C: OpenAI Whisper
`config.json`
```json
{
"voice": {
"model_name": "openai-asr"
},
"model_list": [
{
"model_name": "openai-asr",
"model": "openai/whisper-1"
}
]
}
```
`.security.yml`
```yaml
model_list:
openai-asr:
api_keys:
- "sk-openai-your-key"
```
## Other ASR-Capable Model Types
PicoClaw currently supports three main ASR routes:
| Route | Example models | Behavior |
| --- | --- | --- |
| ElevenLabs ASR | `elevenlabs/scribe_v1` | Uses the ElevenLabs transcription API. |
| Whisper endpoint models | `openai/whisper-1`, `groq/whisper-large-v3` | Uses an OpenAI-compatible `/audio/transcriptions` endpoint. |
| Audio-capable chat models **(Under construction)** | `openai/gpt-4o-audio-preview`, `gemini/gemini-2.5-flash` | Sends audio to a multimodal chat model and asks it to transcribe. |
If you are unsure which one to pick, choose Groq Whisper or ElevenLabs first.
## How PicoClaw Chooses a Transcriber
`DetectTranscriber` resolves ASR in this order:
1. **Preferred path**: resolve `voice.model_name` against `model_list`.
2. If that resolved model is:
- `elevenlabs/...`, PicoClaw uses the ElevenLabs transcriber.
- an OpenAI-compatible Whisper model, PicoClaw uses the Whisper transcriber.
- an audio-capable chat model, PicoClaw uses `AudioModelTranscriber`.
3. **Fallback path**: if `voice.model_name` is not set, PicoClaw performs a compatibility scan through `model_list` for legacy auto-detected ASR entries.
Fallback scanning exists for backward compatibility. New configurations should set `voice.model_name` explicitly.
## Common Mistakes
- Defining an ASR model in `model_list` but forgetting to set `voice.model_name`.
- Putting the API key in `voice` instead of `.security.yml`.
- Using a non-ASR model and expecting Whisper-style transcription behavior.
- Setting a custom `api_base` that points to the wrong provider endpoint.
## Minimal Checklist
Before testing voice input, make sure:
- `voice.model_name` matches a `model_list[].model_name`.
- The matching `.security.yml` entry contains a valid API key.
- The selected model is actually ASR-capable.
- Voice input is enabled for the channel you are using.

166
pkg/audio/asr/README_zh.md Normal file
View file

@ -0,0 +1,166 @@
# ASR自动语音识别
这个目录负责 PicoClaw 的语音转文字能力。
如果你是第一次配置 ASR可以参考如下步骤
1. 在 `model_list` 里添加一个或多个支持 ASR 的模型条目。
2. 用 `voice.model_name` 指向你想使用的那个条目。
3. 在 `.security.yml` 里配置对应的 API Key。
## 快速推荐
对于大多数新用户,建议先从下面两种开始:
| 提供商 | 示例模型 | 推荐理由 |
| --- | --- | --- |
| [Groq](https://console.groq.com/keys) | `groq/whisper-large-v3-turbo` | Whisper 风格转录速度快,并且提供 OpenAI 兼容接口配置比较直接。Groq 目前官方提供2000请求每日的免费套餐。 |
| [ElevenLabs](https://elevenlabs.io/pricing) | `elevenlabs/scribe_v1` | 上手简单语音转文字质量也不错。ElevenLabs 目前官方免费套餐包含 STT 用量。 |
价格和免费额度可能会变化,正式使用前请以官网定价页为准。
## ASR 配置是如何工作的
PicoClaw 不会把 ASR 的 API Key 放在 `voice` 配置里。
推荐的方式是:
- `voice.model_name` 用来选择 `model_list` 里的某个命名模型。
- `model_list` 条目描述真实的提供商和模型。
- `.security.yml` 负责保存该模型条目的 API Key。
这种方式更明确、更安全,也和 PicoClaw 其他模型配置方式保持一致。
## 推荐配置方式
### 方案 AGroq Whisper
`config.json`
```json
{
"voice": {
"model_name": "groq-asr",
"echo_transcription": true
},
"model_list": [
{
"model_name": "groq-asr",
"model": "groq/whisper-large-v3-turbo"
}
]
}
```
`.security.yml`
```yaml
model_list:
groq-asr:
api_keys:
- "gsk_your_groq_key"
```
说明:
- 你可以不写 `api_base`PicoClaw 会自动使用 Groq 默认接口地址。
- 如果你手动设置 Groq Whisper 的 `api_base`,下面两种写法都可以:
- `https://api.groq.com/openai/v1`
- `https://api.groq.com/openai/v1/audio/transcriptions`
- 只要是 OpenAI 兼容、并且模型名里包含 `whisper` 的模型,都可以走 Whisper 转录路径,不仅限于 `whisper-large-v3-turbo`
### 方案 BElevenLabs
`config.json`
```json
{
"voice": {
"model_name": "elevenlabs-asr",
"echo_transcription": true
},
"model_list": [
{
"model_name": "elevenlabs-asr",
"model": "elevenlabs/scribe_v1"
}
]
}
```
`.security.yml`
```yaml
model_list:
elevenlabs-asr:
api_keys:
- "sk-elevenlabs-your-key"
```
### 方案 COpenAI Whisper
`config.json`
```json
{
"voice": {
"model_name": "openai-asr"
},
"model_list": [
{
"model_name": "openai-asr",
"model": "openai/whisper-1"
}
]
}
```
`.security.yml`
```yaml
model_list:
openai-asr:
api_keys:
- "sk-openai-your-key"
```
## 其他支持 ASR 的模型类型
PicoClaw 目前主要支持三种 ASR 路径:
| 路径 | 示例模型 | 行为说明 |
| --- | --- | --- |
| ElevenLabs ASR | `elevenlabs/scribe_v1` | 使用 ElevenLabs 的语音转录接口。 |
| Whisper 接口模型 | `openai/whisper-1``groq/whisper-large-v3` | 使用 OpenAI 兼容的 `/audio/transcriptions` 接口。 |
| 支持音频的聊天模型 **(重构中)** | `openai/gpt-4o-audio-preview``gemini/gemini-2.5-flash` | 把音频发给多模态聊天模型,并要求它返回转录结果。 |
如果你不确定该选哪种,建议优先使用 Groq Whisper 或 ElevenLabs。
## PicoClaw 如何选择转录器
`DetectTranscriber` 会按下面顺序选择 ASR
1. **首选路径**:根据 `voice.model_name``model_list` 中找到对应模型。
2. 如果找到的模型属于以下类型:
- `elevenlabs/...`,则使用 ElevenLabs transcriber。
- OpenAI 兼容的 Whisper 模型,则使用 Whisper transcriber。
- 支持音频输入的聊天模型,则使用 `AudioModelTranscriber`
3. **回退路径**:如果没有设置 `voice.model_name`PicoClaw 会为了兼容旧配置,扫描 `model_list` 中可自动识别的 ASR 条目。
回退扫描只是为了兼容旧行为。新配置建议始终显式设置 `voice.model_name`
## 常见错误
- 在 `model_list` 里定义了 ASR 模型,但忘了设置 `voice.model_name`
- 把 API Key 写进了 `voice`,而不是 `.security.yml`
- 选择了不支持 ASR 的模型,却期望得到 Whisper 风格的转录结果。
- 自定义了错误的 `api_base`,导致请求打到错误的接口地址。
## 最小检查清单
在测试语音输入前,请确认:
- `voice.model_name` 能正确匹配某个 `model_list[].model_name`
- `.security.yml` 中对应条目已经配置了有效 API Key。
- 你选择的模型确实支持 ASR。
- 你当前使用的频道已经启用了语音输入能力。

253
pkg/audio/asr/agent.go Normal file
View file

@ -0,0 +1,253 @@
package asr
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/pion/rtp"
"github.com/pion/webrtc/v3/pkg/media/oggwriter"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/logger"
)
type speechAccumulator struct {
writer *oggwriter.OggWriter
file string
lastAudioAt time.Time
mu sync.Mutex
closed bool
chatID string
speakerID string
sessionID string
channel string
}
func (a *speechAccumulator) Push(chunk bus.AudioChunk) {
a.mu.Lock()
defer a.mu.Unlock()
if a.closed {
return
}
a.lastAudioAt = time.Now()
pkt := &rtp.Packet{
Header: rtp.Header{
SequenceNumber: uint16(chunk.Sequence),
Timestamp: chunk.Timestamp,
SSRC: 1, // Stable arbitrary dummy
},
Payload: chunk.Data,
}
if err := a.writer.WriteRTP(pkt); err != nil {
logger.ErrorCF("voice-agent", "Failed to write RTP", map[string]any{"error": err})
}
}
func (a *speechAccumulator) Close() {
a.mu.Lock()
defer a.mu.Unlock()
if !a.closed {
a.writer.Close()
a.closed = true
}
}
type Agent struct {
bus *bus.MessageBus
transcriber Transcriber
mu sync.Mutex
sessions map[string]*speechAccumulator // keyed by sessionID_speakerID
}
func NewAgent(mb *bus.MessageBus, t Transcriber) *Agent {
return &Agent{
bus: mb,
transcriber: t,
sessions: make(map[string]*speechAccumulator),
}
}
func (a *Agent) Start(ctx context.Context) {
logger.InfoCF("voice-agent", "Started Voice Agent orchestrator", nil)
go a.listenChunks(ctx)
go a.vadTick(ctx)
// Cleanup sessions on shutdown
go func() {
<-ctx.Done()
a.mu.Lock()
for key, acc := range a.sessions {
acc.Close()
os.Remove(acc.file)
delete(a.sessions, key)
}
a.mu.Unlock()
logger.InfoCF("voice-agent", "Cleaned up voice sessions on shutdown", nil)
}()
}
func (a *Agent) listenChunks(ctx context.Context) {
chunks := a.bus.AudioChunksChan()
for {
select {
case <-ctx.Done():
return
case chunk, ok := <-chunks:
if !ok {
return
}
a.handleChunk(chunk)
}
}
}
func (a *Agent) handleChunk(chunk bus.AudioChunk) {
// Only accept Opus-encoded audio
if chunk.Format != "opus" {
logger.DebugCF("voice-agent", "Ignoring unsupported audio format", map[string]any{"format": chunk.Format})
return
}
key := fmt.Sprintf("%s_%s", chunk.SessionID, chunk.SpeakerID)
a.mu.Lock()
acc, exists := a.sessions[key]
if !exists {
filename := filepath.Join(os.TempDir(), fmt.Sprintf("voice_%s_%d.ogg", key, time.Now().UnixNano()))
writer, err := oggwriter.New(filename, uint32(chunk.SampleRate), uint16(chunk.Channels))
if err != nil {
a.mu.Unlock()
logger.ErrorCF("voice-agent", "Failed to create OggWriter", map[string]any{"error": err})
return
}
acc = &speechAccumulator{
writer: writer,
file: filename,
lastAudioAt: time.Now(),
chatID: chunk.ChatID,
speakerID: chunk.SpeakerID,
sessionID: chunk.SessionID,
channel: chunk.Channel,
}
a.sessions[key] = acc
logger.DebugCF("voice-agent", "Started accumulating voice", map[string]any{"key": key, "file": filename})
}
a.mu.Unlock()
acc.Push(chunk)
}
func (a *Agent) vadTick(ctx context.Context) {
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
a.checkSilence(ctx)
}
}
}
func (a *Agent) checkSilence(ctx context.Context) {
a.mu.Lock()
now := time.Now()
var finished []*speechAccumulator
for key, acc := range a.sessions {
acc.mu.Lock()
last := acc.lastAudioAt
acc.mu.Unlock()
if now.Sub(last) > 1500*time.Millisecond {
acc.Close()
delete(a.sessions, key)
finished = append(finished, acc)
}
}
a.mu.Unlock()
for _, acc := range finished {
go a.processUtterance(ctx, acc)
}
}
func (a *Agent) processUtterance(ctx context.Context, acc *speechAccumulator) {
defer os.Remove(acc.file)
logger.InfoCF("voice-agent", "User finished speaking, transcribing...", map[string]any{"file": acc.file})
if a.transcriber == nil {
logger.ErrorCF("voice-agent", "No STT configured!", nil)
return
}
res, err := a.transcriber.Transcribe(ctx, acc.file)
if err != nil {
logger.ErrorCF("voice-agent", "Transcription failed", map[string]any{"error": err})
return
}
if res.Text == "" {
logger.DebugCF("voice-agent", "Ignored empty transcription", map[string]any{"file": acc.file})
return
}
logger.InfoCF("voice-agent", "Transcription result", map[string]any{"text": res.Text, "duration": res.Duration})
channelType := acc.channel
if channelType == "" {
channelType = "discord" // fallback for legacy chunks
}
text := strings.ToLower(strings.TrimSpace(res.Text))
if strings.Contains(text, "leave the voice channel") || strings.Contains(text, "leave voice") ||
strings.Contains(text, "disconnect voice") || strings.Contains(text, "leave the channel") ||
strings.Contains(text, "leave channel") {
logger.InfoCF("voice-agent", "Voice command triggered: leave", nil)
if err := a.bus.PublishVoiceControl(ctx, bus.VoiceControl{
SessionID: acc.sessionID,
Type: "command",
Action: "leave",
}); err != nil {
logger.ErrorCF("voice-agent", "Failed to publish leave control", map[string]any{"error": err})
}
if err := a.bus.PublishOutbound(ctx, bus.OutboundMessage{
Context: bus.NewOutboundContext(channelType, acc.chatID, ""),
Content: "Goodbye! Leaving the voice channel.",
}); err != nil {
logger.ErrorCF("voice-agent", "Failed to publish goodbye message", map[string]any{"error": err})
}
return
}
oralPrompt := "\n\n[SYSTEM]: The user just spoke this to you over voice chat. Please reply in a highly concise, conversational, oral style suitable for text-to-speech. Do not use markdown, emojis, asterisks, or code blocks. Speak naturally."
if err := a.bus.PublishInbound(ctx, bus.InboundMessage{
Context: bus.InboundContext{
Channel: channelType,
ChatID: acc.chatID,
ChatType: "channel",
SenderID: acc.speakerID,
Raw: map[string]string{
"is_voice": "true",
},
},
Content: res.Text + oralPrompt,
}); err != nil {
logger.ErrorCF("voice-agent", "Failed to publish inbound message", map[string]any{"error": err})
}
}

196
pkg/audio/asr/agent_test.go Normal file
View file

@ -0,0 +1,196 @@
package asr
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/pion/webrtc/v3/pkg/media/oggwriter"
"github.com/sipeed/picoclaw/pkg/bus"
)
type fakeTranscriber struct {
text string
err error
lastPath string
}
func (f *fakeTranscriber) Name() string { return "fake" }
func (f *fakeTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) {
f.lastPath = audioFilePath
if f.err != nil {
return nil, f.err
}
return &TranscriptionResponse{Text: f.text}, nil
}
func waitForFileRemoval(t *testing.T, path string, timeout time.Duration) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if _, err := os.Stat(path); os.IsNotExist(err) {
return
}
time.Sleep(10 * time.Millisecond)
}
if _, err := os.Stat(path); err == nil {
t.Fatalf("expected file to be removed: %s", path)
}
}
func TestAgentHandleChunkCreatesSession(t *testing.T) {
t.Parallel()
mb := bus.NewMessageBus()
defer mb.Close()
agent := NewAgent(mb, &fakeTranscriber{})
chunk := bus.AudioChunk{
SessionID: "sess",
SpeakerID: "speaker",
ChatID: "chat",
Channel: "discord",
Sequence: 1,
Timestamp: 1,
SampleRate: 48000,
Channels: 2,
Format: "opus",
Data: []byte{0xF8, 0xFF, 0xFE},
}
agent.handleChunk(chunk)
key := "sess_speaker"
agent.mu.Lock()
acc, ok := agent.sessions[key]
agent.mu.Unlock()
if !ok {
t.Fatal("expected session to be created")
}
acc.Close()
_ = os.Remove(acc.file)
}
func TestAgentHandleChunkIgnoresUnsupportedFormat(t *testing.T) {
t.Parallel()
mb := bus.NewMessageBus()
defer mb.Close()
agent := NewAgent(mb, &fakeTranscriber{})
chunk := bus.AudioChunk{Format: "pcm"}
agent.handleChunk(chunk)
agent.mu.Lock()
count := len(agent.sessions)
agent.mu.Unlock()
if count != 0 {
t.Fatalf("expected no sessions, got %d", count)
}
}
func TestAgentProcessUtteranceLeaveCommand(t *testing.T) {
t.Parallel()
mb := bus.NewMessageBus()
defer mb.Close()
tr := &fakeTranscriber{text: "please leave the voice channel now"}
agent := NewAgent(mb, tr)
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "voice.ogg")
if err := os.WriteFile(filePath, []byte("data"), 0o600); err != nil {
t.Fatalf("write temp file: %v", err)
}
acc := &speechAccumulator{
file: filePath,
chatID: "chat",
speakerID: "speaker",
sessionID: "sess",
channel: "discord",
}
agent.processUtterance(context.Background(), acc)
select {
case ctrl := <-mb.VoiceControlsChan():
if ctrl.Action != "leave" || ctrl.Type != "command" || ctrl.SessionID != "sess" {
t.Fatalf("unexpected voice control: %#v", ctrl)
}
case <-time.After(250 * time.Millisecond):
t.Fatal("expected voice control publish")
}
select {
case out := <-mb.OutboundChan():
if !strings.Contains(out.Content, "Leaving the voice channel") {
t.Fatalf("unexpected outbound content: %q", out.Content)
}
case <-time.After(250 * time.Millisecond):
t.Fatal("expected outbound publish")
}
if _, err := os.Stat(filePath); !os.IsNotExist(err) {
t.Fatalf("expected temp file to be removed")
}
}
func TestAgentCheckSilencePublishesInboundAndCleansUp(t *testing.T) {
t.Parallel()
mb := bus.NewMessageBus()
defer mb.Close()
tr := &fakeTranscriber{text: "hello there"}
agent := NewAgent(mb, tr)
filePath := filepath.Join(t.TempDir(), "voice.ogg")
writer, err := oggwriter.New(filePath, 48000, 2)
if err != nil {
t.Fatalf("create ogg writer: %v", err)
}
acc := &speechAccumulator{
writer: writer,
file: filePath,
lastAudioAt: time.Now().Add(-2 * time.Second),
chatID: "chat",
speakerID: "speaker",
sessionID: "sess",
channel: "slack",
}
agent.mu.Lock()
agent.sessions["sess_speaker"] = acc
agent.mu.Unlock()
agent.checkSilence(context.Background())
select {
case msg := <-mb.InboundChan():
if msg.Channel != "slack" {
t.Fatalf("unexpected inbound channel: %q", msg.Channel)
}
if !strings.Contains(msg.Content, "hello there") {
t.Fatalf("unexpected inbound content: %q", msg.Content)
}
if msg.Context.Raw["is_voice"] != "true" {
t.Fatalf("expected is_voice metadata, got %#v", msg.Context.Raw)
}
case <-time.After(500 * time.Millisecond):
t.Fatal("expected inbound publish")
}
waitForFileRemoval(t, filePath, 500*time.Millisecond)
}

131
pkg/audio/asr/asr.go Normal file
View file

@ -0,0 +1,131 @@
package asr
import (
"context"
"strings"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/providers"
)
type Transcriber interface {
Name() string
Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error)
}
type TranscriptionResponse struct {
Text string `json:"text"`
Language string `json:"language,omitempty"`
Duration float64 `json:"duration,omitempty"`
}
func supportsAudioTranscription(model string) bool {
protocol, _ := providers.ExtractProtocol(model)
switch protocol {
case "openai", "azure", "azure-openai",
"litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia",
"ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras",
"vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl",
"qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita",
"coding-plan", "alibaba-coding", "qwen-coding":
// These protocols all go through the OpenAI-compatible or Azure provider path in
// providers.CreateProviderFromConfig, so they are the only ones that can supply
// the audio media payload shape expected by NewAudioModelTranscriber.
// TODO: Further restrict this by modelID, since not every model under these
// protocols supports audio transcription.
return true
default:
return false
}
}
func supportsWhisperTranscription(model string) bool {
protocol, _ := providers.ExtractProtocol(model)
switch protocol {
case "openai", "litellm", "openrouter", "groq", "zhipu", "gemini", "nvidia",
"ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras",
"vivgrid", "volcengine", "vllm", "qwen", "qwen-intl", "qwen-international", "dashscope-intl",
"qwen-us", "dashscope-us", "mistral", "avian", "minimax", "longcat", "modelscope", "novita",
"coding-plan", "alibaba-coding", "qwen-coding", "mimo":
return true
default:
return false
}
}
func whisperModelID(modelCfg *config.ModelConfig) string {
if modelCfg == nil || modelCfg.APIKey() == "" {
return ""
}
if !supportsWhisperTranscription(modelCfg.Model) {
return ""
}
_, modelID := providers.ExtractProtocol(strings.TrimSpace(modelCfg.Model))
if strings.Contains(strings.ToLower(modelID), "whisper") {
return modelID
}
return ""
}
func transcriberFromModelConfig(modelCfg *config.ModelConfig) Transcriber {
if modelCfg == nil {
return nil
}
protocol, _ := providers.ExtractProtocol(modelCfg.Model)
if protocol == "elevenlabs" && modelCfg.APIKey() != "" {
return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase)
}
if modelID := whisperModelID(modelCfg); modelID != "" {
return NewWhisperTranscriber(modelCfg)
}
if supportsAudioTranscription(modelCfg.Model) {
return NewAudioModelTranscriber(modelCfg)
}
return nil
}
func fallbackTranscriberFromModelConfig(modelCfg *config.ModelConfig) Transcriber {
if modelCfg == nil {
return nil
}
protocol, _ := providers.ExtractProtocol(modelCfg.Model)
if protocol == "elevenlabs" && modelCfg.APIKey() != "" {
return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase)
}
if modelID := whisperModelID(modelCfg); modelID != "" {
return NewWhisperTranscriber(modelCfg)
}
return nil
}
// DetectTranscriber inspects cfg and returns the appropriate Transcriber, or
// nil if no supported transcription provider is configured.
func DetectTranscriber(cfg *config.Config) Transcriber {
if cfg == nil {
return nil
}
if modelName := strings.TrimSpace(cfg.Voice.ModelName); modelName != "" {
modelCfg, err := cfg.GetModelConfig(modelName)
if err == nil {
if tr := transcriberFromModelConfig(modelCfg); tr != nil {
return tr
}
}
}
// Fall back to compatibility scanning for legacy auto-detected ASR providers.
for _, mc := range cfg.ModelList {
if tr := fallbackTranscriberFromModelConfig(mc); tr != nil {
return tr
}
}
return nil
}

View file

@ -1,4 +1,4 @@
package voice
package asr
import (
"testing"
@ -33,26 +33,68 @@ func TestDetectTranscriber(t *testing.T) {
wantName: "audio-model",
},
{
name: "groq via model list",
name: "voice model name alias selects elevenlabs transcriber",
cfg: &config.Config{
Voice: config.VoiceConfig{ModelName: "my-asr-model"},
ModelList: []*config.ModelConfig{
{
ModelName: "my-asr-model",
Model: "elevenlabs/scribe_v1",
APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"),
},
},
},
wantName: "elevenlabs",
},
{
name: "voice model name alias selects whisper transcriber for groq",
cfg: &config.Config{
Voice: config.VoiceConfig{ModelName: "my-asr-model"},
ModelList: []*config.ModelConfig{
{
ModelName: "my-asr-model",
Model: "groq/whisper-large-v3",
APIKeys: config.SimpleSecureStrings("sk-groq-model"),
},
},
},
wantName: "whisper",
},
{
name: "openai whisper alias selects whisper transcriber",
cfg: &config.Config{
Voice: config.VoiceConfig{ModelName: "my-asr-model"},
ModelList: []*config.ModelConfig{
{
ModelName: "my-asr-model",
Model: "openai/whisper-1",
APIKeys: config.SimpleSecureStrings("sk-openai-model"),
},
},
},
wantName: "whisper",
},
{
name: "whisper via model list fallback",
cfg: &config.Config{
ModelList: []*config.ModelConfig{
{ModelName: "openai", Model: "openai/gpt-4o", APIKeys: config.SimpleSecureStrings("sk-openai")},
{
ModelName: "groq",
Model: "groq/llama-3.3-70b",
Model: "groq/whisper-large-v3-turbo",
APIKeys: config.SimpleSecureStrings("sk-groq-model"),
},
},
},
wantName: "groq",
wantName: "whisper",
},
{
name: "voice model name selects non-gemini audio model transcriber",
name: "voice model name alias selects non-gemini audio model transcriber",
cfg: &config.Config{
Voice: config.VoiceConfig{ModelName: "voice-openai-audio"},
Voice: config.VoiceConfig{ModelName: "my-asr-model"},
ModelList: []*config.ModelConfig{
{
ModelName: "voice-openai-audio",
ModelName: "my-asr-model",
Model: "openai/gpt-4o-audio-preview",
APIKeys: config.SimpleSecureStrings("sk-openai"),
},
@ -92,7 +134,7 @@ func TestDetectTranscriber(t *testing.T) {
name: "groq model list entry without key is skipped",
cfg: &config.Config{
ModelList: []*config.ModelConfig{
{Model: "groq/llama-3.3-70b"},
{Model: "groq/whisper-large-v3"},
},
},
wantNil: true,
@ -103,12 +145,12 @@ func TestDetectTranscriber(t *testing.T) {
ModelList: []*config.ModelConfig{
{
ModelName: "groq",
Model: "groq/llama-3.3-70b",
Model: "groq/whisper-large-v3",
APIKeys: config.SimpleSecureStrings("sk-groq-model"),
},
},
},
wantName: "groq",
wantName: "whisper",
},
{
name: "missing voice model name config returns nil",
@ -127,15 +169,17 @@ func TestDetectTranscriber(t *testing.T) {
{
name: "elevenlabs voice config key",
cfg: &config.Config{
Voice: config.VoiceConfig{ElevenLabsAPIKey: "sk_elevenlabs_test"},
ModelList: []*config.ModelConfig{
{Model: "elevenlabs/scribe_v1", APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test")},
},
},
wantName: "elevenlabs",
},
{
name: "elevenlabs takes priority over groq model list",
cfg: &config.Config{
Voice: config.VoiceConfig{ElevenLabsAPIKey: "sk_elevenlabs_test"},
ModelList: []*config.ModelConfig{
{Model: "elevenlabs/scribe_v1", APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test")},
{
ModelName: "groq",
Model: "groq/llama-3.3-70b",
@ -149,10 +193,10 @@ func TestDetectTranscriber(t *testing.T) {
name: "voice model name takes priority over elevenlabs",
cfg: &config.Config{
Voice: config.VoiceConfig{
ModelName: "voice-gemini",
ElevenLabsAPIKey: "sk_elevenlabs_test",
ModelName: "voice-gemini",
},
ModelList: []*config.ModelConfig{
{Model: "elevenlabs", APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test")},
{
ModelName: "voice-gemini",
Model: "gemini/gemini-2.5-flash",

View file

@ -1,4 +1,4 @@
package voice
package asr
import (
"context"

View file

@ -1,4 +1,4 @@
package voice
package asr
import (
"context"

View file

@ -1,4 +1,4 @@
package voice
package asr
import (
"bytes"
@ -23,12 +23,16 @@ type ElevenLabsTranscriber struct {
httpClient *http.Client
}
func NewElevenLabsTranscriber(apiKey string) *ElevenLabsTranscriber {
func NewElevenLabsTranscriber(apiKey, apiBase string) *ElevenLabsTranscriber {
logger.DebugCF("voice", "Creating ElevenLabs transcriber", map[string]any{"has_api_key": apiKey != ""})
if apiBase == "" {
apiBase = "https://api.elevenlabs.io"
}
return &ElevenLabsTranscriber{
apiKey: apiKey,
apiBase: "https://api.elevenlabs.io",
apiBase: apiBase,
httpClient: &http.Client{
Timeout: 120 * time.Second,
},

View file

@ -1,4 +1,4 @@
package voice
package asr
import (
"context"
@ -14,7 +14,7 @@ import (
var _ Transcriber = (*ElevenLabsTranscriber)(nil)
func TestElevenLabsTranscriberName(t *testing.T) {
tr := NewElevenLabsTranscriber("sk_test")
tr := NewElevenLabsTranscriber("sk_test", "")
if got := tr.Name(); got != "elevenlabs" {
t.Errorf("Name() = %q, want %q", got, "elevenlabs")
}
@ -43,7 +43,7 @@ func TestElevenLabsTranscribe(t *testing.T) {
}))
defer srv.Close()
tr := NewElevenLabsTranscriber("sk_test")
tr := NewElevenLabsTranscriber("sk_test", "")
tr.apiBase = srv.URL
resp, err := tr.Transcribe(context.Background(), audioPath)
@ -64,7 +64,7 @@ func TestElevenLabsTranscribe(t *testing.T) {
}))
defer srv.Close()
tr := NewElevenLabsTranscriber("sk_bad")
tr := NewElevenLabsTranscriber("sk_bad", "")
tr.apiBase = srv.URL
_, err := tr.Transcribe(context.Background(), audioPath)
@ -74,7 +74,7 @@ func TestElevenLabsTranscribe(t *testing.T) {
})
t.Run("missing file", func(t *testing.T) {
tr := NewElevenLabsTranscriber("sk_test")
tr := NewElevenLabsTranscriber("sk_test", "")
_, err := tr.Transcribe(context.Background(), filepath.Join(tmpDir, "nonexistent.ogg"))
if err == nil {
t.Fatal("expected error for missing file, got nil")

View file

@ -0,0 +1,245 @@
package asr
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
"github.com/sipeed/picoclaw/pkg/utils"
)
type WhisperTranscriber struct {
apiKey string
apiBase string
modelID string
providerName string
httpClient *http.Client
}
func NewWhisperTranscriber(modelCfg *config.ModelConfig) *WhisperTranscriber {
if modelCfg == nil {
return nil
}
protocol, modelID := providers.ExtractProtocol(modelCfg.Model)
if modelID == "" {
modelID = strings.TrimSpace(modelCfg.Model)
}
tr := newWhisperTranscriber(
modelCfg.APIKey(),
providers.ResolveAPIBase(modelCfg),
modelID,
protocol,
)
if tr == nil {
return nil
}
logger.DebugCF("voice", "Creating whisper transcriber", map[string]any{
"api_base": tr.apiBase,
"has_key": tr.apiKey != "",
"model": tr.modelID,
"provider": tr.providerName,
})
return tr
}
func NewGroqTranscriber(apiKey, modelID string) *WhisperTranscriber {
return newWhisperTranscriber(apiKey, "https://api.groq.com/openai/v1", modelID, "groq")
}
func newWhisperTranscriber(apiKey, apiBase, modelID, providerName string) *WhisperTranscriber {
if modelID == "" {
return nil
}
if providerName == "" {
providerName = "whisper"
}
return &WhisperTranscriber{
apiKey: apiKey,
apiBase: strings.TrimRight(apiBase, "/"),
modelID: modelID,
providerName: providerName,
httpClient: &http.Client{
Timeout: 60 * time.Second,
},
}
}
func (t *WhisperTranscriber) transcriptionURL() string {
base := strings.TrimRight(t.apiBase, "/")
if strings.HasSuffix(base, "/audio/transcriptions") {
return base
}
return base + "/audio/transcriptions"
}
func (t *WhisperTranscriber) TranscribeData(
ctx context.Context,
data []byte,
filename string,
) (*TranscriptionResponse, error) {
logger.InfoCF("voice", "Starting whisper transcription from memory", map[string]any{
"bytes": len(data),
"filename": filename,
"model": t.modelID,
"provider": t.providerName,
})
var requestBody bytes.Buffer
writer := multipart.NewWriter(&requestBody)
part, err := writer.CreateFormFile("file", filename)
if err != nil {
logger.ErrorCF("voice", "Failed to create whisper form file", map[string]any{"error": err})
return nil, fmt.Errorf("failed to create form file: %w", err)
}
if _, copyErr := io.Copy(part, bytes.NewReader(data)); copyErr != nil {
logger.ErrorCF("voice", "Failed to copy whisper file content", map[string]any{"error": copyErr})
return nil, fmt.Errorf("failed to copy file content: %w", copyErr)
}
if err = writer.WriteField("model", t.modelID); err != nil {
logger.ErrorCF("voice", "Failed to write whisper model field", map[string]any{"error": err})
return nil, fmt.Errorf("failed to write model field: %w", err)
}
if err = writer.WriteField("response_format", "json"); err != nil {
logger.ErrorCF("voice", "Failed to write whisper response_format field", map[string]any{"error": err})
return nil, fmt.Errorf("failed to write response_format field: %w", err)
}
if err = writer.Close(); err != nil {
logger.ErrorCF("voice", "Failed to close whisper multipart writer", map[string]any{"error": err})
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
}
return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), int64(len(data)))
}
func (t *WhisperTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) {
logger.InfoCF("voice", "Starting whisper transcription", map[string]any{
"audio_file": audioFilePath,
"model": t.modelID,
"provider": t.providerName,
})
audioFile, err := os.Open(audioFilePath)
if err != nil {
return nil, fmt.Errorf("failed to open audio file %s: %w", audioFilePath, err)
}
defer audioFile.Close()
fileInfo, err := audioFile.Stat()
if err != nil {
return nil, fmt.Errorf("failed to stat audio file %s: %w", audioFilePath, err)
}
var requestBody bytes.Buffer
writer := multipart.NewWriter(&requestBody)
part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath))
if err != nil {
return nil, fmt.Errorf("failed to create form file: %w", err)
}
if _, copyErr := io.Copy(part, audioFile); copyErr != nil {
return nil, fmt.Errorf("failed to copy audio data: %w", copyErr)
}
if err = writer.WriteField("model", t.modelID); err != nil {
return nil, fmt.Errorf("failed to write model field: %w", err)
}
if err = writer.WriteField("response_format", "json"); err != nil {
return nil, fmt.Errorf("failed to write response_format field: %w", err)
}
if err = writer.Close(); err != nil {
return nil, fmt.Errorf("failed to close multipart writer: %w", err)
}
return t.doRequest(ctx, &requestBody, writer.FormDataContentType(), fileInfo.Size())
}
func (t *WhisperTranscriber) doRequest(
ctx context.Context,
requestBody *bytes.Buffer,
contentType string,
fileSize int64,
) (*TranscriptionResponse, error) {
url := t.transcriptionURL()
req, err := http.NewRequestWithContext(ctx, "POST", url, requestBody)
if err != nil {
logger.ErrorCF("voice", "Failed to create whisper request", map[string]any{"error": err})
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", contentType)
if t.apiKey != "" {
req.Header.Set("Authorization", "Bearer "+t.apiKey)
}
logger.DebugCF("voice", "Sending whisper transcription request", map[string]any{
"file_size_bytes": fileSize,
"model": t.modelID,
"provider": t.providerName,
"request_size_bytes": requestBody.Len(),
"url": url,
})
resp, err := t.httpClient.Do(req)
if err != nil {
logger.ErrorCF("voice", "Failed to send whisper request", map[string]any{"error": err})
return nil, fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
logger.ErrorCF("voice", "Failed to read whisper response", map[string]any{"error": err})
return nil, fmt.Errorf("failed to read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
logger.ErrorCF("voice", "Whisper API error", map[string]any{
"provider": t.providerName,
"response": string(body),
"status_code": resp.StatusCode,
})
return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
}
var result TranscriptionResponse
if err := json.Unmarshal(body, &result); err != nil {
logger.ErrorCF("voice", "Failed to unmarshal whisper response", map[string]any{"error": err})
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
}
logger.InfoCF("voice", "Whisper transcription completed successfully", map[string]any{
"duration_seconds": result.Duration,
"language": result.Language,
"provider": t.providerName,
"text_length": len(result.Text),
"transcription_preview": utils.Truncate(result.Text, 50),
})
return &result, nil
}
func (t *WhisperTranscriber) Name() string {
return "whisper"
}

View file

@ -0,0 +1,102 @@
package asr
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/sipeed/picoclaw/pkg/config"
)
func TestWhisperTranscriberTranscribeDataUsesConfiguredModel(t *testing.T) {
var gotModel string
var gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if got := r.Header.Get("Authorization"); got != "Bearer sk-openai-test" {
t.Errorf("Authorization = %q, want %q", got, "Bearer sk-openai-test")
}
reader, err := r.MultipartReader()
if err != nil {
t.Fatalf("MultipartReader() error: %v", err)
}
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("NextPart() error: %v", err)
}
data, err := io.ReadAll(part)
if err != nil {
t.Fatalf("ReadAll() error: %v", err)
}
if part.FormName() == "model" {
gotModel = string(data)
}
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "hello from whisper"}); err != nil {
t.Fatalf("Encode() error: %v", err)
}
}))
defer server.Close()
tr := NewWhisperTranscriber(&config.ModelConfig{
Model: "openai/whisper-1",
APIBase: server.URL,
APIKeys: config.SimpleSecureStrings("sk-openai-test"),
})
tr.httpClient = server.Client()
resp, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.ogg")
if err != nil {
t.Fatalf("TranscribeData() error: %v", err)
}
if resp.Text != "hello from whisper" {
t.Errorf("Text = %q, want %q", resp.Text, "hello from whisper")
}
if gotModel != "whisper-1" {
t.Errorf("model field = %q, want %q", gotModel, "whisper-1")
}
if gotPath != "/audio/transcriptions" {
t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions")
}
}
func TestWhisperTranscriberUsesEndpointAPIBaseWithoutDoubleAppend(t *testing.T) {
var gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(TranscriptionResponse{Text: "ok"}); err != nil {
t.Fatalf("Encode() error: %v", err)
}
}))
defer server.Close()
tr := NewWhisperTranscriber(&config.ModelConfig{
Model: "groq/whisper-large-v3",
APIBase: server.URL + "/audio/transcriptions",
APIKeys: config.SimpleSecureStrings("sk-groq-test"),
})
tr.httpClient = server.Client()
if _, err := tr.TranscribeData(context.Background(), []byte("audio"), "clip.ogg"); err != nil {
t.Fatalf("TranscribeData() error: %v", err)
}
if gotPath != "/audio/transcriptions" {
t.Errorf("path = %q, want %q", gotPath, "/audio/transcriptions")
}
}

57
pkg/audio/ogg.go Normal file
View file

@ -0,0 +1,57 @@
package audio
import (
"bytes"
"fmt"
"io"
)
// DecodeOggOpus reads an Ogg format stream and extracts individual Opus payloads.
// It calls onFrame for every complete Opus frame found in the stream.
func DecodeOggOpus(r io.Reader, onFrame func([]byte) error) error {
var packet bytes.Buffer
header := make([]byte, 27)
segment := make([]byte, 255)
for {
if _, err := io.ReadFull(r, header); err != nil {
if err == io.EOF || err == io.ErrUnexpectedEOF {
return nil
}
return fmt.Errorf("failed to read ogg header: %w", err)
}
if string(header[:4]) != "OggS" {
return fmt.Errorf("invalid ogg magic string")
}
pageSegments := int(header[26])
segmentTable := make([]byte, pageSegments)
if _, err := io.ReadFull(r, segmentTable); err != nil {
return fmt.Errorf("failed to read segment table: %w", err)
}
for _, lacing := range segmentTable {
if _, err := io.ReadFull(r, segment[:lacing]); err != nil {
return fmt.Errorf("failed to read segment data: %w", err)
}
packet.Write(segment[:lacing])
// If lacing is less than 255, the packet is complete
if lacing < 255 {
if packet.Len() > 0 {
packetBytes := packet.Bytes()
// Ignore Ogg Opus headers
if !bytes.HasPrefix(packetBytes, []byte("OpusHead")) &&
!bytes.HasPrefix(packetBytes, []byte("OpusTags")) {
if err := onFrame(packetBytes); err != nil {
return err
}
}
// Start new packet
packet.Reset()
}
}
}
}
}

146
pkg/audio/ogg_test.go Normal file
View file

@ -0,0 +1,146 @@
package audio
import (
"bytes"
"reflect"
"strings"
"testing"
)
// buildOggPage helper creates an Ogg page for testing.
// lacingVals specifies the segment table, and data is the payload.
func buildOggPage(lacingVals []byte, data []byte) []byte {
var buf bytes.Buffer
// 27-byte Ogg header
header := make([]byte, 27)
copy(header[:4], "OggS")
header[5] = 0 // type flag
// For testing, we only care about OggS magic and page_segments (byte 26)
header[26] = byte(len(lacingVals))
buf.Write(header)
buf.Write(lacingVals)
buf.Write(data)
return buf.Bytes()
}
func TestDecodeOggOpus_ValidParsing(t *testing.T) {
var b bytes.Buffer
// Packet 1: Single segment, length 50
pkt1 := bytes.Repeat([]byte{1}, 50)
// Packet 2: Multi-segment (255 + 10 = 265 bytes)
pkt2Part1 := bytes.Repeat([]byte{2}, 255)
pkt2Part2 := bytes.Repeat([]byte{2}, 10)
// Packet 3: Continued across pages. Page 1 gets 255, Page 2 gets 20. Total 275 bytes.
pkt3Part1 := bytes.Repeat([]byte{3}, 255)
pkt3Part2 := bytes.Repeat([]byte{3}, 20)
// Page 1: OpusHead (skip), OpusTags (skip), pkt1, pkt2, pkt3Part1
page1Lacing := []byte{8, 8, 50, 255, 10, 255}
page1Data := bytes.Join([][]byte{
[]byte("OpusHead"),
[]byte("OpusTags"),
pkt1,
pkt2Part1, pkt2Part2,
pkt3Part1,
}, nil)
// Page 2: pkt3Part2, pkt4 (length 10)
pkt4 := bytes.Repeat([]byte{4}, 10)
page2Lacing := []byte{20, 10}
page2Data := bytes.Join([][]byte{
pkt3Part2,
pkt4,
}, nil)
b.Write(buildOggPage(page1Lacing, page1Data))
b.Write(buildOggPage(page2Lacing, page2Data))
var frames [][]byte
err := DecodeOggOpus(&b, func(frame []byte) error {
// making a copy to store as DecodeOggOpus might reuse backing array
cpy := make([]byte, len(frame))
copy(cpy, frame)
frames = append(frames, cpy)
return nil
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectedFrames := [][]byte{
pkt1,
append(pkt2Part1, pkt2Part2...),
append(pkt3Part1, pkt3Part2...),
pkt4,
}
if len(frames) != len(expectedFrames) {
t.Fatalf("expected %d frames, got %d", len(expectedFrames), len(frames))
}
for i, expected := range expectedFrames {
if !reflect.DeepEqual(frames[i], expected) {
t.Errorf("frame %d mismatch:\nexp: %v\ngot: %v", i, expected, frames[i])
}
}
}
func TestDecodeOggOpus_Errors(t *testing.T) {
tests := []struct {
name string
data []byte
errContains string
}{
{
name: "invalid magic string",
data: []byte(
"OggX\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
),
errContains: "invalid ogg magic string",
},
{
name: "short header",
data: []byte("Ogg"),
errContains: "failed to read ogg header",
},
{
name: "eof in segment table",
data: func() []byte {
h := make([]byte, 27)
copy(h, "OggS")
h[26] = 5 // expects 5 bytes of segment table, but none provided
return h
}(),
errContains: "failed to read segment table",
},
{
name: "eof in segment data",
data: func() []byte {
h := make([]byte, 27, 28)
copy(h, "OggS")
h[26] = 1
return append(h, 100) // expects 100 bytes of data, but none provided
}(),
errContains: "failed to read segment data",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := DecodeOggOpus(bytes.NewReader(tt.data), func(b []byte) error { return nil })
if tt.name == "short header" {
if err != nil {
t.Errorf("expected no error (io.EOF/ErrUnexpectedEOF swallowed), got %v", err)
}
return
}
if err == nil {
t.Fatalf("expected error containing %q, got nil", tt.errContains)
}
if !strings.Contains(err.Error(), tt.errContains) {
t.Errorf("expected error to contain %q, got: %q", tt.errContains, err.Error())
}
})
}
}

96
pkg/audio/sentence.go Normal file
View file

@ -0,0 +1,96 @@
package audio
import (
"strings"
"unicode"
)
// SplitSentences splits text into sentence-sized chunks suitable for TTS synthesis.
// It splits on sentence-ending punctuation (.!?\n, as well as CJK 。, , ) while avoiding false splits
// on decimal numbers. Very short fragments are merged with
// the next sentence to prevent choppy playback.
func SplitSentences(text string) []string {
if text == "" {
return nil
}
var sentences []string
var current strings.Builder
runes := []rune(text)
for i := 0; i < len(runes); i++ {
r := runes[i]
if r == '\n' {
s := strings.TrimSpace(current.String())
if s != "" {
sentences = append(sentences, s)
}
current.Reset()
continue
}
current.WriteRune(r)
if r == '.' || r == '!' || r == '?' || r == '。' || r == '' || r == '' {
// Avoid splitting on decimal numbers like "3.14"
if r == '.' && i > 0 && unicode.IsDigit(runes[i-1]) &&
i+1 < len(runes) && unicode.IsDigit(runes[i+1]) {
continue
}
// Consume contiguous punctuation clusters (e.g., "..." or "?!").
for i+1 < len(runes) && (runes[i+1] == '.' || runes[i+1] == '!' || runes[i+1] == '?' || runes[i+1] == '。' || runes[i+1] == '' || runes[i+1] == '') {
i++
current.WriteRune(runes[i])
}
s := strings.TrimSpace(current.String())
if s != "" {
sentences = append(sentences, s)
}
current.Reset()
}
}
// Flush remaining text
if s := strings.TrimSpace(current.String()); s != "" {
sentences = append(sentences, s)
}
// Merge very short fragments with the next sentence
return mergeShorties(sentences, 15)
}
// mergeShorties merges sentences shorter than minLen characters with the following sentence.
func mergeShorties(sentences []string, minLen int) []string {
if len(sentences) <= 1 {
return sentences
}
var merged []string
var buf string
for _, s := range sentences {
if buf != "" {
buf += " " + s
if len([]rune(buf)) >= minLen {
merged = append(merged, buf)
buf = ""
}
} else if len([]rune(s)) < minLen {
buf = s
} else {
merged = append(merged, s)
}
}
if buf != "" {
if len(merged) > 0 {
merged[len(merged)-1] += " " + buf
} else {
merged = append(merged, buf)
}
}
return merged
}

View file

@ -0,0 +1,69 @@
package audio
import (
"reflect"
"testing"
)
func TestSplitSentences(t *testing.T) {
tests := []struct {
name string
in string
want []string
}{
{
name: "empty input",
in: "",
want: nil,
},
{
name: "single sentence",
in: "Hello world.",
want: []string{"Hello world."},
},
{
name: "decimal numbers do not split",
in: "The value is 3.14 today. Keep watching closely.",
want: []string{"The value is 3.14 today.", "Keep watching closely."},
},
{
name: "newline boundary",
in: "This is line number one\nThis is line number two",
want: []string{"This is line number one", "This is line number two"},
},
{
name: "newline with surrounding spaces",
in: " This is the first line \n This is the second line ",
want: []string{"This is the first line", "This is the second line"},
},
{
name: "trailing punctuation consumed",
in: "Please wait a moment... What on earth?! That is perfectly fine.",
want: []string{"Please wait a moment...", "What on earth?!", "That is perfectly fine."},
},
{
name: "short leading fragment merges with next",
in: "Hi. This is a longer sentence.",
want: []string{"Hi. This is a longer sentence."},
},
{
name: "consecutive short fragments keep merging",
in: "A. B. C. This is the real sentence.",
want: []string{"A. B. C. This is the real sentence."},
},
{
name: "short trailing fragment merges back",
in: "This sentence is long enough. End.",
want: []string{"This sentence is long enough. End."},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := SplitSentences(tc.in)
if !reflect.DeepEqual(got, tc.want) {
t.Fatalf("SplitSentences(%q) = %#v, want %#v", tc.in, got, tc.want)
}
})
}
}

137
pkg/audio/tts/README.md Normal file
View file

@ -0,0 +1,137 @@
# TTS (Text-to-Speech)
This package handles speech synthesis for PicoClaw.
If you are new to TTS setup, the simplest workflow is:
1. Add a TTS-capable entry to `model_list`.
2. Point `voice.tts_model_name` at that entry.
3. Put the API key in `.security.yml`.
## Quick Recommendation
For most users, these are the best starting points:
| Provider | Why start here |
| --- | --- |
| [OpenAI](https://platform.openai.com/docs/guides/text-to-speech) | Best-supported path in PicoClaw today. The current TTS implementation is built around the OpenAI-compatible `/audio/speech` API shape, and OpenAI is the safest default. |
| [Xiaomi MiMo](https://platform.xiaomimimo.com) | A good second option if you want an OpenAI-compatible provider endpoint and are already using MiMo models in the rest of your stack. |
## How TTS Configuration Works
PicoClaw does not keep TTS API keys inside `voice`.
Instead:
- `voice.tts_model_name` selects a named entry from `model_list`.
- That `model_list` entry provides the provider, model ID, API base, and proxy settings.
- `.security.yml` stores the API key for the same named model entry.
This is the recommended and supported configuration pattern.
## Recommended Setup
### Option A: OpenAI
`config.json`
```json
{
"voice": {
"tts_model_name": "openai-tts"
},
"model_list": [
{
"model_name": "openai-tts",
"model": "openai/tts-1"
}
]
}
```
`.security.yml`
```yaml
model_list:
openai-tts:
api_keys:
- "sk-openai-your-key"
```
### Option B: Xiaomi MiMo
`config.json`
```json
{
"voice": {
"tts_model_name": "mimo-tts"
},
"model_list": [
{
"model_name": "mimo-tts",
"model": "mimo/mimo-v2-tts"
}
]
}
```
`.security.yml`
```yaml
model_list:
mimo-tts:
api_keys:
- "your-mimo-key"
```
If you use a custom MiMo endpoint, you can also set `api_base` explicitly. Otherwise PicoClaw will use the provider default.
## What PicoClaw Sends Today
The current TTS runtime uses an OpenAI-compatible speech request with these defaults:
- Endpoint: `/audio/speech`
- Response format: `opus`
- Voice: `alloy`
- Model: taken from the selected `model_list` entry
That means:
- `openai/tts-1` works naturally.
- Other OpenAI-compatible providers can work if they accept the same request format.
- PicoClaw currently does not expose a user-facing config field for changing the TTS voice from `alloy`.
## How PicoClaw Chooses a TTS Provider
`DetectTTS` resolves TTS in this order:
1. **Preferred path**: resolve `voice.tts_model_name` against `model_list`.
2. If a matching model entry exists and has an API key, PicoClaw creates an OpenAI-compatible TTS provider using that model's settings.
3. **Fallback path**: if `voice.tts_model_name` is not set or cannot be resolved, PicoClaw scans `model_list` for the first entry whose model string contains `tts` and has an API key.
Fallback scanning exists for compatibility. New configs should set `voice.tts_model_name` explicitly.
## Notes About API Base Handling
PicoClaw normalizes the configured base URL for TTS:
- For OpenAI, a base like `https://api.openai.com` or `https://api.openai.com/v1` becomes `https://api.openai.com/v1/audio/speech`.
- For other OpenAI-compatible providers, PicoClaw preserves the configured base path and ensures it ends with `/audio/speech`.
- If `api_base` is omitted, PicoClaw uses the provider default base when the model prefix is known.
## Common Mistakes
- Setting `voice.tts_model_name` to a name that does not exist in `model_list`.
- Adding a TTS model but forgetting to put its API key in `.security.yml`.
- Assuming PicoClaw will automatically use provider-specific custom voices.
- Using a provider endpoint that is not compatible with the OpenAI `/audio/speech` request format.
## Minimal Checklist
Before testing `send_tts`, make sure:
- `voice.tts_model_name` matches a `model_list[].model_name`.
- The matching `.security.yml` entry contains a valid API key.
- The chosen provider supports an OpenAI-compatible speech synthesis endpoint.
- Your selected model is actually a TTS-capable model.

137
pkg/audio/tts/README_zh.md Normal file
View file

@ -0,0 +1,137 @@
# TTS文本转语音
这个目录负责 PicoClaw 的语音合成能力。
如果你是第一次配置 TTS可以参照下面这个流程
1. 在 `model_list` 里添加一个支持 TTS 的模型。
2. 用 `voice.tts_model_name` 指向这个模型。
3. 在 `.security.yml` 里配置对应的 API Key。
## 快速推荐
对于大多数用户,建议优先从下面两种开始:
| 提供商 | 推荐理由 |
| --- | --- |
| [OpenAI](https://platform.openai.com/docs/guides/text-to-speech) | 这是 PicoClaw 当前最稳定、最直接的 TTS 路径。当前实现就是围绕 OpenAI 兼容的 `/audio/speech` 接口格式构建的,所以 OpenAI 是最稳妥的默认选择。 |
| [Xiaomi MiMo](https://platform.xiaomimimo.com) | 由于响应速度和语音音色对于中国用户更友好MiMo 是一个不错的第二选择。 |
## TTS 配置是如何工作的
PicoClaw 不会把 TTS 的 API Key 放在 `voice` 配置里。
推荐方式是:
- `voice.tts_model_name` 用来选择 `model_list` 里的某个命名模型。
- 对应的 `model_list` 条目提供真实的 provider、model ID、`api_base` 和代理配置。
- `.security.yml` 负责保存该模型条目的 API Key。
这是当前推荐且受支持的配置方式。
## 推荐配置方式
### 方案 AOpenAI
`config.json`
```json
{
"voice": {
"tts_model_name": "openai-tts"
},
"model_list": [
{
"model_name": "openai-tts",
"model": "openai/tts-1"
}
]
}
```
`.security.yml`
```yaml
model_list:
openai-tts:
api_keys:
- "sk-openai-your-key"
```
### 方案 BXiaomi MiMo
`config.json`
```json
{
"voice": {
"tts_model_name": "mimo-tts"
},
"model_list": [
{
"model_name": "mimo-tts",
"model": "mimo/mimo-v2-tts"
}
]
}
```
`.security.yml`
```yaml
model_list:
mimo-tts:
api_keys:
- "your-mimo-key"
```
如果你使用自定义的 MiMo 接口地址,也可以显式设置 `api_base`。如果不设置PicoClaw 会自动使用该 provider 的默认地址。
## PicoClaw 当前实际发送的 TTS 请求
当前 TTS 运行时使用的是 OpenAI 兼容的语音合成请求,并带有以下默认值:
- Endpoint`/audio/speech`
- 返回格式:`opus`
- Voice`alloy`
- Model来自你所选中的 `model_list` 条目
这意味着:
- `openai/tts-1` 可以自然工作。
- 其他 OpenAI 兼容 provider 也可能可用,前提是它们接受相同的请求格式。
- PicoClaw 目前还没有对用户暴露一个配置项来修改 TTS voice当前固定为 `alloy`
## PicoClaw 如何选择 TTS Provider
`DetectTTS` 会按下面顺序选择 TTS
1. **首选路径**:根据 `voice.tts_model_name``model_list` 中找到对应模型。
2. 如果找到了匹配条目,并且它有 API KeyPicoClaw 就会使用这个模型条目的配置创建一个 OpenAI 兼容的 TTS provider。
3. **回退路径**:如果没有设置 `voice.tts_model_name`或者该名字无法解析PicoClaw 会扫描 `model_list`,选中第一个模型字符串里包含 `tts` 且带有 API Key 的条目。
回退扫描只是为了兼容旧行为。新配置建议始终显式设置 `voice.tts_model_name`
## 关于 API Base 的处理方式
PicoClaw 会对 TTS 的 `api_base` 做规范化处理:
- 对 OpenAI 来说,像 `https://api.openai.com``https://api.openai.com/v1` 这样的地址,会自动变成 `https://api.openai.com/v1/audio/speech`
- 对其他 OpenAI 兼容 providerPicoClaw 会尽量保留你提供的基础路径,只确保它最终以 `/audio/speech` 结尾。
- 如果没有设置 `api_base`,并且模型前缀是已知 providerPicoClaw 会自动使用该 provider 的默认地址。
## 常见错误
- `voice.tts_model_name` 指向了一个不存在的 `model_list` 名称。
- 在 `model_list` 里定义了 TTS 模型,但忘了在 `.security.yml` 中配置对应 API Key。
- 误以为 PicoClaw 会自动支持 provider 自定义 voice 参数。
- 使用了不兼容 OpenAI `/audio/speech` 请求格式的接口地址。
## 最小检查清单
在测试 `send_tts` 之前,请确认:
- `voice.tts_model_name` 能正确匹配某个 `model_list[].model_name`
- `.security.yml` 中对应条目已经配置了有效 API Key。
- 你所选的 provider 支持 OpenAI 兼容的语音合成接口。
- 你选择的模型本身确实支持 TTS。

162
pkg/audio/tts/mimo_tts.go Normal file
View file

@ -0,0 +1,162 @@
package tts
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
)
type MimoTTSProvider struct {
apiKey string
apiBase string
voice string
format string
model string
httpClient *http.Client
}
func NewMimoTTSProvider(apiKey string, apiBase string, model string, proxyURL string) *MimoTTSProvider {
if apiBase == "" {
apiBase = "https://api.xiaomimimo.com/v1/chat/completions"
} else {
if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" {
path := u.Path
if u.Host == "api.xiaomimimo.com" {
if path == "" || path == "/" || path == "/v1" || path == "/v1/" {
path = "/v1/chat/completions"
} else {
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
if !strings.HasPrefix(path, "/v1/") {
path = "/v1" + strings.TrimSuffix(path, "/")
}
if !strings.HasSuffix(path, "/chat/completions") {
path = strings.TrimSuffix(path, "/") + "/chat/completions"
}
}
} else {
if !strings.HasSuffix(path, "/chat/completions") {
path = strings.TrimSuffix(path, "/") + "/chat/completions"
}
}
u.Path = path
apiBase = u.String()
} else {
if apiBase == "https://api.xiaomimimo.com/v1" {
apiBase = "https://api.xiaomimimo.com/v1/chat/completions"
} else if !strings.HasSuffix(apiBase, "/chat/completions") {
apiBase = strings.TrimSuffix(apiBase, "/") + "/chat/completions"
}
}
}
model = strings.TrimSpace(model)
if model == "" {
model = "mimo-v2-tts"
}
client := &http.Client{Timeout: 60 * time.Second}
if proxyURL != "" {
if pURL, err := url.Parse(proxyURL); err == nil {
client.Transport = &http.Transport{Proxy: http.ProxyURL(pURL)}
} else {
logger.WarnF(
"NewMimoTTSProvider: invalid proxy URL; proceeding without proxy",
map[string]any{"proxyURL": proxyURL, "error": err},
)
}
}
return &MimoTTSProvider{
apiKey: apiKey,
apiBase: apiBase,
voice: "default_zh", // mimo_default now seems to be an alias for default_en, which is not working for Chinese TTS. default_zh seems to work fine with both English and Chinese, and is likely the intended default for TTS.
format: "mp3",
model: model,
httpClient: client,
}
}
func (t *MimoTTSProvider) Name() string {
return "mimo-tts"
}
func (t *MimoTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) {
logger.DebugCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text), "provider": t.Name()})
reqBody := map[string]any{
"model": t.model,
"messages": []map[string]string{
{"role": "assistant", "content": text},
},
"audio": map[string]string{
"format": t.format,
"voice": t.voice,
},
"stream": false,
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData))
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Api-Key", t.apiKey)
resp, err := t.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
}
var payload struct {
Choices []struct {
Message struct {
Audio struct {
Data string `json:"data"`
} `json:"audio"`
} `json:"message"`
} `json:"choices"`
}
err = json.Unmarshal(body, &payload)
if err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
if len(payload.Choices) == 0 || payload.Choices[0].Message.Audio.Data == "" {
return nil, fmt.Errorf("invalid TTS response: missing audio data")
}
audioBytes, err := base64.StdEncoding.DecodeString(payload.Choices[0].Message.Audio.Data)
if err != nil {
return nil, fmt.Errorf("failed to decode audio data: %w", err)
}
return io.NopCloser(bytes.NewReader(audioBytes)), nil
}

126
pkg/audio/tts/openai_tts.go Normal file
View file

@ -0,0 +1,126 @@
package tts
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers/common"
)
type OpenAITTSProvider struct {
apiKey string
apiBase string
voice string
model string
httpClient *http.Client
}
func NewOpenAITTSProvider(apiKey string, apiBase string, proxyURL string, model string) *OpenAITTSProvider {
// Normalize apiBase to avoid malformed endpoints like
// "https://api.openai.com/audio/speech" when "/v1" is required.
if apiBase == "" {
apiBase = "https://api.openai.com/v1/audio/speech"
} else {
if u, err := url.Parse(apiBase); err == nil && u.Scheme != "" && u.Host != "" {
path := u.Path
if u.Host == "api.openai.com" {
// For the official OpenAI host, ensure exactly one /v1 prefix and
// that the path ends with /audio/speech.
if path == "" || path == "/" || path == "/v1" {
path = "/v1/audio/speech"
} else {
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
if !strings.HasPrefix(path, "/v1/") {
path = "/v1" + strings.TrimSuffix(path, "/")
}
if !strings.HasSuffix(path, "/audio/speech") {
path = strings.TrimSuffix(path, "/") + "/audio/speech"
}
}
} else {
// For non-OpenAI hosts (e.g., proxies), preserve the existing base
// path and only ensure it ends with /audio/speech.
if !strings.HasSuffix(path, "/audio/speech") {
path = strings.TrimSuffix(path, "/") + "/audio/speech"
}
}
u.Path = path
apiBase = u.String()
} else {
// Fallback to the previous string-based behavior if parsing fails.
if apiBase == "https://api.openai.com/v1" {
apiBase = "https://api.openai.com/v1/audio/speech"
} else if !strings.HasSuffix(apiBase, "/audio/speech") {
// Just in case they provide openrouter base or standard base
apiBase = strings.TrimSuffix(apiBase, "/") + "/audio/speech"
}
}
}
client := common.NewHTTPClient(proxyURL)
client.Timeout = 60 * time.Second
model = strings.TrimSpace(model)
if model == "" {
model = "tts-1"
}
return &OpenAITTSProvider{
apiKey: apiKey,
apiBase: apiBase,
voice: "alloy",
model: model,
httpClient: client,
}
}
func (t *OpenAITTSProvider) Name() string {
return "openai-tts"
}
func (t *OpenAITTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) {
logger.DebugCF("voice-tts", "Starting TTS synthesis", map[string]any{"text_len": len(text)})
reqBody := map[string]any{
"model": t.model,
"input": text,
"voice": t.voice,
"response_format": "opus",
}
jsonData, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", t.apiBase, bytes.NewReader(jsonData))
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+t.apiKey)
resp, err := t.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
if resp.StatusCode != http.StatusOK {
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
}
return resp.Body, nil
}

151
pkg/audio/tts/tts.go Normal file
View file

@ -0,0 +1,151 @@
package tts
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/media"
"github.com/sipeed/picoclaw/pkg/providers"
)
type TTSProvider interface {
Name() string
Synthesize(ctx context.Context, text string) (io.ReadCloser, error)
}
func providerFromModelConfig(mc *config.ModelConfig) TTSProvider {
if mc == nil || mc.APIKey() == "" {
return nil
}
protocol, modelID := providers.ExtractProtocol(mc.Model)
if modelID == "" {
modelID = strings.TrimSpace(mc.Model)
}
switch protocol {
case "mimo":
return NewMimoTTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), modelID, mc.Proxy)
default:
return NewOpenAITTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), mc.Proxy, modelID)
}
}
func DetectTTS(cfg *config.Config) TTSProvider {
if cfg == nil {
return nil
}
if modelName := strings.TrimSpace(cfg.Voice.TTSModelName); modelName != "" {
if mc, err := cfg.GetModelConfig(modelName); err == nil {
if provider := providerFromModelConfig(mc); provider != nil {
return provider
}
}
}
for _, mc := range cfg.ModelList {
if strings.Contains(strings.ToLower(mc.Model), "tts") && mc.APIKey() != "" {
if provider := providerFromModelConfig(mc); provider != nil {
return provider
}
}
}
return nil
}
// SynthesizeAndStore synthesizes text to speech and registers it in the media store, returning the media reference.
func SynthesizeAndStore(
ctx context.Context,
provider TTSProvider,
store media.MediaStore,
text string,
filename string,
channel string,
chatID string,
) (string, error) {
if provider == nil {
return "", fmt.Errorf("tts provider is not configured")
}
if store == nil {
return "", fmt.Errorf("media store not configured")
}
if channel == "" || chatID == "" {
return "", fmt.Errorf("no target channel/chat available")
}
if strings.TrimSpace(text) == "" {
return "", fmt.Errorf("text is required")
}
stream, err := provider.Synthesize(ctx, text)
if err != nil {
return "", fmt.Errorf("tts synthesize failed: %w", err)
}
defer stream.Close()
err = os.MkdirAll(media.TempDir(), 0o700)
if err != nil {
return "", fmt.Errorf("failed to create media temp dir: %w", err)
}
fileExt := ".ogg"
contentType := "audio/ogg"
if provider.Name() == "mimo-tts" {
fileExt = ".mp3"
contentType = "audio/mpeg"
}
file, err := os.CreateTemp(media.TempDir(), "tts-*"+fileExt)
if err != nil {
return "", fmt.Errorf("failed to create temp file: %w", err)
}
removeTemp := true
defer func() {
if removeTemp {
_ = os.Remove(file.Name())
}
}()
_, err = io.Copy(file, stream)
if err != nil {
file.Close()
return "", fmt.Errorf("failed to write tts audio: %w", err)
}
err = file.Close()
if err != nil {
return "", fmt.Errorf("failed to close tts audio file: %w", err)
}
filename = strings.TrimSpace(filename)
if filename == "" {
filename = fmt.Sprintf("tts-%d%s", time.Now().Unix(), fileExt)
}
ext := strings.ToLower(filepath.Ext(filename))
if ext == "" {
filename += fileExt
} else if ext != fileExt {
filename = strings.TrimSuffix(filename, filepath.Ext(filename)) + fileExt
}
scope := fmt.Sprintf("tool:send_tts:%s:%s:%d", channel, chatID, time.Now().UnixNano())
ref, err := store.Store(file.Name(), media.MediaMeta{
Filename: filename,
ContentType: contentType,
Source: "tool:send_tts",
}, scope)
if err != nil {
return "", fmt.Errorf("failed to register audio: %w", err)
}
removeTemp = false
return ref, nil
}

247
pkg/audio/tts/tts_test.go Normal file
View file

@ -0,0 +1,247 @@
package tts
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/media"
)
func TestNewOpenAITTSProvider_APIBaseNormalization(t *testing.T) {
t.Parallel()
cases := []struct {
name string
input string
expect string
}{
{
name: "empty base",
input: "",
expect: "https://api.openai.com/v1/audio/speech",
},
{
name: "official host no path",
input: "https://api.openai.com",
expect: "https://api.openai.com/v1/audio/speech",
},
{
name: "official host v1",
input: "https://api.openai.com/v1",
expect: "https://api.openai.com/v1/audio/speech",
},
{
name: "official host v1 slash",
input: "https://api.openai.com/v1/",
expect: "https://api.openai.com/v1/audio/speech",
},
{
name: "non-openai host preserves base path",
input: "https://proxy.example.com/base",
expect: "https://proxy.example.com/base/audio/speech",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
provider := NewOpenAITTSProvider("key", tc.input, "", "")
if provider.apiBase != tc.expect {
t.Fatalf("apiBase mismatch: got %q, want %q", provider.apiBase, tc.expect)
}
})
}
}
func TestOpenAITTSProvider_SynthesizeSuccess(t *testing.T) {
t.Parallel()
var gotPath string
var gotAuth string
var gotContentType string
var gotBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
gotAuth = r.Header.Get("Authorization")
gotContentType = r.Header.Get("Content-Type")
bodyBytes, _ := io.ReadAll(r.Body)
_ = r.Body.Close()
_ = json.Unmarshal(bodyBytes, &gotBody)
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("audio-bytes"))
}))
defer server.Close()
provider := NewOpenAITTSProvider("k123", server.URL, "", "")
stream, err := provider.Synthesize(context.Background(), "hello")
if err != nil {
t.Fatalf("Synthesize failed: %v", err)
}
defer stream.Close()
data, err := io.ReadAll(stream)
if err != nil {
t.Fatalf("read stream failed: %v", err)
}
if gotPath != "/audio/speech" {
t.Fatalf("request path mismatch: got %q", gotPath)
}
if gotAuth != "Bearer k123" {
t.Fatalf("authorization mismatch: got %q", gotAuth)
}
if gotContentType != "application/json" {
t.Fatalf("content-type mismatch: got %q", gotContentType)
}
if gotBody["model"] != "tts-1" || gotBody["voice"] != "alloy" || gotBody["response_format"] != "opus" ||
gotBody["input"] != "hello" {
bodyJSON, _ := json.Marshal(gotBody)
t.Fatalf("request body mismatch: %s", string(bodyJSON))
}
if string(data) != "audio-bytes" {
t.Fatalf("response body mismatch: got %q", string(data))
}
}
func TestOpenAITTSProvider_SynthesizeNon200(t *testing.T) {
t.Parallel()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte("nope"))
}))
defer server.Close()
provider := NewOpenAITTSProvider("k123", server.URL, "", "")
_, err := provider.Synthesize(context.Background(), "hello")
if err == nil {
t.Fatal("expected error")
}
if !strings.Contains(err.Error(), "API error (status 500): nope") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNewOpenAITTSProvider_UsesConfiguredModel(t *testing.T) {
t.Parallel()
provider := NewOpenAITTSProvider("key", "https://api.xiaomimimo.com/v1", "", "mimo-v2-tts")
if provider.model != "mimo-v2-tts" {
t.Fatalf("model mismatch: got %q, want %q", provider.model, "mimo-v2-tts")
}
if provider.apiBase != "https://api.xiaomimimo.com/v1/audio/speech" {
t.Fatalf("apiBase mismatch: got %q", provider.apiBase)
}
}
func TestDetectTTS_UsesMimoProviderForMimoModels(t *testing.T) {
t.Parallel()
provider := DetectTTS(&config.Config{
Voice: config.VoiceConfig{TTSModelName: "mimo-tts"},
ModelList: []*config.ModelConfig{
{
ModelName: "mimo-tts",
Model: "mimo/mimo-v2-tts",
APIKeys: config.SimpleSecureStrings("sk-mimo"),
},
},
})
ttsProvider, ok := provider.(*MimoTTSProvider)
if !ok {
t.Fatalf("DetectTTS() type = %T, want *MimoTTSProvider", provider)
}
if ttsProvider.model != "mimo-v2-tts" {
t.Fatalf("model mismatch: got %q, want %q", ttsProvider.model, "mimo-v2-tts")
}
if ttsProvider.apiBase != "https://api.xiaomimimo.com/v1/chat/completions" {
t.Fatalf("apiBase mismatch: got %q", ttsProvider.apiBase)
}
}
type stubTTSProvider struct {
name string
}
func (s stubTTSProvider) Name() string {
return s.name
}
func (s stubTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("audio")), nil
}
func TestSynthesizeAndStore_UsesOggMetadataByDefault(t *testing.T) {
t.Parallel()
store := media.NewFileMediaStore()
ref, err := SynthesizeAndStore(
context.Background(),
stubTTSProvider{name: "openai-tts"},
store,
"hello",
"",
"discord",
"chat123",
)
if err != nil {
t.Fatalf("SynthesizeAndStore failed: %v", err)
}
path, meta, err := store.ResolveWithMeta(ref)
if err != nil {
t.Fatalf("ResolveWithMeta failed: %v", err)
}
if meta.ContentType != "audio/ogg" {
t.Fatalf("ContentType = %q, want %q", meta.ContentType, "audio/ogg")
}
if filepath.Ext(path) != ".ogg" {
t.Fatalf("stored file extension = %q, want %q", filepath.Ext(path), ".ogg")
}
if filepath.Ext(meta.Filename) != ".ogg" {
t.Fatalf("filename extension = %q, want %q", filepath.Ext(meta.Filename), ".ogg")
}
}
func TestSynthesizeAndStore_UsesMp3MetadataForMimo(t *testing.T) {
t.Parallel()
store := media.NewFileMediaStore()
ref, err := SynthesizeAndStore(
context.Background(),
stubTTSProvider{name: "mimo-tts"},
store,
"hello",
"",
"discord",
"chat123",
)
if err != nil {
t.Fatalf("SynthesizeAndStore failed: %v", err)
}
path, meta, err := store.ResolveWithMeta(ref)
if err != nil {
t.Fatalf("ResolveWithMeta failed: %v", err)
}
if meta.ContentType != "audio/mpeg" {
t.Fatalf("ContentType = %q, want %q", meta.ContentType, "audio/mpeg")
}
if filepath.Ext(path) != ".mp3" {
t.Fatalf("stored file extension = %q, want %q", filepath.Ext(path), ".mp3")
}
if filepath.Ext(meta.Filename) != ".mp3" {
t.Fatalf("filename extension = %q, want %q", filepath.Ext(meta.Filename), ".mp3")
}
}

View file

@ -3,6 +3,7 @@ package discord
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
@ -14,6 +15,8 @@ import (
"github.com/bwmarrin/discordgo"
"github.com/gorilla/websocket"
"github.com/sipeed/picoclaw/pkg/audio"
"github.com/sipeed/picoclaw/pkg/audio/tts"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/channels"
"github.com/sipeed/picoclaw/pkg/config"
@ -42,6 +45,15 @@ type DiscordChannel struct {
typingMu sync.Mutex
typingStop map[string]chan struct{} // chatID → stop signal
botUserID string // stored for mention checking
bus *bus.MessageBus
tts tts.TTSProvider
voiceMu sync.RWMutex
voiceSSRC map[string]map[uint32]string // guildID -> ssrc -> userID
// TTS interruption: cancel active playback when user speaks
ttsMu sync.Mutex
cancelTTS context.CancelFunc
ttsPlayID uint64
}
func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) {
@ -73,6 +85,8 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC
config: cfg,
ctx: context.Background(),
typingStop: make(map[string]chan struct{}),
bus: bus,
voiceSSRC: make(map[string]map[uint32]string),
}, nil
}
@ -90,6 +104,8 @@ func (c *DiscordChannel) Start(ctx context.Context) error {
c.session.AddHandler(c.handleMessage)
go c.listenVoiceControl(c.ctx)
if err := c.session.Open(); err != nil {
return fmt.Errorf("failed to open discord session: %w", err)
}
@ -142,6 +158,25 @@ func (c *DiscordChannel) Send(ctx context.Context, msg bus.OutboundMessage) ([]s
return nil, nil
}
if c.tts != nil {
if ch, err := c.session.State.Channel(channelID); err == nil && ch.GuildID != "" {
if vc, ok := c.session.VoiceConnections[ch.GuildID]; ok && vc != nil {
// Cancel any previous TTS playback
c.ttsMu.Lock()
if c.cancelTTS != nil {
c.cancelTTS()
}
ttsCtx, ttsCancel := context.WithCancel(c.ctx)
c.ttsPlayID++
playID := c.ttsPlayID
c.cancelTTS = ttsCancel
c.ttsMu.Unlock()
go c.playTTS(ttsCtx, vc, msg.Content, playID)
}
}
}
msgID, err := c.sendChunk(ctx, channelID, msg.Content, msg.ReplyToMessageID)
if err != nil {
return nil, err
@ -359,6 +394,10 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag
return
}
if c.handleVoiceCommand(s, m) {
return
}
content := m.Content
// In guild (group) channels, apply unified group trigger filtering
@ -642,3 +681,134 @@ func (c *DiscordChannel) stripBotMention(text string) string {
text = strings.ReplaceAll(text, fmt.Sprintf("<@!%s>", c.botUserID), "")
return strings.TrimSpace(text)
}
func (c *DiscordChannel) listenVoiceControl(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case ctrl, ok := <-c.bus.VoiceControlsChan():
if !ok {
return
}
if ctrl.Type == "command" && ctrl.Action == "leave" {
if strings.HasPrefix(ctrl.SessionID, "discord_vc_") {
guildID := strings.TrimPrefix(ctrl.SessionID, "discord_vc_")
vc, exists := c.session.VoiceConnections[guildID]
if exists && vc != nil {
vc.Disconnect(ctx)
}
}
}
}
}
}
func (c *DiscordChannel) playTTS(ctx context.Context, vc *discordgo.VoiceConnection, text string, playID uint64) {
// Capture the cancel func associated with this playback (if any).
// Clear cancelTTS when playback finishes (normal or interrupted),
// but only if it still refers to this playback's cancel func.
defer func() {
c.ttsMu.Lock()
if c.ttsPlayID == playID {
c.cancelTTS = nil
}
c.ttsMu.Unlock()
}()
sentences := audio.SplitSentences(text)
if len(sentences) == 0 {
return
}
logger.InfoCF("discord", "Starting streamed TTS", map[string]any{"sentences": len(sentences)})
// Pipeline: prefetch next sentence's audio while playing current
type ttResult struct {
stream io.ReadCloser
err error
}
var prefetch chan ttResult
// Ensure any in-flight prefetch is drained on exit to prevent stream leaks,
// but avoid blocking indefinitely if the prefetch goroutine is stuck or never sends.
defer func() {
if prefetch != nil {
select {
case result := <-prefetch:
if result.stream != nil {
result.stream.Close()
}
case <-time.After(100 * time.Millisecond):
// Timed out waiting for a prefetched result; avoid blocking on exit.
}
}
}()
for i, sentence := range sentences {
// Check for cancellation (interruption)
select {
case <-ctx.Done():
logger.InfoCF("discord", "TTS interrupted", map[string]any{"at_sentence": i})
return
default:
}
// Start prefetching the NEXT sentence while we process the current one
var nextPrefetch chan ttResult
if i+1 < len(sentences) {
nextPrefetch = make(chan ttResult, 1)
nextSentence := sentences[i+1]
go func() {
s, e := c.tts.Synthesize(ctx, nextSentence)
nextPrefetch <- ttResult{s, e}
}()
}
// Get the current sentence's audio
var stream io.ReadCloser
var err error
if prefetch != nil {
// Use prefetched result from previous iteration, but be responsive to cancellation.
var result ttResult
select {
case result = <-prefetch:
stream, err = result.stream, result.err
case <-ctx.Done():
// Context canceled while waiting for prefetched audio; abort playback.
logger.InfoCF(
"discord",
"TTS interrupted while waiting for prefetched audio",
map[string]any{"at_sentence": i},
)
return
}
} else {
// First sentence: synthesize directly
stream, err = c.tts.Synthesize(ctx, sentence)
}
if err != nil {
if stream != nil {
stream.Close()
}
logger.ErrorCF("discord", "TTS synthesize failed", map[string]any{"error": err.Error(), "sentence": i})
prefetch = nextPrefetch
continue
}
if err := streamOggOpusToDiscord(ctx, vc, stream); err != nil {
logger.ErrorCF("discord", "TTS playback failed", map[string]any{"error": err.Error(), "sentence": i})
}
stream.Close()
prefetch = nextPrefetch
}
}
// VoiceCapabilities returns the voice capabilities of the channel.
func (c *DiscordChannel) VoiceCapabilities() channels.VoiceCapabilities {
return channels.VoiceCapabilities{ASR: true, TTS: true}
}

View file

@ -1,6 +1,7 @@
package discord
import (
"github.com/sipeed/picoclaw/pkg/audio/tts"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/channels"
"github.com/sipeed/picoclaw/pkg/config"
@ -8,6 +9,10 @@ import (
func init() {
channels.RegisterFactory("discord", func(cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) {
return NewDiscordChannel(cfg.Channels.Discord, b)
ch, err := NewDiscordChannel(cfg.Channels.Discord, b)
if err == nil {
ch.tts = tts.DetectTTS(cfg)
}
return ch, err
})
}

View file

@ -0,0 +1,314 @@
package discord
import (
"context"
"fmt"
"io"
"time"
"github.com/bwmarrin/discordgo"
"github.com/sipeed/picoclaw/pkg/audio"
"github.com/sipeed/picoclaw/pkg/bus"
"github.com/sipeed/picoclaw/pkg/identity"
"github.com/sipeed/picoclaw/pkg/logger"
)
func (c *DiscordChannel) setVoiceUserID(guildID string, ssrc uint32, userID string) {
if userID == "" {
return
}
c.voiceMu.Lock()
defer c.voiceMu.Unlock()
ssrcMap, ok := c.voiceSSRC[guildID]
if !ok {
ssrcMap = make(map[uint32]string)
c.voiceSSRC[guildID] = ssrcMap
}
ssrcMap[ssrc] = userID
}
func (c *DiscordChannel) voiceUserID(guildID string, ssrc uint32) string {
c.voiceMu.RLock()
defer c.voiceMu.RUnlock()
ssrcMap, ok := c.voiceSSRC[guildID]
if !ok {
return ""
}
return ssrcMap[ssrc]
}
func (c *DiscordChannel) handleVoiceCommand(s *discordgo.Session, m *discordgo.MessageCreate) bool {
if m.Content == "!vc join" {
vs, err := s.State.VoiceState(m.GuildID, m.Author.ID)
if err != nil || vs == nil {
if _, sendErr := s.ChannelMessageSend(
m.ChannelID,
"You need to be in a voice channel first!",
); sendErr != nil {
logger.InfoCF("discord", "Failed to send voice channel requirement message", map[string]any{
"channel": m.ChannelID,
"error": sendErr,
})
}
return true
}
logger.InfoCF("discord", "Joining voice channel", map[string]any{"channel": vs.ChannelID})
vc, err := s.ChannelVoiceJoin(c.ctx, m.GuildID, vs.ChannelID, false, false)
if err != nil {
if _, sendErr := s.ChannelMessageSend(
m.ChannelID,
fmt.Sprintf("Failed to join voice channel: %v", err),
); sendErr != nil {
logger.InfoCF("discord", "Failed to send voice join error message", map[string]any{
"channel": m.ChannelID,
"error": sendErr,
})
}
return true
}
go c.receiveVoice(vc, m.GuildID, m.ChannelID)
if _, sendErr := s.ChannelMessageSend(
m.ChannelID,
"Joined Voice Channel! Listening for audio...",
); sendErr != nil {
logger.InfoCF("discord", "Failed to send voice join success message", map[string]any{
"channel": m.ChannelID,
"error": sendErr,
})
}
return true
} else if m.Content == "!vc leave" {
vc, exists := s.VoiceConnections[m.GuildID]
if exists && vc != nil {
if err := vc.Disconnect(c.ctx); err != nil {
logger.InfoCF("discord", "Failed to disconnect from voice channel", map[string]any{
"guild": m.GuildID,
"error": err,
})
}
if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Left Voice Channel."); sendErr != nil {
logger.InfoCF("discord", "Failed to send voice leave success message", map[string]any{
"channel": m.ChannelID,
"error": sendErr,
})
}
} else {
if _, sendErr := s.ChannelMessageSend(m.ChannelID, "Not in a voice channel."); sendErr != nil {
logger.InfoCF("discord", "Failed to send voice not-in-channel message", map[string]any{
"channel": m.ChannelID,
"error": sendErr,
})
}
}
return true
}
return false
}
func VoiceReceiveActive(vc *discordgo.VoiceConnection) bool {
return vc != nil && vc.OpusRecv != nil
}
func streamOggOpusToDiscord(ctx context.Context, vc *discordgo.VoiceConnection, r io.Reader) (retErr error) {
// Recover from panic if vc.OpusSend is closed mid-send (e.g. on disconnect)
defer func() {
if rec := recover(); rec != nil {
retErr = fmt.Errorf("voice connection closed during playback")
logger.RecoverPanicNoExit(rec)
}
}()
// Wait for the speaking transition to register
vc.Speaking(true)
defer vc.Speaking(false)
return audio.DecodeOggOpus(r, func(frame []byte) error {
select {
case <-ctx.Done():
return ctx.Err()
case vc.OpusSend <- frame:
return nil
}
})
}
func (c *DiscordChannel) receiveVoice(vc *discordgo.VoiceConnection, guildID string, chatID string) {
logger.InfoCF("discord", "Started listening for voice", map[string]any{"guild": guildID})
vc.AddHandler(func(_ *discordgo.VoiceConnection, vs *discordgo.VoiceSpeakingUpdate) {
if vs == nil {
return
}
c.setVoiceUserID(guildID, uint32(vs.SSRC), vs.UserID)
})
defer func() {
c.voiceMu.Lock()
delete(c.voiceSSRC, guildID)
c.voiceMu.Unlock()
}()
go func(ctx context.Context, vc *discordgo.VoiceConnection) {
// Recover from potential panics if OpusSend is closed mid-send.
defer func() {
if rec := recover(); rec != nil {
logger.WarnCF("discord", "Recovered from panic while sending wake-up frames", map[string]any{
"error": rec,
"guild": guildID,
})
}
}()
// If the voice connection or OpusSend are not available, nothing to do.
if vc == nil || vc.OpusSend == nil {
return
}
time.Sleep(250 * time.Millisecond) // Wait a bit for connection to settle
// Abort if the context has already been canceled.
select {
case <-ctx.Done():
return
default:
}
vc.Speaking(true)
defer vc.Speaking(false)
silenceFrame := []byte{0xF8, 0xFF, 0xFE}
for i := 0; i < 5; i++ {
select {
case <-ctx.Done():
return
case vc.OpusSend <- silenceFrame:
}
time.Sleep(20 * time.Millisecond)
}
logger.DebugCF("discord", "Sent wake-up silence frames", map[string]any{"guild": guildID})
}(c.ctx, vc)
sessionID := fmt.Sprintf("discord_vc_%s", guildID)
c.bus.PublishVoiceControl(c.ctx, bus.VoiceControl{
SessionID: sessionID,
Type: "state",
Action: "listening",
})
var sequence uint64 = 0
var interruptCount int
var lastInterruptAt time.Time
for {
select {
case <-c.ctx.Done():
return
case p, ok := <-vc.OpusRecv:
if !ok {
logger.InfoCF("discord", "Voice channel closed", map[string]any{"guild": guildID})
// Cancel any TTS that may still be playing
c.ttsMu.Lock()
if c.cancelTTS != nil {
c.cancelTTS()
c.cancelTTS = nil
}
c.ttsMu.Unlock()
return
}
if p == nil {
logger.DebugCF("discord", "Received nil Opus packet", nil)
continue
}
if len(p.Opus) == 0 {
logger.DebugCF("discord", "Received empty Opus packet", map[string]any{
"seq": p.Sequence,
"ssrc": p.SSRC,
})
continue
}
logger.DebugCF("discord", "Received Opus packet", map[string]any{
"seq": p.Sequence,
"len": len(p.Opus),
"ssrc": p.SSRC,
})
// Interruption detection: if user sends voice while TTS is playing,
// cancel TTS after a short debounce (3 packets in 200ms)
now := time.Now()
if now.Sub(lastInterruptAt) > 500*time.Millisecond {
interruptCount = 0
}
interruptCount++
lastInterruptAt = now
if interruptCount >= 3 {
c.ttsMu.Lock()
if c.cancelTTS != nil {
c.cancelTTS()
c.cancelTTS = nil
logger.InfoCF("discord", "TTS interrupted by user voice", nil)
}
c.ttsMu.Unlock()
interruptCount = 0
}
userID := c.voiceUserID(guildID, p.SSRC)
if userID == "" {
logger.DebugCF("discord", "Dropping voice packet without user mapping", map[string]any{
"ssrc": p.SSRC,
"guild": guildID,
})
continue
}
sender := bus.SenderInfo{
Platform: "discord",
PlatformID: userID,
CanonicalID: identity.BuildCanonicalID("discord", userID),
}
if !c.IsAllowedSender(sender) {
logger.DebugCF("discord", "Voice packet rejected by allowlist", map[string]any{
"user_id": userID,
"guild": guildID,
})
continue
}
sequence++
chunk := bus.AudioChunk{
SessionID: sessionID,
SpeakerID: userID,
ChatID: chatID,
Channel: "discord",
Sequence: sequence,
Timestamp: p.Timestamp,
SampleRate: 48000,
Channels: 2,
Format: "opus",
Data: p.Opus,
}
ctx, cancel := context.WithTimeout(c.ctx, 100*time.Millisecond)
err := c.bus.PublishAudioChunk(ctx, chunk)
cancel()
if err != nil {
logger.ErrorCF("discord", "Failed to publish audio chunk", map[string]any{
"guild": guildID,
"sessionID": sessionID,
"sequence": sequence,
"error": err.Error(),
})
}
}
}
}

View file

@ -6,6 +6,8 @@ import (
"strings"
larkim "github.com/larksuite/oapi-sdk-go/v3/service/im/v1"
"github.com/sipeed/picoclaw/pkg/channels"
)
// mentionPlaceholderRegex matches @_user_N placeholders inserted by Feishu for mentions.
@ -145,3 +147,8 @@ func extractImageKeysRecursive(v any, feishuKeys, externalURLs *[]string) {
}
}
}
// VoiceCapabilities returns the voice capabilities of the channel.
func (c *FeishuChannel) VoiceCapabilities() channels.VoiceCapabilities {
return channels.VoiceCapabilities{ASR: true, TTS: true}
}

View file

@ -696,3 +696,8 @@ func (c *LINEChannel) downloadContent(messageID, filename string) string {
},
})
}
// VoiceCapabilities returns the voice capabilities of the channel.
func (c *LINEChannel) VoiceCapabilities() channels.VoiceCapabilities {
return channels.VoiceCapabilities{ASR: true, TTS: true}
}

View file

@ -444,6 +444,23 @@ func (m *Manager) initChannels(channels *config.ChannelsConfig) error {
m.initChannel("irc", "IRC")
}
if channels.VK.Enabled && channels.VK.Token.String() != "" && channels.VK.GroupID != 0 {
m.initChannel("vk", "VK")
}
if channels.TeamsWebhook.Enabled && len(channels.TeamsWebhook.Webhooks) > 0 {
hasValidTarget := false
for _, target := range channels.TeamsWebhook.Webhooks {
if target.WebhookURL.String() != "" {
hasValidTarget = true
break
}
}
if hasValidTarget {
m.initChannel("teams_webhook", "Teams Webhook")
}
}
logger.InfoCF("channels", "Channel initialization completed", map[string]any{
"enabled_channels": len(m.channels),
})

Some files were not shown because too many files have changed in this diff Show more