diff --git a/README.md b/README.md index 30ac67d8f..2fa71230d 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,14 @@ ## 📢 News +2026-05-11 🛒 **LicheeRV-Claw on AliExpress!** You can now purchase LicheeRV-Claw from [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), making it easier to try PicoClaw on compact RISC-V hardware. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Android Support!** PicoClaw now runs on Android! Download the APK at [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 Released!** Agent architecture overhaul (SubTurn, Hooks, Steering, EventBus), WeChat/WeCom integration, security hardening (.security.yml, sensitive data filtering), new providers (AWS Bedrock, Azure, Xiaomi MiMo), and 35 bug fixes. PicoClaw has reached **26K Stars**! @@ -447,7 +455,7 @@ For full provider configuration details, see [Providers & Models](docs/guides/pr ## 💬 Channels (Chat Apps) -Talk to your PicoClaw through 18+ messaging platforms: +Talk to your PicoClaw through 19+ messaging platforms: | Channel | Setup | Protocol | Docs | |---------|-------|----------|------| @@ -465,6 +473,7 @@ Talk to your PicoClaw through 18+ messaging platforms: | **VK** | Easy (group token) | Long Poll | [Guide](docs/channels/vk/README.md) | | **IRC** | Medium (server + nick) | IRC protocol | [Guide](docs/guides/chat-apps.md#irc) | | **OneBot** | Medium (WebSocket URL) | OneBot v11 | [Guide](docs/channels/onebot/README.md) | +| **MQTT** | Easy (broker + agent_id) | MQTT pub/sub | [Guide](docs/channels/mqtt/README.md) | | **MaixCam** | Easy (enable) | TCP socket | [Guide](docs/channels/maixcam/README.md) | | **Pico** | Easy (enable) | Native protocol | Built-in | | **Pico Client** | Easy (WebSocket URL) | WebSocket | Built-in | @@ -484,7 +493,8 @@ PicoClaw can search the web to provide up-to-date information. Configure in `too | Search Engine | API Key | Free Tier | Link | |--------------|---------|-----------|------| | DuckDuckGo | Not needed | Unlimited | Built-in fallback | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Required | 1000 queries/day | AI-powered, China-optimized | +| [Gemini Google Search](https://aistudio.google.com/apikey) | Required | Varies | Gemini with Google Search grounding | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Required | 1500/month (daily allocation) | AI-powered, China-optimized | | [Tavily](https://tavily.com) | Required | 1000 queries/month | Optimized for AI Agents | | [Brave Search](https://brave.com/search/api) | Required | 2000 queries/month | Fast and private | | [Perplexity](https://www.perplexity.ai) | Required | Paid | AI-powered search | @@ -617,7 +627,7 @@ For detailed guides beyond this README: | Topic | Description | |-------|-------------| | [Docker & Quick Start](docs/guides/docker.md) | Docker Compose setup, Launcher/Agent modes | -| [Chat Apps](docs/guides/chat-apps.md) | All 17+ channel setup guides | +| [Chat Apps](docs/guides/chat-apps.md) | All 18+ channel setup guides | | [Configuration](docs/guides/configuration.md) | Environment variables, workspace layout, security sandbox | | [MCP Server CLI](docs/reference/mcp-cli.md) | Add, list, test, edit, and remove MCP server entries from the CLI | | [Scheduled Tasks and Cron Jobs](docs/reference/cron.md) | Cron schedule types, deliver modes, command gates, job storage | diff --git a/assets/licheerv-claw.jpg b/assets/licheerv-claw.jpg new file mode 100644 index 000000000..afcf6b8d3 Binary files /dev/null and b/assets/licheerv-claw.jpg differ diff --git a/assets/wechat.png b/assets/wechat.png index b368f75d3..8fb8e0d8f 100644 Binary files a/assets/wechat.png and b/assets/wechat.png differ diff --git a/config/config.example.json b/config/config.example.json index 4205b8e8a..d87f711d0 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -11,6 +11,8 @@ "summarize_message_threshold": 20, "summarize_token_percent": 75, "split_on_marker": false, + "max_llm_retries": 2, + "llm_retry_backoff_secs": 2, "tool_feedback": { "enabled": false, "max_args_length": 300, @@ -18,6 +20,15 @@ } } }, + "evolution": { + "enabled": false, + "mode": "observe", + "state_dir": "", + "min_task_count": 2, + "min_success_ratio": 0.7, + "cold_path_trigger": "after_turn", + "cold_path_times": [] + }, "model_list": [ { "model_name": "gpt-5.4", @@ -41,6 +52,7 @@ }, { "model_name": "gemini", + "_comment": "Optional: set \"tool_schema_transform\": \"simple\" for providers that reject complex tool JSON Schema.", "model": "antigravity/gemini-2.0-flash", "auth_method": "oauth" }, @@ -279,6 +291,12 @@ "enabled": false, "max_results": 5 }, + "gemini": { + "enabled": false, + "api_key": "", + "model": "gemini-2.5-flash", + "max_results": 5 + }, "perplexity": { "enabled": false, "api_key": "pplx-xxx", @@ -479,6 +497,15 @@ "approval_timeout_ms": 60000 } }, + "events": { + "logging": { + "enabled": true, + "include": ["agent.*"], + "exclude": [], + "min_severity": "info", + "include_payload": false + } + }, "gateway": { "_comment": "Default log level is set to 'fatal'. Other available options are 'debug', 'info', 'warn' and 'error'.", "host": "localhost", diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 6df7447a7..17e144ebd 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -6,6 +6,8 @@ Internal architecture notes for major runtime mechanisms and subsystem design. - [SubTurn Mechanism](subturn.md): sub-agent coordination, concurrency control, and lifecycle handling. - [Session System](session-system.md): session scope allocation, JSONL persistence, alias compatibility, and migration. ([ZH](session-system.zh.md)) - [Routing System](routing-system.md): agent dispatch, session policy selection, and light/heavy model routing. ([ZH](routing-system.zh.md)) +- [Runtime Events](runtime-events.md): runtime event envelope, centralized event logging, filters, and examples. ([ZH](runtime-events.zh.md)) +- [Agent Self-Evolution](agent-self-evolution.md): learning records, draft generation, application modes, and state layout. - [Hook System Guide](hooks/README.md): current hook architecture and protocol details. - [Agent Refactor](agent-refactor/README.md): notes and checkpoints for the agent refactor work. diff --git a/docs/architecture/agent-self-evolution.md b/docs/architecture/agent-self-evolution.md new file mode 100644 index 000000000..40e040fa0 --- /dev/null +++ b/docs/architecture/agent-self-evolution.md @@ -0,0 +1,47 @@ +# Agent Self-Evolution + +Agent self-evolution lets PicoClaw learn from completed turns and turn repeated successful behavior into skill improvements. The runtime is controlled by the top-level `evolution` config block. + +## Flow + +The hot path runs at the end of an agent turn. When `evolution.enabled` is true, it records a learning record with the turn summary, success state, used skills, tool executions, and session/workspace metadata. Heartbeat turns are skipped. + +The cold path groups related task records, checks the configured success threshold, and prepares skill drafts for patterns that have enough evidence. Drafts can target new skills or append/replace/merge existing workspace skills. + +The apply path validates generated `SKILL.md` content before writing. Invalid drafts are rejected before a skill directory or file is created. + +## Safety Considerations + +Evolution creates a persistent feedback loop: user input can become a task record, task records can be clustered into an LLM-generated draft, and an accepted draft can become `SKILL.md` content that is loaded into future agent prompts. Treat generated skill content as prompt-sensitive material, especially in `apply` mode. + +The current local scanner is a narrow guardrail, not a complete safety boundary. It rejects structurally invalid drafts and a small set of obvious secret-like substrings, but it does not reliably detect prompt injection, unsafe instructions, or every form of sensitive data. Use `observe` or `draft` when human review is required before skill changes reach disk. + +In `apply` mode, accepted drafts can update workspace skills automatically. Existing skills are backed up before replacement, but recovery is manual: an operator must restore the desired backup if an applied skill should be rolled back. + +## Modes + +| Mode | Behavior | +|------|----------| +| `observe` | Record learning data only. No cold-path draft generation runs automatically. | +| `draft` | Record learning data and generate candidate skill drafts when the cold path runs. | +| `apply` | Generate drafts and allow accepted drafts to update workspace skills. | + +When `evolution.enabled` is false, `mode` is treated as disabled at runtime. + +## Cold Path Trigger + +`cold_path_trigger` only matters in `draft` and `apply` modes. + +| Trigger | Behavior | +|---------|----------| +| `after_turn` | Run the cold path after eligible turns. | +| `scheduled` | Run the cold path at configured `cold_path_times`. | +| `manual` | Do not run automatically. There is no user-facing Web/API/CLI trigger yet; code can still invoke `Runtime.RunColdPathOnce`. | + +`cold_path_times` uses `HH:MM` strings and is ignored unless the trigger is `scheduled`. + +## State + +By default, evolution state is stored under the workspace. `state_dir` can redirect that state to another directory. The state includes learning records, clustered pattern records, drafts, and skill profiles. + +For user-facing configuration fields, see the [Configuration Guide](../guides/configuration.md#agent-self-evolution). diff --git a/docs/architecture/hooks/README.md b/docs/architecture/hooks/README.md index 5be0f30b5..06f1a2c07 100644 --- a/docs/architecture/hooks/README.md +++ b/docs/architecture/hooks/README.md @@ -13,7 +13,7 @@ The repository no longer ships standalone example source files. The Go and Pytho | Type | Interface | Stage | Can modify data | | --- | --- | --- | --- | -| Observer | `EventObserver` | EventBus broadcast | No | +| Observer | `RuntimeEventObserver` | Runtime event bus broadcast | No | | LLM interceptor | `LLMInterceptor` | `before_llm` / `after_llm` | Yes | | Tool interceptor | `ToolInterceptor` | `before_tool` / `after_tool` | Yes | | Tool approver | `ToolApprover` | `approve_tool` | No, returns allow/deny | @@ -136,9 +136,9 @@ Example: "/tmp/review_gate.py" ], "observe": [ - "tool_exec_start", - "tool_exec_end", - "tool_exec_skipped" + "agent.tool.exec_start", + "agent.tool.exec_end", + "agent.tool.exec_skipped" ], "intercept": [ "before_tool", @@ -174,7 +174,7 @@ Both examples are intentionally safe: they only log, never rewrite, and never de The following is a minimal logging hook for in-process use. It implements: -1. `EventObserver` +1. `RuntimeEventObserver` 2. `LLMInterceptor` 3. `ToolInterceptor` 4. `ToolApprover` @@ -196,6 +196,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/agent" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -217,12 +218,12 @@ func NewExampleLoggerHook(opts ExampleLoggerHookOptions) *ExampleLoggerHook { } } -func (h *ExampleLoggerHook) OnEvent(ctx context.Context, evt agent.Event) error { +func (h *ExampleLoggerHook) OnRuntimeEvent(ctx context.Context, evt runtimeevents.Event) error { _ = ctx if h == nil || !h.logEvents { return nil } - h.record("event", evt.Meta, map[string]any{ + h.record("event", evt.Scope, map[string]any{ "event": evt.Kind.String(), "payload": evt.Payload, }, nil) @@ -275,7 +276,7 @@ func (h *ExampleLoggerHook) ApproveTool( return decision, nil } -func (h *ExampleLoggerHook) record(stage string, meta agent.EventMeta, payload any, decision any) { +func (h *ExampleLoggerHook) record(stage string, refs any, payload any, decision any) { logger.InfoCF("hooks", "Example hook observed", map[string]any{ "stage": stage, }) @@ -286,7 +287,7 @@ func (h *ExampleLoggerHook) record(stage string, meta agent.EventMeta, payload a entry := map[string]any{ "ts": time.Now().UTC(), "stage": stage, - "meta": meta, + "refs": refs, "payload": payload, "decision": decision, } @@ -428,7 +429,7 @@ If you only see `before_llm` and `after_llm`, that usually means the request did The following script is a minimal process-hook example. It uses only the Python standard library and supports: 1. `hook.hello` -2. `hook.event` +2. `hook.runtime_event` 3. `hook.before_tool` 4. `hook.approve_tool` @@ -564,8 +565,8 @@ def main() -> int: }) if not message_id: - if method == "hook.event" and LOG_EVENTS: - log_stderr(f"observed event: {params.get('Kind')}") + if method == "hook.runtime_event" and LOG_EVENTS: + log_stderr(f"observed event: {params.get('kind')}") continue try: @@ -606,9 +607,9 @@ if __name__ == "__main__": "/abs/path/to/review_gate.py" ], "observe": [ - "tool_exec_start", - "tool_exec_end", - "tool_exec_skipped" + "agent.tool.exec_start", + "agent.tool.exec_end", + "agent.tool.exec_skipped" ], "intercept": [ "before_tool", @@ -626,7 +627,7 @@ if __name__ == "__main__": ### Environment Variables - `PICOCLAW_HOOK_LOG_EVENTS` - Whether to write `hook.event` summaries to `stderr`, enabled by default + Whether to write `hook.runtime_event` summaries to `stderr`, enabled by default - `PICOCLAW_HOOK_LOG_FILE` Path to an external log file. When set, the script appends inbound hook requests, notifications, and outbound responses as JSON Lines @@ -645,7 +646,7 @@ Typical interpretation: - Only `hook.hello` The process started and completed the handshake, but no business hook request has arrived yet -- `hook.event` +- `hook.runtime_event` The `observe` configuration is working - `hook.before_tool` The `intercept: ["before_tool", ...]` configuration is working @@ -664,7 +665,7 @@ A complete sample: ```json {"ts":"2026-03-21T14:12:00+00:00","direction":"in","id":1,"method":"hook.hello","params":{"name":"py_review_gate","version":1,"modes":["observe","tool","approve"]},"notification":false} {"ts":"2026-03-21T14:12:00+00:00","direction":"out","id":1,"response":{"ok":true,"name":"python-review-gate"},"error":null} -{"ts":"2026-03-21T14:12:05+00:00","direction":"in","id":0,"method":"hook.event","params":{"Kind":"tool_exec_start"},"notification":true} +{"ts":"2026-03-21T14:12:05+00:00","direction":"in","id":0,"method":"hook.runtime_event","params":{"kind":"agent.tool.exec_start"},"notification":true} {"ts":"2026-03-21T14:12:05+00:00","direction":"in","id":7,"method":"hook.before_tool","params":{"tool":"echo_text","arguments":{"text":"hello"}},"notification":false} {"ts":"2026-03-21T14:12:05+00:00","direction":"out","id":7,"response":{"action":"continue"},"error":null} ``` @@ -672,7 +673,7 @@ A complete sample: Additional notes: - Timestamps are UTC -- `notification=true` means it was a notification such as `hook.event`, which does not expect a response +- `notification=true` means it was a notification such as `hook.runtime_event`, which does not expect a response - `id` increases within a single hook process; if the process restarts, the counter starts over ## Process-Hook Protocol @@ -681,7 +682,7 @@ Current process hooks use `JSON-RPC over stdio`: - PicoClaw starts the external process - Requests and responses are exchanged as one JSON message per line -- `hook.event` is a notification and does not need a response +- `hook.runtime_event` is a notification and does not need a response - `hook.before_llm`, `hook.after_llm`, `hook.before_tool`, `hook.after_tool`, and `hook.approve_tool` are request/response calls The host does not currently accept new RPCs initiated by the process hook. In practice, that means an external hook can only respond to PicoClaw calls; it cannot call back into the host to send channel messages. diff --git a/docs/architecture/hooks/README.zh.md b/docs/architecture/hooks/README.zh.md index 2170d45c8..1fff40832 100644 --- a/docs/architecture/hooks/README.zh.md +++ b/docs/architecture/hooks/README.zh.md @@ -13,7 +13,7 @@ | 类型 | 接口 | 作用阶段 | 能否改写 | | --- | --- | --- | --- | -| 观察型 | `EventObserver` | EventBus 广播事件时 | 否 | +| 观察型 | `RuntimeEventObserver` | runtime event bus 广播事件时 | 否 | | LLM 拦截型 | `LLMInterceptor` | `before_llm` / `after_llm` | 是 | | Tool 拦截型 | `ToolInterceptor` | `before_tool` / `after_tool` | 是 | | Tool 审批型 | `ToolApprover` | `approve_tool` | 否，返回批准/拒绝 | @@ -136,9 +136,9 @@ HookManager 的排序规则是： "/tmp/review_gate.py" ], "observe": [ - "tool_exec_start", - "tool_exec_end", - "tool_exec_skipped" + "agent.tool.exec_start", + "agent.tool.exec_end", + "agent.tool.exec_skipped" ], "intercept": [ "before_tool", @@ -174,7 +174,7 @@ tail -f /tmp/picoclaw-hook-review-gate.log 下面这段代码是一个最小的“记录型” in-process hook。它实现了： -1. `EventObserver` +1. `RuntimeEventObserver` 2. `LLMInterceptor` 3. `ToolInterceptor` 4. `ToolApprover` @@ -196,6 +196,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/agent" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -217,12 +218,12 @@ func NewExampleLoggerHook(opts ExampleLoggerHookOptions) *ExampleLoggerHook { } } -func (h *ExampleLoggerHook) OnEvent(ctx context.Context, evt agent.Event) error { +func (h *ExampleLoggerHook) OnRuntimeEvent(ctx context.Context, evt runtimeevents.Event) error { _ = ctx if h == nil || !h.logEvents { return nil } - h.record("event", evt.Meta, map[string]any{ + h.record("event", evt.Scope, map[string]any{ "event": evt.Kind.String(), "payload": evt.Payload, }, nil) @@ -275,7 +276,7 @@ func (h *ExampleLoggerHook) ApproveTool( return decision, nil } -func (h *ExampleLoggerHook) record(stage string, meta agent.EventMeta, payload any, decision any) { +func (h *ExampleLoggerHook) record(stage string, refs any, payload any, decision any) { logger.InfoCF("hooks", "Example hook observed", map[string]any{ "stage": stage, }) @@ -286,7 +287,7 @@ func (h *ExampleLoggerHook) record(stage string, meta agent.EventMeta, payload a entry := map[string]any{ "ts": time.Now().UTC(), "stage": stage, - "meta": meta, + "refs": refs, "payload": payload, "decision": decision, } @@ -428,7 +429,7 @@ func init() { 下面这段脚本是一个最小的 `process hook` 示例。它只使用 Python 标准库，支持： 1. `hook.hello` -2. `hook.event` +2. `hook.runtime_event` 3. `hook.before_tool` 4. `hook.approve_tool` @@ -564,8 +565,8 @@ def main() -> int: }) if not message_id: - if method == "hook.event" and LOG_EVENTS: - log_stderr(f"observed event: {params.get('Kind')}") + if method == "hook.runtime_event" and LOG_EVENTS: + log_stderr(f"observed event: {params.get('kind')}") continue try: @@ -606,9 +607,9 @@ if __name__ == "__main__": "/abs/path/to/review_gate.py" ], "observe": [ - "tool_exec_start", - "tool_exec_end", - "tool_exec_skipped" + "agent.tool.exec_start", + "agent.tool.exec_end", + "agent.tool.exec_skipped" ], "intercept": [ "before_tool", @@ -626,7 +627,7 @@ if __name__ == "__main__": ### 环境变量 - `PICOCLAW_HOOK_LOG_EVENTS` - 是否把 `hook.event` 写到 `stderr`，默认开启 + 是否把 `hook.runtime_event` 写到 `stderr`，默认开启 - `PICOCLAW_HOOK_LOG_FILE` 外部日志文件路径。设置后，脚本会把收到的 hook 请求、notification 和返回结果按 JSON Lines 追加到该文件 @@ -645,7 +646,7 @@ if __name__ == "__main__": - 只看到 `hook.hello` 说明进程启动并完成握手了，但还没有新的业务 hook 请求真正打进来 -- 看到 `hook.event` +- 看到 `hook.runtime_event` 说明 `observe` 配置生效了 - 看到 `hook.before_tool` 说明 `intercept: ["before_tool", ...]` 生效了 @@ -664,7 +665,7 @@ if __name__ == "__main__": ```json {"ts":"2026-03-21T14:12:00+00:00","direction":"in","id":1,"method":"hook.hello","params":{"name":"py_review_gate","version":1,"modes":["observe","tool","approve"]},"notification":false} {"ts":"2026-03-21T14:12:00+00:00","direction":"out","id":1,"response":{"ok":true,"name":"python-review-gate"},"error":null} -{"ts":"2026-03-21T14:12:05+00:00","direction":"in","id":0,"method":"hook.event","params":{"Kind":"tool_exec_start"},"notification":true} +{"ts":"2026-03-21T14:12:05+00:00","direction":"in","id":0,"method":"hook.runtime_event","params":{"kind":"agent.tool.exec_start"},"notification":true} {"ts":"2026-03-21T14:12:05+00:00","direction":"in","id":7,"method":"hook.before_tool","params":{"tool":"echo_text","arguments":{"text":"hello"}},"notification":false} {"ts":"2026-03-21T14:12:05+00:00","direction":"out","id":7,"response":{"action":"continue"},"error":null} ``` @@ -672,7 +673,7 @@ if __name__ == "__main__": 补充说明： - 时间戳是 UTC，不是本地时区 -- `notification=true` 表示这是 `hook.event` 这类不需要响应的通知 +- `notification=true` 表示这是 `hook.runtime_event` 这类不需要响应的通知 - `id` 会随着当前进程内的请求递增；如果 hook 进程重启，计数会重新开始 ## Process Hook 协议约定 @@ -681,7 +682,7 @@ if __name__ == "__main__": - PicoClaw 启动外部进程 - 请求和响应都按“一行一个 JSON 消息”传输 -- `hook.event` 是 notification，不需要响应 +- `hook.runtime_event` 是 notification，不需要响应 - `hook.before_llm` / `hook.after_llm` / `hook.before_tool` / `hook.after_tool` / `hook.approve_tool` 是 request/response 当前宿主不会接受 process hook 主动发起的新 RPC。也就是说，外部 hook 现在只能“响应 PicoClaw 的调用”，不能反向调用宿主去发送 channel 消息。 diff --git a/docs/architecture/hooks/hook-json-protocol.md b/docs/architecture/hooks/hook-json-protocol.md index 58b6e323b..b04606777 100644 --- a/docs/architecture/hooks/hook-json-protocol.md +++ b/docs/architecture/hooks/hook-json-protocol.md @@ -437,21 +437,28 @@ Approval hook for deciding whether to allow execution of sensitive tools. --- -## 7. `hook.event` (notification) +## 7. `hook.runtime_event` (notification) -Observer event, broadcast only, no response required. `id` is `0` or absent. +Runtime observer event, broadcast only, no response required. `id` is `0` or absent. ```json { "jsonrpc": "2.0", - "method": "hook.event", + "method": "hook.runtime_event", "params": { - "Kind": "tool_exec_start", - "Meta": { - "AgentID": "agent-1", - "TurnID": "turn-1" + "kind": "agent.tool.exec_start", + "source": { + "component": "agent", + "name": "agent-1" }, - "Payload": { + "scope": { + "agent_id": "agent-1", + "session_key": "session-1", + "turn_id": "turn-1", + "channel": "cli", + "chat_id": "chat-1" + }, + "payload": { "Tool": "echo_text", "Arguments": {"text": "hello"} } @@ -460,12 +467,14 @@ Observer event, broadcast only, no response required. `id` is `0` or absent. ``` Common `Kind` values: -- `turn_start` / `turn_end` -- `llm_request` / `llm_response` -- `tool_exec_start` / `tool_exec_end` / `tool_exec_skipped` -- `steering_injected` -- `interrupt_received` -- `error` +- `agent.turn.start` / `agent.turn.end` +- `agent.llm.request` / `agent.llm.response` +- `agent.tool.exec_start` / `agent.tool.exec_end` / `agent.tool.exec_skipped` +- `agent.steering.injected` +- `agent.interrupt.received` +- `agent.error` + +Legacy observe configuration names such as `turn_end` and `tool_exec_start` are still accepted and normalized to runtime event names. New process hook notifications use `hook.runtime_event`. --- @@ -513,7 +522,7 @@ Standard flow for plugin tool injection: ```python def handle_before_llm(params: dict) -> dict: tools = params.get("tools", []) - + # Add plugin tool definition tools.append({ "type": "function", @@ -529,7 +538,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + return { "action": "modify", "request": { @@ -546,12 +555,12 @@ def handle_before_llm(params: dict) -> dict: ```python def handle_before_tool(params: dict) -> dict: tool = params.get("tool", "") - + if tool == "my_plugin_tool": # Implement tool logic here args = params.get("arguments", {}) input_text = args.get("input", "") - + # Return result directly, no need to register in ToolRegistry return { "action": "respond", @@ -561,8 +570,8 @@ def handle_before_tool(params: dict) -> dict: "is_error": False } } - + return {"action": "continue"} ``` -This way, external hooks can fully implement plugin tools without registering any tool implementation inside PicoClaw. \ No newline at end of file +This way, external hooks can fully implement plugin tools without registering any tool implementation inside PicoClaw. diff --git a/docs/architecture/hooks/hook-json-protocol.zh.md b/docs/architecture/hooks/hook-json-protocol.zh.md index 675e0a429..dac010a3e 100644 --- a/docs/architecture/hooks/hook-json-protocol.zh.md +++ b/docs/architecture/hooks/hook-json-protocol.zh.md @@ -437,21 +437,28 @@ --- -## 7. `hook.event`（notification） +## 7. `hook.runtime_event`（notification） -观察型事件，仅广播，无需响应。`id` 为 `0` 或不存在。 +runtime 观察型事件，仅广播，无需响应。`id` 为 `0` 或不存在。 ```json { "jsonrpc": "2.0", - "method": "hook.event", + "method": "hook.runtime_event", "params": { - "Kind": "tool_exec_start", - "Meta": { - "AgentID": "agent-1", - "TurnID": "turn-1" + "kind": "agent.tool.exec_start", + "source": { + "component": "agent", + "name": "agent-1" }, - "Payload": { + "scope": { + "agent_id": "agent-1", + "session_key": "session-1", + "turn_id": "turn-1", + "channel": "cli", + "chat_id": "chat-1" + }, + "payload": { "Tool": "echo_text", "Arguments": {"text": "hello"} } @@ -460,12 +467,14 @@ ``` 常见 `Kind` 值： -- `turn_start` / `turn_end` -- `llm_request` / `llm_response` -- `tool_exec_start` / `tool_exec_end` / `tool_exec_skipped` -- `steering_injected` -- `interrupt_received` -- `error` +- `agent.turn.start` / `agent.turn.end` +- `agent.llm.request` / `agent.llm.response` +- `agent.tool.exec_start` / `agent.tool.exec_end` / `agent.tool.exec_skipped` +- `agent.steering.injected` +- `agent.interrupt.received` +- `agent.error` + +旧 observe 配置名如 `turn_end`、`tool_exec_start` 仍然可用，并会归一化为 runtime event 名称。新的 process hook 通知使用 `hook.runtime_event`。 --- @@ -513,7 +522,7 @@ ```python def handle_before_llm(params: dict) -> dict: tools = params.get("tools", []) - + # 添加插件工具定义 tools.append({ "type": "function", @@ -529,7 +538,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + return { "action": "modify", "request": { @@ -546,12 +555,12 @@ def handle_before_llm(params: dict) -> dict: ```python def handle_before_tool(params: dict) -> dict: tool = params.get("tool", "") - + if tool == "my_plugin_tool": # 在这里实现工具逻辑 args = params.get("arguments", {}) input_text = args.get("input", "") - + # 直接返回结果，无需在 ToolRegistry 注册 return { "action": "respond", @@ -561,8 +570,8 @@ def handle_before_tool(params: dict) -> dict: "is_error": False } } - + return {"action": "continue"} ``` -通过这种方式，外部 hook 可以完全实现插件工具，无需在 PicoClaw 内部注册任何工具实现。 \ No newline at end of file +通过这种方式，外部 hook 可以完全实现插件工具，无需在 PicoClaw 内部注册任何工具实现。 diff --git a/docs/architecture/hooks/plugin-tool-injection.md b/docs/architecture/hooks/plugin-tool-injection.md index 9e699867b..b0436bc66 100644 --- a/docs/architecture/hooks/plugin-tool-injection.md +++ b/docs/architecture/hooks/plugin-tool-injection.md @@ -67,7 +67,7 @@ def handle_hello(params: dict) -> dict: def handle_before_llm(params: dict) -> dict: """Inject weather query tool definition""" tools = params.get("tools", []) - + # Add weather query tool tools.append({ "type": "function", @@ -86,7 +86,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + return { "action": "modify", "request": { @@ -102,17 +102,17 @@ def handle_before_tool(params: dict) -> dict: """Handle tool call, return result directly""" tool = params.get("tool", "") args = params.get("arguments", {}) - + if tool == "get_weather": city = args.get("city", "") result = get_weather(city) - + # Use respond action to return result directly, skip ToolRegistry return { "action": "respond", "result": result, } - + # Other tools continue normal flow return {"action": "continue"} @@ -142,7 +142,7 @@ def send_response(message_id: int, result: Any | None = None, error: str | None payload["error"] = {"code": -32000, "message": error} else: payload["result"] = result if result is not None else {} - + sys.stdout.write(json.dumps(payload, ensure_ascii=True) + "\n") sys.stdout.flush() @@ -152,19 +152,19 @@ def main() -> int: line = raw_line.strip() if not line: continue - + try: message = json.loads(line) except json.JSONDecodeError: continue - + method = message.get("method") message_id = message.get("id", 0) params = message.get("params") or {} - + if not message_id: continue - + try: result = handle_request(str(method or ""), params) send_response(int(message_id), result=result) @@ -172,7 +172,7 @@ def main() -> int: send_response(int(message_id), error=str(exc)) except Exception as exc: send_response(int(message_id), error=f"unexpected error: {exc}") - + return 0 @@ -375,7 +375,7 @@ Multiple tools can be injected simultaneously: ```python def handle_before_llm(params: dict) -> dict: tools = params.get("tools", []) - + # Tool 1: Weather query tools.append({ "type": "function", @@ -391,7 +391,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + # Tool 2: Calculator tools.append({ "type": "function", @@ -407,7 +407,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + return { "action": "modify", "request": { @@ -422,13 +422,13 @@ def handle_before_llm(params: dict) -> dict: def handle_before_tool(params: dict) -> dict: tool = params.get("tool", "") args = params.get("arguments", {}) - + if tool == "get_weather": return { "action": "respond", "result": get_weather(args.get("city", "")), } - + if tool == "calculate": # Simple calculation example try: @@ -451,7 +451,7 @@ def handle_before_tool(params: dict) -> dict: "is_error": True, }, } - + return {"action": "continue"} ``` @@ -504,7 +504,7 @@ func (h *WeatherPluginHook) BeforeLLM( }, }, }) - + return req, agent.HookDecision{Action: agent.HookActionContinue}, nil } @@ -514,7 +514,7 @@ func (h *WeatherPluginHook) BeforeTool( ) (*agent.ToolCallHookRequest, agent.HookDecision, error) { if call.Tool == "get_weather" { city := call.Arguments["city"].(string) - + // Set HookResult, use respond action next := call.Clone() next.HookResult = &tools.ToolResult{ @@ -522,10 +522,10 @@ func (h *WeatherPluginHook) BeforeTool( Silent: false, IsError: false, } - + return next, agent.HookDecision{Action: agent.HookActionRespond}, nil } - + return call, agent.HookDecision{Action: agent.HookActionContinue}, nil } @@ -572,14 +572,14 @@ This means: def handle_before_tool(params: dict) -> dict: tool = params.get("tool", "") args = params.get("arguments", {}) - + # Security check: only handle plugin tools if tool in ["get_weather", "calculate"]: return { "action": "respond", "result": execute_plugin_tool(tool, args), } - + # Other tools continue normal flow (will go through approval) return {"action": "continue"} ``` diff --git a/docs/architecture/hooks/plugin-tool-injection.zh.md b/docs/architecture/hooks/plugin-tool-injection.zh.md index ccc7ff7f6..0448ec1a8 100644 --- a/docs/architecture/hooks/plugin-tool-injection.zh.md +++ b/docs/architecture/hooks/plugin-tool-injection.zh.md @@ -67,7 +67,7 @@ def handle_hello(params: dict) -> dict: def handle_before_llm(params: dict) -> dict: """注入天气查询工具定义""" tools = params.get("tools", []) - + # 添加天气查询工具 tools.append({ "type": "function", @@ -86,7 +86,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + return { "action": "modify", "request": { @@ -102,17 +102,17 @@ def handle_before_tool(params: dict) -> dict: """处理工具调用，直接返回结果""" tool = params.get("tool", "") args = params.get("arguments", {}) - + if tool == "get_weather": city = args.get("city", "") result = get_weather(city) - + # 使用 respond action 直接返回结果，跳过 ToolRegistry return { "action": "respond", "result": result, } - + # 其他工具继续正常流程 return {"action": "continue"} @@ -142,7 +142,7 @@ def send_response(message_id: int, result: Any | None = None, error: str | None payload["error"] = {"code": -32000, "message": error} else: payload["result"] = result if result is not None else {} - + sys.stdout.write(json.dumps(payload, ensure_ascii=True) + "\n") sys.stdout.flush() @@ -152,19 +152,19 @@ def main() -> int: line = raw_line.strip() if not line: continue - + try: message = json.loads(line) except json.JSONDecodeError: continue - + method = message.get("method") message_id = message.get("id", 0) params = message.get("params") or {} - + if not message_id: continue - + try: result = handle_request(str(method or ""), params) send_response(int(message_id), result=result) @@ -172,7 +172,7 @@ def main() -> int: send_response(int(message_id), error=str(exc)) except Exception as exc: send_response(int(message_id), error=f"unexpected error: {exc}") - + return 0 @@ -375,7 +375,7 @@ media:// ```python def handle_before_llm(params: dict) -> dict: tools = params.get("tools", []) - + # 工具1：天气查询 tools.append({ "type": "function", @@ -391,7 +391,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + # 工具2：计算器 tools.append({ "type": "function", @@ -407,7 +407,7 @@ def handle_before_llm(params: dict) -> dict: } } }) - + return { "action": "modify", "request": { @@ -422,13 +422,13 @@ def handle_before_llm(params: dict) -> dict: def handle_before_tool(params: dict) -> dict: tool = params.get("tool", "") args = params.get("arguments", {}) - + if tool == "get_weather": return { "action": "respond", "result": get_weather(args.get("city", "")), } - + if tool == "calculate": # 简单计算示例 try: @@ -451,7 +451,7 @@ def handle_before_tool(params: dict) -> dict: "is_error": True, }, } - + return {"action": "continue"} ``` @@ -504,7 +504,7 @@ func (h *WeatherPluginHook) BeforeLLM( }, }, }) - + return req, agent.HookDecision{Action: agent.HookActionContinue}, nil } @@ -514,7 +514,7 @@ func (h *WeatherPluginHook) BeforeTool( ) (*agent.ToolCallHookRequest, agent.HookDecision, error) { if call.Tool == "get_weather" { city := call.Arguments["city"].(string) - + // 设置 HookResult，使用 respond action next := call.Clone() next.HookResult = &tools.ToolResult{ @@ -522,10 +522,10 @@ func (h *WeatherPluginHook) BeforeTool( Silent: false, IsError: false, } - + return next, agent.HookDecision{Action: agent.HookActionRespond}, nil } - + return call, agent.HookDecision{Action: agent.HookActionContinue}, nil } @@ -572,14 +572,14 @@ func getWeatherData(city string) string { def handle_before_tool(params: dict) -> dict: tool = params.get("tool", "") args = params.get("arguments", {}) - + # 安全检查：只处理插件工具 if tool in ["get_weather", "calculate"]: return { "action": "respond", "result": execute_plugin_tool(tool, args), } - + # 其他工具继续正常流程（会经过审批） return {"action": "continue"} ``` diff --git a/docs/architecture/runtime-events.md b/docs/architecture/runtime-events.md new file mode 100644 index 000000000..5d625a34b --- /dev/null +++ b/docs/architecture/runtime-events.md @@ -0,0 +1,216 @@ +# Runtime Events And Event Logging + +PicoClaw runtime events are the read-only observation surface for agent, channel, gateway, message bus, and MCP activity. Publishing events and printing logs are separate responsibilities: + +- Event publishing: components publish `pkg/events.Event` values to the runtime event bus for hooks, tests, diagnostics, and future UI consumers. +- Event logging: the built-in runtime event logger subscribes to the same bus and prints only the events selected by configuration. + +This keeps runtime code focused on publishing events while log policy stays centralized. + +## Default Behavior + +By default, only `agent.*` events are printed: + +```json +{ + "events": { + "logging": { + "enabled": true, + "include": ["agent.*"], + "min_severity": "info", + "include_payload": false + } + } +} +``` + +This preserves the previous behavior: agent turn, LLM, tool, steering, subturn, and error events appear in logs. Channel, gateway, bus, and MCP events are still published to the runtime event bus, but they are not printed unless configured. + +## Configuration + +The configuration lives under `events.logging` in `config.json`: + +| Field | Type | Default | Description | +| ----- | ---- | ------- | ----------- | +| `enabled` | bool | `true` | Enables the built-in event logger subscription | +| `include` | string[] | `["agent.*"]` | Event kinds to print; supports exact matches, `*`, and patterns such as `agent.*` | +| `exclude` | string[] | `[]` | Event kinds to suppress after include matching | +| `min_severity` | string | `info` | Minimum severity: `debug`, `info`, `warn`, or `error` | +| `include_payload` | bool | `false` | Adds raw event payloads to log fields | + +`include_payload` is disabled by default. Agent events print safe summary fields such as `user_len`, `args_count`, and `content_len` instead of full user messages or tool arguments. Enable raw payload logging only for short-lived diagnostics in a trusted log environment. + +## Matching Rules + +`include` and `exclude` match the `Event.Kind` string: + +```json +{ + "events": { + "logging": { + "include": ["gateway.*", "channel.lifecycle.*", "agent.error"], + "exclude": ["gateway.ready"], + "min_severity": "info" + } + } +} +``` + +Common patterns: + +- `["agent.*"]`: print agent events only. +- `["*"]`: print all runtime events. +- `["gateway.*", "channel.*"]`: print gateway and channel events only. +- `exclude: ["agent.llm.delta"]`: suppress high-volume streaming delta events. +- `min_severity: "warn"`: print warn and error events only. + +## Environment Variables + +The same settings can be overridden with environment variables: + +```bash +PICOCLAW_EVENTS_LOGGING_ENABLED=true +PICOCLAW_EVENTS_LOGGING_INCLUDE="gateway.*,channel.lifecycle.*" +PICOCLAW_EVENTS_LOGGING_EXCLUDE="gateway.ready" +PICOCLAW_EVENTS_LOGGING_MIN_SEVERITY=info +PICOCLAW_EVENTS_LOGGING_INCLUDE_PAYLOAD=false +``` + +`include` and `exclude` use comma-separated values. + +## Event Names And Triggers + +The table below lists the current runtime event kinds, when they are emitted, and the most useful event details. `Source`, `Scope`, and `Correlation` are shared envelope fields that may appear on every event. The "Details" column refers to useful payload fields or log summary fields. + +### Agent + +| Event | Trigger | Details | +| ----- | ------- | ------- | +| `agent.turn.start` | An agent starts processing one user or system input after the turn scope has been created. | `user_len`, `media_count`; scope usually includes `agent_id`, `session_key`, `turn_id`, `channel`, `chat_id`, `message_id` | +| `agent.turn.end` | A turn exits, whether it completed, errored, or was hard-aborted. | `status` (`completed`/`error`/`aborted`), `iterations_total`, `duration_ms`, `final_len` | +| `agent.llm.request` | Before each LLM provider request. | `model`, `messages`, `tools`, `max_tokens` | +| `agent.llm.delta` | Reserved for streaming LLM deltas; the kind is defined, but the current implementation has no natural emit site. | `content_delta_len`, `reasoning_delta_len` | +| `agent.llm.response` | After the LLM provider returns a complete response. | `content_len`, `tool_calls`, `has_reasoning` | +| `agent.llm.retry` | Before retrying an LLM request after context, rate-limit, transient provider, or fallback handling. | `attempt`, `max_retries`, `reason`, `error`, `backoff_ms` | +| `agent.context.compress` | Agent context history is compressed, for example during proactive budget checks or LLM retry handling. | `reason`, `dropped_messages`, `remaining_messages` | +| `agent.session.summarize` | Async session history summarization completes. | `summarized_messages`, `kept_messages`, `summary_len`, `omitted_oversized` | +| `agent.tool.exec_start` | Before the agent executes a tool call. | `tool`, `args_count`; full arguments are not logged by default | +| `agent.tool.exec_end` | After a tool call completes, including successful results, tool errors, and async results. | `tool`, `duration_ms`, `for_llm_len`, `for_user_len`, `is_error`, `async` | +| `agent.tool.exec_skipped` | A tool call is skipped because the tool is unavailable, arguments are invalid, or turn control logic requires skipping it. | `tool`, `reason` | +| `agent.steering.injected` | Queued steering messages are injected into the next LLM context. | `count`, `total_content_len` | +| `agent.follow_up.queued` | An async tool result is queued back into the inbound/follow-up flow. | `source_tool`, `content_len` | +| `agent.interrupt.received` | A turn accepts steering, graceful interrupt, or hard-abort input. | `interrupt_kind`, `role`, `content_len`, `queue_depth`, `hint_len` | +| `agent.subturn.spawn` | A parent turn creates a child turn/subagent. | `child_agent_id`, `label`, `parent_turn_id` | +| `agent.subturn.end` | A child turn ends. | `child_agent_id`, `status` | +| `agent.subturn.result_delivered` | A child turn result is delivered to the target channel/chat. | `target_channel`, `target_chat_id`, `content_len` | +| `agent.subturn.orphan` | A child turn result cannot be delivered or cannot be associated back to its parent turn. | `parent_turn_id`, `child_turn_id`, `reason` | +| `agent.error` | Agent execution reports an error. | `stage`, `error` | + +### Channel + +| Event | Trigger | Details | +| ----- | ------- | ------- | +| `channel.lifecycle.initialized` | The channel manager creates and registers a channel instance from config. | `type`; scope includes `channel` | +| `channel.lifecycle.started` | Channel `Start()` succeeds and worker goroutines have been started; added channels during hot reload also emit it. | `type` | +| `channel.lifecycle.start_failed` | Channel `Start()` fails. | `type`, `error`; severity is `error` | +| `channel.lifecycle.stopped` | Channel `Stop()` succeeds. | `type` | +| `channel.webhook.registered` | A channel webhook handler is registered on the shared HTTP mux. | `type`; scope includes `channel` | +| `channel.webhook.unregistered` | A channel webhook handler is removed from the shared HTTP mux. | `type`; scope includes `channel` | +| `channel.message.outbound_queued` | An outbound text or media message is queued into its channel worker. | `media`, `content_len`, `reply_to_message_id`; scope comes from the original inbound context | +| `channel.message.outbound_sent` | An outbound text or media message is sent successfully, or a placeholder edit handled the response. | `media`, `content_len`, `message_ids`, `reply_to_message_id` | +| `channel.message.outbound_failed` | An outbound text or media message exhausts retries or hits a permanent failure. | `media`, `content_len`, `retries`, `error`, `reply_to_message_id`; severity is `error` | +| `channel.rate_limited` | A channel worker is waiting for a rate-limit token and the context is canceled, interrupting this delivery. | `media`, `content_len`, `error`, `reply_to_message_id`; severity is `warn` | + +### Message Bus + +| Event | Trigger | Details | +| ----- | ------- | ------- | +| `bus.publish.failed` | Publishing inbound, outbound, media, audio, or voice-control data fails, or required context is missing. | `stream`, `error`; scope is derived from message context when possible | +| `bus.close.started` | Message bus shutdown begins. | `drained` is usually `0` | +| `bus.close.drained` | Shutdown waits for buffered messages to drain and at least one buffered message was drained. | `drained` | +| `bus.close.completed` | Message bus shutdown completes. | `drained` | + +### Gateway + +| Event | Trigger | Details | +| ----- | ------- | ------- | +| `gateway.start` | Gateway startup reaches the agent/runtime event bus/bootstrap binding point. | `duration_ms` | +| `gateway.ready` | Gateway services, channel manager, HTTP server, and other core services are ready. | `duration_ms` | +| `gateway.shutdown` | Gateway shutdown begins. | No fixed payload; envelope fields may be the only fields | +| `gateway.reload.started` | Hot reload execution starts. | `duration_ms` | +| `gateway.reload.completed` | Hot reload completes successfully. | `duration_ms` | +| `gateway.reload.failed` | Hot reload fails. | `duration_ms`, `error`; severity is `error` | + +### MCP + +| Event | Trigger | Details | +| ----- | ------- | ------- | +| `mcp.server.connecting` | The MCP manager is about to connect to a server. | `server`, `type`, `url`, `command` | +| `mcp.server.connected` | An MCP server connects and its tool list has been initialized. | `server`, `type`, `url`, `command`, `tool_count` | +| `mcp.server.failed` | An MCP server connection fails, or the manager is closed before connecting. | `server`, `type`, `url`, `command`, `error`; severity is `error` | +| `mcp.tool.discovered` | A tool from an MCP server is discovered and registered. | `server`, `type`, `url`, `command`, `tool` | +| `mcp.tool.call.start` | The MCP tool wrapper starts a remote tool call. | `server`, `tool`; when emitted inside an agent turn, scope includes turn/chat information | +| `mcp.tool.call.end` | The MCP tool wrapper finishes a remote tool call, including failures. | `server`, `tool`, `duration_ms`, `is_error`, `error` | + +## Log Fields + +Runtime event logs include stable envelope fields when available: + +- `event_id` +- `event_kind` +- `severity` +- `event_time` +- `source_component` +- `source_name` +- `agent_id` +- `session_key` +- `turn_id` +- `channel` +- `account` +- `chat_id` +- `topic_id` +- `space_id` +- `space_type` +- `chat_type` +- `sender_id` +- `message_id` +- `trace_id` +- `parent_turn_id` +- `request_id` +- `reply_to_id` + +Agent events add safe payload summaries: + +| Event | Summary fields | +| ----- | -------------- | +| `agent.turn.start` | `user_len`, `media_count` | +| `agent.turn.end` | `status`, `iterations_total`, `duration_ms`, `final_len` | +| `agent.llm.request` | `model`, `messages`, `tools`, `max_tokens` | +| `agent.llm.delta` | `content_delta_len`, `reasoning_delta_len` | +| `agent.llm.response` | `content_len`, `tool_calls`, `has_reasoning` | +| `agent.llm.retry` | `attempt`, `max_retries`, `reason`, `error`, `backoff_ms` | +| `agent.context.compress` | `reason`, `dropped_messages`, `remaining_messages` | +| `agent.session.summarize` | `summarized_messages`, `kept_messages`, `summary_len`, `omitted_oversized` | +| `agent.tool.exec_start` | `tool`, `args_count` | +| `agent.tool.exec_end` | `tool`, `duration_ms`, `for_llm_len`, `for_user_len`, `is_error`, `async` | +| `agent.tool.exec_skipped` | `tool`, `reason` | +| `agent.steering.injected` | `count`, `total_content_len` | +| `agent.follow_up.queued` | `source_tool`, `content_len` | +| `agent.interrupt.received` | `interrupt_kind`, `role`, `content_len`, `queue_depth`, `hint_len` | +| `agent.subturn.spawn` | `child_agent_id`, `label` | +| `agent.subturn.end` | `child_agent_id`, `status` | +| `agent.subturn.result_delivered` | `target_channel`, `target_chat_id`, `content_len` | +| `agent.subturn.orphan` | `parent_turn_id`, `child_turn_id`, `reason` | +| `agent.error` | `stage`, `error` | + +## Event Domains + +Runtime event kinds are defined in `pkg/events/kind.go`. Event logging can select these domains: + +- `agent.*`: agent turn, LLM, tool, context, steering, interrupt, subturn, and error events. +- `channel.*`: channel lifecycle, webhook registration, outbound queued/sent/failed, and rate limiting. +- `bus.*`: publish failures and close lifecycle. +- `gateway.*`: start, ready, shutdown, and reload lifecycle. +- `mcp.*`: MCP server connection, tool discovery, and tool call events. + +See [`../../config/config.example.json`](../../config/config.example.json) for the default event logging example. diff --git a/docs/architecture/runtime-events.zh.md b/docs/architecture/runtime-events.zh.md new file mode 100644 index 000000000..3ed384537 --- /dev/null +++ b/docs/architecture/runtime-events.zh.md @@ -0,0 +1,216 @@ +# Runtime Events 与事件日志 + +PicoClaw 的 runtime event 是运行时观察面，用来描述 agent、channel、gateway、message bus、MCP 等组件发生了什么。事件发布和日志打印是两件事： + +- 事件发布：组件把 `pkg/events.Event` 发布到 runtime event bus，供 hook、测试、调试工具或后续 UI 消费。 +- 事件日志：内置 runtime event logger 订阅同一个 bus，并按配置把匹配的事件打印到日志。 + +这样可以让业务流程继续只负责发布事件，日志策略统一收口到一个地方。 + +## 默认行为 + +默认配置只打印 `agent.*` 事件： + +```json +{ + "events": { + "logging": { + "enabled": true, + "include": ["agent.*"], + "min_severity": "info", + "include_payload": false + } + } +} +``` + +这个默认值保持了旧行为：agent turn、LLM、tool、steering、subturn、error 等事件会出现在日志中；channel、gateway、bus、MCP 事件仍会发布到 runtime event bus，但默认不打印，避免网关启动和消息投递日志过于嘈杂。 + +## 配置项 + +配置位于 `config.json` 的 `events.logging`： + +| 字段 | 类型 | 默认值 | 说明 | +| ---- | ---- | ------ | ---- | +| `enabled` | bool | `true` | 是否启用内置事件日志订阅器 | +| `include` | string[] | `["agent.*"]` | 允许打印的事件 kind，支持精确匹配、`*`、`agent.*` 这类 glob/prefix | +| `exclude` | string[] | `[]` | 在 include 命中后排除的事件 kind，匹配规则同 include | +| `min_severity` | string | `info` | 最低打印级别：`debug`、`info`、`warn`、`error` | +| `include_payload` | bool | `false` | 是否把原始 payload 放进日志字段 | + +`include_payload` 默认关闭。agent 事件日志会输出安全摘要字段，例如 `user_len`、`args_count`、`content_len`，不会默认输出完整用户消息或工具参数。只有在排查问题、并且确认日志存储环境可信时，才建议临时打开 `include_payload`。 + +## 匹配规则 + +`include` 和 `exclude` 都匹配 `Event.Kind` 字符串： + +```json +{ + "events": { + "logging": { + "include": ["gateway.*", "channel.lifecycle.*", "agent.error"], + "exclude": ["gateway.ready"], + "min_severity": "info" + } + } +} +``` + +常用写法： + +- `["agent.*"]`：只打印 agent 事件。 +- `["*"]`：打印所有 runtime events。 +- `["gateway.*", "channel.*"]`：只打印 gateway 和 channel 事件。 +- `exclude: ["agent.llm.delta"]`：排除高频流式 delta 事件。 +- `min_severity: "warn"`：只打印 warn/error 事件。 + +## 环境变量 + +同一组配置也可以通过环境变量覆盖，适合临时调试： + +```bash +PICOCLAW_EVENTS_LOGGING_ENABLED=true +PICOCLAW_EVENTS_LOGGING_INCLUDE="gateway.*,channel.lifecycle.*" +PICOCLAW_EVENTS_LOGGING_EXCLUDE="gateway.ready" +PICOCLAW_EVENTS_LOGGING_MIN_SEVERITY=info +PICOCLAW_EVENTS_LOGGING_INCLUDE_PAYLOAD=false +``` + +`include` 和 `exclude` 的环境变量使用逗号分隔。 + +## 事件名称与触发时机 + +下面列出当前 runtime event kind、触发时机和主要事件详情。`Source`、`Scope`、`Correlation` 是所有事件都可能携带的 envelope 字段；表里的“主要详情”指 payload 或日志摘要中最有用的字段。 + +### Agent + +| 事件名 | 触发时机 | 主要详情 | +| ------ | -------- | -------- | +| `agent.turn.start` | agent 开始处理一次用户输入或系统输入，turn scope 已创建时 | `user_len`, `media_count`; scope 通常包含 `agent_id`, `session_key`, `turn_id`, `channel`, `chat_id`, `message_id` | +| `agent.turn.end` | 一次 turn 退出时，无论完成、报错还是 hard abort | `status` (`completed`/`error`/`aborted`), `iterations_total`, `duration_ms`, `final_len` | +| `agent.llm.request` | 每次调用 LLM provider 前 | `model`, `messages`, `tools`, `max_tokens` | +| `agent.llm.delta` | 预留给流式 LLM delta；当前实现已定义但没有自然发送点 | `content_delta_len`, `reasoning_delta_len` | +| `agent.llm.response` | LLM provider 返回完整响应后 | `content_len`, `tool_calls`, `has_reasoning` | +| `agent.llm.retry` | LLM 请求因上下文、限流、临时错误等原因准备重试前 | `attempt`, `max_retries`, `reason`, `error`, `backoff_ms` | +| `agent.context.compress` | 上下文历史被压缩时，例如主动预算检查或 LLM retry 处理 | `reason`, `dropped_messages`, `remaining_messages` | +| `agent.session.summarize` | 会话历史异步摘要完成时 | `summarized_messages`, `kept_messages`, `summary_len`, `omitted_oversized` | +| `agent.tool.exec_start` | agent 准备执行一个工具调用前 | `tool`, `args_count`; 默认不打印完整参数 | +| `agent.tool.exec_end` | 工具调用完成后，包括成功、工具错误和 async 结果 | `tool`, `duration_ms`, `for_llm_len`, `for_user_len`, `is_error`, `async` | +| `agent.tool.exec_skipped` | 工具调用被跳过时，例如工具不可用、参数无效或 turn 控制逻辑要求跳过 | `tool`, `reason` | +| `agent.steering.injected` | queued steering message 被注入下一轮 LLM 上下文时 | `count`, `total_content_len` | +| `agent.follow_up.queued` | async 工具结果被重新排入 inbound/follow-up 流程时 | `source_tool`, `content_len` | +| `agent.interrupt.received` | turn 接受 steering、graceful interrupt 或 hard abort 指令时 | `interrupt_kind`, `role`, `content_len`, `queue_depth`, `hint_len` | +| `agent.subturn.spawn` | 父 turn 创建子 turn/subagent 时 | `child_agent_id`, `label`, `parent_turn_id` | +| `agent.subturn.end` | 子 turn 结束时 | `child_agent_id`, `status` | +| `agent.subturn.result_delivered` | 子 turn 结果成功投递到目标 channel/chat 时 | `target_channel`, `target_chat_id`, `content_len` | +| `agent.subturn.orphan` | 子 turn 结果无法投递或无法关联回父 turn 时 | `parent_turn_id`, `child_turn_id`, `reason` | +| `agent.error` | agent 执行流程报告错误时 | `stage`, `error` | + +### Channel + +| 事件名 | 触发时机 | 主要详情 | +| ------ | -------- | -------- | +| `channel.lifecycle.initialized` | channel manager 根据配置创建并注册 channel 实例后 | `type`; scope 包含 `channel` | +| `channel.lifecycle.started` | channel `Start()` 成功，worker 已启动时；热重载新增 channel 也会触发 | `type` | +| `channel.lifecycle.start_failed` | channel `Start()` 失败时 | `type`, `error`; severity 为 `error` | +| `channel.lifecycle.stopped` | channel `Stop()` 成功后 | `type` | +| `channel.webhook.registered` | channel 的 webhook handler 被注册到共享 HTTP mux 时 | `type`; scope 包含 `channel` | +| `channel.webhook.unregistered` | channel 的 webhook handler 从共享 HTTP mux 移除时 | `type`; scope 包含 `channel` | +| `channel.message.outbound_queued` | outbound 文本或媒体消息被放入对应 channel worker 队列时 | `media`, `content_len`, `reply_to_message_id`; scope 来自原 inbound context | +| `channel.message.outbound_sent` | outbound 文本或媒体消息成功发送，或 placeholder edit 已处理响应时 | `media`, `content_len`, `message_ids`, `reply_to_message_id` | +| `channel.message.outbound_failed` | outbound 文本或媒体消息重试耗尽或遇到永久失败时 | `media`, `content_len`, `retries`, `error`, `reply_to_message_id`; severity 为 `error` | +| `channel.rate_limited` | channel worker 等待 rate limiter token 时被 context 取消，导致本次发送被限流/中断 | `media`, `content_len`, `error`, `reply_to_message_id`; severity 为 `warn` | + +### Message Bus + +| 事件名 | 触发时机 | 主要详情 | +| ------ | -------- | -------- | +| `bus.publish.failed` | inbound、outbound、media、audio 或 voice control 发布失败，或缺少必要 context 时 | `stream`, `error`; scope 尽量来自消息 context | +| `bus.close.started` | message bus 开始关闭时 | `drained` 通常为 `0` | +| `bus.close.drained` | close 期间等待队列 drain，并且 drain 到至少一条 buffered message 时 | `drained` | +| `bus.close.completed` | message bus 完成关闭时 | `drained` | + +### Gateway + +| 事件名 | 触发时机 | 主要详情 | +| ------ | -------- | -------- | +| `gateway.start` | gateway 完成 agent/runtime event bus/bootstrap 绑定后 | `duration_ms` | +| `gateway.ready` | gateway 服务、channel manager、HTTP 等关键服务启动完成后 | `duration_ms` | +| `gateway.shutdown` | gateway 开始关闭流程时 | 无固定 payload，可能只有 envelope 字段 | +| `gateway.reload.started` | 热重载开始执行时 | `duration_ms` | +| `gateway.reload.completed` | 热重载成功完成时 | `duration_ms` | +| `gateway.reload.failed` | 热重载失败时 | `duration_ms`, `error`; severity 为 `error` | + +### MCP + +| 事件名 | 触发时机 | 主要详情 | +| ------ | -------- | -------- | +| `mcp.server.connecting` | MCP manager 准备连接某个 server 前 | `server`, `type`, `url`, `command` | +| `mcp.server.connected` | MCP server 连接成功并完成工具列表初始化后 | `server`, `type`, `url`, `command`, `tool_count` | +| `mcp.server.failed` | MCP server 连接失败，或 manager 已关闭导致无法连接时 | `server`, `type`, `url`, `command`, `error`; severity 为 `error` | +| `mcp.tool.discovered` | MCP server 的某个工具被发现并注册时 | `server`, `type`, `url`, `command`, `tool` | +| `mcp.tool.call.start` | MCP tool wrapper 开始执行一次远端工具调用前 | `server`, `tool`; 如果在 agent turn 内触发，scope 会带上对应 turn/chat 信息 | +| `mcp.tool.call.end` | MCP tool wrapper 完成一次远端工具调用后，包括失败结果 | `server`, `tool`, `duration_ms`, `is_error`, `error` | + +## 日志字段 + +所有事件日志都会尽量包含稳定 envelope 字段： + +- `event_id` +- `event_kind` +- `severity` +- `event_time` +- `source_component` +- `source_name` +- `agent_id` +- `session_key` +- `turn_id` +- `channel` +- `account` +- `chat_id` +- `topic_id` +- `space_id` +- `space_type` +- `chat_type` +- `sender_id` +- `message_id` +- `trace_id` +- `parent_turn_id` +- `request_id` +- `reply_to_id` + +agent 事件还会追加 payload 摘要字段： + +| 事件 | 摘要字段 | +| ---- | -------- | +| `agent.turn.start` | `user_len`, `media_count` | +| `agent.turn.end` | `status`, `iterations_total`, `duration_ms`, `final_len` | +| `agent.llm.request` | `model`, `messages`, `tools`, `max_tokens` | +| `agent.llm.delta` | `content_delta_len`, `reasoning_delta_len` | +| `agent.llm.response` | `content_len`, `tool_calls`, `has_reasoning` | +| `agent.llm.retry` | `attempt`, `max_retries`, `reason`, `error`, `backoff_ms` | +| `agent.context.compress` | `reason`, `dropped_messages`, `remaining_messages` | +| `agent.session.summarize` | `summarized_messages`, `kept_messages`, `summary_len`, `omitted_oversized` | +| `agent.tool.exec_start` | `tool`, `args_count` | +| `agent.tool.exec_end` | `tool`, `duration_ms`, `for_llm_len`, `for_user_len`, `is_error`, `async` | +| `agent.tool.exec_skipped` | `tool`, `reason` | +| `agent.steering.injected` | `count`, `total_content_len` | +| `agent.follow_up.queued` | `source_tool`, `content_len` | +| `agent.interrupt.received` | `interrupt_kind`, `role`, `content_len`, `queue_depth`, `hint_len` | +| `agent.subturn.spawn` | `child_agent_id`, `label` | +| `agent.subturn.end` | `child_agent_id`, `status` | +| `agent.subturn.result_delivered` | `target_channel`, `target_chat_id`, `content_len` | +| `agent.subturn.orphan` | `parent_turn_id`, `child_turn_id`, `reason` | +| `agent.error` | `stage`, `error` | + +## 可打印的事件域 + +当前 runtime event kind 定义在 `pkg/events/kind.go`。事件日志配置可以选择这些域： + +- `agent.*`：agent turn、LLM、tool、context、steering、interrupt、subturn、error。 +- `channel.*`：channel lifecycle、webhook 注册、outbound queued/sent/failed、rate limited。 +- `bus.*`：publish failed、close started/drained/completed。 +- `gateway.*`：start、ready、shutdown、reload started/completed/failed。 +- `mcp.*`：server connecting/connected/failed、tool discovered、tool call start/end。 + +默认事件日志示例见 [`../../config/config.example.json`](../../config/config.example.json)。 diff --git a/docs/architecture/subturn.md b/docs/architecture/subturn.md index 0a927b56d..31a56902c 100644 --- a/docs/architecture/subturn.md +++ b/docs/architecture/subturn.md @@ -135,16 +135,16 @@ The agent loop polls for async SubTurn results at two points per iteration: All active turns are registered in `AgentLoop.activeTurnStates` (`sync.Map`, keyed by session key). A reservation sentinel is stored atomically via `LoadOrStore` before the worker starts, then replaced with the real `*turnState` when `runTurn` registers. This prevents a TOCTOU race where multiple messages for the same session could spawn concurrent workers. The sentinel is cleaned up by the worker's deferred cleanup. This allows `HardAbort` and `/subagents` observability commands to find and operate on active turns. -## Event Bus Integration +## Runtime Event Integration -SubTurns emit specific events to the PicoClaw `EventBus` for observability and debugging: +SubTurns emit runtime events through `pkg/events` for observability and debugging: | Event Kind | When Emitted | Payload | |:------|:-------------|:--------| -| `subturn_spawn` | Sub-turn successfully initialized | `SubTurnSpawnPayload{AgentID, Label, ParentTurnID}` | -| `subturn_end` | Sub-turn finishes (success or error) | `SubTurnEndPayload{AgentID, Status}` | -| `subturn_result_delivered` | Async result successfully delivered to parent | `SubTurnResultDeliveredPayload{TargetChannel, TargetChatID, ContentLen}` | -| `subturn_orphan` | Result cannot be delivered (parent finished or channel full) | `SubTurnOrphanPayload{ParentTurnID, ChildTurnID, Reason}` | +| `agent.subturn.spawn` | Sub-turn successfully initialized | `SubTurnSpawnPayload{AgentID, Label, ParentTurnID}` | +| `agent.subturn.end` | Sub-turn finishes (success or error) | `SubTurnEndPayload{AgentID, Status}` | +| `agent.subturn.result_delivered` | Async result successfully delivered to parent | `SubTurnResultDeliveredPayload{TargetChannel, TargetChatID, ContentLen}` | +| `agent.subturn.orphan` | Result cannot be delivered (parent finished or channel full) | `SubTurnOrphanPayload{ParentTurnID, ChildTurnID, Reason}` | ## API Reference @@ -240,13 +240,13 @@ An orphan result occurs when: 2. The `pendingResults` channel is full (buffer size: 16) When a result becomes orphan: -- `SubTurnOrphanResultEvent` is emitted to EventBus +- `agent.subturn.orphan` is emitted to the runtime event bus - The result is **NOT** delivered to the LLM context - External systems can listen to this event for custom handling ### Preventing Orphan Results - Use `Critical: true` for important SubTurns that must complete -- Monitor `SubTurnOrphanResultEvent` for observability +- Monitor `agent.subturn.orphan` for observability - Consider the 16-buffer limit when spawning many async SubTurns ## Tool Inheritance diff --git a/docs/channels/mqtt/README.fr.md b/docs/channels/mqtt/README.fr.md new file mode 100644 index 000000000..c16868a32 --- /dev/null +++ b/docs/channels/mqtt/README.fr.md @@ -0,0 +1,140 @@ +# 📡 Canal MQTT + +PicoClaw prend en charge n'importe quel client MQTT comme canal de messagerie. Les appareils ou services publient des requêtes vers un broker ; PicoClaw s'abonne, les traite et publie les réponses en retour. + +## 🚀 Démarrage rapide + +**1. Ajouter le canal dans `~/.picoclaw/config.json` :** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "tcp://localhost:1883", + "agent_id": "assistant" + } + } + } +} +``` + +**2. Démarrer la passerelle :** + +```bash +picoclaw gateway +``` + +**3. Envoyer un message depuis n'importe quel client MQTT :** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Quel est l'\''usage CPU ?"}' +``` + +**4. S'abonner pour recevoir la réponse :** + +```bash +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +--- + +## 📨 Structure des topics + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +| Segment | Description | +|---------|-------------| +| `prefix` | Préfixe de topic configuré côté serveur. Défaut : `/picoclaw` | +| `agent_id` | Identifiant de l'instance PicoClaw, défini dans le champ `agent_id` | +| `client_id` | Identifiant de session défini par le client — utiliser un ID stable par appareil pour maintenir le contexte | + +### Payload du message (JSON) + +```json +{ "text": "votre message ici" } +``` + +--- + +## ⚙️ Configuration + +### config.json + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://votre-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "client_id": "", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +### .security.yml (identifiants) + +Le nom d'utilisateur et le mot de passe sont stockés dans `~/.picoclaw/.security.yml`, pas dans `config.json` : + +```yaml +channel_list: + mqtt: + settings: + username: votre_utilisateur + password: votre_mot_de_passe +``` + +### Champs de configuration + +| Champ | Emplacement | Requis | Défaut | Description | +|-------|-------------|--------|--------|-------------| +| `broker` | `settings` | Oui | — | URL du broker MQTT, ex. `tcp://host:1883`, `ssl://host:8883` | +| `agent_id` | `settings` | Oui | — | Identifiant de l'agent, utilisé dans le chemin du topic | +| `topic_prefix` | `settings` | Non | `/picoclaw` | Préfixe de l'espace de noms des topics | +| `username` | `.security.yml` | Non | — | Nom d'utilisateur pour l'authentification au broker | +| `password` | `.security.yml` | Non | — | Mot de passe pour l'authentification au broker | +| `client_id` | `settings` | Non | auto-généré | ID client paho envoyé au broker. Auto-généré sous la forme `picoclaw-mqtt-{agent_id}-{8 hex}` ; fixe pour la durée du processus, réutilisé à la reconnexion | +| `keep_alive` | `settings` | Non | `60` | Intervalle keepalive MQTT en secondes | +| `qos` | `settings` | Non | `0` | Niveau QoS pour la publication et l'abonnement : `0`, `1` ou `2` | + +### Variables d'environnement + +| Variable | Champ | +|----------|-------| +| `PICOCLAW_CHANNELS_MQTT_BROKER` | `broker` | +| `PICOCLAW_CHANNELS_MQTT_AGENT_ID` | `agent_id` | +| `PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX` | `topic_prefix` | +| `PICOCLAW_CHANNELS_MQTT_USERNAME` | `username` | +| `PICOCLAW_CHANNELS_MQTT_PASSWORD` | `password` | +| `PICOCLAW_CHANNELS_MQTT_CLIENT_ID` | `client_id` | +| `PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE` | `keep_alive` | +| `PICOCLAW_CHANNELS_MQTT_QOS` | `qos` | + +--- + +## 🔄 Reconnexion + +PicoClaw se reconnecte automatiquement au broker en cas de perte de connexion, avec un intervalle de 5 secondes. L'abonnement est rétabli automatiquement. L'ID client côté broker reste identique à chaque reconnexion. + +--- + +## ⚠️ Remarques + +- **TLS** : SSL/TLS est supporté (URL broker en `ssl://`). La vérification du certificat est désactivée par défaut. +- **Réponses en streaming** : Les réponses en streaming envoient plusieurs messages vers le topic de réponse ; les concaténer dans l'ordre pour obtenir la réponse complète. +- **client_id vs ID de session** : Le `client_id` dans le chemin du topic est défini par votre application cliente. Il est distinct de l'ID client paho utilisé par PicoClaw pour se connecter au broker. +- **Instances multiples** : Si plusieurs instances PicoClaw utilisent le même `agent_id` sur le même broker, définir des `client_id` distincts pour éviter les conflits. diff --git a/docs/channels/mqtt/README.ja.md b/docs/channels/mqtt/README.ja.md new file mode 100644 index 000000000..80ccafdc5 --- /dev/null +++ b/docs/channels/mqtt/README.ja.md @@ -0,0 +1,140 @@ +# 📡 MQTT チャンネル + +PicoClaw は任意の MQTT クライアントをメッセージチャンネルとして使用できます。デバイスやサービスがブローカーにリクエストをパブリッシュし、PicoClaw がサブスクライブして処理し、レスポンスをパブリッシュして返します。 + +## 🚀 クイックスタート + +**1. `~/.picoclaw/config.json` にチャンネルを追加：** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "tcp://localhost:1883", + "agent_id": "assistant" + } + } + } +} +``` + +**2. ゲートウェイを起動：** + +```bash +picoclaw gateway +``` + +**3. 任意の MQTT クライアントからメッセージを送信：** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "CPU使用率を確認してください"}' +``` + +**4. レスポンスを受信するためにサブスクライブ：** + +```bash +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +--- + +## 📨 トピック構造 + +``` +{prefix}/{agent_id}/{client_id}/request # クライアント → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → クライアント +``` + +| セグメント | 説明 | +|-----------|------| +| `prefix` | トピックのプレフィックス。サーバー側で設定。デフォルト：`/picoclaw` | +| `agent_id` | PicoClaw インスタンスの識別子。`agent_id` フィールドに設定 | +| `client_id` | クライアントが定義するセッション識別子。デバイスごとに同一の ID を使用するとコンテキストが維持される | + +### メッセージペイロード（JSON） + +```json +{ "text": "メッセージ内容" } +``` + +--- + +## ⚙️ 設定 + +### config.json + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "client_id": "", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +### .security.yml（認証情報） + +ユーザー名とパスワードは `config.json` ではなく `~/.picoclaw/.security.yml` に保存します： + +```yaml +channel_list: + mqtt: + settings: + username: your_username + password: your_password +``` + +### 設定フィールド + +| フィールド | 場所 | 必須 | デフォルト | 説明 | +|-----------|------|------|-----------|------| +| `broker` | `settings` | はい | — | MQTT ブローカー URL。例：`tcp://host:1883`、`ssl://host:8883` | +| `agent_id` | `settings` | はい | — | エージェント識別子。トピックパスの一部として使用される | +| `topic_prefix` | `settings` | いいえ | `/picoclaw` | トピックの名前空間プレフィックス | +| `username` | `.security.yml` | いいえ | — | ブローカー認証のユーザー名 | +| `password` | `.security.yml` | いいえ | — | ブローカー認証のパスワード | +| `client_id` | `settings` | いいえ | 自動生成 | ブローカーに送信する paho クライアント ID。未設定の場合 `picoclaw-mqtt-{agent_id}-{8桁hex}` で自動生成。プロセスの生存期間中は固定され、再接続時も同じ ID を使用 | +| `keep_alive` | `settings` | いいえ | `60` | MQTT キープアライブ間隔（秒） | +| `qos` | `settings` | いいえ | `0` | パブリッシュおよびサブスクライブの QoS レベル：`0`、`1`、`2` | + +### 環境変数 + +| 変数 | フィールド | +|------|----------| +| `PICOCLAW_CHANNELS_MQTT_BROKER` | `broker` | +| `PICOCLAW_CHANNELS_MQTT_AGENT_ID` | `agent_id` | +| `PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX` | `topic_prefix` | +| `PICOCLAW_CHANNELS_MQTT_USERNAME` | `username` | +| `PICOCLAW_CHANNELS_MQTT_PASSWORD` | `password` | +| `PICOCLAW_CHANNELS_MQTT_CLIENT_ID` | `client_id` | +| `PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE` | `keep_alive` | +| `PICOCLAW_CHANNELS_MQTT_QOS` | `qos` | + +--- + +## 🔄 再接続 + +接続が切断された場合、PicoClaw は 5 秒間隔で自動的にブローカーに再接続します。再接続後はサブスクリプションも自動的に再確立されます。再接続時はブローカー側のクライアント ID が同一に保たれるため、ブローカーは同じセッションとして認識します。 + +--- + +## ⚠️ 注意事項 + +- **TLS**：SSL/TLS をサポートしています（ブローカー URL に `ssl://` を使用）。デフォルトでは証明書検証をスキップします。 +- **ストリーミングレスポンス**：ストリーミング出力時はレスポンストピックに複数のメッセージが送信されます。順番に結合すると完全なレスポンスになります。 +- **client_id とセッション ID の違い**：トピックパスの `client_id` はクライアントアプリケーションが設定するセッション識別子です。PicoClaw がブローカーへの接続に使用する paho クライアント ID とは別の概念です。 +- **複数インスタンス**：同じ `agent_id` で複数の PicoClaw インスタンスを同一ブローカーに接続する場合、ブローカーレベルの競合を避けるために各インスタンスに異なる `client_id` を設定してください。 diff --git a/docs/channels/mqtt/README.md b/docs/channels/mqtt/README.md new file mode 100644 index 000000000..c894d77f7 --- /dev/null +++ b/docs/channels/mqtt/README.md @@ -0,0 +1,142 @@ +# 📡 MQTT Channel + +PicoClaw supports any MQTT client as a chat channel. Devices or services publish requests to a broker; PicoClaw subscribes, processes them, and publishes responses back. + +## 🚀 Quick Start + +**1. Add the channel to `~/.picoclaw/config.json`:** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "tcp://localhost:1883", + "agent_id": "assistant" + } + } + } +} +``` + +**2. Start the gateway:** + +```bash +picoclaw gateway +``` + +**3. Send a message from any MQTT client:** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "What is the CPU usage?"}' +``` + +**4. Subscribe to receive the response:** + +```bash +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +--- + +## 📨 Topic Structure + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +| Segment | Description | +|---------|-------------| +| `prefix` | Topic prefix, configured server-side. Default: `/picoclaw` | +| `agent_id` | PicoClaw instance identifier, set in `agent_id` config field | +| `client_id` | Client-defined session identifier — use a stable ID per device to maintain conversation context | + +### Message Payload (JSON) + +```json +{ "text": "your message here" } +``` + +--- + +## ⚙️ Configuration + +### config.json + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "client_id": "", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +### .security.yml (credentials) + +Username and password are stored in `~/.picoclaw/.security.yml`, not in `config.json`: + +```yaml +channel_list: + mqtt: + settings: + username: your_username + password: your_password +``` + +### Configuration Fields + +| Field | Location | Required | Default | Description | +|-------|----------|----------|---------|-------------| +| `broker` | `settings` | Yes | — | MQTT broker URL, e.g. `tcp://host:1883`, `ssl://host:8883` | +| `agent_id` | `settings` | Yes | — | Agent identifier, used as part of the topic path | +| `topic_prefix` | `settings` | No | `/picoclaw` | Topic namespace prefix | +| `username` | `.security.yml` | No | — | Broker authentication username | +| `password` | `.security.yml` | No | — | Broker authentication password | +| `client_id` | `settings` | No | auto-generated | Paho client ID sent to the broker. Auto-generated as `picoclaw-mqtt-{agent_id}-{8-char hex}` if not set; stays fixed for the process lifetime so reconnects reuse the same ID | +| `keep_alive` | `settings` | No | `60` | MQTT keepalive interval in seconds | +| `qos` | `settings` | No | `0` | QoS level for publish and subscribe: `0`, `1`, or `2` | + +### Environment Variables + +All fields can be set via environment variables: + +| Variable | Field | +|----------|-------| +| `PICOCLAW_CHANNELS_MQTT_BROKER` | `broker` | +| `PICOCLAW_CHANNELS_MQTT_AGENT_ID` | `agent_id` | +| `PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX` | `topic_prefix` | +| `PICOCLAW_CHANNELS_MQTT_USERNAME` | `username` | +| `PICOCLAW_CHANNELS_MQTT_PASSWORD` | `password` | +| `PICOCLAW_CHANNELS_MQTT_CLIENT_ID` | `client_id` | +| `PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE` | `keep_alive` | +| `PICOCLAW_CHANNELS_MQTT_QOS` | `qos` | + +--- + +## 🔄 Reconnection + +PicoClaw automatically reconnects to the broker if the connection is lost, with a 5-second retry interval. On reconnect, the subscription is re-established automatically. The broker-side client ID stays the same across reconnects so the broker correctly identifies it as the same session. + +--- + +## ⚠️ Notes + +- **TLS**: SSL/TLS is supported (`ssl://` broker URL). Certificate verification is skipped by default. +- **Streaming**: Streaming responses send multiple messages to the response topic; concatenate them in order. +- **client_id vs session ID**: The `client_id` in the topic path is set by your client application and identifies the conversation session. It is separate from the broker-level client ID used by PicoClaw's paho connection. +- **Multiple instances**: If you run multiple PicoClaw instances against the same broker with the same `agent_id`, set distinct `client_id` values to avoid broker-level conflicts. diff --git a/docs/channels/mqtt/README.pt-br.md b/docs/channels/mqtt/README.pt-br.md new file mode 100644 index 000000000..da95b6ba6 --- /dev/null +++ b/docs/channels/mqtt/README.pt-br.md @@ -0,0 +1,140 @@ +# 📡 Canal MQTT + +O PicoClaw suporta qualquer cliente MQTT como canal de mensagens. Dispositivos ou serviços publicam requisições para um broker; o PicoClaw assina, processa e publica as respostas de volta. + +## 🚀 Início rápido + +**1. Adicione o canal ao `~/.picoclaw/config.json`:** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "tcp://localhost:1883", + "agent_id": "assistant" + } + } + } +} +``` + +**2. Inicie o gateway:** + +```bash +picoclaw gateway +``` + +**3. Envie uma mensagem de qualquer cliente MQTT:** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Qual é o uso de CPU?"}' +``` + +**4. Assine para receber a resposta:** + +```bash +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +--- + +## 📨 Estrutura de tópicos + +``` +{prefix}/{agent_id}/{client_id}/request # Cliente → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Cliente +``` + +| Segmento | Descrição | +|----------|-----------| +| `prefix` | Prefixo do tópico configurado no servidor. Padrão: `/picoclaw` | +| `agent_id` | Identificador da instância do PicoClaw, definido no campo `agent_id` | +| `client_id` | Identificador de sessão definido pelo cliente — use um ID estável por dispositivo para manter o contexto da conversa | + +### Payload da mensagem (JSON) + +```json +{ "text": "sua mensagem aqui" } +``` + +--- + +## ⚙️ Configuração + +### config.json + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://seu-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "client_id": "", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +### .security.yml (credenciais) + +O nome de usuário e a senha são armazenados em `~/.picoclaw/.security.yml`, não no `config.json`: + +```yaml +channel_list: + mqtt: + settings: + username: seu_usuario + password: sua_senha +``` + +### Campos de configuração + +| Campo | Local | Obrigatório | Padrão | Descrição | +|-------|-------|-------------|--------|-----------| +| `broker` | `settings` | Sim | — | URL do broker MQTT, ex. `tcp://host:1883`, `ssl://host:8883` | +| `agent_id` | `settings` | Sim | — | Identificador do agente, usado como parte do caminho do tópico | +| `topic_prefix` | `settings` | Não | `/picoclaw` | Prefixo do namespace dos tópicos | +| `username` | `.security.yml` | Não | — | Nome de usuário para autenticação no broker | +| `password` | `.security.yml` | Não | — | Senha para autenticação no broker | +| `client_id` | `settings` | Não | gerado automaticamente | ID de cliente paho enviado ao broker. Gerado automaticamente como `picoclaw-mqtt-{agent_id}-{8 hex}` se não definido; fixo durante o tempo de vida do processo e reutilizado nas reconexões | +| `keep_alive` | `settings` | Não | `60` | Intervalo de keepalive MQTT em segundos | +| `qos` | `settings` | Não | `0` | Nível de QoS para publicação e assinatura: `0`, `1` ou `2` | + +### Variáveis de ambiente + +| Variável | Campo | +|----------|-------| +| `PICOCLAW_CHANNELS_MQTT_BROKER` | `broker` | +| `PICOCLAW_CHANNELS_MQTT_AGENT_ID` | `agent_id` | +| `PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX` | `topic_prefix` | +| `PICOCLAW_CHANNELS_MQTT_USERNAME` | `username` | +| `PICOCLAW_CHANNELS_MQTT_PASSWORD` | `password` | +| `PICOCLAW_CHANNELS_MQTT_CLIENT_ID` | `client_id` | +| `PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE` | `keep_alive` | +| `PICOCLAW_CHANNELS_MQTT_QOS` | `qos` | + +--- + +## 🔄 Reconexão + +O PicoClaw reconecta automaticamente ao broker se a conexão for perdida, com intervalo de 5 segundos. Após a reconexão, a assinatura é restabelecida automaticamente. O ID de cliente no broker permanece o mesmo nas reconexões, permitindo que o broker identifique corretamente a mesma sessão. + +--- + +## ⚠️ Observações + +- **TLS**: SSL/TLS é suportado (URL do broker com `ssl://`). A verificação de certificado é ignorada por padrão. +- **Respostas em streaming**: Respostas em streaming enviam múltiplas mensagens para o tópico de resposta; concatene-as na ordem recebida para obter a resposta completa. +- **client_id vs ID de sessão**: O `client_id` no caminho do tópico é definido pela sua aplicação cliente e identifica a sessão. É separado do ID de cliente paho usado pelo PicoClaw para se conectar ao broker. +- **Múltiplas instâncias**: Se várias instâncias do PicoClaw usarem o mesmo `agent_id` no mesmo broker, defina `client_id` distintos para evitar conflitos no nível do broker. diff --git a/docs/channels/mqtt/README.vi.md b/docs/channels/mqtt/README.vi.md new file mode 100644 index 000000000..f680c78bb --- /dev/null +++ b/docs/channels/mqtt/README.vi.md @@ -0,0 +1,140 @@ +# 📡 Kênh MQTT + +PicoClaw hỗ trợ bất kỳ client MQTT nào làm kênh nhắn tin. Thiết bị hoặc dịch vụ publish yêu cầu lên broker; PicoClaw subscribe, xử lý và publish phản hồi trở lại. + +## 🚀 Bắt đầu nhanh + +**1. Thêm kênh vào `~/.picoclaw/config.json`:** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "tcp://localhost:1883", + "agent_id": "assistant" + } + } + } +} +``` + +**2. Khởi động gateway:** + +```bash +picoclaw gateway +``` + +**3. Gửi tin nhắn từ bất kỳ client MQTT nào:** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "CPU đang dùng bao nhiêu phần trăm?"}' +``` + +**4. Subscribe để nhận phản hồi:** + +```bash +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +--- + +## 📨 Cấu trúc topic + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +| Phân đoạn | Mô tả | +|-----------|-------| +| `prefix` | Tiền tố topic, cấu hình phía server. Mặc định: `/picoclaw` | +| `agent_id` | Định danh instance PicoClaw, đặt trong trường `agent_id` | +| `client_id` | Định danh phiên do client xác định — dùng ID ổn định cho mỗi thiết bị để duy trì ngữ cảnh hội thoại | + +### Payload tin nhắn (JSON) + +```json +{ "text": "nội dung tin nhắn" } +``` + +--- + +## ⚙️ Cấu hình + +### config.json + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "client_id": "", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +### .security.yml (thông tin xác thực) + +Tên người dùng và mật khẩu được lưu trong `~/.picoclaw/.security.yml`, không phải trong `config.json`: + +```yaml +channel_list: + mqtt: + settings: + username: ten_nguoi_dung + password: mat_khau +``` + +### Các trường cấu hình + +| Trường | Vị trí | Bắt buộc | Mặc định | Mô tả | +|--------|--------|----------|----------|-------| +| `broker` | `settings` | Có | — | URL của MQTT broker, ví dụ `tcp://host:1883`, `ssl://host:8883` | +| `agent_id` | `settings` | Có | — | Định danh agent, dùng làm một phần của đường dẫn topic | +| `topic_prefix` | `settings` | Không | `/picoclaw` | Tiền tố không gian tên topic | +| `username` | `.security.yml` | Không | — | Tên người dùng xác thực với broker | +| `password` | `.security.yml` | Không | — | Mật khẩu xác thực với broker | +| `client_id` | `settings` | Không | tự động tạo | Client ID paho gửi đến broker. Tự động tạo dạng `picoclaw-mqtt-{agent_id}-{8 hex}` nếu không đặt; cố định trong suốt vòng đời tiến trình, tái sử dụng khi kết nối lại | +| `keep_alive` | `settings` | Không | `60` | Khoảng thời gian keepalive MQTT (giây) | +| `qos` | `settings` | Không | `0` | Mức QoS cho publish và subscribe: `0`, `1` hoặc `2` | + +### Biến môi trường + +| Biến | Trường | +|------|--------| +| `PICOCLAW_CHANNELS_MQTT_BROKER` | `broker` | +| `PICOCLAW_CHANNELS_MQTT_AGENT_ID` | `agent_id` | +| `PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX` | `topic_prefix` | +| `PICOCLAW_CHANNELS_MQTT_USERNAME` | `username` | +| `PICOCLAW_CHANNELS_MQTT_PASSWORD` | `password` | +| `PICOCLAW_CHANNELS_MQTT_CLIENT_ID` | `client_id` | +| `PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE` | `keep_alive` | +| `PICOCLAW_CHANNELS_MQTT_QOS` | `qos` | + +--- + +## 🔄 Kết nối lại + +PicoClaw tự động kết nối lại với broker nếu mất kết nối, với khoảng thời gian thử lại 5 giây. Sau khi kết nối lại, subscription được tái thiết lập tự động. Client ID phía broker giữ nguyên qua các lần kết nối lại, giúp broker nhận diện chính xác cùng một phiên. + +--- + +## ⚠️ Lưu ý + +- **TLS**: Hỗ trợ SSL/TLS (URL broker dùng `ssl://`). Mặc định bỏ qua xác minh chứng chỉ. +- **Phản hồi streaming**: Phản hồi streaming gửi nhiều tin nhắn đến topic response; ghép nối chúng theo thứ tự để có phản hồi đầy đủ. +- **client_id và ID phiên**: `client_id` trong đường dẫn topic được đặt bởi ứng dụng client của bạn và xác định phiên hội thoại. Nó khác với client ID paho mà PicoClaw dùng để kết nối broker. +- **Nhiều instance**: Nếu nhiều instance PicoClaw dùng cùng `agent_id` trên cùng broker, hãy đặt `client_id` riêng biệt cho từng instance để tránh xung đột ở tầng broker. diff --git a/docs/channels/mqtt/README.zh.md b/docs/channels/mqtt/README.zh.md new file mode 100644 index 000000000..e7e529cde --- /dev/null +++ b/docs/channels/mqtt/README.zh.md @@ -0,0 +1,142 @@ +# 📡 MQTT 渠道 + +PicoClaw 支持将任意 MQTT 客户端作为消息渠道。设备或服务向 Broker 发布请求，PicoClaw 订阅后处理并将响应发布回去。 + +## 🚀 快速开始 + +**1. 在 `~/.picoclaw/config.json` 中添加渠道：** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "tcp://localhost:1883", + "agent_id": "assistant" + } + } + } +} +``` + +**2. 启动网关：** + +```bash +picoclaw gateway +``` + +**3. 用任意 MQTT 客户端发送消息：** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "查一下CPU使用率"}' +``` + +**4. 订阅响应：** + +```bash +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +--- + +## 📨 Topic 结构 + +``` +{prefix}/{agent_id}/{client_id}/request # 客户端 → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → 客户端 +``` + +| 段 | 说明 | +|----|------| +| `prefix` | Topic 前缀，由服务端配置，默认 `/picoclaw` | +| `agent_id` | PicoClaw 实例标识，对应配置中的 `agent_id` 字段 | +| `client_id` | 客户端自定义会话标识——同一设备保持相同 ID 可维持上下文连续性 | + +### 消息体（JSON） + +```json +{ "text": "你的消息内容" } +``` + +--- + +## ⚙️ 配置说明 + +### config.json + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "client_id": "", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +### .security.yml（用户名和密码） + +用户名和密码存储于 `~/.picoclaw/.security.yml`，不写入 `config.json`： + +```yaml +channel_list: + mqtt: + settings: + username: your_username + password: your_password +``` + +### 字段说明 + +| 字段 | 位置 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `broker` | `settings` | 是 | — | MQTT Broker 地址，如 `tcp://host:1883`、`ssl://host:8883` | +| `agent_id` | `settings` | 是 | — | Agent 标识，作为 topic 路径的一部分 | +| `topic_prefix` | `settings` | 否 | `/picoclaw` | Topic 命名空间前缀 | +| `username` | `.security.yml` | 否 | — | Broker 认证用户名 | +| `password` | `.security.yml` | 否 | — | Broker 认证密码 | +| `client_id` | `settings` | 否 | 自动生成 | 发送给 Broker 的 paho 客户端 ID。未配置时自动生成为 `picoclaw-mqtt-{agent_id}-{8位hex}`，进程生命周期内固定不变，断线重连时复用同一 ID | +| `keep_alive` | `settings` | 否 | `60` | MQTT 心跳间隔（秒） | +| `qos` | `settings` | 否 | `0` | 发布和订阅的 QoS 级别：`0`、`1` 或 `2` | + +### 环境变量 + +所有字段均可通过环境变量配置： + +| 环境变量 | 对应字段 | +|----------|----------| +| `PICOCLAW_CHANNELS_MQTT_BROKER` | `broker` | +| `PICOCLAW_CHANNELS_MQTT_AGENT_ID` | `agent_id` | +| `PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX` | `topic_prefix` | +| `PICOCLAW_CHANNELS_MQTT_USERNAME` | `username` | +| `PICOCLAW_CHANNELS_MQTT_PASSWORD` | `password` | +| `PICOCLAW_CHANNELS_MQTT_CLIENT_ID` | `client_id` | +| `PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE` | `keep_alive` | +| `PICOCLAW_CHANNELS_MQTT_QOS` | `qos` | + +--- + +## 🔄 断线重连 + +连接断开后 PicoClaw 会自动以 5 秒间隔重连 Broker，重连成功后自动重新订阅。断线重连时复用相同的 Broker 客户端 ID，Broker 能正确识别为同一连接。 + +--- + +## ⚠️ 注意事项 + +- **TLS**：支持 SSL/TLS（Broker 地址使用 `ssl://`），默认跳过证书验证。 +- **流式响应**：流式输出时会向 response topic 发送多条消息，客户端按顺序拼接即为完整回复。 +- **client_id 与会话 ID 的区别**：topic 路径中的 `client_id` 由客户端应用自行设置，用于区分会话；它与 PicoClaw paho 连接 Broker 时使用的客户端 ID 是两个独立的概念。 +- **多实例部署**：若多个 PicoClaw 实例使用相同 `agent_id` 连接同一 Broker，需为每个实例配置不同的 `client_id` 以避免 Broker 层面的冲突。 diff --git a/docs/channels/telegram/README.md b/docs/channels/telegram/README.md index a4138009e..215d80afe 100644 --- a/docs/channels/telegram/README.md +++ b/docs/channels/telegram/README.md @@ -15,7 +15,8 @@ The Telegram channel uses long polling via the Telegram Bot API for bot-based co "token": "123456789:ABCdefGHIjklMNOpqrsTUVwxyz", "allow_from": ["123456789"], "proxy": "", - "use_markdown_v2": false + "use_markdown_v2": false, + "media_group_delay_ms": 500 } } } @@ -28,6 +29,7 @@ The Telegram channel uses long polling via the Telegram Bot API for bot-based co | allow_from | array | No | Allowlist of user IDs; empty means all users are allowed | | proxy | string | No | Proxy URL for connecting to the Telegram API (e.g. http://127.0.0.1:7890) | | use_markdown_v2 | bool | No | Enable Telegram MarkdownV2 formatting | +| media_group_delay_ms | int | No | Idle delay before processing Telegram media groups/albums. Defaults to 500 ms | ## Setup diff --git a/docs/design/hook-system-design.zh.md b/docs/design/hook-system-design.zh.md index ab5566bec..090437c20 100644 --- a/docs/design/hook-system-design.zh.md +++ b/docs/design/hook-system-design.zh.md @@ -1,11 +1,15 @@ # PicoClaw Hook 系统设计（基于 `refactor/agent`） +> 当前状态：本文是 hook 系统的早期设计记录。事件系统升级后，观察型 hook 的主路径已经切到 +> `pkg/events.Event`、`RuntimeEventObserver` 和进程 hook 的 `hook.runtime_event`。 +> 旧 `agent.Event`、`EventKind`、`hook.event` 兼容层已经删除。 + ## 背景 本设计围绕两个议题展开： - `#1316`：把 agent loop 重构为事件驱动、可中断、可追加、可观测 -- `#1796`：在 EventBus 稳定后，把 hooks 设计为 EventBus 的 consumer，而不是重新发明一套事件模型 +- `#1796`：在 runtime event bus 稳定后，把 hooks 设计为事件 consumer，而不是重新发明一套事件模型 当前分支已经完成了第一步里的“事件系统基础”，但还没有真正的 hook 挂载层。因此这里的目标不是重新设计 event，而是在已有实现上补出一层可扩展、可拦截、可外挂的 HookManager。 @@ -52,20 +56,18 @@ pi-mono 的核心思路更接近当前分支： 当前分支已经具备 hook 系统的地基： -- `pkg/agent/events.go` 定义了稳定的 `EventKind`、`EventMeta` 和 payload -- `pkg/agent/eventbus.go` 提供了非阻塞 fan-out 的 `EventBus` +- `pkg/events` 定义 runtime event envelope、kind、scope、source、severity 和 fan-out bus +- `pkg/agent/event_payloads.go` 保留 agent domain payload +- agent domain payload 保留在 `pkg/agent/event_payloads.go` - `pkg/agent/loop.go` 中的 `runTurn()` 已在 turn、llm、tool、interrupt、follow-up、summary 等节点发射事件 - `pkg/agent/steering.go` 已支持 steering、graceful interrupt、hard abort - `pkg/agent/turn.go` 已维护 turn phase、恢复点、active turn、abort 状态 ### 现有缺口 -当前分支还缺四件事： - -- 没有 HookManager，只有 EventBus -- 没有 Before/After LLM、Before/After Tool 这种同步拦截点 -- 没有审批型 hook -- 子 agent 仍走 `pkg/tools/SubagentManager + RunToolLoop`，没有接入 `pkg/agent` 的 turn tree 和事件流 +早期设计时的缺口包括 HookManager、Before/After LLM、Before/After Tool、审批型 hook +以及 sub-turn 接入。当前实现已经覆盖主 turn 的 HookManager、LLM/Tool 拦截和审批； +sub-turn 事件已接入 runtime event bus。 ### 一个关键现实 @@ -73,19 +75,19 @@ pi-mono 的核心思路更接近当前分支： ## 设计原则 -- Hook 必须建立在 `pkg/agent` 的 EventBus 和 turn 上下文之上 -- EventBus 负责广播，HookManager 负责拦截，两者职责分离 +- Hook 必须建立在 `pkg/events` runtime event bus 和 turn 上下文之上 +- runtime event bus 负责广播，HookManager 负责拦截，两者职责分离 - 项目内挂载要简单，项目外挂载必须走 IPC - 观察型 hook 不能阻塞 loop；拦截型 hook 必须有超时 - 先覆盖主 turn，不把 sub-turn 一次做满 -- 不新增第二套用户事件命名系统，优先复用 `EventKind.String()` +- 不新增第二套用户事件命名系统，新观察点统一使用 `pkg/events.Kind` ## 总体架构 分成三层： -1. `EventBus` - 负责广播只读事件，现有实现直接复用 +1. `pkg/events` runtime event bus + 负责广播只读事件，覆盖 agent、channel、gateway、bus、MCP 等运行时组件 2. `HookManager` 负责管理 hook、排序、超时、错误隔离，并在 `runTurn()` 的明确检查点执行同步拦截 @@ -97,7 +99,7 @@ pi-mono 的核心思路更接近当前分支： 换句话说： -- EventBus 是“发生了什么” +- runtime event bus 是“发生了什么” - HookManager 是“谁能介入” - HookMount 是“这些 hook 从哪里来” @@ -113,11 +115,11 @@ pi-mono 的核心思路更接近当前分支： ```go type EventObserver interface { - OnEvent(ctx context.Context, evt agent.Event) error + OnRuntimeEvent(ctx context.Context, evt events.Event) error } ``` -这类 hook 直接订阅 EventBus 即可。 +这类 hook 直接订阅 runtime event bus 即可。 适用场景： @@ -156,7 +158,7 @@ type ToolApprover interface { ## 对外暴露的最小 hook 面 -V1 不需要把所有 EventKind 都变成可拦截点。 +V1 不需要把所有 runtime event kind 都变成可拦截点。 建议只开放这些同步 hook： @@ -168,19 +170,19 @@ V1 不需要把所有 EventKind 都变成可拦截点。 其余节点继续作为只读事件暴露： -- `turn_start` -- `turn_end` -- `llm_request` -- `llm_response` -- `tool_exec_start` -- `tool_exec_end` -- `tool_exec_skipped` -- `steering_injected` -- `follow_up_queued` -- `interrupt_received` -- `context_compress` -- `session_summarize` -- `error` +- `agent.turn.start` +- `agent.turn.end` +- `agent.llm.request` +- `agent.llm.response` +- `agent.tool.exec_start` +- `agent.tool.exec_end` +- `agent.tool.exec_skipped` +- `agent.steering.injected` +- `agent.follow_up.queued` +- `agent.interrupt.received` +- `agent.context.compress` +- `agent.session.summarize` +- `agent.error` `subturn_*` 在 V1 中保留名字，但不承诺一定触发，直到子 turn 迁移完成。 @@ -369,7 +371,7 @@ PicoClaw 启动外部进程，并在其 stdin/stdout 上跑协议。 ### 观察链路 ```text -runTurn() -> emitEvent() -> EventBus -> observers +runTurn() -> emitEvent() -> runtime event bus -> observers ``` ### 拦截链路 @@ -453,7 +455,7 @@ V1 不做复杂自动发现。 ### Phase 3 - 把 `SubagentManager` 迁移到 `runTurn/sub-turn` -- 接通 `subturn_spawn` / `subturn_end` / `subturn_result_delivered` +- 接通 `agent.subturn.spawn` / `agent.subturn.end` / `agent.subturn.result_delivered` ### Phase 4 @@ -464,13 +466,13 @@ V1 不做复杂自动发现。 最适合 PicoClaw 当前分支的方案，不是直接复制 OpenClaw 的 hooks，也不是完整照搬 pi-mono 的 extension system，而是： -- 以现有 `EventBus` 为只读观察面 +- 以 `pkg/events` runtime event bus 为只读观察面 - 以新增 `HookManager` 为同步拦截面 - 项目内通过 Go 对象直接挂载 - 项目外通过 `stdio JSON-RPC` 进程通信挂载 这样做有三个好处： -- 和 `#1796` 一致，hooks 只是 EventBus 之上的消费层 +- 和 `#1796` 一致，hooks 只是 runtime event bus 之上的消费层 - 和当前 `refactor/agent` 实现一致，不需要推翻已有事件系统 - 同时满足“仓内简单挂载”和“仓外进程通信挂载”两个硬需求 diff --git a/docs/guides/chat-apps.fr.md b/docs/guides/chat-apps.fr.md index d9112c595..a03141e5e 100644 --- a/docs/guides/chat-apps.fr.md +++ b/docs/guides/chat-apps.fr.md @@ -4,7 +4,7 @@ ## 💬 Applications de Chat -Communiquez avec votre PicoClaw via Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot ou MaixCam. +Communiquez avec votre PicoClaw via Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MQTT ou MaixCam. > **Note** : Tous les canaux basés sur les webhooks (LINE, WeCom, etc.) sont servis sur un seul serveur HTTP Gateway partagé (`gateway.host`:`gateway.port`, par défaut `127.0.0.1:18790`). Il n'y a pas de ports par canal à configurer. Note : Feishu utilise le mode WebSocket/SDK et n'utilise pas le serveur HTTP webhook partagé. @@ -23,6 +23,7 @@ Communiquez avec votre PicoClaw via Telegram, Discord, WhatsApp, Matrix, QQ, Din | **Feishu (飞书)** | ⭐⭐⭐ Avancé | Collaboration entreprise, fonctionnalités riches | [Documentation](../channels/feishu/README.fr.md) | | **IRC** | ⭐⭐ Moyen | Serveur + configuration TLS | [Documentation](#irc) | | **OneBot** | ⭐⭐ Moyen | Compatible NapCat/Go-CQHTTP, écosystème communautaire | [Documentation](../channels/onebot/README.fr.md) | +| **MQTT** | ⭐ Facile | N'importe quel client MQTT via broker pub/sub | [Documentation](../channels/mqtt/README.fr.md) | | **MaixCam** | ⭐ Facile | Canal d'intégration matérielle pour caméras AI Sipeed | [Documentation](../channels/maixcam/README.fr.md) | | **Pico** | ⭐ Facile | Canal protocole natif PicoClaw | | @@ -681,3 +682,67 @@ picoclaw gateway ``` + + +
+MQTT + +N'importe quel client MQTT peut communiquer avec PicoClaw via un broker. Les appareils ou services publient des requêtes vers le broker ; PicoClaw s'abonne, les traite et publie les réponses en retour. + +**1. Configurer** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://votre-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +Nom d'utilisateur et mot de passe dans `~/.picoclaw/.security.yml` : + +```yaml +channel_list: + mqtt: + settings: + username: votre_utilisateur + password: votre_mot_de_passe +``` + +**Format des topics** + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +Le `client_id` est défini par votre application cliente pour identifier les appareils ou sessions. + +**2. Lancer** + +```bash +picoclaw gateway +``` + +**3. Tester** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Bonjour"}' + +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +Pour les options complètes, voir [Documentation du canal MQTT](../channels/mqtt/README.fr.md). + +
diff --git a/docs/guides/chat-apps.ja.md b/docs/guides/chat-apps.ja.md index 49c41a66e..cc9671bd5 100644 --- a/docs/guides/chat-apps.ja.md +++ b/docs/guides/chat-apps.ja.md @@ -25,6 +25,7 @@ PicoClaw は複数のチャットプラットフォームをサポートして | **Feishu (飛書)** | ⭐⭐⭐ やや難 | エンタープライズコラボレーション、機能豊富 | [ドキュメント](../channels/feishu/README.ja.md) | | **IRC** | ⭐⭐ 中程度 | サーバー + TLS 設定 | [ドキュメント](#irc) | | **OneBot** | ⭐⭐ 中程度 | NapCat/Go-CQHTTP 互換、コミュニティエコシステム充実 | [ドキュメント](../channels/onebot/README.ja.md) | +| **MQTT** | ⭐ 簡単 | ブローカー経由で任意の MQTT クライアントと通信 | [ドキュメント](../channels/mqtt/README.ja.md) | | **MaixCam** | ⭐ 簡単 | Sipeed AI カメラハードウェア統合チャネル | [ドキュメント](../channels/maixcam/README.ja.md) | | **Pico** | ⭐ 簡単 | PicoClaw ネイティブプロトコルチャネル | | @@ -670,3 +671,67 @@ picoclaw gateway ``` + + +
+MQTT + +任意の MQTT クライアントがブローカーを介して PicoClaw と通信できます。デバイスやサービスがブローカーにリクエストをパブリッシュし、PicoClaw がサブスクライブして処理し、レスポンスをパブリッシュして返します。 + +**1. 設定** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +ユーザー名とパスワードは `~/.picoclaw/.security.yml` に記載します： + +```yaml +channel_list: + mqtt: + settings: + username: your_username + password: your_password +``` + +**トピック形式** + +``` +{prefix}/{agent_id}/{client_id}/request # クライアント → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → クライアント +``` + +`client_id` はクライアントアプリケーションがデバイスやセッションを識別するために設定します。 + +**2. 起動** + +```bash +picoclaw gateway +``` + +**3. テスト** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "こんにちは"}' + +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +完全な設定オプションは [MQTT チャンネルドキュメント](../channels/mqtt/README.ja.md) を参照してください。 + +
diff --git a/docs/guides/chat-apps.md b/docs/guides/chat-apps.md index 62418f91a..4fcf12653 100644 --- a/docs/guides/chat-apps.md +++ b/docs/guides/chat-apps.md @@ -4,7 +4,7 @@ ## 💬 Chat Apps -Talk to your picoclaw through Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MaixCam, or Pico (native protocol) +Talk to your picoclaw through Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MQTT, MaixCam, or Pico (native protocol) > **Note**: Channels that rely on HTTP callbacks share a single Gateway HTTP server (`gateway.host`:`gateway.port`, default `127.0.0.1:18790`). Socket/stream-based channels such as Feishu, DingTalk, and WeCom do not rely on the shared webhook server for inbound delivery. @@ -23,6 +23,7 @@ Talk to your picoclaw through Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, | **Feishu (飞书)** | ⭐⭐⭐ Advanced | Enterprise collaboration, feature-rich | [Docs](../channels/feishu/README.md) | | **IRC** | ⭐⭐ Medium | Server + TLS configuration | [Docs](#irc) | | **OneBot** | ⭐⭐ Medium | NapCat/Go-CQHTTP compatible, community ecosystem | [Docs](../channels/onebot/README.md) | +| **MQTT** | ⭐ Easy | Any MQTT client via broker pub/sub | [Docs](../channels/mqtt/README.md) | | **MaixCam** | ⭐ Easy | Hardware integration channel for Sipeed AI cameras | [Docs](../channels/maixcam/README.md) | | **Pico** | ⭐ Easy | Native PicoClaw protocol channel | | @@ -587,3 +588,69 @@ picoclaw gateway ``` + + +
+MQTT + +Any MQTT client can communicate with PicoClaw via a broker. Devices or services publish requests to the broker; PicoClaw subscribes, processes them, and publishes responses back. + +**1. Configure** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +Username and password go in `~/.picoclaw/.security.yml`: + +```yaml +channel_list: + mqtt: + settings: + username: your_username + password: your_password +``` + +**Topic format** + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +`client_id` is set by your client application to identify different devices or sessions. + +**2. Run** + +```bash +picoclaw gateway +``` + +**3. Test** + +```bash +# Send a message +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Hello"}' + +# Subscribe to responses +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +For full configuration options see [MQTT Channel Docs](../channels/mqtt/README.md). + +
diff --git a/docs/guides/chat-apps.ms.md b/docs/guides/chat-apps.ms.md index 6bfa7565e..03e8d36ca 100644 --- a/docs/guides/chat-apps.ms.md +++ b/docs/guides/chat-apps.ms.md @@ -4,7 +4,7 @@ ## 💬 Aplikasi Sembang -Berbual dengan picoclaw anda melalui Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MaixCam, atau Pico (protokol asli) +Berbual dengan picoclaw anda melalui Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MQTT, MaixCam, atau Pico (protokol asli) > **Nota**: Semua saluran berasaskan webhook (LINE, WeCom, dan sebagainya) diservis pada satu pelayan HTTP Gateway yang dikongsi (`gateway.host`:`gateway.port`, lalai `127.0.0.1:18790`). Tiada port khusus per saluran untuk dikonfigurasikan. Nota: Feishu menggunakan mod WebSocket/SDK dan tidak menggunakan pelayan HTTP webhook yang dikongsi. @@ -22,6 +22,7 @@ Berbual dengan picoclaw anda melalui Telegram, Discord, WhatsApp, Matrix, QQ, Di | **Slack** | Sederhana (Bot token + App token) | | **IRC** | Sederhana (pelayan + konfigurasi TLS) | | **OneBot** | Sederhana (QQ melalui protokol OneBot) | +| **MQTT** | Mudah (broker + agent_id) | | **MaixCam** | Mudah (integrasi perkakasan Sipeed) | | **Pico** | Protokol PicoClaw asli | @@ -445,3 +446,67 @@ picoclaw gateway > **Nota**: WeCom AI Bot menggunakan protokol streaming pull — tiada isu timeout balasan. Tugasan panjang (>30 saat) akan bertukar secara automatik kepada penghantaran push `response_url`. + + +
+MQTT + +Mana-mana client MQTT boleh berkomunikasi dengan PicoClaw melalui broker. Peranti atau perkhidmatan menerbitkan permintaan ke broker; PicoClaw melanggan, memproses dan menerbitkan respons kembali. + +**1. Konfigurasi** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +Nama pengguna dan kata laluan dalam `~/.picoclaw/.security.yml`: + +```yaml +channel_list: + mqtt: + settings: + username: nama_pengguna + password: kata_laluan +``` + +**Format topik** + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +`client_id` ditetapkan oleh aplikasi client anda untuk mengenal pasti peranti atau sesi. + +**2. Jalankan** + +```bash +picoclaw gateway +``` + +**3. Uji** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Helo"}' + +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +Untuk semua pilihan konfigurasi, lihat [Dokumentasi Saluran MQTT](../channels/mqtt/README.md). + +
diff --git a/docs/guides/chat-apps.pt-br.md b/docs/guides/chat-apps.pt-br.md index 6d4fbdc23..f6b89ca3b 100644 --- a/docs/guides/chat-apps.pt-br.md +++ b/docs/guides/chat-apps.pt-br.md @@ -4,7 +4,7 @@ ## 💬 Aplicativos de Chat -Converse com seu picoclaw através do Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot ou MaixCam +Converse com seu picoclaw através do Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MQTT ou MaixCam > **Nota**: Todos os canais baseados em webhook (LINE, WeCom, etc.) são servidos em um único servidor HTTP Gateway compartilhado (`gateway.host`:`gateway.port`, padrão `127.0.0.1:18790`). Não há portas por canal para configurar. Nota: Feishu usa o modo WebSocket/SDK e não utiliza o servidor HTTP webhook compartilhado. @@ -23,6 +23,7 @@ Converse com seu picoclaw através do Telegram, Discord, WhatsApp, Matrix, QQ, D | **Feishu (飞书)** | ⭐⭐⭐ Avançado | Colaboração empresarial, rico em recursos | [Documentação](../channels/feishu/README.pt-br.md) | | **IRC** | ⭐⭐ Médio | Servidor + configuração TLS | [Documentação](#irc) | | **OneBot** | ⭐⭐ Médio | Compatível com NapCat/Go-CQHTTP, ecossistema comunitário | [Documentação](../channels/onebot/README.pt-br.md) | +| **MQTT** | ⭐ Fácil | Qualquer cliente MQTT via broker pub/sub | [Documentação](../channels/mqtt/README.pt-br.md) | | **MaixCam** | ⭐ Fácil | Canal de integração de hardware para câmeras AI Sipeed | [Documentação](../channels/maixcam/README.pt-br.md) | | **Pico** | ⭐ Fácil | Canal de protocolo nativo PicoClaw | | @@ -695,3 +696,67 @@ picoclaw gateway ``` + + +
+MQTT + +Qualquer cliente MQTT pode se comunicar com o PicoClaw via broker. Dispositivos ou serviços publicam requisições para o broker; o PicoClaw assina, processa e publica as respostas de volta. + +**1. Configurar** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://seu-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +Nome de usuário e senha em `~/.picoclaw/.security.yml`: + +```yaml +channel_list: + mqtt: + settings: + username: seu_usuario + password: sua_senha +``` + +**Formato dos tópicos** + +``` +{prefix}/{agent_id}/{client_id}/request # Cliente → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Cliente +``` + +O `client_id` é definido pela sua aplicação cliente para identificar dispositivos ou sessões. + +**2. Iniciar** + +```bash +picoclaw gateway +``` + +**3. Testar** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Olá"}' + +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +Para todas as opções de configuração, veja a [Documentação do Canal MQTT](../channels/mqtt/README.pt-br.md). + +
diff --git a/docs/guides/chat-apps.vi.md b/docs/guides/chat-apps.vi.md index 8d0b4ee32..8071c9d3d 100644 --- a/docs/guides/chat-apps.vi.md +++ b/docs/guides/chat-apps.vi.md @@ -4,7 +4,7 @@ ## 💬 Ứng Dụng Chat -Trò chuyện với picoclaw của bạn qua Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot hoặc MaixCam +Trò chuyện với picoclaw của bạn qua Telegram, Discord, WhatsApp, Matrix, QQ, DingTalk, LINE, WeCom, Feishu, Slack, IRC, OneBot, MQTT hoặc MaixCam > **Lưu ý**: Tất cả các kênh dựa trên webhook (LINE, WeCom, v.v.) được phục vụ trên một máy chủ HTTP Gateway chung (`gateway.host`:`gateway.port`, mặc định `127.0.0.1:18790`). Không có port riêng cho từng kênh. Lưu ý: Feishu sử dụng chế độ WebSocket/SDK và không sử dụng máy chủ HTTP webhook chung. @@ -23,6 +23,7 @@ Trò chuyện với picoclaw của bạn qua Telegram, Discord, WhatsApp, Matrix | **Feishu (飞书)** | ⭐⭐⭐ Nâng cao | Cộng tác doanh nghiệp, nhiều tính năng | [Tài liệu](../channels/feishu/README.vi.md) | | **IRC** | ⭐⭐ Trung bình | Máy chủ + cấu hình TLS | [Tài liệu](#irc) | | **OneBot** | ⭐⭐ Trung bình | Tương thích NapCat/Go-CQHTTP, hệ sinh thái cộng đồng | [Tài liệu](../channels/onebot/README.vi.md) | +| **MQTT** | ⭐ Dễ | Bất kỳ client MQTT nào qua broker pub/sub | [Tài liệu](../channels/mqtt/README.vi.md) | | **MaixCam** | ⭐ Dễ | Kênh tích hợp phần cứng cho camera AI Sipeed | [Tài liệu](../channels/maixcam/README.vi.md) | | **Pico** | ⭐ Dễ | Kênh giao thức bản địa PicoClaw | | @@ -696,3 +697,67 @@ picoclaw gateway ``` + + +
+MQTT + +Bất kỳ client MQTT nào đều có thể giao tiếp với PicoClaw qua broker. Thiết bị hoặc dịch vụ publish yêu cầu lên broker; PicoClaw subscribe, xử lý và publish phản hồi trở lại. + +**1. Cấu hình** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +Tên người dùng và mật khẩu trong `~/.picoclaw/.security.yml`: + +```yaml +channel_list: + mqtt: + settings: + username: ten_nguoi_dung + password: mat_khau +``` + +**Định dạng topic** + +``` +{prefix}/{agent_id}/{client_id}/request # Client → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → Client +``` + +`client_id` do ứng dụng client đặt để phân biệt thiết bị hoặc phiên. + +**2. Khởi động** + +```bash +picoclaw gateway +``` + +**3. Kiểm tra** + +```bash +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "Xin chào"}' + +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +Xem đầy đủ tùy chọn cấu hình tại [Tài liệu Kênh MQTT](../channels/mqtt/README.vi.md). + +
diff --git a/docs/guides/chat-apps.zh.md b/docs/guides/chat-apps.zh.md index b5891dc69..d7400cd83 100644 --- a/docs/guides/chat-apps.zh.md +++ b/docs/guides/chat-apps.zh.md @@ -4,7 +4,7 @@ ## 💬 聊天应用集成 (Chat Apps) -PicoClaw 支持多种聊天平台，使您的 Agent 能够连接到任何地方。 +PicoClaw 支持多种聊天平台，使您的 Agent 能够连接到任何地方，包括 Telegram、Discord、WhatsApp、微信、QQ、钉钉、LINE、企业微信、飞书、Slack、IRC、OneBot、MQTT、MaixCam 等。 > **注意**: 依赖 HTTP 回调的渠道共用同一个 Gateway HTTP 服务器（`gateway.host`:`gateway.port`，默认 `127.0.0.1:18790`），无需为每个渠道单独配置端口。飞书、钉钉、企业微信这类 Socket/Stream 模式渠道不依赖共享 webhook 服务器来接收入站消息。 @@ -25,6 +25,7 @@ PicoClaw 支持多种聊天平台，使您的 Agent 能够连接到任何地方 | **飞书 (Feishu)** | ⭐⭐⭐ 较难 | 企业级协作，功能丰富 | [查看文档](../channels/feishu/README.zh.md) | | **IRC** | ⭐⭐ 中等 | 服务器 + TLS 配置 | [查看文档](#irc) | | **OneBot** | ⭐⭐ 中等 | 兼容 NapCat/Go-CQHTTP，社区生态丰富 | [查看文档](../channels/onebot/README.zh.md) | +| **MQTT** | ⭐ 简单 | 任意 MQTT 客户端通过 Broker 收发消息 | [查看文档](../channels/mqtt/README.zh.md) | | **MaixCam** | ⭐ 简单 | 专为 AI 摄像头设计的硬件集成通道 | [查看文档](../channels/maixcam/README.zh.md) | | **Pico** | ⭐ 简单 | PicoClaw 原生协议通道 | | @@ -610,3 +611,69 @@ picoclaw gateway ``` + + +
+MQTT + +任意 MQTT 客户端均可通过 Broker 与 PicoClaw 通信。设备或服务向 Broker 发布请求，PicoClaw 订阅后处理并将响应发布回去。 + +**1. 配置** + +```json +{ + "channel_list": { + "mqtt": { + "enabled": true, + "type": "mqtt", + "settings": { + "broker": "ssl://your-broker:8883", + "agent_id": "assistant", + "topic_prefix": "/picoclaw", + "keep_alive": 60, + "qos": 0 + } + } + } +} +``` + +用户名和密码存储于 `~/.picoclaw/.security.yml`： + +```yaml +channel_list: + mqtt: + settings: + username: your_username + password: your_password +``` + +**Topic 格式** + +``` +{prefix}/{agent_id}/{client_id}/request # 客户端 → PicoClaw +{prefix}/{agent_id}/{client_id}/response # PicoClaw → 客户端 +``` + +`client_id` 由客户端自行指定，用于区分不同设备或会话。 + +**2. 运行** + +```bash +picoclaw gateway +``` + +**3. 测试** + +```bash +# 发送消息 +mosquitto_pub -t "/picoclaw/assistant/device1/request" \ + -m '{"text": "你好"}' + +# 订阅响应 +mosquitto_sub -t "/picoclaw/assistant/device1/response" +``` + +完整配置选项请参考 [MQTT 渠道文档](../channels/mqtt/README.zh.md)。 + +
diff --git a/docs/guides/configuration.it.md b/docs/guides/configuration.it.md new file mode 100644 index 000000000..d7de46895 --- /dev/null +++ b/docs/guides/configuration.it.md @@ -0,0 +1,281 @@ +# ⚙️ Guida alla Configurazione + +> Torna al [README](../../README.md) + +## ⚙️ Configurazione + +File di configurazione: `~/.picoclaw/config.json` + +### Variabili d'Ambiente + +Puoi sovrascrivere i percorsi predefiniti usando variabili d'ambiente. Questo è utile per installazioni portatili, distribuzioni containerizzate, o per eseguire picoclaw come servizio di sistema. Queste variabili sono indipendenti e controllano percorsi diversi. + +| Variabile | Descrizione | Percorso Predefinito | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| `PICOCLAW_CONFIG` | Sovrascrive il percorso al file di configurazione. Indica direttamente a picoclaw quale `config.json` caricare, ignorando tutte le altre posizioni. | `~/.picoclaw/config.json` | +| `PICOCLAW_HOME` | Sovrascrive la directory radice per i dati di picoclaw. Modifica la posizione predefinita del `workspace` e delle altre directory dati. | `~/.picoclaw` | + +**Esempi:** + +```bash +# Esegui picoclaw usando un file di configurazione specifico +# Il percorso del workspace verrà letto da quel file di configurazione +PICOCLAW_CONFIG=/etc/picoclaw/production.json picoclaw gateway + +# Esegui picoclaw con tutti i dati salvati in /opt/picoclaw +# La configurazione verrà caricata dal percorso predefinito ~/.picoclaw/config.json +# Il workspace verrà creato in /opt/picoclaw/workspace +PICOCLAW_HOME=/opt/picoclaw picoclaw agent + +# Usa entrambi per un setup completamente personalizzato +PICOCLAW_HOME=/srv/picoclaw PICOCLAW_CONFIG=/srv/picoclaw/main.json picoclaw gateway +``` + +### Struttura del Workspace + +PicoClaw salva i dati nel workspace configurato (predefinito: `~/.picoclaw/workspace`): + +``` +~/.picoclaw/workspace/ +├── sessions/ # Sessioni di conversazione e cronologia +├── memory/ # Memoria a lungo termine (MEMORY.md) +├── state/ # Stato persistente (ultimo canale, ecc.) +├── cron/ # Database dei job pianificati +├── skills/ # Skill personalizzate +├── AGENT.md # Guida al comportamento dell'agent +├── HEARTBEAT.md # Prompt per task periodici (controllato ogni 30 min) +├── SOUL.md # Anima dell'agent +└── USER.md # Preferenze dell'utente +``` + +> **Nota:** Le modifiche a `AGENT.md`, `SOUL.md`, `USER.md` e `memory/MEMORY.md` vengono rilevate automaticamente a runtime tramite il tracciamento della data di modifica (mtime). **Non è necessario riavviare il gateway** dopo aver modificato questi file — l'agent caricherà il nuovo contenuto alla prossima richiesta. + +### Sorgenti delle Skill + +Per impostazione predefinita, le skill vengono caricate da: + +1. `~/.picoclaw/workspace/skills` (workspace) +2. `~/.picoclaw/skills` (globale) +3. `/skills` (builtin) + +Per configurazioni avanzate/di test, puoi sovrascrivere la directory radice delle skill builtin con: + +```bash +export PICOCLAW_BUILTIN_SKILLS=/path/to/skills +``` + +### Politica Unificata di Esecuzione dei Comandi + +- I comandi slash generici vengono eseguiti tramite un unico percorso in `pkg/agent/loop.go` via `commands.Executor`. +- Gli adattatori dei canali non consumano più localmente i comandi generici; inoltrano il testo in entrata al percorso bus/agent. Telegram registra ancora automaticamente i comandi supportati all'avvio. +- Un comando slash sconosciuto (ad esempio `/foo`) viene passato all'elaborazione LLM come se fosse un messaggio dell'utente. +- Un comando registrato ma non supportato sul canale corrente (ad esempio `/show` su WhatsApp) restituisce un errore esplicito all'utente e interrompe l'elaborazione. + +### Allowlist dei Tool per Agent + +La dichiarazione dei tool per-agent vive nel frontmatter di `AGENT.md`, non in `config.json`. + +Se `tools` è omesso nel frontmatter, l'agent riceve il normale set globale dei tool abilitati. Se `tools` è presente, PicoClaw registra per quell'agent solo i tool runtime elencati. + +```md +--- +name: Research Agent +description: Specialista per ricerca web e analisi approfondita. +tools: [read_file, write_file, web_search, web_fetch, message] +skills: [deep-research] +mcpServers: [web-index] +--- + +Sei l'agent di ricerca. +``` + +Note: + +- È una allowlist reale, non un suggerimento per l'LLM. +- I nomi dei tool fanno match 1:1 con il nome runtime del tool. +- Se ti serve controllo preciso, usa i nomi runtime effettivi come `web_search`, `web_fetch`, `spawn`, `subagent`, `send_file`. +- Le dichiarazioni dei tool in `AGENT.md` sono usate dal runtime e dai tool, ma non vengono iniettate nel prompt di discovery. + +### Discovery Multi-Agent (Automatica) + +Quando un agent ha peer spawnabili, PicoClaw inietta automaticamente nel suo system prompt un registry strutturato dei peer. Non serve una chiamata aggiuntiva a un tool `list_agents`. + +Questa discovery serve soprattutto a rendere affidabile la delega tramite `spawn` con `agent_id` esplicito. + +Ogni entry include: + +| Campo | Significato | +|-------|-------------| +| `id` | ID stabile dell'agent | +| `name` | Nome identitario da `AGENT.md` frontmatter | +| `description` | Descrizione identitaria da `AGENT.md` frontmatter | + +Dettagli importanti: + +- La sezione include solo i peer che l'agent corrente può spawnare tramite `subagents.allow_agents`. +- L'agent corrente e i peer non spawnabili vengono omessi, così il modello non pianifica contro agent non disponibili. +- La discovery è volutamente leggera. Fornisce al modello solo l'identità necessaria per scegliere un peer: `id`, `name`, `description`. +- `config.json` resta il layer infrastrutturale: workspace, agent di default, routing e permessi di subagent. Questi permessi controllano anche la visibilità nella discovery. +- `AGENT.md` resta il layer di identità. Il codice runtime e i tool possono comunque usare `tools`, `skills`, `mcpServers` e `model` quando avviene la delega. + +Forma dell'oggetto iniettato: + +```json +{ + "agents": [ + { + "id": "research", + "name": "Research Agent", + "description": "Specialista per investigazioni e lavoro web." + } + ] +} +``` + +In pratica, un agent generalista sceglie un peer in base alla descrizione del suo ruolo, poi chiama `spawn` con l'`agent_id` del peer. Il runtime risolve il resto. + +### 🔒 Sandbox di Sicurezza + +PicoClaw esegue in un ambiente sandboxed per impostazione predefinita. L'agent può accedere solo ai file ed eseguire comandi all'interno del workspace configurato. + +#### Configurazione Predefinita + +```json +{ + "agents": { + "defaults": { + "workspace": "~/.picoclaw/workspace", + "restrict_to_workspace": true + } + } +} +``` + +| Opzione | Predefinito | Descrizione | +| ----------------------- | ----------------------- | ---------------------------------------------------- | +| `workspace` | `~/.picoclaw/workspace` | Directory di lavoro dell'agent | +| `restrict_to_workspace` | `true` | Limita l'accesso a file/comandi al workspace | + +#### Strumenti Protetti + +Quando `restrict_to_workspace: true`, i seguenti strumenti sono in sandbox: + +| Strumento | Funzione | Restrizione | +| ------------- | ------------------------- | ---------------------------------------------------- | +| `read_file` | Legge file | Solo file all'interno del workspace | +| `write_file` | Scrive file | Solo file all'interno del workspace | +| `list_dir` | Elenca directory | Solo directory all'interno del workspace | +| `edit_file` | Modifica file | Solo file all'interno del workspace | +| `append_file` | Aggiunge ai file | Solo file all'interno del workspace | +| `exec` | Esegue comandi | I percorsi dei comandi devono essere nel workspace | + +#### Protezione Exec Aggiuntiva + +Anche con `restrict_to_workspace: false`, lo strumento `exec` blocca questi comandi pericolosi: + +* `rm -rf`, `del /f`, `rmdir /s` — Cancellazione di massa +* `format`, `mkfs`, `diskpart` — Formattazione del disco +* `dd if=` — Imaging del disco +* Scrittura su `/dev/sd[a-z]` — Scritture dirette su disco +* `shutdown`, `reboot`, `poweroff` — Spegnimento del sistema +* Fork bomb `:(){ :|:& };:` + +### Controllo Accesso ai File + +| Chiave di configurazione | Tipo | Predefinito | Descrizione | +|--------------------------|------|-------------|-------------| +| `tools.allow_read_paths` | string[] | `[]` | Percorsi aggiuntivi consentiti per la lettura al di fuori del workspace | +| `tools.allow_write_paths` | string[] | `[]` | Percorsi aggiuntivi consentiti per la scrittura al di fuori del workspace | + +### Sicurezza Exec + +| Chiave di configurazione | Tipo | Predefinito | Descrizione | +|--------------------------|------|-------------|-------------| +| `tools.exec.allow_remote` | bool | `false` | Consente lo strumento exec da canali remoti (Telegram/Discord ecc.) | +| `tools.exec.enable_deny_patterns` | bool | `true` | Abilita l'intercettazione dei comandi pericolosi | +| `tools.exec.custom_deny_patterns` | string[] | `[]` | Pattern regex personalizzati da bloccare | +| `tools.exec.custom_allow_patterns` | string[] | `[]` | Pattern regex personalizzati da consentire | + +> **Nota di sicurezza:** La protezione dei symlink è abilitata per impostazione predefinita — tutti i percorsi file vengono risolti tramite `filepath.EvalSymlinks` prima del confronto con la whitelist, prevenendo attacchi di escape tramite symlink. + +#### Limitazione Nota: Processi Figlio degli Strumenti di Build + +Il controllo di sicurezza exec ispeziona solo la riga di comando avviata direttamente da PicoClaw. Non ispeziona ricorsivamente i processi figlio generati da strumenti di sviluppo consentiti come `make`, `go run`, `cargo`, `npm run` o script di build personalizzati. + +Ciò significa che un comando di primo livello può comunque compilare o avviare altri binari dopo aver superato il controllo iniziale. In pratica, tratta gli script di build, i Makefile, gli script di pacchetti e i binari generati come codice eseguibile che richiede lo stesso livello di revisione di un comando shell diretto. + +Per ambienti ad alto rischio: + +* Esamina gli script di build prima dell'esecuzione. +* Preferisci l'approvazione/revisione manuale per i workflow di compilazione ed esecuzione. +* Esegui PicoClaw in un container o VM se hai bisogno di un isolamento più forte di quello fornito dal controllo integrato. + +#### Esempi di Errore + +``` +[ERROR] tool: Tool execution failed +{tool=exec, error=Command blocked by safety guard (path outside working dir)} +``` + +``` +[ERROR] tool: Tool execution failed +{tool=exec, error=Command blocked by safety guard (dangerous pattern detected)} +``` + +#### Disabilitare le Restrizioni (Rischio di Sicurezza) + +Se hai bisogno che l'agent acceda a percorsi al di fuori del workspace: + +**Metodo 1: File di configurazione** + +```json +{ + "agents": { + "defaults": { + "restrict_to_workspace": false + } + } +} +``` + +**Metodo 2: Variabile d'ambiente** + +```bash +export PICOCLAW_AGENTS_DEFAULTS_RESTRICT_TO_WORKSPACE=false +``` + +> ⚠️ **Attenzione**: Disabilitare questa restrizione consente all'agent di accedere a qualsiasi percorso sul tuo sistema. Usare con cautela solo in ambienti controllati. + +#### Coerenza dei Confini di Sicurezza + +L'impostazione `restrict_to_workspace` si applica in modo coerente a tutti i percorsi di esecuzione: + +| Percorso di esecuzione | Confine di sicurezza | +| ---------------------- | --------------------------------- | +| Main Agent | `restrict_to_workspace` ✅ | +| Subagent / Spawn | Eredita la stessa restrizione ✅ | +| Heartbeat tasks | Eredita la stessa restrizione ✅ | + +Tutti i percorsi condividono la stessa restrizione del workspace — non è possibile aggirare il confine di sicurezza tramite subagent o task pianificati. + +### Heartbeat (Task Periodici) + +PicoClaw può eseguire task periodici automaticamente. Crea un file `HEARTBEAT.md` nel tuo workspace: + +```markdown +# Periodic Tasks + +- Check my email for important messages +- Review my calendar for upcoming events +- Check the weather forecast +``` + +L'agent leggerà questo file ogni 30 minuti (configurabile) ed eseguirà tutti i task usando gli strumenti disponibili. + +#### Task Asincroni con Spawn + +Per task di lunga durata (ricerca web, chiamate API), usa lo strumento `spawn` per creare un **subagent**: + +```markdown +# Periodic Tasks +``` diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 28fc7b775..64a61f241 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -69,6 +69,36 @@ PicoClaw stores data in your configured workspace (default: `~/.picoclaw/workspa > **Note:** Changes to `AGENT.md`, `SOUL.md`, `USER.md` and `memory/MEMORY.md` are automatically detected at runtime via file modification time (mtime) tracking. You do **not** need to restart the gateway after editing these files — the agent picks up the new content on the next request. +### Agent Self-Evolution + +The `evolution` block controls PicoClaw's self-evolution runtime. When enabled, the agent records completed turns as learning records. In higher modes it can group repeated successful patterns, generate skill drafts, and optionally apply accepted drafts into workspace skills. + +```json +{ + "evolution": { + "enabled": false, + "mode": "observe", + "state_dir": "", + "min_task_count": 2, + "min_success_ratio": 0.7, + "cold_path_trigger": "after_turn", + "cold_path_times": [] + } +} +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `enabled` | `false` | Enables learning-record capture for completed agent turns. Heartbeat turns are ignored. | +| `mode` | `observe` | `observe` records data only. `draft` can generate candidate skill drafts. `apply` can apply accepted drafts to workspace skills. | +| `state_dir` | `""` | Optional directory for evolution state. Leave empty to use the default under the workspace. | +| `min_task_count` | `2` | Minimum related task records required before a pattern is eligible for draft generation. | +| `min_success_ratio` | `0.7` | Minimum success ratio for a task cluster. Use a value greater than `0` and up to `1`. | +| `cold_path_trigger` | `after_turn` | Runs draft generation `after_turn`, on a `scheduled` cadence, or disables automatic cold-path runs when set to `manual`. There is no user-facing manual trigger yet. Applies only in `draft` and `apply` modes. | +| `cold_path_times` | `[]` | Scheduled run times used when `cold_path_trigger` is `scheduled`, written as `HH:MM` strings. | + +Use `observe` first if you want to inspect learning records without generating skill changes. Use `draft` when you want PicoClaw to prepare reviewable improvements. Use `apply` only when you are comfortable letting accepted drafts update workspace skills. + ### Web launcher dashboard **picoclaw-launcher** serves a browser UI that requires password sign-in first. On first run, open `/launcher-setup` to create the dashboard password. Later manual sign-ins use `/launcher-login`. @@ -211,6 +241,69 @@ earlier and broader fallback rules later. For more complete routing and model-tier examples, see the [Routing Guide](routing-guide.md). +### Agent Tool Allowlist + +Per-agent tool declarations live in `AGENT.md` frontmatter, not in `config.json`. + +If `tools` is omitted from frontmatter, the agent gets the normal globally enabled tool set. If `tools` is present, PicoClaw registers only the listed runtime tools for that agent. + +```md +--- +name: Research Agent +description: Specialist for web research and in-depth analysis. +tools: [read_file, write_file, web_search, web_fetch, message] +skills: [deep-research] +mcpServers: [web-index] +--- + +You are the research agent. +``` + +Notes: + +- This is an allowlist, not a preference hint. +- Tool names are matched against the runtime tool name 1:1. +- Use runtime tool names such as `web_search`, `web_fetch`, `spawn`, `subagent`, `send_file`. +- Tool declarations in `AGENT.md` are used by runtime/tooling, but they are not injected into the discovery prompt. + +### Agent Discovery (Automatic) + +When an agent has spawnable peers and can call `spawn`, PicoClaw injects a structured agent registry into that agent's system prompt on every turn. No extra `list_agents` tool call is required. + +This registry is intended to make delegation concrete and reliable, especially when using `spawn` with a target `agent_id`. + +Each entry includes: + +| Field | Meaning | +|-------|---------| +| `id` | Stable agent id | +| `name` | Agent identity name from `AGENT.md` frontmatter | +| `description` | Agent identity description from `AGENT.md` frontmatter | + +Important behavior: + +- The discovery section appears only when the current agent has the `spawn` tool and includes only peer agents it is permitted to spawn via `subagents.allow_agents`. +- The current agent and non-spawnable peers are omitted, so the model does not plan against unavailable agents. +- Discovery is intentionally lightweight. It gives the model only the identity it needs to choose a peer: `id`, `name`, and `description`. +- `config.json` remains the infrastructure layer: workspace, default agent selection, routing, and subagent permissions. Those permissions also gate discovery visibility. +- `AGENT.md` remains the identity layer. Runtime/tool code can still use its `tools`, `skills`, `mcpServers`, and `model` fields when delegation happens. + +Example injected shape: + +```json +{ + "agents": [ + { + "id": "research", + "name": "Research Agent", + "description": "Specialist for long-form investigation and web work." + } + ] +} +``` + +In practice, this means a generalist agent can choose a peer based on its role description, then call `spawn` with the peer's `agent_id`. The runtime resolves the rest. + ### 🔒 Security Sandbox PicoClaw runs in a sandboxed environment by default. The agent can only access files and execute commands within the configured workspace. diff --git a/docs/guides/configuration.zh.md b/docs/guides/configuration.zh.md index dbc853d98..a6e4d42b9 100644 --- a/docs/guides/configuration.zh.md +++ b/docs/guides/configuration.zh.md @@ -67,6 +67,36 @@ PicoClaw 将数据存储在您配置的工作区中（默认：`~/.picoclaw/work > **提示：** 对 `AGENT.md`、`SOUL.md`、`USER.md` 和 `memory/MEMORY.md` 的修改会通过文件修改时间（mtime）在运行时自动检测。**无需重启 gateway**，Agent 将在下一次请求时自动加载最新内容。 +### Agent 自进化 + +`evolution` 配置块控制 PicoClaw 的自进化运行时。启用后，Agent 会把已完成的回合记录为学习记录。在更高模式下，它可以聚类重复出现的成功模式、生成技能草稿，并可选择把已接受的草稿应用到工作区技能中。 + +```json +{ + "evolution": { + "enabled": false, + "mode": "observe", + "state_dir": "", + "min_task_count": 2, + "min_success_ratio": 0.7, + "cold_path_trigger": "after_turn", + "cold_path_times": [] + } +} +``` + +| 字段 | 默认值 | 说明 | +|------|--------|------| +| `enabled` | `false` | 启用已完成 Agent 回合的学习记录采集。Heartbeat 回合会被忽略。 | +| `mode` | `observe` | `observe` 只记录数据；`draft` 可生成候选技能草稿；`apply` 可将已接受草稿应用到工作区技能。 | +| `state_dir` | `""` | 自进化状态的可选目录。留空时使用工作区下的默认位置。 | +| `min_task_count` | `2` | 一个模式具备生成草稿资格前所需的最小相关任务记录数。 | +| `min_success_ratio` | `0.7` | 任务聚类所需的最小成功率，取值需大于 `0`，且不超过 `1`。 | +| `cold_path_trigger` | `after_turn` | 草稿生成可在 `after_turn` 后运行、按 `scheduled` 定时运行；设置为 `manual` 时会关闭自动冷路径运行。目前还没有用户可用的手动触发入口。仅在 `draft` 和 `apply` 模式下生效。 | +| `cold_path_times` | `[]` | 当 `cold_path_trigger` 为 `scheduled` 时使用的运行时间，格式为 `HH:MM` 字符串。 | + +如果你只想先检查学习记录，建议从 `observe` 开始。需要生成可审查改进时使用 `draft`。只有在你接受让已通过的草稿更新工作区技能时，才使用 `apply`。 + ### Web 启动器控制台 用 **picoclaw-launcher** 打开浏览器控制台前需要先使用密码登录。首次启动时打开 `/launcher-setup` 创建 dashboard 登录密码；后续手动登录使用 `/launcher-login`。 @@ -753,6 +783,42 @@ PicoClaw 按协议族路由提供商： +### 事件日志 + +PicoClaw 的 runtime events 会覆盖 agent、channel、gateway、message bus 和 MCP 等运行时组件。默认只打印 `agent.*` 事件，其他事件仍会发布到 runtime event bus，但不会进入日志。 + +```json +{ + "events": { + "logging": { + "enabled": true, + "include": ["agent.*"], + "exclude": [], + "min_severity": "info", + "include_payload": false + } + } +} +``` + +常用配置： + +```json +{ + "events": { + "logging": { + "include": ["*"], + "exclude": ["agent.llm.delta"], + "min_severity": "warn" + } + } +} +``` + +`include` / `exclude` 支持精确事件名和 `gateway.*`、`channel.lifecycle.*` 这类模式。`include_payload` 默认关闭，避免把完整用户消息或工具参数写入日志；agent 事件会默认输出长度、计数、状态等摘要字段。 + +更多字段说明和示例见 [Runtime Events 与事件日志](../architecture/runtime-events.zh.md)。 + ### 定时任务 / 提醒 PicoClaw 通过 `cron` 工具支持 cron 风格的定时任务。Agent 可以设置、列出和取消在指定时间触发的提醒或周期性任务。 @@ -775,6 +841,7 @@ PicoClaw 通过 `cron` 工具支持 cron 风格的定时任务。Agent 可以设 | 主题 | 说明 | | ---- | ---- | | [敏感数据过滤](../security/sensitive_data_filtering.zh.md) | 在发送给 LLM 前，从工具结果中过滤 API 密钥和令牌 | +| [Runtime Events 与事件日志](../architecture/runtime-events.zh.md) | 统一运行时事件、日志过滤和调试配置 | | [Hook 系统](../architecture/hooks/README.zh.md) | 事件驱动 Hook：观察者、拦截器、审批 Hook | | [Steering](../architecture/steering.md) | 在工具调用间向运行中的 Agent 注入消息 | | [SubTurn](../architecture/subturn.md) | 子 Agent 协调、并发控制、生命周期管理 | diff --git a/docs/guides/providers.md b/docs/guides/providers.md index d99d8c016..7b078373d 100644 --- a/docs/guides/providers.md +++ b/docs/guides/providers.md @@ -116,23 +116,47 @@ This design also enables **multi-agent support** with flexible provider selectio #### `model_list` Entry Fields -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `model_name` | string | Yes | Unique name used to reference this model in agent config | -| `provider` | string | No | Preferred provider identifier. When present, PicoClaw sends `model` unchanged to that provider | -| `model` | string | Yes | Native model ID when `provider` is set. If `provider` is omitted, the legacy `provider/model` form is still supported | -| `api_keys` | string[] | Yes* | API key(s) for authentication. Multiple keys enable per-request rotation. Not required for local providers (Ollama, LM Studio, VLLM) | -| `api_base` | string | No | Override the default API endpoint URL | -| `proxy` | string | No | HTTP proxy URL for this model entry | -| `user_agent` | string | No | Custom `User-Agent` header sent with API requests (supported by OpenAI-compatible, Gemini, Anthropic, and Azure providers) | -| `request_timeout` | int | No | Request timeout in seconds (default varies by provider) | -| `max_tokens_field` | string | No | Override the max tokens field name in request body (e.g., `max_completion_tokens` for o1 models) | -| `thinking_level` | string | No | Extended thinking level: `off`, `low`, `medium`, `high`, `xhigh`, or `adaptive` | -| `extra_body` | object | No | Additional fields to inject into every request body | +| Field | Type | Required | Description | +|-------|------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `model_name` | string | Yes | Unique name used to reference this model in agent config | +| `provider` | string | No | Preferred provider identifier. When present, PicoClaw sends `model` unchanged to that provider | +| `model` | string | Yes | Native model ID when `provider` is set. If `provider` is omitted, the legacy `provider/model` form is still supported | +| `api_keys` | string[] | Yes* | API key(s) for authentication. Multiple keys enable per-request rotation. Not required for local providers (Ollama, LM Studio, VLLM) | +| `api_base` | string | No | Override the default API endpoint URL | +| `proxy` | string | No | HTTP proxy URL for this model entry | +| `user_agent` | string | No | Custom `User-Agent` header sent with API requests (supported by OpenAI-compatible, Gemini, Anthropic, and Azure providers) | +| `request_timeout` | int | No | Request timeout in seconds (default varies by provider) | +| `max_tokens_field` | string | No | Override the max tokens field name in request body (e.g., `max_completion_tokens` for o1 models) | +| `thinking_level` | string | No | Extended thinking level: `off`, `low`, `medium`, `high`, `xhigh`, or `adaptive` | +| `tool_schema_transform` | string | No | Optional compatibility transform for tool parameter schemas. Default: disabled. Supported values: `simple`. | +| `extra_body` | object | No | Additional fields to inject into every request body | | `custom_headers` | object | No | Additional HTTP headers to inject into every request (e.g., `{"X-Source":"coding-plan"}`). If a key matches a built-in header, the custom value overrides the built-in one (e.g., `Authorization`, `User-Agent`, `Content-Type`, `Accept`). | -| `rpm` | int | No | Per-minute request rate limit | -| `fallbacks` | string[] | No | Fallback model names for automatic failover | -| `enabled` | bool | No | Whether this model entry is active (default: `true`) | +| `rpm` | int | No | Per-minute request rate limit | +| `fallbacks` | string[] | No | Fallback model names for automatic failover | +| `enabled` | bool | No | Whether this model entry is active (default: `true`) | + +#### Tool Schema Compatibility + +By default, PicoClaw now forwards tool JSON Schemas unchanged. + +Some providers reject advanced JSON Schema features such as `$ref`, `$defs`, `anyOf`, `oneOf`, `allOf`, `pattern`, or numeric/string constraints inside tool declarations. For those models, you can opt into a compatibility transform per model entry with `tool_schema_transform`. + +Use `simple` when the upstream provider expects the conservative style function schema subset: + +```json +{ + "model_name": "gemini-2.5-flash-safe-tools", + "provider": "gemini", + "model": "gemini-2.5-flash", + "api_keys": ["your-gemini-key"], + "tool_schema_transform": "simple" +} +``` + +Notes: + +- Default behavior is disabled. If you omit `tool_schema_transform`, PicoClaw sends the original tool schema. +- The setting is per model entry, so you can enable it only for the providers that need it. #### Provider / Model Resolution @@ -393,10 +417,8 @@ It also applies cooldown tracking per candidate to avoid immediately retrying a ], "agents": { "defaults": { - "model": { - "primary": "qwen-main", - "fallbacks": ["deepseek-backup", "gemini-backup"] - } + "model_name": "qwen-main", + "model_fallbacks": ["deepseek-backup", "gemini-backup"] } } } diff --git a/docs/guides/providers.zh.md b/docs/guides/providers.zh.md index 1302407a3..4bab65f6b 100644 --- a/docs/guides/providers.zh.md +++ b/docs/guides/providers.zh.md @@ -362,10 +362,8 @@ PicoClaw 按下面的规则解析 `provider` 和最终发给上游的模型 ID ], "agents": { "defaults": { - "model": { - "primary": "qwen-main", - "fallbacks": ["deepseek-backup", "gemini-backup"] - } + "model_name": "qwen-main", + "model_fallbacks": ["deepseek-backup", "gemini-backup"] } } } diff --git a/docs/project/README.fr.md b/docs/project/README.fr.md index b02067d2a..392c5321c 100644 --- a/docs/project/README.fr.md +++ b/docs/project/README.fr.md @@ -57,6 +57,14 @@ ## 📢 Actualités +2026-05-11 🛒 **LicheeRV-Claw disponible sur AliExpress !** Vous pouvez désormais acheter le LicheeRV-Claw sur [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), ce qui facilite l'essai de PicoClaw sur du matériel RISC-V compact. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Support Android !** PicoClaw fonctionne maintenant sur Android ! Téléchargez l'APK sur [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 publiée !** Refonte de l'architecture Agent (SubTurn, Hooks, Steering, EventBus), intégration WeChat/WeCom, renforcement de la sécurité (.security.yml, filtrage des données sensibles), nouveaux providers (AWS Bedrock, Azure, Xiaomi MiMo), et 35 corrections de bugs. PicoClaw a atteint **26K Stars** ! @@ -479,7 +487,7 @@ PicoClaw peut effectuer des recherches sur le web pour fournir des informations | Moteur de recherche | Clé API | Niveau gratuit | Lien | |--------------------|---------|----------------|------| | DuckDuckGo | Non requise | Illimité | Fallback intégré | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Requise | 1000 requêtes/jour | IA, optimisé pour le chinois | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Requise | 1500 requêtes/mois (allocation journalière) | IA, optimisé pour le chinois | | [Tavily](https://tavily.com) | Requise | 1000 requêtes/mois | Optimisé pour les Agents IA | | [Brave Search](https://brave.com/search/api) | Requise | 2000 requêtes/mois | Rapide et privé | | [Perplexity](https://www.perplexity.ai) | Requise | Payant | Recherche propulsée par IA | diff --git a/docs/project/README.id.md b/docs/project/README.id.md index 49c64e74c..49568f654 100644 --- a/docs/project/README.id.md +++ b/docs/project/README.id.md @@ -56,6 +56,14 @@ ## 📢 Berita +2026-05-11 🛒 **LicheeRV-Claw tersedia di AliExpress!** Kini Anda dapat membeli LicheeRV-Claw di [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), sehingga lebih mudah mencoba PicoClaw di hardware RISC-V ringkas. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Dukungan Android!** PicoClaw sekarang berjalan di Android! Unduh APK di [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 Dirilis!** Perombakan arsitektur Agent (SubTurn, Hooks, Steering, EventBus), integrasi WeChat/WeCom, penguatan keamanan (.security.yml, penyaringan data sensitif), provider baru (AWS Bedrock, Azure, Xiaomi MiMo), dan 35 perbaikan bug. PicoClaw telah mencapai **26K Stars**! @@ -474,7 +482,7 @@ PicoClaw dapat mencari web untuk memberikan informasi terkini. Konfigurasi di `t | Mesin Pencari | API Key | Tier Gratis | Tautan | |--------------|---------|-------------|--------| | DuckDuckGo | Tidak perlu | Tidak terbatas | Fallback bawaan | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Diperlukan | 1000 kueri/hari | Bertenaga AI, dioptimalkan untuk bahasa Mandarin | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Diperlukan | 1500 kueri/bulan (alokasi harian) | Bertenaga AI, dioptimalkan untuk bahasa Mandarin | | [Tavily](https://tavily.com) | Diperlukan | 1000 kueri/bulan | Dioptimalkan untuk AI Agent | | [Brave Search](https://brave.com/search/api) | Diperlukan | 2000 kueri/bulan | Cepat dan privat | | [Perplexity](https://www.perplexity.ai) | Diperlukan | Berbayar | Pencarian bertenaga AI | diff --git a/docs/project/README.it.md b/docs/project/README.it.md index 0cf6cf8db..4d04718f4 100644 --- a/docs/project/README.it.md +++ b/docs/project/README.it.md @@ -56,6 +56,14 @@ ## 📢 Novità +2026-05-11 🛒 **LicheeRV-Claw disponibile su AliExpress!** Ora puoi acquistare LicheeRV-Claw su [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), rendendo più semplice provare PicoClaw su hardware RISC-V compatto. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Supporto Android!** PicoClaw ora funziona su Android! Scarica l'APK su [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 rilasciata!** Revisione dell'architettura Agent (SubTurn, Hooks, Steering, EventBus), integrazione WeChat/WeCom, rafforzamento della sicurezza (.security.yml, filtraggio dati sensibili), nuovi provider (AWS Bedrock, Azure, Xiaomi MiMo) e 35 correzioni di bug. PicoClaw raggiunge **26K Stars**! @@ -474,7 +482,7 @@ PicoClaw può cercare sul web per fornire informazioni aggiornate. Configura in | Motore di Ricerca | API Key | Piano Gratuito | Link | |-------------------|---------|----------------|------| | DuckDuckGo | Non necessaria | Illimitato | Fallback integrato | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Richiesta | 1000 query/giorno | IA, ottimizzato per il cinese | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Richiesta | 1500 query/mese (allocazione giornaliera) | IA, ottimizzato per il cinese | | [Tavily](https://tavily.com) | Richiesta | 1000 query/mese | Ottimizzato per AI Agent | | [Brave Search](https://brave.com/search/api) | Richiesta | 2000 query/mese | Veloce e privato | | [Perplexity](https://www.perplexity.ai) | Richiesta | A pagamento | Ricerca potenziata dall'IA | diff --git a/docs/project/README.ja.md b/docs/project/README.ja.md index 6e3060688..1a1e9f469 100644 --- a/docs/project/README.ja.md +++ b/docs/project/README.ja.md @@ -56,6 +56,14 @@ ## 📢 ニュース +2026-05-11 🛒 **LicheeRV-Claw が AliExpress で購入可能に！** [AliExpress](https://www.aliexpress.com/item/1005006519668532.html) から LicheeRV-Claw を購入できるようになり、コンパクトな RISC-V ハードウェアで PicoClaw を試しやすくなりました。 + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Android サポート！** PicoClawがAndroidで動作！APKは[picoclaw.io](https://picoclaw.io/download)からダウンロード 2026-03-25 🚀 **v0.2.4 リリース！** Agent アーキテクチャ全面刷新（SubTurn、Hooks、Steering、EventBus）、WeChat/WeCom 統合、セキュリティ強化（.security.yml、機密データフィルタリング）、新プロバイダー（AWS Bedrock、Azure、Xiaomi MiMo）、35 件のバグ修正。PicoClaw **26K ⭐** 達成！ @@ -475,7 +483,7 @@ PicoClaw は最新情報を提供するために Web を検索できます。`to | 検索エンジン | API キー | 無料枠 | リンク | |------------|---------|--------|-------| | DuckDuckGo | 不要 | 無制限 | 内蔵フォールバック | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | 必須 | 1000 クエリ/日 | AI 搭載、中国語に最適化 | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | 必須 | 1500 クエリ/月（日次割り当て） | AI 搭載、中国語に最適化 | | [Tavily](https://tavily.com) | 必須 | 1000 クエリ/月 | AI Agent 向けに最適化 | | [Brave Search](https://brave.com/search/api) | 必須 | 2000 クエリ/月 | 高速でプライベート | | [Perplexity](https://www.perplexity.ai) | 必須 | 有料 | AI 搭載検索 | diff --git a/docs/project/README.ko.md b/docs/project/README.ko.md index dfefa67fe..2c79d161a 100644 --- a/docs/project/README.ko.md +++ b/docs/project/README.ko.md @@ -56,6 +56,14 @@ ## 📢 뉴스 +2026-05-11 🛒 **LicheeRV-Claw를 AliExpress에서 구매할 수 있습니다!** 이제 [AliExpress](https://www.aliexpress.com/item/1005006519668532.html)에서 LicheeRV-Claw를 구매해 소형 RISC-V 하드웨어에서 PicoClaw를 더 쉽게 사용해 볼 수 있습니다. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Android 지원!** PicoClaw가 이제 Android에서 실행됩니다! APK는 [picoclaw.io](https://picoclaw.io/download)에서 다운로드하세요. 2026-03-25 🚀 **v0.2.4 출시!** 에이전트 아키텍처 전면 개편(SubTurn, Hooks, Steering, EventBus), WeChat/WeCom 통합, 보안 강화(`.security.yml`, 민감 정보 필터링), 새 프로바이더(AWS Bedrock, Azure, Xiaomi MiMo), 그리고 35건의 버그 수정이 포함되었습니다. PicoClaw는 **26K 스타**를 달성했습니다! @@ -480,7 +488,7 @@ PicoClaw는 최신 정보를 제공하기 위해 웹 검색을 수행할 수 있 | 검색 엔진 | API Key | 무료 제공량 | 링크 | |-----------|---------|-------------|------| | DuckDuckGo | 불필요 | 무제한 | 내장 백업 검색 | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | 필수 | 하루 1000회 쿼리 | AI 기반, 중국 시장 최적화 | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | 필수 | 월 1500회 쿼리 (일할 할당) | AI 기반, 중국 시장 최적화 | | [Tavily](https://tavily.com) | 필수 | 월 1000회 쿼리 | AI 에이전트에 최적화 | | [Brave Search](https://brave.com/search/api) | 필수 | 월 2000회 쿼리 | 빠르고 프라이빗함 | | [Perplexity](https://www.perplexity.ai) | 필수 | 유료 | AI 기반 검색 | diff --git a/docs/project/README.ms.md b/docs/project/README.ms.md index 73c428f11..068208fac 100644 --- a/docs/project/README.ms.md +++ b/docs/project/README.ms.md @@ -56,6 +56,14 @@ ## 📢 Berita +2026-05-11 🛒 **LicheeRV-Claw tersedia di AliExpress!** Anda kini boleh membeli LicheeRV-Claw di [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), menjadikannya lebih mudah untuk mencuba PicoClaw pada perkakasan RISC-V yang kompak. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Sokongan Android!** PicoClaw sekarang berjalan di Android! Muat turun APK di [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 Dikeluarkan!** Penstrukturan semula seni bina Agent (SubTurn, Hooks, Steering, EventBus), integrasi WeChat/WeCom, penguatan keselamatan (.security.yml, penapisan data sensitif), penyedia baharu (AWS Bedrock, Azure, Xiaomi MiMo), dan 35 pembetulan pepijat. PicoClaw mencapai **26K Stars**! @@ -474,7 +482,7 @@ PicoClaw boleh mencari web untuk menyediakan maklumat terkini. Konfigurasikan da | Enjin Carian | Kunci API | Peringkat Percuma | Pautan | |-------------|-----------|-------------------|--------| | DuckDuckGo | Tidak perlu | Tanpa had | Sandaran terbina dalam | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Diperlukan | 1000 pertanyaan/hari | Dikuasai AI, dioptimumkan untuk China | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Diperlukan | 1500 pertanyaan/bulan (peruntukan harian) | Dikuasai AI, dioptimumkan untuk China | | [Tavily](https://tavily.com) | Diperlukan | 1000 pertanyaan/bulan | Dioptimumkan untuk AI Agent | | [Brave Search](https://brave.com/search/api) | Diperlukan | 2000 pertanyaan/bulan | Pantas dan peribadi | | [Perplexity](https://www.perplexity.ai) | Diperlukan | Berbayar | Carian dikuasai AI | diff --git a/docs/project/README.pt-br.md b/docs/project/README.pt-br.md index 74cb967de..b69b04caf 100644 --- a/docs/project/README.pt-br.md +++ b/docs/project/README.pt-br.md @@ -56,6 +56,14 @@ ## 📢 Novidades +2026-05-11 🛒 **LicheeRV-Claw no AliExpress!** Agora você pode comprar o LicheeRV-Claw no [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), facilitando testar o PicoClaw em hardware RISC-V compacto. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Suporte Android!** PicoClaw agora roda no Android! Baixe o APK em [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 Lançada!** Reformulação da arquitetura Agent (SubTurn, Hooks, Steering, EventBus), integração WeChat/WeCom, fortalecimento de segurança (.security.yml, filtragem de dados sensíveis), novos providers (AWS Bedrock, Azure, Xiaomi MiMo) e 35 correções de bugs. O PicoClaw atingiu **26K Stars**! @@ -475,7 +483,7 @@ O PicoClaw pode pesquisar na web para fornecer informações atualizadas. Config | Motor de Busca | API Key | Nível Gratuito | Link | |----------------|---------|----------------|------| | DuckDuckGo | Não necessária | Ilimitado | Fallback integrado | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Obrigatória | 1000 consultas/dia | IA, otimizado para chinês | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Obrigatória | 1500 consultas/mês (alocação diária) | IA, otimizado para chinês | | [Tavily](https://tavily.com) | Obrigatória | 1000 consultas/mês | Otimizado para AI Agents | | [Brave Search](https://brave.com/search/api) | Obrigatória | 2000 consultas/mês | Rápido e privado | | [Perplexity](https://www.perplexity.ai) | Obrigatória | Pago | Busca com IA | diff --git a/docs/project/README.vi.md b/docs/project/README.vi.md index 743069021..7311fb21e 100644 --- a/docs/project/README.vi.md +++ b/docs/project/README.vi.md @@ -56,6 +56,14 @@ ## 📢 Tin tức +2026-05-11 🛒 **LicheeRV-Claw đã có trên AliExpress!** Bạn hiện có thể mua LicheeRV-Claw trên [AliExpress](https://www.aliexpress.com/item/1005006519668532.html), giúp việc thử PicoClaw trên phần cứng RISC-V nhỏ gọn dễ dàng hơn. + +

+ + LicheeRV-Claw on AliExpress + +

+ 2026-03-31 📱 **Hỗ trợ Android!** PicoClaw giờ chạy trên Android! Tải APK tại [picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 đã phát hành!** Tái cấu trúc kiến trúc Agent (SubTurn, Hooks, Steering, EventBus), tích hợp WeChat/WeCom, tăng cường bảo mật (.security.yml, lọc dữ liệu nhạy cảm), provider mới (AWS Bedrock, Azure, Xiaomi MiMo) và 35 bản vá lỗi. PicoClaw đã đạt **26K Stars**! @@ -475,7 +483,7 @@ PicoClaw có thể tìm kiếm web để cung cấp thông tin cập nhật. C | Công cụ Tìm kiếm | API Key | Gói miễn phí | Liên kết | |------------------|---------|--------------|----------| | DuckDuckGo | Không cần | Không giới hạn | Dự phòng tích hợp sẵn | -| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Bắt buộc | 1000 truy vấn/ngày | AI, tối ưu cho tiếng Trung | +| [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Bắt buộc | 1500 truy vấn/tháng (phân bổ hàng ngày) | AI, tối ưu cho tiếng Trung | | [Tavily](https://tavily.com) | Bắt buộc | 1000 truy vấn/tháng | Tối ưu cho AI Agent | | [Brave Search](https://brave.com/search/api) | Bắt buộc | 2000 truy vấn/tháng | Nhanh và riêng tư | | [Perplexity](https://www.perplexity.ai) | Bắt buộc | Trả phí | Tìm kiếm hỗ trợ AI | diff --git a/docs/project/README.zh.md b/docs/project/README.zh.md index 253bb84ed..66e2f7ebe 100644 --- a/docs/project/README.zh.md +++ b/docs/project/README.zh.md @@ -56,6 +56,14 @@ ## 📢 新闻 +2026-05-11 🛒 **LicheeRV-Claw 已上架淘宝！** 现在可以在 [淘宝](https://item.taobao.com/item.htm?abbucket=20&id=764939520376) 购买 LicheeRV-Claw，更方便地在小型 RISC-V 硬件上体验 PicoClaw。 + +

+ + LicheeRV-Claw on Taobao + +

+ 2026-03-31 📱 **Android 支持！** PicoClaw 现可在 Android 上运行！APK 下载地址：[picoclaw.io](https://picoclaw.io/download) 2026-03-25 🚀 **v0.2.4 发布！** Agent 架构全面重构（SubTurn、Hook、Steering、EventBus）、微信/企业微信深度集成、安全体系升级（.security.yml、敏感数据过滤）、新增 Provider（AWS Bedrock、Azure、小米 MiMo），以及 35 项 Bug 修复。PicoClaw 已达 **26K ⭐**！ @@ -144,9 +152,9 @@ _*近期版本因快速合并 PR 可能占用 10–20MB，资源优化已列入 PicoClaw 几乎可以部署在任何 Linux 设备上！ -- $9.9 [LicheeRV-Nano](https://www.aliexpress.com/item/1005006519668532.html) E(网口) 或 W(WiFi6) 版本，用于极简家庭助手 -- $30~50 [NanoKVM](https://www.aliexpress.com/item/1005007369816019.html)，或 $100 [NanoKVM-Pro](https://www.aliexpress.com/item/1005010048471263.html)，用于自动化服务器运维 -- $50 [MaixCAM](https://www.aliexpress.com/item/1005008053333693.html) 或 $100 [MaixCAM2](https://www.kickstarter.com/projects/zepan/maixcam2-build-your-next-gen-4k-ai-camera)，用于智能监控 +- $9.9 [LicheeRV-Nano](https://item.taobao.com/item.htm?id=764939520376) E(网口) 或 W(WiFi6) 版本，用于极简家庭助手 +- $30~50 [NanoKVM](https://item.taobao.com/item.htm?id=811206560480)，或 $100 [NanoKVM-Pro](https://item.taobao.com/item.htm?id=994419942411)，用于自动化服务器运维 +- $50 [MaixCAM](https://item.taobao.com/item.htm?id=784724795837) 或 $100 [MaixCAM2](https://item.taobao.com/item.htm?id=1050380368975)，用于智能监控 @@ -475,7 +483,7 @@ PicoClaw 可以搜索网络以提供最新信息。在 `tools.web` 中配置： | 搜索引擎 | API Key | 免费额度 | 链接 | |---------|---------|---------|------| -| [百度搜索](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | 必填 | 1000 次/天 | AI 搜索，国内首选 | +| [百度搜索](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | 必填 | 1500 次/月（按天发放） | AI 搜索，国内首选 | | [Tavily](https://tavily.com) | 必填 | 1000 次/月 | 专为 AI Agent 优化 | | [GLM Search](https://open.bigmodel.cn/) | 必填 | 视情况 | 智谱网络搜索 | | DuckDuckGo | 无需 | 无限制 | 内置备用（国内访问困难） | diff --git a/docs/reference/config-versioning.md b/docs/reference/config-versioning.md index 36f327e8c..74a5bbd89 100644 --- a/docs/reference/config-versioning.md +++ b/docs/reference/config-versioning.md @@ -282,4 +282,3 @@ New config (version 3): - Check that the migration doesn't overwrite values with defaults unnecessarily - Review the conversion logic in the loader functions - Check the auto-backup files (e.g., `config.json.20260330.bak`) to recover original data - diff --git a/docs/reference/tools_configuration.md b/docs/reference/tools_configuration.md index 810d91ef2..d92f58476 100644 --- a/docs/reference/tools_configuration.md +++ b/docs/reference/tools_configuration.md @@ -66,6 +66,32 @@ General settings for fetching and processing webpage content. | `enabled` | bool | true | Enable DuckDuckGo search | | `max_results` | int | 5 | Maximum number of results | +### Gemini Google Search + +Gemini search uses Gemini with Google Search grounding. It returns an AI-synthesized answer with citations from Google Search. + +| Config | Type | Default | Description | +|---------------|--------|----------------------|-----------------------------------| +| `enabled` | bool | false | Enable Gemini Google Search | +| `api_key` | string | - | Google Gemini API key | +| `model` | string | `gemini-2.5-flash` | Gemini model used for search | +| `max_results` | int | 5 | Maximum number of citations | + +```json +{ + "tools": { + "web": { + "gemini": { + "enabled": true, + "api_key": "YOUR_GEMINI_API_KEY", + "model": "gemini-2.5-flash", + "max_results": 5 + } + } + } +} +``` + ### Baidu Search Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5), which is AI-powered and optimized for Chinese-language queries. diff --git a/docs/security/security_configuration.md b/docs/security/security_configuration.md index 065eb1e76..ad4b4f183 100644 --- a/docs/security/security_configuration.md +++ b/docs/security/security_configuration.md @@ -89,6 +89,13 @@ channels: nickserv_password: "your-irc-nickserv-password" sasl_password: "your-irc-sasl-password" +# Channel Settings (nested format for channels that use settings block) +channel_list: + mqtt: + settings: + username: "your-mqtt-username" + password: "your-mqtt-password" + # Web Tool API Keys web: brave: @@ -226,6 +233,19 @@ channels: - `channels.feishu.app_secret` → `config.channels.feishu.app_secret` - etc. +Channels that use a `settings` block (e.g. MQTT) use the `channel_list` key instead: + +```yaml +channel_list: + mqtt: + settings: + username: "value" + password: "value" +``` + +- `channel_list.mqtt.settings.username` → `config.channel_list.mqtt.settings.username` +- `channel_list.mqtt.settings.password` → `config.channel_list.mqtt.settings.password` + ### Web Tools **Brave, Tavily, Perplexity:** diff --git a/go.mod b/go.mod index c7e77c0f9..adf944424 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ module github.com/sipeed/picoclaw -go 1.25.9 +go 1.25.10 require ( - fyne.io/systray v1.12.0 + fyne.io/systray v1.12.1 github.com/SevereCloud/vksdk/v3 v3.3.1 github.com/adhocore/gronx v1.19.6 github.com/anthropics/anthropic-sdk-go v1.26.0 github.com/atc0005/go-teams-notify/v2 v2.14.0 - github.com/aws/aws-sdk-go-v2 v1.41.6 - github.com/aws/aws-sdk-go-v2/config v1.32.16 - github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.5 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.6 github.com/bwmarrin/discordgo v0.29.0 github.com/caarlos0/env/v11 v11.4.0 github.com/charmbracelet/lipgloss v1.1.0 @@ -21,7 +21,8 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/h2non/filetype v1.1.3 - github.com/larksuite/oapi-sdk-go/v3 v3.5.4 + github.com/larksuite/oapi-sdk-go/v3 v3.6.1 + github.com/line/line-bot-sdk-go/v8 v8.19.0 github.com/mdp/qrterminal/v3 v3.2.1 github.com/minio/selfupdate v0.6.0 github.com/modelcontextprotocol/go-sdk v1.5.0 @@ -52,19 +53,19 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.15 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 // indirect - github.com/aws/smithy-go v1.25.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beeper/argo-go v1.1.2 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect @@ -75,6 +76,7 @@ require ( github.com/coder/websocket v1.8.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eclipse/paho.mqtt.golang v1.5.1 // indirect github.com/elliotchance/orderedmap/v3 v3.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -118,7 +120,7 @@ require ( github.com/github/copilot-sdk/go v0.2.0 github.com/go-resty/resty/v2 v2.17.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/jsonschema-go v0.4.2 + github.com/google/jsonschema-go v0.4.3 github.com/grbit/go-json v0.11.0 // indirect github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 5cd39ec8d..b44469e21 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -fyne.io/systray v1.12.0 h1:CA1Kk0e2zwFlxtc02L3QFSiIbxJ/P0n582YrZHT7aTM= -fyne.io/systray v1.12.0/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= +fyne.io/systray v1.12.1 h1:ygBD6aZXwiOmZoY5N+ukbH9pih0Kq6fYgVeMYbr5skQ= +fyne.io/systray v1.12.1/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/SevereCloud/vksdk/v3 v3.3.1 h1:O86zsp5LQnHE+O5acvuXM/s6S1LyxzVTkF6+Lup0Jyg= @@ -21,38 +21,38 @@ github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAf github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q= github.com/atc0005/go-teams-notify/v2 v2.14.0 h1:7N+xw+COnYANLREaAveQ65rsNQ12nIZJED9nMLyscCo= github.com/atc0005/go-teams-notify/v2 v2.14.0/go.mod h1:EECsWM2b0Hvoz7O+QdlsvyN2KCUOFQCGj8bUBXv3A3Q= -github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg= -github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 h1:adBsCIIpLbLmYnkQU+nAChU5yhVTvu5PerROm+/Kq2A= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9/go.mod h1:uOYhgfgThm/ZyAuJGNQ5YgNyOlYfqnGpTHXvk3cpykg= -github.com/aws/aws-sdk-go-v2/config v1.32.16 h1:Q0iQ7quUgJP0F/SCRTieScnaMdXr9h/2+wze1u3cNeM= -github.com/aws/aws-sdk-go-v2/config v1.32.16/go.mod h1:duCCnJEFqpt2RC6no1iK6q+8HpwOAkiUua0pY507dQc= -github.com/aws/aws-sdk-go-v2/credentials v1.19.15 h1:fyvgWTszojq8hEnMi8PPBTvZdTtEVmAVyo+NFLHBhH4= -github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 h1:IOGsJ1xVWhsi+ZO7/NW8OuZZBtMJLZbk4P5HDjJO0jQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22/go.mod h1:b+hYdbU+jGKfXE8kKM6g1+h+L/Go3vMvzlxBsiuGsxg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 h1:GmLa5Kw1ESqtFpXsx5MmC84QWa/ZrLZvlJGa2y+4kcQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 h1:dY4kWZiSaXIzxnKlj17nHnBcXXBfac6UlsAx2qL6XrU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22/go.mod h1:KIpEUx0JuRZLO7U6cbV204cWAEco2iC3l061IxlwLtI= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 h1:FPXsW9+gMuIeKmz7j6ENWcWtBGTe1kH8r9thNt5Uxx4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23/go.mod h1:7J8iGMdRKk6lw2C+cMIphgAnT8uTwBwNOsGkyOCm80U= -github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.5 h1:ZGTl4Rxft1uyENAlGESY04hMzE4cLLNUPI7dGw08haw= -github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.5/go.mod h1:jnugA+VgESQGgXuEKK6zVToET/DtODq7LQYpe+BkKT4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcbVcGABLOVuPYaIihj6IlkqubBwFj10K5fxRek= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6Y1Lbvt9WFuJ0ugUK2xn6hIWUBBbKuOWF30s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 h1:a1Fq/KXn75wSzoJaPQTgZO0wHGqE9mjFnylnqEPTchA= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.10/go.mod h1:p6+MXNxW7IA6dMgHfTAzljuwSKD0NCm/4lbS4t6+7vI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 h1:x6bKbmDhsgSZwv6q19wY/u3rLk/3FGjJWyqKcIRufpE= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.16/go.mod h1:CudnEVKRtLn0+3uMV0yEXZ+YZOKnAtUJ5DmDhilVnIw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 h1:oK/njaL8GtyEihkWMD4k3VgHCT64RQKkZwh0DG5j8ak= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20/go.mod h1:JHs8/y1f3zY7U5WcuzoJ/yAYGYtNIVPKLIbp61euvmg= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 h1:ks8KBcZPh3PYISr5dAiXCM5/Thcuxk8l+PG4+A0exds= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.0/go.mod h1:pFw33T0WLvXU3rw1WBkpMlkgIn54eCB5FYLhjDc9Foo= -github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U= -github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.6 h1:Wbo1WlWyGaAXlr6C7OGXq9avbdJhIV9cQ4M6E34b5x8= +github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.50.6/go.mod h1:uY1fJe6m3I3w/m8UAkQ89Cm/ZAt/um6LW+AOZU33LDI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/beeper/argo-go v1.1.2 h1:UQI2G8F+NLfGTOmTUI0254pGKx/HUU/etbUGTJv91Fs= @@ -95,6 +95,8 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= +github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= github.com/elliotchance/orderedmap/v3 v3.1.0 h1:j4DJ5ObEmMBt/lcwIecKcoRxIQUEnw0L804lXYDt/pg= github.com/elliotchance/orderedmap/v3 v3.1.0/go.mod h1:G+Hc2RwaZvJMcS4JpGCOyViCnGeKf0bTYCGTO4uhjSo= github.com/ergochat/irc-go v0.6.0 h1:Y0AGV76aeihJfCtLaQh+OyJKFiKGrYC0VTkeMZ6XW28= @@ -142,8 +144,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -177,8 +179,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/larksuite/oapi-sdk-go/v3 v3.5.4 h1:U2S9x9LrfH++ZqJ+YAiUlqzCWJmVXhFdS8Z7rIBH8H0= -github.com/larksuite/oapi-sdk-go/v3 v3.5.4/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/line/line-bot-sdk-go/v8 v8.19.0 h1:5FD/1SprRZ8Y0FiUI6syYiBewOs0ak2tuUBMYN0wzE4= +github.com/line/line-bot-sdk-go/v8 v8.19.0/go.mod h1:AeSRUuu7WGgveGDJb6DyKyFUOst2UB2aF6LO2cQeuXs= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 2c456dca7..5749149c1 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -21,6 +21,7 @@ import ( "github.com/sipeed/picoclaw/pkg/commands" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/constants" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/providers" @@ -37,9 +38,13 @@ type AgentLoop struct { registry *AgentRegistry state *state.Manager - // Event system (from Incoming) - eventBus *EventBus - hooks *HookManager + // Runtime event system + runtimeEvents runtimeevents.Bus + ownsRuntimeEvents bool + runtimeEventLogMu sync.RWMutex + runtimeEventLogger *runtimeEventLogger + runtimeEventLogSub runtimeevents.Subscription + hooks *HookManager // Runtime state running atomic.Bool @@ -50,9 +55,11 @@ type AgentLoop struct { transcriber asr.Transcriber cmdRegistry *commands.Registry mcp mcpRuntime + evolution *evolutionBridge hookRuntime hookRuntime steering *steeringQueue pendingSkills sync.Map + pendingStops sync.Map mu sync.RWMutex // workerSem limits concurrent turn processing workers. @@ -172,6 +179,12 @@ func (al *AgentLoop) Run(ctx context.Context) error { phase: TurnPhaseSetup, } if _, loaded := al.activeTurnStates.LoadOrStore(sessionKey, placeholder); loaded { + if al.tryHandleStopCommand(ctx, msg, sessionKey) { + continue + } + + msg = al.prepareInboundMessageForAgent(ctx, msg) + // Another turn is already active (or reserved) for this session — enqueue if err := al.enqueueSteeringMessage(sessionKey, agentID, providers.Message{ Role: "user", @@ -235,6 +248,24 @@ func (al *AgentLoop) Run(ctx context.Context) error { defer al.channelManager.InvokeTypingStop(m.Channel, m.ChatID) } + if al.takePendingStop(sessionKey) { + al.activeTurnStates.Delete(sessionKey) + target := &continuationTarget{ + SessionKey: sessionKey, + Channel: m.Channel, + ChatID: m.ChatID, + } + continued, continueErr := al.drainQueuedSteeringContinuations(ctx, target) + if continueErr != nil { + al.maybePublishError(ctx, m.Channel, m.ChatID, sessionKey, continueErr) + return + } + if continued != "" { + al.PublishResponseIfNeeded(ctx, target.Channel, target.ChatID, target.SessionKey, continued) + } + return + } + al.runTurnWithSteering(ctx, m) }(msg) @@ -280,13 +311,28 @@ func (al *AgentLoop) Close() { }) } } + evolution := al.currentEvolutionBridge() + if evolution != nil { + if err := evolution.Close(); err != nil { + logger.ErrorCF("agent", "Failed to close evolution bridge", + map[string]any{ + "error": err.Error(), + }) + } + } al.GetRegistry().Close() if al.hooks != nil { al.hooks.Close() } - if al.eventBus != nil { - al.eventBus.Close() + al.closeRuntimeEventLogger() + if al.runtimeEvents != nil && al.ownsRuntimeEvents { + if err := al.runtimeEvents.Close(); err != nil { + logger.ErrorCF("agent", "Failed to close runtime event bus", + map[string]any{ + "error": err.Error(), + }) + } } } @@ -294,12 +340,6 @@ func (al *AgentLoop) Close() { // UnmountHook removes a previously registered in-process hook. -// SubscribeEvents registers a subscriber for agent-loop events. - -// UnsubscribeEvents removes a previously registered event subscriber. - -// EventDrops returns the number of dropped events for the given kind. - type turnEventScope struct { agentID string sessionKey string @@ -364,14 +404,29 @@ func (al *AgentLoop) ReloadProviderAndConfig( // Ensure shared tools are re-registered on the new registry registerSharedTools(al, cfg, al.bus, registry, provider) + newEvolution, evolutionErr := newEvolutionBridge(registry, cfg, provider) + if evolutionErr != nil { + logger.WarnCF("agent", "Failed to reinitialize evolution bridge during reload", + map[string]any{"error": evolutionErr.Error()}) + } + if newEvolution != nil { + newEvolution.setCurrentCheck(al.isCurrentEvolutionBridge) + if err := newEvolution.subscribeRuntimeEvents(al.runtimeEvents.Channel()); err != nil { + logger.WarnCF("agent", "Failed to subscribe reloaded evolution bridge to runtime events", + map[string]any{"error": err.Error()}) + } + } + // Atomically swap the config and registry under write lock // This ensures readers see a consistent pair al.mu.Lock() oldRegistry := al.registry + oldEvolution := al.evolution // Store new values al.cfg = cfg al.registry = registry + al.evolution = newEvolution // Also update fallback chain with new config; rebuild rate limiter registry. newRL := providers.NewRateLimiterRegistry() @@ -384,6 +439,7 @@ func (al *AgentLoop) ReloadProviderAndConfig( al.fallback = providers.NewFallbackChain(providers.NewCooldownTracker(), newRL) al.mu.Unlock() + al.refreshRuntimeEventLogger(cfg) oldMCPManager := al.mcp.reset() al.hookRuntime.reset(al) @@ -398,6 +454,12 @@ func (al *AgentLoop) ReloadProviderAndConfig( map[string]any{"error": err.Error()}) } } + if oldEvolution != nil { + if err := oldEvolution.Close(); err != nil { + logger.WarnCF("agent", "Failed to close previous evolution bridge during reload", + map[string]any{"error": err.Error()}) + } + } if err := al.ensureMCPInitialized(ctx); err != nil { logger.WarnCF("agent", "MCP failed to reinitialize after reload", map[string]any{"error": err.Error()}) diff --git a/pkg/agent/agent_command.go b/pkg/agent/agent_command.go index a2ed068d6..ae0293d71 100644 --- a/pkg/agent/agent_command.go +++ b/pkg/agent/agent_command.go @@ -274,6 +274,12 @@ func (al *AgentLoop) buildCommandsRuntime( return nil }, } + rt.StopActiveTurn = func() (commands.StopResult, error) { + if opts == nil { + return commands.StopResult{}, fmt.Errorf("process options not available") + } + return al.stopActiveTurnForSession(opts.Dispatch.SessionKey) + } if agent != nil && agent.ContextBuilder != nil { rt.ListSkillNames = agent.ContextBuilder.ListSkillNames } diff --git a/pkg/agent/agent_event.go b/pkg/agent/agent_event.go index 9b8625df1..174b93e38 100644 --- a/pkg/agent/agent_event.go +++ b/pkg/agent/agent_event.go @@ -5,7 +5,7 @@ package agent import ( "fmt" - "github.com/sipeed/picoclaw/pkg/logger" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" ) func (al *AgentLoop) newTurnEventScope(agentID, sessionKey string, turnCtx *TurnContext) turnEventScope { @@ -18,8 +18,8 @@ func (al *AgentLoop) newTurnEventScope(agentID, sessionKey string, turnCtx *Turn } } -func (ts turnEventScope) meta(iteration int, source, tracePath string) EventMeta { - return EventMeta{ +func (ts turnEventScope) meta(iteration int, source, tracePath string) HookMeta { + return HookMeta{ AgentID: ts.agentID, TurnID: ts.turnID, SessionKey: ts.sessionKey, @@ -30,119 +30,55 @@ func (ts turnEventScope) meta(iteration int, source, tracePath string) EventMeta } } -func (al *AgentLoop) emitEvent(kind EventKind, meta EventMeta, payload any) { - clonedMeta := cloneEventMeta(meta) - evt := Event{ - Kind: kind, - Meta: clonedMeta, - Context: cloneTurnContext(clonedMeta.turnContext), - Payload: payload, +func (al *AgentLoop) emitEvent(kind runtimeevents.Kind, meta HookMeta, payload any) { + clonedMeta := cloneHookMeta(meta) + eventCtx := cloneTurnContext(clonedMeta.turnContext) + evt := runtimeevents.Event{ + Kind: kind, + Source: runtimeevents.Source{Component: "agent", Name: clonedMeta.AgentID}, + Scope: runtimeScopeFromHookMeta(clonedMeta, eventCtx), + Correlation: runtimeCorrelationFromHookMeta(clonedMeta), + Severity: runtimeSeverityForAgentEvent(kind, payload), + Payload: payload, + Attrs: runtimeAttrsFromHookMeta(clonedMeta), } - if al == nil || al.eventBus == nil { + if al == nil { return } - al.logEvent(evt) - - al.eventBus.Emit(evt) + deliveredToEvolution := false + if kind == runtimeevents.KindAgentTurnEnd { + evolution := al.currentEvolutionBridge() + if evolution != nil { + deliveredToEvolution = evolution.handleRuntimeTurnEnd(evt) + } + } + if deliveredToEvolution { + if evt.Attrs == nil { + evt.Attrs = make(map[string]any, 1) + } + evt.Attrs[evolutionDirectDeliveryAttr] = true + } + al.publishRuntimeEvent(evt) } -func (al *AgentLoop) logEvent(evt Event) { - fields := map[string]any{ - "event_kind": evt.Kind.String(), - "agent_id": evt.Meta.AgentID, - "turn_id": evt.Meta.TurnID, - "session_key": evt.Meta.SessionKey, - "iteration": evt.Meta.Iteration, +func (al *AgentLoop) currentEvolutionBridge() *evolutionBridge { + if al == nil { + return nil } + al.mu.RLock() + defer al.mu.RUnlock() + return al.evolution +} - if evt.Meta.TracePath != "" { - fields["trace"] = evt.Meta.TracePath +func (al *AgentLoop) isCurrentEvolutionBridge(bridge *evolutionBridge) bool { + if al == nil || bridge == nil { + return false } - if evt.Meta.Source != "" { - fields["source"] = evt.Meta.Source - } - - appendEventContextFields(fields, evt.Context) - - switch payload := evt.Payload.(type) { - case TurnStartPayload: - fields["user_len"] = len(payload.UserMessage) - fields["media_count"] = payload.MediaCount - case TurnEndPayload: - fields["status"] = payload.Status - fields["iterations_total"] = payload.Iterations - fields["duration_ms"] = payload.Duration.Milliseconds() - fields["final_len"] = payload.FinalContentLen - case LLMRequestPayload: - fields["model"] = payload.Model - fields["messages"] = payload.MessagesCount - fields["tools"] = payload.ToolsCount - fields["max_tokens"] = payload.MaxTokens - case LLMDeltaPayload: - fields["content_delta_len"] = payload.ContentDeltaLen - fields["reasoning_delta_len"] = payload.ReasoningDeltaLen - case LLMResponsePayload: - fields["content_len"] = payload.ContentLen - fields["tool_calls"] = payload.ToolCalls - fields["has_reasoning"] = payload.HasReasoning - case LLMRetryPayload: - fields["attempt"] = payload.Attempt - fields["max_retries"] = payload.MaxRetries - fields["reason"] = payload.Reason - fields["error"] = payload.Error - fields["backoff_ms"] = payload.Backoff.Milliseconds() - case ContextCompressPayload: - fields["reason"] = payload.Reason - fields["dropped_messages"] = payload.DroppedMessages - fields["remaining_messages"] = payload.RemainingMessages - case SessionSummarizePayload: - fields["summarized_messages"] = payload.SummarizedMessages - fields["kept_messages"] = payload.KeptMessages - fields["summary_len"] = payload.SummaryLen - fields["omitted_oversized"] = payload.OmittedOversized - case ToolExecStartPayload: - fields["tool"] = payload.Tool - fields["args_count"] = len(payload.Arguments) - case ToolExecEndPayload: - fields["tool"] = payload.Tool - fields["duration_ms"] = payload.Duration.Milliseconds() - fields["for_llm_len"] = payload.ForLLMLen - fields["for_user_len"] = payload.ForUserLen - fields["is_error"] = payload.IsError - fields["async"] = payload.Async - case ToolExecSkippedPayload: - fields["tool"] = payload.Tool - fields["reason"] = payload.Reason - case SteeringInjectedPayload: - fields["count"] = payload.Count - fields["total_content_len"] = payload.TotalContentLen - case FollowUpQueuedPayload: - fields["source_tool"] = payload.SourceTool - fields["content_len"] = payload.ContentLen - case InterruptReceivedPayload: - fields["interrupt_kind"] = payload.Kind - fields["role"] = payload.Role - fields["content_len"] = payload.ContentLen - fields["queue_depth"] = payload.QueueDepth - fields["hint_len"] = payload.HintLen - case SubTurnSpawnPayload: - fields["child_agent_id"] = payload.AgentID - fields["label"] = payload.Label - case SubTurnEndPayload: - fields["child_agent_id"] = payload.AgentID - fields["status"] = payload.Status - case SubTurnResultDeliveredPayload: - fields["target_channel"] = payload.TargetChannel - fields["target_chat_id"] = payload.TargetChatID - fields["content_len"] = payload.ContentLen - case ErrorPayload: - fields["stage"] = payload.Stage - fields["error"] = payload.Message - } - - logger.InfoCF("eventbus", fmt.Sprintf("Agent event: %s", evt.Kind.String()), fields) + al.mu.RLock() + defer al.mu.RUnlock() + return al.evolution == bridge } // MountHook registers an in-process hook on the agent loop. @@ -161,28 +97,26 @@ func (al *AgentLoop) UnmountHook(name string) { al.hooks.Unmount(name) } -// SubscribeEvents registers a subscriber for agent-loop events. -func (al *AgentLoop) SubscribeEvents(buffer int) EventSubscription { - if al == nil || al.eventBus == nil { - ch := make(chan Event) - close(ch) - return EventSubscription{C: ch} +// RuntimeEvents returns the root runtime event channel. +func (al *AgentLoop) RuntimeEvents() runtimeevents.EventChannel { + if al == nil || al.runtimeEvents == nil { + return nil } - return al.eventBus.Subscribe(buffer) + return al.runtimeEvents.Channel() } -// UnsubscribeEvents removes a previously registered event subscriber. -func (al *AgentLoop) UnsubscribeEvents(id uint64) { - if al == nil || al.eventBus == nil { - return +// RuntimeEventStats returns runtime event bus counters. +func (al *AgentLoop) RuntimeEventStats() runtimeevents.Stats { + if al == nil || al.runtimeEvents == nil { + return runtimeevents.Stats{Closed: true} } - al.eventBus.Unsubscribe(id) + return al.runtimeEvents.Stats() } -// EventDrops returns the number of dropped events for the given kind. -func (al *AgentLoop) EventDrops(kind EventKind) int64 { - if al == nil || al.eventBus == nil { - return 0 +// RuntimeEventBus returns the runtime event bus used by the agent loop. +func (al *AgentLoop) RuntimeEventBus() runtimeevents.Bus { + if al == nil { + return nil } - return al.eventBus.Dropped(kind) + return al.runtimeEvents } diff --git a/pkg/agent/agent_init.go b/pkg/agent/agent_init.go index 335fd8537..50f0227a1 100644 --- a/pkg/agent/agent_init.go +++ b/pkg/agent/agent_init.go @@ -13,6 +13,7 @@ import ( "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/commands" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/skills" @@ -24,6 +25,7 @@ func NewAgentLoop( cfg *config.Config, msgBus *bus.MessageBus, provider providers.LLMProvider, + opts ...AgentLoopOption, ) *AgentLoop { registry := NewAgentRegistry(cfg, provider) @@ -47,7 +49,12 @@ func NewAgentLoop( stateManager = state.NewManager(defaultAgent.Workspace) } - eventBus := NewEventBus() + bridge, err := newEvolutionBridge(registry, cfg, provider) + if err != nil { + logger.WarnCF("agent", "Failed to initialize evolution bridge", map[string]any{ + "error": err.Error(), + }) + } // Determine worker pool size from config (default: 1 = sequential) workerPoolSize := cfg.Agents.Defaults.MaxParallelTurns @@ -56,18 +63,37 @@ func NewAgentLoop( } al := &AgentLoop{ - bus: msgBus, - cfg: cfg, - registry: registry, - state: stateManager, - eventBus: eventBus, - fallback: fallbackChain, - cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()), - steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)), - workerSem: make(chan struct{}, workerPoolSize), + bus: msgBus, + cfg: cfg, + registry: registry, + state: stateManager, + fallback: fallbackChain, + cmdRegistry: commands.NewRegistry(commands.BuiltinDefinitions()), + evolution: bridge, + steering: newSteeringQueue(parseSteeringMode(cfg.Agents.Defaults.SteeringMode)), + workerSem: make(chan struct{}, workerPoolSize), + ownsRuntimeEvents: true, } + for _, opt := range opts { + if opt != nil { + opt(al) + } + } + if al.runtimeEvents == nil { + al.runtimeEvents = runtimeevents.NewBus() + al.ownsRuntimeEvents = true + } + if bridge != nil { + bridge.setCurrentCheck(al.isCurrentEvolutionBridge) + if err := bridge.subscribeRuntimeEvents(al.runtimeEvents.Channel()); err != nil { + logger.WarnCF("agent", "Failed to subscribe evolution bridge to runtime events", map[string]any{ + "error": err.Error(), + }) + } + } + al.refreshRuntimeEventLogger(cfg) al.providerFactory = providers.CreateProviderFromConfig - al.hooks = NewHookManager(eventBus) + al.hooks = NewHookManager(al.runtimeEvents.Channel()) configureHookManagerFromConfig(al.hooks, cfg) al.contextManager = al.resolveContextManager() @@ -327,5 +353,22 @@ func registerSharedTools( } else if (spawnEnabled || spawnStatusEnabled) && !cfg.Tools.IsToolEnabled("subagent") { logger.WarnCF("agent", "spawn/spawn_status tools require subagent to be enabled", nil) } + + // Register delegate tool for multi-agent setups. + // Auto-enabled when multiple agents exist. Delegation uses the SubTurn + // mechanism directly (not SubagentManager) and is independent of the + // subagent tool. + if len(registry.ListAgentIDs()) > 1 { + delegateTool := tools.NewDelegateTool() + delegateTool.SetSpawner(NewSubTurnSpawner(al)) + currentAgentID := agentID + delegateTool.SetSelfAgentID(currentAgentID) + delegateTool.SetAllowlistChecker(func(targetAgentID string) bool { + return registry.CanSpawnSubagent(currentAgentID, targetAgentID) + }) + agent.Tools.Register(delegateTool) + } + + warnOnUnknownAgentToolDeclarations(agentID, agent.Workspace, agent.Definition, agent.Tools) } } diff --git a/pkg/agent/agent_mcp.go b/pkg/agent/agent_mcp.go index fcb57a5d4..e8cdf81c8 100644 --- a/pkg/agent/agent_mcp.go +++ b/pkg/agent/agent_mcp.go @@ -85,8 +85,18 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { return nil } + mcpCfg := filterMCPConfigServers(al.cfg.Tools.MCP, al.registry.allowedMCPServers()) + if mcpCfg.Servers == nil || len(mcpCfg.Servers) == 0 { + logger.InfoCF( + "agent", + "No MCP servers selected after applying per-agent mcpServers allowlists", + nil, + ) + return nil + } + findValidServer := false - for _, serverCfg := range al.cfg.Tools.MCP.Servers { + for _, serverCfg := range mcpCfg.Servers { if serverCfg.Enabled { findValidServer = true } @@ -97,7 +107,7 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { } al.mcp.initOnce.Do(func() { - mcpManager := mcp.NewManager() + mcpManager := mcp.NewManager(mcp.WithRuntimeEvents(al.runtimeEvents)) defaultAgent := al.registry.GetDefaultAgent() workspacePath := al.cfg.WorkspacePath() @@ -105,7 +115,7 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { workspacePath = defaultAgent.Workspace } - if err := mcpManager.LoadFromMCPConfig(ctx, al.cfg.Tools.MCP, workspacePath); err != nil { + if err := mcpManager.LoadFromMCPConfig(ctx, mcpCfg, workspacePath); err != nil { al.mcp.setInitErr(fmt.Errorf("failed to load MCP servers: %w", err)) logger.WarnCF("agent", "Failed to load MCP servers, MCP tools will not be available", map[string]any{ @@ -132,27 +142,9 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { // Determine whether this server's tools should be deferred (hidden). // Per-server "deferred" field takes precedence over the global Discovery.Enabled. - serverCfg := al.cfg.Tools.MCP.Servers[serverName] + serverCfg := mcpCfg.Servers[serverName] registerAsHidden := serverIsDeferred(al.cfg.Tools.MCP.Discovery.Enabled, serverCfg) - - for _, agentID := range agentIDs { - agent, ok := al.registry.GetAgent(agentID) - if !ok || agent.ContextBuilder == nil { - continue - } - if err := agent.ContextBuilder.RegisterPromptContributor(mcpServerPromptContributor{ - serverName: serverName, - toolCount: len(conn.Tools), - deferred: registerAsHidden, - }); err != nil { - logger.WarnCF("agent", "Failed to register MCP prompt contributor", - map[string]any{ - "agent_id": agentID, - "server": serverName, - "error": err.Error(), - }) - } - } + registeredToolsByAgent := make(map[string]map[string]struct{}, len(agentIDs)) for _, tool := range conn.Tools { for _, agentID := range agentIDs { @@ -160,28 +152,57 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { if !ok { continue } + if !agent.AllowsMCPServer(serverName) { + logger.DebugCF("agent", "Skipped MCP tool registration by agent mcpServers allowlist", + map[string]any{ + "agent_id": agentID, + "server": serverName, + "tool": tool.Name, + }) + continue + } mcpTool := tools.NewMCPTool(mcpManager, serverName, tool) + toolName := mcpTool.Name() mcpTool.SetWorkspace(agent.Workspace) mcpTool.SetMaxInlineTextRunes(al.cfg.Tools.MCP.GetMaxInlineTextChars()) + mcpTool.SetEventPublisher(al.runtimeEvents) if registerAsHidden { agent.Tools.RegisterHidden(mcpTool) } else { agent.Tools.Register(mcpTool) } + if !toolRegistryIncludes(agent.Tools, toolName) { + continue + } + recordRegisteredMCPTool(registeredToolsByAgent, agentID, toolName) totalRegistrations++ logger.DebugCF("agent", "Registered MCP tool", map[string]any{ "agent_id": agentID, "server": serverName, "tool": tool.Name, - "name": mcpTool.Name(), + "name": toolName, "deferred": registerAsHidden, }) } } + + for _, agentID := range agentIDs { + agent, ok := al.registry.GetAgent(agentID) + if !ok { + continue + } + registerMCPServerPromptContributor( + agentID, + agent, + serverName, + len(registeredToolsByAgent[agentID]), + registerAsHidden, + ) + } } logger.InfoCF("agent", "MCP tools registered successfully", map[string]any{ @@ -229,6 +250,9 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { if !ok { continue } + if !agentHasDiscoverableMCPServers(al.cfg, agent.MCPServerAllowlist) { + continue + } if useRegex { agent.Tools.Register(tools.NewRegexSearchTool(agent.Tools, ttl, maxSearchResults)) @@ -245,6 +269,89 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { return al.mcp.getInitErr() } +func registerMCPServerPromptContributor( + agentID string, + agent *AgentInstance, + serverName string, + toolCount int, + registerAsHidden bool, +) { + if agent == nil || agent.ContextBuilder == nil || toolCount <= 0 { + return + } + if err := agent.ContextBuilder.RegisterPromptContributor(mcpServerPromptContributor{ + serverName: serverName, + toolCount: toolCount, + deferred: registerAsHidden, + }); err != nil { + logger.WarnCF("agent", "Failed to register MCP prompt contributor", + map[string]any{ + "agent_id": agentID, + "server": serverName, + "error": err.Error(), + }) + } +} + +func recordRegisteredMCPTool( + registeredToolsByAgent map[string]map[string]struct{}, + agentID, toolName string, +) { + if registeredToolsByAgent[agentID] == nil { + registeredToolsByAgent[agentID] = make(map[string]struct{}) + } + registeredToolsByAgent[agentID][toolName] = struct{}{} +} + +func toolRegistryIncludes(registry *tools.ToolRegistry, name string) bool { + if registry == nil { + return false + } + return registry.HasRegistered(name) +} + +func filterMCPConfigServers( + mcpCfg config.MCPConfig, + allowed map[string]struct{}, +) config.MCPConfig { + if allowed == nil { + return mcpCfg + } + + filtered := mcpCfg + filtered.Servers = make(map[string]config.MCPServerConfig) + normalizedAllowed := make(map[string]struct{}, len(allowed)) + for serverName := range allowed { + name := normalizeMCPServerName(serverName) + if name == "" { + continue + } + normalizedAllowed[name] = struct{}{} + } + for serverName, serverCfg := range mcpCfg.Servers { + if _, ok := normalizedAllowed[normalizeMCPServerName(serverName)]; ok { + filtered.Servers[serverName] = serverCfg + } + } + + return filtered +} + +func agentHasDiscoverableMCPServers(cfg *config.Config, allowed map[string]struct{}) bool { + if cfg == nil || !cfg.Tools.MCP.Enabled || !cfg.Tools.MCP.Discovery.Enabled { + return false + } + + filtered := filterMCPConfigServers(cfg.Tools.MCP, allowed) + for _, serverCfg := range filtered.Servers { + if serverCfg.Enabled && serverIsDeferred(cfg.Tools.MCP.Discovery.Enabled, serverCfg) { + return true + } + } + + return false +} + // serverIsDeferred reports whether an MCP server's tools should be registered // as hidden (deferred/discovery mode). // diff --git a/pkg/agent/agent_mcp_test.go b/pkg/agent/agent_mcp_test.go index b68fcc2c1..f85861146 100644 --- a/pkg/agent/agent_mcp_test.go +++ b/pkg/agent/agent_mcp_test.go @@ -14,6 +14,7 @@ import ( "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/mcp" + agenttools "github.com/sipeed/picoclaw/pkg/tools" ) func boolPtr(b bool) *bool { return &b } @@ -135,6 +136,139 @@ func TestServerIsDeferred(t *testing.T) { } } +func TestRegisterMCPServerPromptContributorUsesActualRegisteredToolCount(t *testing.T) { + cb := NewContextBuilder(t.TempDir()) + agent := &AgentInstance{ContextBuilder: cb} + + registerMCPServerPromptContributor("research", agent, "github", 0, false) + messages := cb.BuildMessagesFromPrompt(PromptBuildRequest{CurrentMessage: "hello"}) + if prompt := messages[0].Content; strings.Contains(prompt, "MCP server `github`") { + t.Fatalf("expected no MCP prompt when no tools were registered, got %q", prompt) + } + + registerMCPServerPromptContributor("research", agent, "github", 2, false) + messages = cb.BuildMessagesFromPrompt(PromptBuildRequest{CurrentMessage: "hello"}) + prompt := messages[0].Content + if !strings.Contains(prompt, "MCP server `github` is connected") { + t.Fatalf("expected MCP prompt for registered tools, got %q", prompt) + } + if !strings.Contains(prompt, "It contributes 2 tool(s)") { + t.Fatalf("expected actual registered tool count in prompt, got %q", prompt) + } +} + +func TestToolRegistryIncludesReportsOnlyRegisteredTools(t *testing.T) { + registry := agenttools.NewToolRegistry() + registry.SetAllowlist([]string{"mcp_github_search"}) + + registry.RegisterHidden(&allowlistTestTool{name: "mcp_github_search"}) + registry.RegisterHidden(&allowlistTestTool{name: "mcp_github_create_issue"}) + + if !toolRegistryIncludes(registry, "mcp_github_search") { + t.Fatal("expected hidden registered MCP tool to be included") + } + if toolRegistryIncludes(registry, "mcp_github_create_issue") { + t.Fatal("blocked MCP tool should not be included") + } +} + +func TestFilterMCPConfigServersCaseInsensitivePreservesOriginalKeys(t *testing.T) { + mcpCfg := config.MCPConfig{ + Servers: map[string]config.MCPServerConfig{ + "GitHub": {Enabled: true}, + "filesystem": {Enabled: true}, + "Slack": {Enabled: true}, + }, + } + allowed := map[string]struct{}{ + "github": {}, + "FILESYSTEM": {}, + } + + filtered := filterMCPConfigServers(mcpCfg, allowed) + + if len(filtered.Servers) != 2 { + t.Fatalf("filtered.Servers = %v, want 2 entries", filtered.Servers) + } + if _, ok := filtered.Servers["GitHub"]; !ok { + t.Fatal("expected original GitHub config key to be preserved") + } + if _, ok := filtered.Servers["filesystem"]; !ok { + t.Fatal("expected filesystem config key to be preserved") + } + if _, ok := filtered.Servers["github"]; ok { + t.Fatal("did not expect normalized github key to replace original config key") + } + if _, ok := filtered.Servers["Slack"]; ok { + t.Fatal("did not expect unallowed Slack server") + } +} + +func TestAgentHasDiscoverableMCPServers(t *testing.T) { + deferredFalse := false + cfg := &config.Config{ + Tools: config.ToolsConfig{ + MCP: config.MCPConfig{ + ToolConfig: config.ToolConfig{Enabled: true}, + Discovery: config.ToolDiscoveryConfig{ + Enabled: true, + UseBM25: true, + UseRegex: false, + }, + Servers: map[string]config.MCPServerConfig{ + "github": {Enabled: true}, + "filesystem": {Enabled: true, Deferred: &deferredFalse}, + }, + }, + }, + } + + tests := []struct { + name string + allowed map[string]struct{} + want bool + }{ + { + name: "nil allowlist includes discoverable enabled server", + want: true, + }, + { + name: "empty allowlist denies all servers", + allowed: map[string]struct{}{}, + want: false, + }, + { + name: "selected server discoverable", + allowed: map[string]struct{}{ + "github": {}, + }, + want: true, + }, + { + name: "selected server opted out of discovery", + allowed: map[string]struct{}{ + "filesystem": {}, + }, + want: false, + }, + { + name: "unknown allowlist server matches nothing", + allowed: map[string]struct{}{ + "slack": {}, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := agentHasDiscoverableMCPServers(cfg, tt.allowed); got != tt.want { + t.Fatalf("agentHasDiscoverableMCPServers() = %v, want %v", got, tt.want) + } + }) + } +} + func TestEnsureMCPInitialized_LoadFailureSetsInitErr(t *testing.T) { al, cfg, _, _, cleanup := newTestAgentLoop(t) defer cleanup() diff --git a/pkg/agent/agent_message.go b/pkg/agent/agent_message.go index 96b0b0817..4d2886a80 100644 --- a/pkg/agent/agent_message.go +++ b/pkg/agent/agent_message.go @@ -102,9 +102,27 @@ func (al *AgentLoop) ProcessHeartbeat( }) } -func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) (string, error) { +func (al *AgentLoop) prepareInboundMessageForAgent( + ctx context.Context, + msg bus.InboundMessage, +) bus.InboundMessage { msg = bus.NormalizeInboundMessage(msg) + var hadAudio bool + msg, hadAudio = al.transcribeAudioInMessage(ctx, msg) + + // For audio messages the placeholder was deferred by the channel. + // Now that transcription (and optional feedback) is done, send it. + if hadAudio && al.channelManager != nil { + al.channelManager.SendPlaceholder(ctx, msg.Channel, msg.ChatID) + } + + return msg +} + +func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) (string, error) { + msg = al.prepareInboundMessageForAgent(ctx, msg) + // Add message preview to log (show full content for error messages) var logContent string if strings.Contains(msg.Content, "Error:") || strings.Contains(msg.Content, "error") { @@ -123,15 +141,6 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage) }, ) - var hadAudio bool - msg, hadAudio = al.transcribeAudioInMessage(ctx, msg) - - // For audio messages the placeholder was deferred by the channel. - // Now that transcription (and optional feedback) is done, send it. - if hadAudio && al.channelManager != nil { - al.channelManager.SendPlaceholder(ctx, msg.Channel, msg.ChatID) - } - // Route system messages to processSystemMessage if msg.Channel == "system" { return al.processSystemMessage(ctx, msg) diff --git a/pkg/agent/agent_options.go b/pkg/agent/agent_options.go new file mode 100644 index 000000000..224062a3f --- /dev/null +++ b/pkg/agent/agent_options.go @@ -0,0 +1,20 @@ +package agent + +import runtimeevents "github.com/sipeed/picoclaw/pkg/events" + +// AgentLoopOption configures an AgentLoop at construction time. +type AgentLoopOption func(*AgentLoop) + +// WithRuntimeEvents injects the runtime event bus used for new observation APIs. +// +// The injected bus is treated as externally owned and will not be closed by +// AgentLoop.Close. Passing nil leaves the default owned runtime bus enabled. +func WithRuntimeEvents(bus runtimeevents.Bus) AgentLoopOption { + return func(al *AgentLoop) { + if bus == nil { + return + } + al.runtimeEvents = bus + al.ownsRuntimeEvents = false + } +} diff --git a/pkg/agent/agent_outbound.go b/pkg/agent/agent_outbound.go index 1728f6f79..f4a01adfd 100644 --- a/pkg/agent/agent_outbound.go +++ b/pkg/agent/agent_outbound.go @@ -56,6 +56,16 @@ func (al *AgentLoop) PublishResponseIfNeeded(ctx context.Context, channel, chatI } if alreadySentToSameChat { + if al.channelManager != nil && channel != "" && chatID != "" { + dismissCtx, dismissCancel := context.WithTimeout(ctx, 5*time.Second) + al.channelManager.DismissToolFeedback( + dismissCtx, + channel, + chatID, + nil, + ) + dismissCancel() + } logger.DebugCF( "agent", "Skipped outbound (message tool already sent to same chat)", diff --git a/pkg/agent/agent_steering.go b/pkg/agent/agent_steering.go index c674bcafa..9b136e7cd 100644 --- a/pkg/agent/agent_steering.go +++ b/pkg/agent/agent_steering.go @@ -44,11 +44,36 @@ func (al *AgentLoop) runTurnWithSteering(ctx context.Context, initialMsg bus.Inb return } - // Drain steering queue using existing Continue mechanism + continued, continueErr := al.drainQueuedSteeringContinuations(ctx, target) + if continueErr != nil { + logger.WarnCF("agent", "Failed to continue queued steering", + map[string]any{ + "channel": target.Channel, + "chat_id": target.ChatID, + "error": continueErr.Error(), + }) + } else if continued != "" { + finalResponse = continued + } + + // Publish final response + if finalResponse != "" { + al.PublishResponseIfNeeded(ctx, target.Channel, target.ChatID, target.SessionKey, finalResponse) + } +} + +func (al *AgentLoop) drainQueuedSteeringContinuations( + ctx context.Context, + target *continuationTarget, +) (string, error) { + if target == nil { + return "", nil + } + + finalResponse := "" for al.pendingSteeringCountForScope(target.SessionKey) > 0 { - // Check for context cancellation between iterations - if ctx.Err() != nil { - return + if err := ctx.Err(); err != nil { + return finalResponse, err } logger.InfoCF("agent", "Continuing queued steering after turn end", @@ -61,13 +86,7 @@ func (al *AgentLoop) runTurnWithSteering(ctx context.Context, initialMsg bus.Inb continued, continueErr := al.Continue(ctx, target.SessionKey, target.Channel, target.ChatID) if continueErr != nil { - logger.WarnCF("agent", "Failed to continue queued steering", - map[string]any{ - "channel": target.Channel, - "chat_id": target.ChatID, - "error": continueErr.Error(), - }) - break + return finalResponse, continueErr } if continued == "" { break @@ -75,10 +94,7 @@ func (al *AgentLoop) runTurnWithSteering(ctx context.Context, initialMsg bus.Inb finalResponse = continued } - // Publish final response - if finalResponse != "" { - al.PublishResponseIfNeeded(ctx, target.Channel, target.ChatID, target.SessionKey, finalResponse) - } + return finalResponse, nil } func (al *AgentLoop) resolveSteeringTarget(msg bus.InboundMessage) (string, string, bool) { diff --git a/pkg/agent/agent_stop.go b/pkg/agent/agent_stop.go new file mode 100644 index 000000000..54cd51477 --- /dev/null +++ b/pkg/agent/agent_stop.go @@ -0,0 +1,122 @@ +package agent + +import ( + "context" + "fmt" + "strings" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/commands" +) + +func (al *AgentLoop) tryHandleStopCommand( + ctx context.Context, + msg bus.InboundMessage, + sessionKey string, +) bool { + cmdName, ok := commands.CommandName(msg.Content) + if !ok || cmdName != "stop" { + return false + } + + result, err := al.stopActiveTurnForSession(sessionKey) + + // This function is only called when loaded=true (another turn already + // claimed this session). If stopActiveTurnForSession found a pending + // placeholder but didn't stop it, that placeholder belongs to the other + // message's worker which hasn't started yet — arm a pending stop so the + // worker will bail when it checks before running. + if err == nil && !result.Stopped { + if ts := al.getActiveTurnState(sessionKey); ts != nil { + snap := ts.snapshot() + if strings.HasPrefix(snap.TurnID, pendingTurnPrefix) { + al.markPendingStop(sessionKey) + result.Stopped = true + } + } + } + + reply := commands.FormatStopReply(result) + if err != nil { + reply = "Failed to stop task: " + err.Error() + } + + if al.channelManager != nil { + al.channelManager.InvokeTypingStop(msg.Channel, msg.ChatID) + } + al.resetMessageToolRound(sessionKey) + al.PublishResponseIfNeeded(ctx, msg.Channel, msg.ChatID, sessionKey, reply) + return true +} + +func (al *AgentLoop) stopActiveTurnForSession(sessionKey string) (commands.StopResult, error) { + sessionKey = strings.TrimSpace(sessionKey) + if sessionKey == "" { + return commands.StopResult{}, fmt.Errorf("session key is required") + } + + result := commands.StopResult{} + cleared := al.clearSteeringMessagesForScope(sessionKey) + al.clearPendingSkills(sessionKey) + + ts := al.getActiveTurnState(sessionKey) + if ts == nil { + result.Stopped = cleared > 0 + return result, nil + } + + snap := ts.snapshot() + result.TaskName = snap.UserMessage + + if strings.HasPrefix(snap.TurnID, pendingTurnPrefix) { + // A pending placeholder means this session is either idle (our own + // placeholder from the /stop command) or another message is queued but + // hasn't started yet. In both cases, we don't arm a pending stop here; + // the caller (tryHandleStopCommand) handles the "another message queued" + // case explicitly, since it knows loaded=true. + return result, nil + } + + if err := al.HardAbort(sessionKey); err != nil { + if al.getActiveTurnState(sessionKey) == nil { + result.Stopped = cleared > 0 + return result, nil + } + return commands.StopResult{}, err + } + + result.Stopped = true + return result, nil +} + +func (al *AgentLoop) markPendingStop(sessionKey string) { + sessionKey = strings.TrimSpace(sessionKey) + if sessionKey == "" { + return + } + al.pendingStops.Store(sessionKey, struct{}{}) +} + +func (al *AgentLoop) takePendingStop(sessionKey string) bool { + sessionKey = strings.TrimSpace(sessionKey) + if sessionKey == "" { + return false + } + _, ok := al.pendingStops.LoadAndDelete(sessionKey) + return ok +} + +func (al *AgentLoop) resetMessageToolRound(sessionKey string) { + if strings.TrimSpace(sessionKey) == "" { + return + } + if registry := al.GetRegistry(); registry != nil { + if agent := registry.GetDefaultAgent(); agent != nil { + if tool, ok := agent.Tools.Get("message"); ok { + if resetter, ok := tool.(interface{ ResetSentInRound(sessionKey string) }); ok { + resetter.ResetSentInRound(sessionKey) + } + } + } + } +} diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index f326e2acb..7a869ec94 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -19,6 +19,7 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/routing" @@ -56,6 +57,38 @@ func (f *fakeMediaChannel) SendMedia(ctx context.Context, msg bus.OutboundMediaM return nil, nil } +type recordingChannelManager struct { + dismissed []string +} + +func (m *recordingChannelManager) GetChannel(name string) (channels.Channel, bool) { + return nil, false +} + +func (m *recordingChannelManager) GetEnabledChannels() []string { + return nil +} + +func (m *recordingChannelManager) InvokeTypingStop(channel, chatID string) {} + +func (m *recordingChannelManager) SendMessage(ctx context.Context, msg bus.OutboundMessage) error { + return nil +} + +func (m *recordingChannelManager) SendMedia(ctx context.Context, msg bus.OutboundMediaMessage) error { + return nil +} + +func (m *recordingChannelManager) SendPlaceholder(ctx context.Context, channel, chatID string) bool { + return false +} + +func (m *recordingChannelManager) DismissToolFeedback( + ctx context.Context, channel, chatID string, outboundCtx *bus.InboundContext, +) { + m.dismissed = append(m.dismissed, fmt.Sprintf("%s:%s", channel, chatID)) +} + func newStartedTestChannelManager( t *testing.T, msgBus *bus.MessageBus, @@ -213,6 +246,44 @@ func TestNewAgentLoop_DoesNotRegisterWebSearchTool_WhenNoReadyProviders(t *testi } } +func TestPublishResponseIfNeeded_DismissesToolFeedbackWhenMessageToolAlreadySent(t *testing.T) { + al, msgBus, provider, sessions, cleanup := newTestAgentLoop(t) + defer cleanup() + _ = msgBus + _ = provider + _ = sessions + + cm := &recordingChannelManager{} + al.channelManager = cm + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + mt := tools.NewMessageTool() + mt.SetSendCallback(func(ctx context.Context, channel, chatID, content, replyToMessageID string) error { + return nil + }) + defaultAgent.Tools.Register(mt) + + result := mt.Execute( + tools.WithToolSessionContext(context.Background(), "main", "session-1", nil), + map[string]any{ + "content": "ack", + "channel": "telegram", + "chat_id": "-100123", + }, + ) + if result == nil || result.IsError { + t.Fatalf("message tool execute failed: %+v", result) + } + al.PublishResponseIfNeeded(context.Background(), "telegram", "-100123", "session-1", "final reply") + + if got := cm.dismissed; len(got) != 1 || got[0] != "telegram:-100123" { + t.Fatalf("dismissed = %v, want [telegram:-100123]", got) + } +} + func TestProcessMessage_IncludesCurrentSenderInDynamicContext(t *testing.T) { tmpDir, err := os.MkdirTemp("", "agent-test-*") if err != nil { @@ -5456,6 +5527,7 @@ func TestParallelMessageProcessing_SameSessionProcessedSequentially(t *testing.T var mu sync.Mutex turnIDs := make(map[string]bool) var wg sync.WaitGroup + var firstResponse sync.Once wg.Add(1) // Only 1 turn should be created for same session cfg := &config.Config{ @@ -5478,19 +5550,27 @@ func TestParallelMessageProcessing_SameSessionProcessedSequentially(t *testing.T al := NewAgentLoop(cfg, msgBus, &concurrentMockProvider{ responseFunc: func(callID int) string { - wg.Done() + firstResponse.Do(func() { + wg.Done() + }) return "ok" }, }) defer al.Close() - sub := al.SubscribeEvents(64) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 64, + runtimeevents.KindAgentTurnStart, + ) + defer closeRuntimeEvents() go func() { - for evt := range sub.C { - if evt.Kind == EventKindTurnStart { + for evt := range runtimeCh { + if evt.Kind == runtimeevents.KindAgentTurnStart { mu.Lock() - turnIDs[evt.Meta.TurnID] = true + turnIDs[evt.Scope.TurnID] = true mu.Unlock() } } diff --git a/pkg/agent/agent_utils.go b/pkg/agent/agent_utils.go index bbfb3f2ae..9228b6d55 100644 --- a/pkg/agent/agent_utils.go +++ b/pkg/agent/agent_utils.go @@ -4,7 +4,6 @@ package agent import ( "context" - "encoding/json" "fmt" "maps" "path/filepath" @@ -171,15 +170,8 @@ func toolFeedbackExplanationFromMessages(messages []providers.Message) string { } func toolFeedbackArgsPreview(args map[string]any, maxLen int) string { - if args == nil { - args = map[string]any{} - } - - argsJSON, err := json.MarshalIndent(args, "", " ") - if err != nil { - return utils.Truncate(fmt.Sprintf("%v", args), maxLen) - } - return utils.Truncate(string(argsJSON), maxLen) + argsJSON := utils.FormatArgsJSON(args, true, false) + return utils.Truncate(argsJSON, maxLen) } func shouldPublishToolFeedback(cfg *config.Config, ts *turnState) bool { @@ -293,6 +285,12 @@ func inferMediaType(filename, contentType string) string { ct := strings.ToLower(contentType) fn := strings.ToLower(filename) + // SVG is an image MIME type, but raster-only delivery endpoints such as + // Telegram SendPhoto reject it. Treat it as a file/document instead. + if strings.HasPrefix(ct, "image/svg") || filepath.Ext(fn) == ".svg" { + return "file" + } + if strings.HasPrefix(ct, "image/") { return "image" } @@ -306,7 +304,7 @@ func inferMediaType(filename, contentType string) string { // Fallback: infer from extension ext := filepath.Ext(fn) switch ext { - case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg": + case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp": return "image" case ".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma", ".opus": return "audio" diff --git a/pkg/agent/agent_utils_test.go b/pkg/agent/agent_utils_test.go new file mode 100644 index 000000000..6612a60b3 --- /dev/null +++ b/pkg/agent/agent_utils_test.go @@ -0,0 +1,76 @@ +package agent + +import "testing" + +func TestInferMediaType(t *testing.T) { + tests := []struct { + name string + filename string + contentType string + want string + }{ + { + name: "png content type", + filename: "diagram", + contentType: "image/png", + want: "image", + }, + { + name: "jpeg extension fallback", + filename: "photo.JPG", + contentType: "", + want: "image", + }, + { + name: "svg content type is file", + filename: "diagram", + contentType: "image/svg+xml", + want: "file", + }, + { + name: "svg content type with parameters is file", + filename: "diagram", + contentType: "image/svg+xml; charset=utf-8", + want: "file", + }, + { + name: "svg extension fallback is file", + filename: "diagram.SVG", + contentType: "", + want: "file", + }, + { + name: "audio content type", + filename: "voice", + contentType: "audio/ogg", + want: "audio", + }, + { + name: "ogg application content type", + filename: "voice.ogg", + contentType: "application/ogg", + want: "audio", + }, + { + name: "video extension fallback", + filename: "clip.MP4", + contentType: "", + want: "video", + }, + { + name: "unknown type", + filename: "archive.bin", + contentType: "application/octet-stream", + want: "file", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := inferMediaType(tt.filename, tt.contentType) + if got != tt.want { + t.Fatalf("inferMediaType(%q, %q) = %q, want %q", tt.filename, tt.contentType, got, tt.want) + } + }) + } +} diff --git a/pkg/agent/context.go b/pkg/agent/context.go index ecde7c33e..87bdd6b41 100644 --- a/pkg/agent/context.go +++ b/pkg/agent/context.go @@ -26,6 +26,7 @@ type ContextBuilder struct { skillsLoader *skills.SkillsLoader memory *MemoryStore splitOnMarker bool + agentDiscovery func(agentID string) []AgentDescriptor promptRegistry *PromptRegistry // Cache for system prompt to avoid rebuilding on every call. @@ -66,6 +67,24 @@ func (cb *ContextBuilder) WithSplitOnMarker(enabled bool) *ContextBuilder { return cb } +func (cb *ContextBuilder) WithAgentDiscovery( + agentID string, + discover func(agentID string) []AgentDescriptor, +) *ContextBuilder { + cb.agentDiscovery = discover + if discover != nil { + if err := cb.RegisterPromptContributor(agentDiscoveryPromptContributor{ + agentID: agentID, + discover: discover, + }); err != nil { + logger.WarnCF("agent", "Failed to register agent discovery prompt contributor", map[string]any{ + "error": err.Error(), + }) + } + } + return cb +} + func getGlobalConfigDir() string { return config.GetHome() } @@ -625,7 +644,9 @@ func formatCurrentSenderLine(senderID, senderDisplayName string) string { } } -func (cb *ContextBuilder) buildDynamicContext(channel, chatID, senderID, senderDisplayName string) string { +func (cb *ContextBuilder) buildDynamicContext( + channel, chatID, senderID, senderDisplayName string, +) string { now := time.Now().Format("2006-01-02 15:04 (Monday)") rt := fmt.Sprintf("%s %s, Go %s", runtime.GOOS, runtime.GOARCH, runtime.Version()) @@ -854,7 +875,11 @@ func sanitizeHistoryForProvider(history []providers.Message) []providers.Message case "assistant": if len(msg.ToolCalls) > 0 { if len(sanitized) == 0 { - logger.DebugCF("agent", "Dropping assistant tool-call turn at history start", map[string]any{}) + logger.DebugCF( + "agent", + "Dropping assistant tool-call turn at history start", + map[string]any{}, + ) continue } prev := sanitized[len(sanitized)-1] @@ -999,10 +1024,28 @@ func (cb *ContextBuilder) AddAssistantMessage( } func (cb *ContextBuilder) buildActiveSkillsContext(skillNames []string) string { - if cb.skillsLoader == nil || len(skillNames) == 0 { + ordered := cb.ResolveActiveSkillsForContext(skillNames) + if len(ordered) == 0 { return "" } + content := cb.skillsLoader.LoadSkillsForContext(ordered) + if strings.TrimSpace(content) == "" { + return "" + } + + return fmt.Sprintf(`# Active Skills + +The following skills are active for this request. Follow them when relevant. + +%s`, content) +} + +func (cb *ContextBuilder) ResolveActiveSkillsForContext(skillNames []string) []string { + if cb.skillsLoader == nil || len(skillNames) == 0 { + return nil + } + var ordered []string seen := make(map[string]struct{}, len(skillNames)) for _, name := range skillNames { @@ -1017,19 +1060,9 @@ func (cb *ContextBuilder) buildActiveSkillsContext(skillNames []string) string { ordered = append(ordered, canonical) } if len(ordered) == 0 { - return "" + return nil } - - content := cb.skillsLoader.LoadSkillsForContext(ordered) - if strings.TrimSpace(content) == "" { - return "" - } - - return fmt.Sprintf(`# Active Skills - -The following skills are active for this request. Follow them when relevant. - -%s`, content) + return ordered } func (cb *ContextBuilder) buildActiveSkillsPromptParts(skillNames []string) []PromptPart { diff --git a/pkg/agent/context_legacy.go b/pkg/agent/context_legacy.go index 5644571fb..94ef5367d 100644 --- a/pkg/agent/context_legacy.go +++ b/pkg/agent/context_legacy.go @@ -7,6 +7,7 @@ import ( "sync" "time" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -41,7 +42,7 @@ func (m *legacyContextManager) Compact(_ context.Context, req *CompactRequest) e // Sync emergency compression — budget exceeded. if result, ok := m.forceCompression(req.SessionKey); ok { m.al.emitEvent( - EventKindContextCompress, + runtimeevents.KindAgentContextCompress, m.al.newTurnEventScope("", req.SessionKey, nil).meta(0, "forceCompression", "turn.context.compress"), ContextCompressPayload{ Reason: req.Reason, @@ -246,7 +247,7 @@ func (m *legacyContextManager) summarizeSession(agent *AgentInstance, sessionKey agent.Sessions.TruncateHistory(sessionKey, keepCount) agent.Sessions.Save(sessionKey) m.al.emitEvent( - EventKindSessionSummarize, + runtimeevents.KindAgentSessionSummarize, m.al.newTurnEventScope(agent.ID, sessionKey, nil).meta(0, "summarizeSession", "turn.session.summarize"), SessionSummarizePayload{ SummarizedMessages: len(validMessages), diff --git a/pkg/agent/context_manager_test.go b/pkg/agent/context_manager_test.go index 629d11fcb..46e521be4 100644 --- a/pkg/agent/context_manager_test.go +++ b/pkg/agent/context_manager_test.go @@ -12,6 +12,7 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -305,8 +306,13 @@ func TestLegacyCompact_Overflow(t *testing.T) { } defaultAgent.Sessions.SetHistory("session-overflow", history) - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentContextCompress, + ) + defer closeRuntimeEvents() err := al.contextManager.Compact(context.Background(), &CompactRequest{ SessionKey: "session-overflow", @@ -329,8 +335,8 @@ func TestLegacyCompact_Overflow(t *testing.T) { } // Event should carry the proactive reason - events := collectEventStream(sub.C) - compressEvt, ok := findEvent(events, EventKindContextCompress) + events := collectRuntimeEventStream(runtimeCh) + compressEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentContextCompress) if !ok { t.Fatal("expected context compress event") } @@ -361,8 +367,13 @@ func TestLegacyCompact_Overflow_ProactiveReason(t *testing.T) { } defaultAgent.Sessions.SetHistory("session-proactive", history) - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentContextCompress, + ) + defer closeRuntimeEvents() err := al.contextManager.Compact(context.Background(), &CompactRequest{ SessionKey: "session-proactive", @@ -372,8 +383,8 @@ func TestLegacyCompact_Overflow_ProactiveReason(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - events := collectEventStream(sub.C) - compressEvt, ok := findEvent(events, EventKindContextCompress) + events := collectRuntimeEventStream(runtimeCh) + compressEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentContextCompress) if !ok { t.Fatal("expected context compress event") } @@ -483,6 +494,14 @@ func TestLegacyCompact_PostTurn_ExceedsMessageThreshold(t *testing.T) { } defaultAgent.Sessions.SetHistory("session-threshold", history) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentSessionSummarize, + ) + defer closeRuntimeEvents() + err := al.contextManager.Compact(context.Background(), &CompactRequest{ SessionKey: "session-threshold", Reason: ContextCompressReasonSummarize, @@ -491,12 +510,8 @@ func TestLegacyCompact_PostTurn_ExceedsMessageThreshold(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - // Wait for async summarization to complete via event - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) - - waitForEvent(t, sub.C, 5*time.Second, func(evt Event) bool { - return evt.Kind == EventKindSessionSummarize + waitForRuntimeEvent(t, runtimeCh, 5*time.Second, func(evt runtimeevents.Event) bool { + return evt.Kind == runtimeevents.KindAgentSessionSummarize }) newHistory := defaultAgent.Sessions.GetHistory("session-threshold") diff --git a/pkg/agent/definition.go b/pkg/agent/definition.go index cf73d607c..5b0e29137 100644 --- a/pkg/agent/definition.go +++ b/pkg/agent/definition.go @@ -35,7 +35,7 @@ type AgentFrontmatter struct { MaxTurns *int `json:"maxTurns,omitempty"` Skills []string `json:"skills,omitempty"` MCPServers []string `json:"mcpServers,omitempty"` - Fields map[string]any `json:"fields,omitempty"` + Fields map[string]any `json:"-"` } // AgentPromptDefinition represents the parsed AGENT.md or AGENTS.md prompt file. @@ -45,6 +45,7 @@ type AgentPromptDefinition struct { Body string `json:"body"` RawFrontmatter string `json:"raw_frontmatter,omitempty"` Frontmatter AgentFrontmatter `json:"frontmatter"` + FrontmatterErr string `json:"frontmatter_error,omitempty"` } // SoulDefinition represents the resolved SOUL.md file linked to the agent. @@ -146,19 +147,21 @@ func loadUserDefinition(workspace string) *UserDefinition { func parseAgentPromptDefinition(path, content string) AgentPromptDefinition { frontmatter, body := splitAgentFrontmatter(content) + parsedFrontmatter, err := parseAgentFrontmatter(path, frontmatter) return AgentPromptDefinition{ Path: path, Raw: content, Body: body, RawFrontmatter: frontmatter, - Frontmatter: parseAgentFrontmatter(path, frontmatter), + Frontmatter: parsedFrontmatter, + FrontmatterErr: errorString(err), } } -func parseAgentFrontmatter(path, frontmatter string) AgentFrontmatter { +func parseAgentFrontmatter(path, frontmatter string) (AgentFrontmatter, error) { frontmatter = strings.TrimSpace(frontmatter) if frontmatter == "" { - return AgentFrontmatter{} + return AgentFrontmatter{}, nil } rawFields := make(map[string]any) @@ -167,7 +170,7 @@ func parseAgentFrontmatter(path, frontmatter string) AgentFrontmatter { "path": path, "error": err.Error(), }) - return AgentFrontmatter{} + return AgentFrontmatter{}, err } var typed struct { @@ -184,7 +187,7 @@ func parseAgentFrontmatter(path, frontmatter string) AgentFrontmatter { "path": path, "error": err.Error(), }) - return AgentFrontmatter{} + return AgentFrontmatter{}, err } return AgentFrontmatter{ @@ -196,7 +199,7 @@ func parseAgentFrontmatter(path, frontmatter string) AgentFrontmatter { Skills: append([]string(nil), typed.Skills...), MCPServers: append([]string(nil), typed.MCPServers...), Fields: rawFields, - } + }, nil } func splitAgentFrontmatter(content string) (frontmatter, body string) { @@ -253,3 +256,10 @@ func fileExists(path string) bool { _, err := os.Stat(path) return err == nil } + +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/pkg/agent/discovery.go b/pkg/agent/discovery.go new file mode 100644 index 000000000..d2f63bc1f --- /dev/null +++ b/pkg/agent/discovery.go @@ -0,0 +1,263 @@ +package agent + +import ( + "encoding/json" + "path/filepath" + "sort" + "strings" + + "github.com/sipeed/picoclaw/pkg/routing" +) + +// AgentDescriptor is the structured discovery payload injected into each +// agent's system prompt so the LLM can choose a peer by identity. +type AgentDescriptor struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` +} + +// ListAgents returns structured descriptors for every agent in the current +// PicoClaw instance. The current workspace, when provided, is used only to +// order the matching agent first for prompt readability. +func (r *AgentRegistry) ListAgents(workspace string) []AgentDescriptor { + r.mu.RLock() + defer r.mu.RUnlock() + + ids := make([]string, 0, len(r.agents)) + for id := range r.agents { + ids = append(ids, id) + } + sort.Strings(ids) + + selfWorkspace := cleanWorkspacePath(workspace) + descriptors := make([]AgentDescriptor, 0, len(ids)) + for _, id := range ids { + agent := r.agents[id] + if agent == nil { + continue + } + descriptors = append(descriptors, r.buildAgentDescriptorLocked(agent)) + } + + if selfWorkspace == "" { + return descriptors + } + + sort.SliceStable(descriptors, func(i, j int) bool { + leftSelf := cleanWorkspacePath( + r.workspaceForAgentIDLocked(descriptors[i].ID), + ) == selfWorkspace + rightSelf := cleanWorkspacePath( + r.workspaceForAgentIDLocked(descriptors[j].ID), + ) == selfWorkspace + if leftSelf != rightSelf { + return leftSelf + } + return descriptors[i].ID < descriptors[j].ID + }) + + return descriptors +} + +// ListSpawnableAgents returns descriptors only when the current agent can call +// spawn, and only for peers it is allowed to spawn. Restricted peers are +// intentionally omitted from discovery. +func (r *AgentRegistry) ListSpawnableAgents(agentID string) []AgentDescriptor { + r.mu.RLock() + defer r.mu.RUnlock() + + parentID := routing.NormalizeAgentID(agentID) + parent, ok := r.agents[parentID] + if !ok || parent == nil { + return nil + } + if !agentHasSpawnTool(parent) { + return nil + } + + ids := make([]string, 0, len(r.agents)) + for id := range r.agents { + if id == parentID { + continue + } + if !agentAllowsSubagent(parent, id) { + continue + } + ids = append(ids, id) + } + sort.Strings(ids) + + descriptors := make([]AgentDescriptor, 0, len(ids)) + for _, id := range ids { + agent := r.agents[id] + if agent == nil { + continue + } + descriptors = append(descriptors, r.buildAgentDescriptorLocked(agent)) + } + return descriptors +} + +// GetAgentDescriptor returns the structured discovery payload for one agent. +func (r *AgentRegistry) GetAgentDescriptor(agentID string) (*AgentDescriptor, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + id := routing.NormalizeAgentID(agentID) + agent, ok := r.agents[id] + if !ok || agent == nil { + return nil, false + } + + descriptor := r.buildAgentDescriptorLocked(agent) + return &descriptor, true +} + +func (r *AgentRegistry) buildAgentDescriptorLocked(agent *AgentInstance) AgentDescriptor { + definition := loadAgentDefinition(agent.Workspace) + name, description := descriptorIdentity(agent.ID, definition) + + return AgentDescriptor{ + ID: agent.ID, + Name: name, + Description: description, + } +} + +func descriptorIdentity(agentID string, definition AgentContextDefinition) (string, string) { + name := agentID + description := "" + if definition.Agent != nil { + if trimmed := strings.TrimSpace(definition.Agent.Frontmatter.Name); trimmed != "" { + name = trimmed + } + if trimmed := strings.TrimSpace(definition.Agent.Frontmatter.Description); trimmed != "" { + description = trimmed + } + } + + if description == "" && + definition.Agent != nil { + if definition.Source == AgentDefinitionSourceAgent { + description = firstNonEmptyLine(definition.Agent.Body) + } else if definition.Source == AgentDefinitionSourceAgents { + description = firstMeaningfulParagraph(definition.Agent.Body) + } + } + + return name, description +} + +func firstNonEmptyLine(content string) string { + content = strings.ReplaceAll(content, "\r\n", "\n") + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + return trimmed + } + } + return "" +} + +func firstMeaningfulParagraph(content string) string { + content = strings.ReplaceAll(content, "\r\n", "\n") + paragraphs := strings.Split(content, "\n\n") + for _, paragraph := range paragraphs { + lines := strings.Split(paragraph, "\n") + parts := make([]string, 0, len(lines)) + inFence := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + inFence = !inFence + continue + } + if inFence || trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, "#") { + continue + } + if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") { + trimmed = strings.TrimSpace(trimmed[2:]) + } + parts = append(parts, trimmed) + } + if len(parts) == 0 { + continue + } + return strings.Join(parts, " ") + } + return "" +} + +func (r *AgentRegistry) workspaceForAgentIDLocked(agentID string) string { + agent, ok := r.agents[routing.NormalizeAgentID(agentID)] + if !ok || agent == nil { + return "" + } + return agent.Workspace +} + +func (r *AgentRegistry) defaultAgentIDLocked() string { + if _, ok := r.agents[routing.DefaultAgentID]; ok { + return routing.DefaultAgentID + } + if r.cfg != nil && len(r.cfg.Agents.List) > 0 { + for _, agentCfg := range r.cfg.Agents.List { + if !agentCfg.Default { + continue + } + id := routing.NormalizeAgentID(agentCfg.ID) + if _, ok := r.agents[id]; ok { + return id + } + } + id := routing.NormalizeAgentID(r.cfg.Agents.List[0].ID) + if _, ok := r.agents[id]; ok { + return id + } + } + for id := range r.agents { + return id + } + return "" +} + +func cleanWorkspacePath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + return filepath.Clean(path) +} + +func formatAgentDiscoverySection(agents []AgentDescriptor) string { + if len(agents) == 0 { + return "" + } + + payload := struct { + Agents []AgentDescriptor `json:"agents"` + }{ + Agents: agents, + } + + encoded, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return "" + } + + var header strings.Builder + header.WriteString("# Agent Discovery\n\n") + header.WriteString("This registry lists the peer agents this agent is permitted to spawn.\n") + header.WriteString( + "Choose a peer based on its description. Use only agent IDs listed here when calling spawn.\n\n", + ) + header.WriteString("```json\n") + header.Write(encoded) + header.WriteString("\n```") + + return header.String() +} diff --git a/pkg/agent/discovery_test.go b/pkg/agent/discovery_test.go new file mode 100644 index 000000000..f31a113d8 --- /dev/null +++ b/pkg/agent/discovery_test.go @@ -0,0 +1,420 @@ +package agent + +import ( + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" +) + +func TestAgentRegistry_ListAgentsBuildsStructuredDescriptors(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +name: Main Frontmatter Name +description: Structured main agent +--- +# Agent + +Handle general requests. +`, + }) + defer cleanupWorkspace(t, mainWorkspace) + + supportWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +name: Support Frontmatter Name +description: Support frontmatter description +--- +# Agent + +Handle support tickets carefully. +`, + }) + defer cleanupWorkspace(t, supportWorkspace) + + cfg := testCfg([]config.AgentConfig{ + {ID: "main", Default: true, Name: "Configured Main", Workspace: mainWorkspace}, + {ID: "support", Workspace: supportWorkspace}, + }) + + registry := NewAgentRegistry(cfg, &mockRegistryProvider{}) + + descriptors := registry.ListAgents(mainWorkspace) + if len(descriptors) != 2 { + t.Fatalf("expected 2 descriptors, got %d", len(descriptors)) + } + + if descriptors[0].ID != "main" { + t.Fatalf("expected current workspace agent first, got %q", descriptors[0].ID) + } + if descriptors[0].Name != "Main Frontmatter Name" { + t.Fatalf("expected frontmatter name to drive discovery, got %q", descriptors[0].Name) + } + if descriptors[0].Description != "Structured main agent" { + t.Fatalf("expected frontmatter description, got %q", descriptors[0].Description) + } + + support, ok := registry.GetAgentDescriptor("support") + if !ok || support == nil { + t.Fatal("expected support descriptor lookup to succeed") + } + if support.Name != "Support Frontmatter Name" { + t.Fatalf("expected support frontmatter name, got %q", support.Name) + } + if support.Description != "Support frontmatter description" { + t.Fatalf("expected support frontmatter description, got %q", support.Description) + } +} + +func TestAgentRegistry_ListSpawnableAgentsRespectsPermissions(t *testing.T) { + cfg := testCfg([]config.AgentConfig{ + { + ID: "parent", + Default: true, + Subagents: &config.SubagentsConfig{ + AllowAgents: []string{"child2", "child1"}, + }, + }, + {ID: "child1"}, + {ID: "child2"}, + {ID: "restricted"}, + }) + cfg.Tools.Spawn.Enabled = true + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + descriptors := al.GetRegistry().ListSpawnableAgents("parent") + if len(descriptors) != 2 { + t.Fatalf("expected 2 spawnable descriptors, got %d: %+v", len(descriptors), descriptors) + } + if descriptors[0].ID != "child1" || descriptors[1].ID != "child2" { + t.Fatalf("expected sorted spawnable peers only, got %+v", descriptors) + } +} + +func TestAgentRegistry_ListSpawnableAgentsRequiresSpawnTool(t *testing.T) { + cfg := testCfg([]config.AgentConfig{ + { + ID: "parent", + Default: true, + Subagents: &config.SubagentsConfig{ + AllowAgents: []string{"child"}, + }, + }, + {ID: "child"}, + }) + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + if descriptors := al.GetRegistry().ListSpawnableAgents("parent"); len(descriptors) != 0 { + t.Fatalf("expected no spawnable descriptors without spawn tool, got %+v", descriptors) + } +} + +func TestContextBuilder_BuildMessagesIncludesAgentDiscoverySection(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +description: Main agent +--- +# Agent + +Generalist. +`, + }) + defer cleanupWorkspace(t, mainWorkspace) + + researchWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +name: Research Agent +description: Research specialist +--- +# Agent + +Investigate deeply. +`, + }) + defer cleanupWorkspace(t, researchWorkspace) + + restrictedWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +name: Restricted Agent +description: Restricted specialist +--- +# Agent + +Handle restricted work. +`, + }) + defer cleanupWorkspace(t, restrictedWorkspace) + + cfg := testCfg([]config.AgentConfig{ + { + ID: "main", + Default: true, + Workspace: mainWorkspace, + Subagents: &config.SubagentsConfig{ + AllowAgents: []string{"research"}, + }, + }, + {ID: "research", Workspace: researchWorkspace}, + {ID: "restricted", Workspace: restrictedWorkspace}, + }) + cfg.Tools.ReadFile.Enabled = true + cfg.Tools.WriteFile.Enabled = true + cfg.Tools.Spawn.Enabled = true + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + mainAgent, ok := al.GetRegistry().GetAgent("main") + if !ok || mainAgent == nil { + t.Fatal("expected main agent") + } + + messages := mainAgent.ContextBuilder.BuildMessages( + nil, + "", + "delegate wisely", + nil, + "telegram", + "chat-1", + "", + "", + ) + if len(messages) == 0 { + t.Fatal("expected messages") + } + + systemPrompt := messages[0].Content + if !strings.Contains(systemPrompt, "# Agent Discovery") { + t.Fatalf("expected discovery section in system prompt, got %q", systemPrompt) + } + if strings.Contains(systemPrompt, `"id": "main"`) { + t.Fatalf("did not expect self descriptor in discovery section, got %q", systemPrompt) + } + if !strings.Contains(systemPrompt, `"id": "research"`) || + !strings.Contains(systemPrompt, `"description": "Research specialist"`) { + t.Fatalf("expected allowed peer descriptor in discovery section, got %q", systemPrompt) + } + if strings.Contains(systemPrompt, `"id": "restricted"`) || + strings.Contains(systemPrompt, `"description": "Restricted specialist"`) { + t.Fatalf("did not expect restricted peer descriptor in discovery section, got %q", systemPrompt) + } + for _, forbidden := range []string{`"current_agent_id"`, `"available_tools"`, `"model"`, `"channels"`, `"skills"`, `"mcpServers"`, `"tools"`} { + if strings.Contains(systemPrompt, forbidden) { + t.Fatalf("did not expect %s in discovery section, got %q", forbidden, systemPrompt) + } + } +} + +func TestContextBuilder_BuildMessagesOmitsAgentDiscoveryWithoutSpawnPermissions(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +description: Main agent +--- +# Agent + +Generalist. +`, + }) + defer cleanupWorkspace(t, mainWorkspace) + + researchWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +description: Research specialist +--- +# Agent + +Investigate deeply. +`, + }) + defer cleanupWorkspace(t, researchWorkspace) + + cfg := testCfg([]config.AgentConfig{ + {ID: "main", Default: true, Workspace: mainWorkspace}, + {ID: "research", Workspace: researchWorkspace}, + }) + cfg.Tools.ReadFile.Enabled = true + cfg.Tools.Spawn.Enabled = true + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + mainAgent, ok := al.GetRegistry().GetAgent("main") + if !ok || mainAgent == nil { + t.Fatal("expected main agent") + } + + messages := mainAgent.ContextBuilder.BuildMessages( + nil, + "", + "handle locally", + nil, + "telegram", + "chat-1", + "", + "", + ) + if len(messages) == 0 { + t.Fatal("expected messages") + } + + systemPrompt := messages[0].Content + if strings.Contains(systemPrompt, "# Agent Discovery") { + t.Fatalf("did not expect discovery section without spawn permissions, got %q", systemPrompt) + } + if strings.Contains(systemPrompt, `"id": "research"`) { + t.Fatalf("did not expect unauthorized peer identity in system prompt, got %q", systemPrompt) + } +} + +func TestContextBuilder_BuildMessagesOmitsAgentDiscoveryWithoutSpawnTool(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +description: Main agent +tools: [read_file] +--- +# Agent + +Generalist. +`, + }) + defer cleanupWorkspace(t, mainWorkspace) + + researchWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +description: Research specialist +--- +# Agent + +Investigate deeply. +`, + }) + defer cleanupWorkspace(t, researchWorkspace) + + cfg := testCfg([]config.AgentConfig{ + { + ID: "main", + Default: true, + Workspace: mainWorkspace, + Subagents: &config.SubagentsConfig{ + AllowAgents: []string{"research"}, + }, + }, + {ID: "research", Workspace: researchWorkspace}, + }) + cfg.Tools.ReadFile.Enabled = true + cfg.Tools.Spawn.Enabled = true + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + mainAgent, ok := al.GetRegistry().GetAgent("main") + if !ok || mainAgent == nil { + t.Fatal("expected main agent") + } + + messages := mainAgent.ContextBuilder.BuildMessages( + nil, + "", + "handle locally", + nil, + "telegram", + "chat-1", + "", + "", + ) + if len(messages) == 0 { + t.Fatal("expected messages") + } + + systemPrompt := messages[0].Content + if strings.Contains(systemPrompt, "# Agent Discovery") { + t.Fatalf("did not expect discovery section without spawn tool, got %q", systemPrompt) + } + if strings.Contains(systemPrompt, `"id": "research"`) { + t.Fatalf("did not expect peer identity without spawn tool, got %q", systemPrompt) + } +} + +func TestContextBuilder_BuildMessagesOmitsAgentDiscoverySectionForSingleton(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +description: Main agent +--- +# Agent + +Generalist. +`, + }) + defer cleanupWorkspace(t, mainWorkspace) + + cfg := testCfg([]config.AgentConfig{ + {ID: "main", Default: true, Workspace: mainWorkspace}, + }) + cfg.Tools.ReadFile.Enabled = true + cfg.Tools.Spawn.Enabled = true + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + mainAgent, ok := al.GetRegistry().GetAgent("main") + if !ok || mainAgent == nil { + t.Fatal("expected main agent") + } + + messages := mainAgent.ContextBuilder.BuildMessages( + nil, + "", + "handle locally", + nil, + "telegram", + "chat-1", + "", + "", + ) + if len(messages) == 0 { + t.Fatal("expected messages") + } + + systemPrompt := messages[0].Content + if strings.Contains(systemPrompt, "# Agent Discovery") { + t.Fatalf("did not expect discovery section for singleton registry, got %q", systemPrompt) + } +} + +func TestAgentRegistry_ListAgentsFallsBackToFirstNonEmptyAgentLine(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +name: Research Agent +--- + + +First useful line. +Second line. +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := testCfg([]config.AgentConfig{ + {ID: "research", Default: true, Workspace: workspace}, + }) + + registry := NewAgentRegistry(cfg, &mockRegistryProvider{}) + descriptor, ok := registry.GetAgentDescriptor("research") + if !ok || descriptor == nil { + t.Fatal("expected research descriptor lookup to succeed") + } + if descriptor.Description != "First useful line." { + t.Fatalf("descriptor.Description = %q, want %q", descriptor.Description, "First useful line.") + } +} diff --git a/pkg/agent/event_payloads.go b/pkg/agent/event_payloads.go new file mode 100644 index 000000000..dee3e620a --- /dev/null +++ b/pkg/agent/event_payloads.go @@ -0,0 +1,198 @@ +package agent + +import "time" + +// TurnEndStatus describes the terminal state of a turn. +type TurnEndStatus string + +const ( + // TurnEndStatusCompleted indicates the turn finished normally. + TurnEndStatusCompleted TurnEndStatus = "completed" + // TurnEndStatusError indicates the turn ended because of an error. + TurnEndStatusError TurnEndStatus = "error" + // TurnEndStatusAborted indicates the turn was hard-aborted and rolled back. + TurnEndStatusAborted TurnEndStatus = "aborted" +) + +// TurnStartPayload describes the start of a turn. +type TurnStartPayload struct { + UserMessage string + MediaCount int +} + +const ( + skillContextTriggerInitialBuild = "initial_build" + skillContextTriggerContextRetryRebuild = "context_retry_rebuild" +) + +type SkillContextSnapshot struct { + Sequence int `json:"sequence"` + Trigger string `json:"trigger"` + SkillNames []string `json:"skill_names,omitempty"` +} + +type ToolExecutionRecord struct { + Name string `json:"name"` + Success bool `json:"success"` + ErrorSummary string `json:"error_summary,omitempty"` + SkillNames []string `json:"skill_names,omitempty"` +} + +// TurnEndPayload describes the completion of a turn. +type TurnEndPayload struct { + Status TurnEndStatus + Workspace string + Iterations int + Duration time.Duration + FinalContentLen int + UserMessage string + FinalContent string + ActiveSkills []string + AttemptedSkills []string + FinalSuccessfulPath []string + SkillContextSnapshots []SkillContextSnapshot + ToolKinds []string + ToolExecutions []ToolExecutionRecord +} + +// LLMRequestPayload describes an outbound LLM request. +type LLMRequestPayload struct { + Model string + MessagesCount int + ToolsCount int + MaxTokens int + Temperature float64 +} + +// LLMResponsePayload describes an inbound LLM response. +type LLMResponsePayload struct { + ContentLen int + ToolCalls int + HasReasoning bool +} + +// LLMDeltaPayload describes a streamed LLM delta. +type LLMDeltaPayload struct { + ContentDeltaLen int + ReasoningDeltaLen int +} + +// LLMRetryPayload describes a retry of an LLM request. +type LLMRetryPayload struct { + Attempt int + MaxRetries int + Reason string + Error string + Backoff time.Duration +} + +// ContextCompressReason identifies why emergency compression ran. +type ContextCompressReason string + +const ( + // ContextCompressReasonProactive indicates compression before the first LLM call. + ContextCompressReasonProactive ContextCompressReason = "proactive_budget" + // ContextCompressReasonRetry indicates compression during context-error retry handling. + ContextCompressReasonRetry ContextCompressReason = "llm_retry" + // ContextCompressReasonSummarize indicates post-turn async summarization. + ContextCompressReasonSummarize ContextCompressReason = "summarize" +) + +// ContextCompressPayload describes a forced history compression. +type ContextCompressPayload struct { + Reason ContextCompressReason + DroppedMessages int + RemainingMessages int +} + +// SessionSummarizePayload describes a completed async session summarization. +type SessionSummarizePayload struct { + SummarizedMessages int + KeptMessages int + SummaryLen int + OmittedOversized bool +} + +// ToolExecStartPayload describes a tool execution request. +type ToolExecStartPayload struct { + Tool string + Arguments map[string]any +} + +// ToolExecEndPayload describes the outcome of a tool execution. +type ToolExecEndPayload struct { + Tool string + Duration time.Duration + ForLLMLen int + ForUserLen int + IsError bool + Async bool +} + +// ToolExecSkippedPayload describes a skipped tool call. +type ToolExecSkippedPayload struct { + Tool string + Reason string +} + +// SteeringInjectedPayload describes steering messages appended before the next LLM call. +type SteeringInjectedPayload struct { + Count int + TotalContentLen int +} + +// FollowUpQueuedPayload describes an async follow-up queued back into the inbound bus. +type FollowUpQueuedPayload struct { + SourceTool string + ContentLen int +} + +type InterruptKind string + +const ( + InterruptKindSteering InterruptKind = "steering" + InterruptKindGraceful InterruptKind = "graceful" + InterruptKindHard InterruptKind = "hard_abort" +) + +// InterruptReceivedPayload describes accepted turn-control input. +type InterruptReceivedPayload struct { + Kind InterruptKind + Role string + ContentLen int + QueueDepth int + HintLen int +} + +// SubTurnSpawnPayload describes the creation of a child turn. +type SubTurnSpawnPayload struct { + AgentID string + Label string + ParentTurnID string +} + +// SubTurnEndPayload describes the completion of a child turn. +type SubTurnEndPayload struct { + AgentID string + Status string +} + +// SubTurnResultDeliveredPayload describes delivery of a sub-turn result. +type SubTurnResultDeliveredPayload struct { + TargetChannel string + TargetChatID string + ContentLen int +} + +// SubTurnOrphanPayload describes a sub-turn result that could not be delivered. +type SubTurnOrphanPayload struct { + ParentTurnID string + ChildTurnID string + Reason string +} + +// ErrorPayload describes an execution error inside the agent loop. +type ErrorPayload struct { + Stage string + Message string +} diff --git a/pkg/agent/eventbus.go b/pkg/agent/eventbus.go deleted file mode 100644 index 546d8436d..000000000 --- a/pkg/agent/eventbus.go +++ /dev/null @@ -1,121 +0,0 @@ -package agent - -import ( - "sync" - "sync/atomic" - "time" -) - -const defaultEventSubscriberBuffer = 16 - -// EventSubscription identifies a subscriber channel returned by EventBus.Subscribe. -type EventSubscription struct { - ID uint64 - C <-chan Event -} - -type eventSubscriber struct { - ch chan Event -} - -// EventBus is a lightweight multi-subscriber broadcaster for agent-loop events. -type EventBus struct { - mu sync.RWMutex - subs map[uint64]eventSubscriber - nextID uint64 - closed bool - dropped [eventKindCount]atomic.Int64 -} - -// NewEventBus creates a new in-process event broadcaster. -func NewEventBus() *EventBus { - return &EventBus{ - subs: make(map[uint64]eventSubscriber), - } -} - -// Subscribe registers a new subscriber with the requested channel buffer size. -// A non-positive buffer uses the default size. -func (b *EventBus) Subscribe(buffer int) EventSubscription { - if buffer <= 0 { - buffer = defaultEventSubscriberBuffer - } - - b.mu.Lock() - defer b.mu.Unlock() - - if b.closed { - ch := make(chan Event) - close(ch) - return EventSubscription{C: ch} - } - - b.nextID++ - id := b.nextID - ch := make(chan Event, buffer) - b.subs[id] = eventSubscriber{ch: ch} - return EventSubscription{ID: id, C: ch} -} - -// Unsubscribe removes a subscriber and closes its channel. -func (b *EventBus) Unsubscribe(id uint64) { - b.mu.Lock() - defer b.mu.Unlock() - - sub, ok := b.subs[id] - if !ok { - return - } - - delete(b.subs, id) - close(sub.ch) -} - -// Emit broadcasts an event to all current subscribers without blocking. -// When a subscriber channel is full, the event is dropped for that subscriber. -func (b *EventBus) Emit(evt Event) { - if evt.Time.IsZero() { - evt.Time = time.Now() - } - - b.mu.RLock() - defer b.mu.RUnlock() - - if b.closed { - return - } - - for _, sub := range b.subs { - select { - case sub.ch <- evt: - default: - if evt.Kind < eventKindCount { - b.dropped[evt.Kind].Add(1) - } - } - } -} - -// Dropped returns the number of dropped events for a given kind. -func (b *EventBus) Dropped(kind EventKind) int64 { - if kind >= eventKindCount { - return 0 - } - return b.dropped[kind].Load() -} - -// Close closes all subscriber channels and stops future broadcasts. -func (b *EventBus) Close() { - b.mu.Lock() - defer b.mu.Unlock() - - if b.closed { - return - } - - b.closed = true - for id, sub := range b.subs { - close(sub.ch) - delete(b.subs, id) - } -} diff --git a/pkg/agent/eventbus_test.go b/pkg/agent/eventbus_test.go index 31b996260..86d7f4afa 100644 --- a/pkg/agent/eventbus_test.go +++ b/pkg/agent/eventbus_test.go @@ -9,61 +9,94 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/routing" "github.com/sipeed/picoclaw/pkg/session" "github.com/sipeed/picoclaw/pkg/tools" ) -func TestEventBus_SubscribeEmitUnsubscribeClose(t *testing.T) { - eventBus := NewEventBus() - sub := eventBus.Subscribe(1) - - eventBus.Emit(Event{ - Kind: EventKindTurnStart, - Meta: EventMeta{TurnID: "turn-1"}, - }) - - select { - case evt := <-sub.C: - if evt.Kind != EventKindTurnStart { - t.Fatalf("expected %v, got %v", EventKindTurnStart, evt.Kind) +func TestAgentLoop_PublishesRuntimeEvents(t *testing.T) { + runtimeBus := runtimeevents.NewBus() + al := &AgentLoop{ + runtimeEvents: runtimeBus, + } + defer func() { + if err := runtimeBus.Close(); err != nil { + t.Errorf("runtime bus close failed: %v", err) } - if evt.Meta.TurnID != "turn-1" { - t.Fatalf("expected turn id turn-1, got %q", evt.Meta.TurnID) + }() + + runtimeSub, runtimeCh, err := al.RuntimeEvents().OfKind(runtimeevents.KindAgentToolExecStart).SubscribeChan( + context.Background(), + runtimeevents.SubscribeOptions{Name: "runtime", Buffer: 1}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + defer func() { + if err := runtimeSub.Close(); err != nil { + t.Errorf("runtime subscription close failed: %v", err) } - case <-time.After(time.Second): - t.Fatal("timed out waiting for event") + }() + + al.emitEvent( + runtimeevents.KindAgentToolExecStart, + HookMeta{ + AgentID: "main", + TurnID: "turn-1", + ParentTurnID: "parent-turn", + SessionKey: "session-1", + Iteration: 2, + TracePath: "trace/root", + Source: "pipeline_execute", + turnContext: &TurnContext{ + Inbound: &bus.InboundContext{ + Channel: "cli", + Account: "default", + ChatID: "direct", + ChatType: "direct", + SenderID: "tester", + MessageID: "msg-1", + TopicID: "topic-1", + }, + }, + }, + ToolExecStartPayload{Tool: "mock_custom", Arguments: map[string]any{"task": "ping"}}, + ) + + runtimeEvt := receiveRuntimeEvent(t, runtimeCh) + if runtimeEvt.Kind != runtimeevents.KindAgentToolExecStart { + t.Fatalf("runtime kind = %q, want %q", runtimeEvt.Kind, runtimeevents.KindAgentToolExecStart) } - - eventBus.Unsubscribe(sub.ID) - if _, ok := <-sub.C; ok { - t.Fatal("expected subscriber channel to be closed after unsubscribe") + if runtimeEvt.Source != (runtimeevents.Source{Component: "agent", Name: "main"}) { + t.Fatalf("runtime source = %+v", runtimeEvt.Source) } - - eventBus.Close() - closedSub := eventBus.Subscribe(1) - if _, ok := <-closedSub.C; ok { - t.Fatal("expected closed bus to return a closed subscriber channel") + if runtimeEvt.Scope.AgentID != "main" || + runtimeEvt.Scope.SessionKey != "session-1" || + runtimeEvt.Scope.TurnID != "turn-1" || + runtimeEvt.Scope.Channel != "cli" || + runtimeEvt.Scope.Account != "default" || + runtimeEvt.Scope.ChatID != "direct" || + runtimeEvt.Scope.TopicID != "topic-1" || + runtimeEvt.Scope.ChatType != "direct" || + runtimeEvt.Scope.SenderID != "tester" || + runtimeEvt.Scope.MessageID != "msg-1" { + t.Fatalf("runtime scope = %+v", runtimeEvt.Scope) } -} - -func TestEventBus_DropsWhenSubscriberIsFull(t *testing.T) { - eventBus := NewEventBus() - sub := eventBus.Subscribe(1) - defer eventBus.Unsubscribe(sub.ID) - - start := time.Now() - for i := 0; i < 1000; i++ { - eventBus.Emit(Event{Kind: EventKindLLMRequest}) + if runtimeEvt.Correlation.TraceID != "trace/root" || + runtimeEvt.Correlation.ParentTurnID != "parent-turn" { + t.Fatalf("runtime correlation = %+v", runtimeEvt.Correlation) } - - if elapsed := time.Since(start); elapsed > 100*time.Millisecond { - t.Fatalf("Emit took too long with a blocked subscriber: %s", elapsed) + if runtimeEvt.Attrs["agent_source"] != "pipeline_execute" || runtimeEvt.Attrs["iteration"] != 2 { + t.Fatalf("runtime attrs = %+v", runtimeEvt.Attrs) } - - if got := eventBus.Dropped(EventKindLLMRequest); got != 999 { - t.Fatalf("expected 999 dropped events, got %d", got) + payload, ok := runtimeEvt.Payload.(ToolExecStartPayload) + if !ok { + t.Fatalf("runtime payload = %T, want ToolExecStartPayload", runtimeEvt.Payload) + } + if payload.Tool != "mock_custom" { + t.Fatalf("runtime payload tool = %q, want mock_custom", payload.Tool) } } @@ -127,8 +160,18 @@ func TestAgentLoop_EmitsMinimalTurnEvents(t *testing.T) { t.Fatal("expected default agent") } - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + expectedKinds := []runtimeevents.Kind{ + runtimeevents.KindAgentTurnStart, + runtimeevents.KindAgentLLMRequest, + runtimeevents.KindAgentLLMResponse, + runtimeevents.KindAgentToolExecStart, + runtimeevents.KindAgentToolExecEnd, + runtimeevents.KindAgentLLMRequest, + runtimeevents.KindAgentLLMResponse, + runtimeevents.KindAgentTurnEnd, + } + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest(t, al, 16, expectedKinds...) + defer closeRuntimeEvents() response, err := al.runAgentLoop(context.Background(), defaultAgent, processOptions{ SessionKey: "session-1", @@ -171,49 +214,36 @@ func TestAgentLoop_EmitsMinimalTurnEvents(t *testing.T) { t.Fatalf("expected final response 'done', got %q", response) } - events := collectEventStream(sub.C) + events := collectRuntimeEventStream(runtimeCh) if len(events) != 8 { t.Fatalf("expected 8 events, got %d", len(events)) } - kinds := make([]EventKind, 0, len(events)) + kinds := make([]runtimeevents.Kind, 0, len(events)) for _, evt := range events { kinds = append(kinds, evt.Kind) } - expectedKinds := []EventKind{ - EventKindTurnStart, - EventKindLLMRequest, - EventKindLLMResponse, - EventKindToolExecStart, - EventKindToolExecEnd, - EventKindLLMRequest, - EventKindLLMResponse, - EventKindTurnEnd, - } if !slices.Equal(kinds, expectedKinds) { t.Fatalf("unexpected event sequence: got %v want %v", kinds, expectedKinds) } - turnID := events[0].Meta.TurnID + turnID := events[0].Scope.TurnID + if turnID == "" { + t.Fatal("expected runtime events to include turn id") + } for i, evt := range events { - if evt.Meta.TurnID != turnID { - t.Fatalf("event %d has mismatched turn id %q, want %q", i, evt.Meta.TurnID, turnID) + if evt.Scope.TurnID != turnID { + t.Fatalf("event %d has mismatched turn id %q, want %q", i, evt.Scope.TurnID, turnID) } - if evt.Meta.SessionKey != "session-1" { - t.Fatalf("event %d has session key %q, want session-1", i, evt.Meta.SessionKey) + if evt.Scope.SessionKey != "session-1" { + t.Fatalf("event %d has session key %q, want session-1", i, evt.Scope.SessionKey) } - if evt.Context == nil || evt.Context.Inbound == nil { - t.Fatalf("event %d missing inbound turn context", i) + if evt.Scope.Channel != "cli" || evt.Scope.ChatID != "direct" || evt.Scope.SenderID != "tester" { + t.Fatalf("event %d scope = %+v", i, evt.Scope) } - if evt.Context.Inbound.Channel != "cli" || evt.Context.Inbound.SenderID != "tester" { - t.Fatalf("event %d inbound context = %+v", i, evt.Context.Inbound) - } - if evt.Context.Route == nil || evt.Context.Route.AgentID != "main" { - t.Fatalf("event %d missing route context: %+v", i, evt.Context.Route) - } - if evt.Context.Scope == nil || evt.Context.Scope.Values["sender"] != "tester" { - t.Fatalf("event %d missing session scope: %+v", i, evt.Context.Scope) + if evt.Scope.AgentID != "main" { + t.Fatalf("event %d has agent id %q, want main", i, evt.Scope.AgentID) } } @@ -309,8 +339,15 @@ func TestAgentLoop_EmitsSteeringAndSkippedToolEvents(t *testing.T) { al.RegisterTool(tool1) al.RegisterTool(tool2) - sub := al.SubscribeEvents(32) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 32, + runtimeevents.KindAgentSteeringInjected, + runtimeevents.KindAgentToolExecSkipped, + runtimeevents.KindAgentInterruptReceived, + ) + defer closeRuntimeEvents() resultCh := make(chan string, 1) go func() { @@ -337,8 +374,8 @@ func TestAgentLoop_EmitsSteeringAndSkippedToolEvents(t *testing.T) { t.Fatal("timeout waiting for steered response") } - events := collectEventStream(sub.C) - steeringEvt, ok := findEvent(events, EventKindSteeringInjected) + events := collectRuntimeEventStream(runtimeCh) + steeringEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentSteeringInjected) if !ok { t.Fatal("expected steering injected event") } @@ -350,7 +387,7 @@ func TestAgentLoop_EmitsSteeringAndSkippedToolEvents(t *testing.T) { t.Fatalf("expected 1 steering message, got %d", steeringPayload.Count) } - skippedEvt, ok := findEvent(events, EventKindToolExecSkipped) + skippedEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentToolExecSkipped) if !ok { t.Fatal("expected skipped tool event") } @@ -362,7 +399,7 @@ func TestAgentLoop_EmitsSteeringAndSkippedToolEvents(t *testing.T) { t.Fatalf("expected skipped tool_two, got %q", skippedPayload.Tool) } - interruptEvt, ok := findEvent(events, EventKindInterruptReceived) + interruptEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentInterruptReceived) if !ok { t.Fatal("expected interrupt received event") } @@ -420,8 +457,14 @@ func TestAgentLoop_EmitsContextCompressEventOnRetry(t *testing.T) { {Role: "user", Content: "Trigger message"}, }) - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentLLMRetry, + runtimeevents.KindAgentContextCompress, + ) + defer closeRuntimeEvents() resp, err := al.runAgentLoop(context.Background(), defaultAgent, processOptions{ SessionKey: "session-1", @@ -439,8 +482,8 @@ func TestAgentLoop_EmitsContextCompressEventOnRetry(t *testing.T) { t.Fatalf("expected retry success, got %q", resp) } - events := collectEventStream(sub.C) - retryEvt, ok := findEvent(events, EventKindLLMRetry) + events := collectRuntimeEventStream(runtimeCh) + retryEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentLLMRetry) if !ok { t.Fatal("expected llm retry event") } @@ -455,7 +498,7 @@ func TestAgentLoop_EmitsContextCompressEventOnRetry(t *testing.T) { t.Fatalf("expected retry attempt 1, got %d", retryPayload.Attempt) } - compressEvt, ok := findEvent(events, EventKindContextCompress) + compressEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentContextCompress) if !ok { t.Fatal("expected context compress event") } @@ -508,14 +551,19 @@ func TestAgentLoop_EmitsSessionSummarizeEvent(t *testing.T) { {Role: "assistant", Content: "Answer three"}, }) - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentSessionSummarize, + ) + defer closeRuntimeEvents() lcm := &legacyContextManager{al: al} lcm.summarizeSession(defaultAgent, "session-1") - events := collectEventStream(sub.C) - summaryEvt, ok := findEvent(events, EventKindSessionSummarize) + events := collectRuntimeEventStream(runtimeCh) + summaryEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentSessionSummarize) if !ok { t.Fatal("expected session summarize event") } @@ -575,8 +623,13 @@ func TestAgentLoop_EmitsFollowUpQueuedEvent(t *testing.T) { t.Fatal("expected default agent") } - sub := al.SubscribeEvents(32) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 32, + runtimeevents.KindAgentFollowUpQueued, + ) + defer closeRuntimeEvents() resp, err := al.runAgentLoop(context.Background(), defaultAgent, processOptions{ SessionKey: "session-1", @@ -600,8 +653,8 @@ func TestAgentLoop_EmitsFollowUpQueuedEvent(t *testing.T) { t.Fatal("timeout waiting for async tool completion") } - followUpEvt := waitForEvent(t, sub.C, 2*time.Second, func(evt Event) bool { - return evt.Kind == EventKindFollowUpQueued + followUpEvt := waitForRuntimeEvent(t, runtimeCh, 2*time.Second, func(evt runtimeevents.Event) bool { + return evt.Kind == runtimeevents.KindAgentFollowUpQueued }) payload, ok := followUpEvt.Payload.(FollowUpQueuedPayload) if !ok { @@ -613,59 +666,29 @@ func TestAgentLoop_EmitsFollowUpQueuedEvent(t *testing.T) { if payload.ContentLen != len("background result") { t.Fatalf("expected content len %d, got %d", len("background result"), payload.ContentLen) } - if followUpEvt.Meta.SessionKey != "session-1" { - t.Fatalf("expected session key session-1, got %q", followUpEvt.Meta.SessionKey) + if followUpEvt.Scope.SessionKey != "session-1" { + t.Fatalf("expected session key session-1, got %q", followUpEvt.Scope.SessionKey) } - if followUpEvt.Meta.TurnID == "" { + if followUpEvt.Scope.TurnID == "" { t.Fatal("expected follow-up event to include turn id") } } -func collectEventStream(ch <-chan Event) []Event { - var events []Event - for { - select { - case evt, ok := <-ch: - if !ok { - return events - } - events = append(events, evt) - default: - return events - } - } -} - -func waitForEvent(t *testing.T, ch <-chan Event, timeout time.Duration, match func(Event) bool) Event { +func receiveRuntimeEvent(t *testing.T, ch <-chan runtimeevents.Event) runtimeevents.Event { t.Helper() - timer := time.NewTimer(timeout) - defer timer.Stop() - - for { - select { - case evt, ok := <-ch: - if !ok { - t.Fatal("event stream closed before expected event arrived") - } - if match(evt) { - return evt - } - case <-timer.C: - t.Fatal("timed out waiting for expected event") + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("runtime event stream closed before expected event arrived") } + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for runtime event") + return runtimeevents.Event{} } } -func findEvent(events []Event, kind EventKind) (Event, bool) { - for _, evt := range events { - if evt.Kind == kind { - return evt, true - } - } - return Event{}, false -} - type stringError string func (e stringError) Error() string { diff --git a/pkg/agent/events.go b/pkg/agent/events.go index f68d3eab5..b23350774 100644 --- a/pkg/agent/events.go +++ b/pkg/agent/events.go @@ -1,97 +1,14 @@ package agent import ( - "fmt" "time" + + runtimeevents "github.com/sipeed/picoclaw/pkg/events" ) -// EventKind identifies a structured agent-loop event. -type EventKind uint8 - -const ( - // EventKindTurnStart is emitted when a turn begins processing. - EventKindTurnStart EventKind = iota - // EventKindTurnEnd is emitted when a turn finishes, successfully or with an error. - EventKindTurnEnd - // EventKindLLMRequest is emitted before a provider chat request is made. - EventKindLLMRequest - // EventKindLLMDelta is emitted when a streaming provider yields a partial delta. - EventKindLLMDelta - // EventKindLLMResponse is emitted after a provider chat response is received. - EventKindLLMResponse - // EventKindLLMRetry is emitted when an LLM request is retried. - EventKindLLMRetry - // EventKindContextCompress is emitted when session history is forcibly compressed. - EventKindContextCompress - // EventKindSessionSummarize is emitted when asynchronous summarization completes. - EventKindSessionSummarize - // EventKindToolExecStart is emitted immediately before a tool executes. - EventKindToolExecStart - // EventKindToolExecEnd is emitted immediately after a tool finishes executing. - EventKindToolExecEnd - // EventKindToolExecSkipped is emitted when a queued tool call is skipped. - EventKindToolExecSkipped - // EventKindSteeringInjected is emitted when queued steering is injected into context. - EventKindSteeringInjected - // EventKindFollowUpQueued is emitted when an async tool queues a follow-up system message. - EventKindFollowUpQueued - // EventKindInterruptReceived is emitted when a soft interrupt message is accepted. - EventKindInterruptReceived - // EventKindSubTurnSpawn is emitted when a sub-turn is spawned. - EventKindSubTurnSpawn - // EventKindSubTurnEnd is emitted when a sub-turn finishes. - EventKindSubTurnEnd - // EventKindSubTurnResultDelivered is emitted when a sub-turn result is delivered. - EventKindSubTurnResultDelivered - // EventKindSubTurnOrphan is emitted when a sub-turn result cannot be delivered. - EventKindSubTurnOrphan - // EventKindError is emitted when a turn encounters an execution error. - EventKindError - - eventKindCount -) - -var eventKindNames = [...]string{ - "turn_start", - "turn_end", - "llm_request", - "llm_delta", - "llm_response", - "llm_retry", - "context_compress", - "session_summarize", - "tool_exec_start", - "tool_exec_end", - "tool_exec_skipped", - "steering_injected", - "follow_up_queued", - "interrupt_received", - "subturn_spawn", - "subturn_end", - "subturn_result_delivered", - "subturn_orphan", - "error", -} - -// String returns the stable string form of an EventKind. -func (k EventKind) String() string { - if k >= eventKindCount { - return fmt.Sprintf("event_kind(%d)", k) - } - return eventKindNames[k] -} - -// Event is the structured envelope broadcast by the agent EventBus. -type Event struct { - Kind EventKind - Time time.Time - Meta EventMeta - Context *TurnContext - Payload any -} - -// EventMeta contains correlation fields shared by all agent-loop events. -type EventMeta struct { +// HookMeta contains correlation fields shared by agent hook requests and +// runtime events emitted from turn processing. +type HookMeta struct { AgentID string TurnID string ParentTurnID string @@ -102,170 +19,41 @@ type EventMeta struct { turnContext *TurnContext } -// TurnEndStatus describes the terminal state of a turn. -type TurnEndStatus string +// EventKind is the legacy in-agent event kind alias kept for tests and +// compatibility shims on top of the runtime event bus. +type EventKind = runtimeevents.Kind const ( - // TurnEndStatusCompleted indicates the turn finished normally. - TurnEndStatusCompleted TurnEndStatus = "completed" - // TurnEndStatusError indicates the turn ended because of an error. - TurnEndStatusError TurnEndStatus = "error" - // TurnEndStatusAborted indicates the turn was hard-aborted and rolled back. - TurnEndStatusAborted TurnEndStatus = "aborted" + EventKindTurnStart EventKind = runtimeevents.KindAgentTurnStart + EventKindTurnEnd EventKind = runtimeevents.KindAgentTurnEnd + EventKindLLMRequest EventKind = runtimeevents.KindAgentLLMRequest + EventKindLLMDelta EventKind = runtimeevents.KindAgentLLMDelta + EventKindLLMResponse EventKind = runtimeevents.KindAgentLLMResponse + EventKindLLMRetry EventKind = runtimeevents.KindAgentLLMRetry + EventKindContextCompress EventKind = runtimeevents.KindAgentContextCompress + EventKindSessionSummarize EventKind = runtimeevents.KindAgentSessionSummarize + EventKindToolExecStart EventKind = runtimeevents.KindAgentToolExecStart + EventKindToolExecEnd EventKind = runtimeevents.KindAgentToolExecEnd + EventKindToolExecSkipped EventKind = runtimeevents.KindAgentToolExecSkipped + EventKindSteeringInjected EventKind = runtimeevents.KindAgentSteeringInjected + EventKindFollowUpQueued EventKind = runtimeevents.KindAgentFollowUpQueued + EventKindInterruptReceived EventKind = runtimeevents.KindAgentInterruptReceived + EventKindSubTurnSpawn EventKind = runtimeevents.KindAgentSubTurnSpawn + EventKindSubTurnEnd EventKind = runtimeevents.KindAgentSubTurnEnd + EventKindSubTurnResultDelivered EventKind = runtimeevents.KindAgentSubTurnResultDelivered + EventKindSubTurnOrphan EventKind = runtimeevents.KindAgentSubTurnOrphan + EventKindError EventKind = runtimeevents.KindAgentError ) -// TurnStartPayload describes the start of a turn. -type TurnStartPayload struct { - UserMessage string - MediaCount int -} - -// TurnEndPayload describes the completion of a turn. -type TurnEndPayload struct { - Status TurnEndStatus - Iterations int - Duration time.Duration - FinalContentLen int -} - -// LLMRequestPayload describes an outbound LLM request. -type LLMRequestPayload struct { - Model string - MessagesCount int - ToolsCount int - MaxTokens int - Temperature float64 -} - -// LLMResponsePayload describes an inbound LLM response. -type LLMResponsePayload struct { - ContentLen int - ToolCalls int - HasReasoning bool -} - -// LLMDeltaPayload describes a streamed LLM delta. -type LLMDeltaPayload struct { - ContentDeltaLen int - ReasoningDeltaLen int -} - -// LLMRetryPayload describes a retry of an LLM request. -type LLMRetryPayload struct { - Attempt int - MaxRetries int - Reason string - Error string - Backoff time.Duration -} - -// ContextCompressReason identifies why emergency compression ran. -type ContextCompressReason string - -const ( - // ContextCompressReasonProactive indicates compression before the first LLM call. - ContextCompressReasonProactive ContextCompressReason = "proactive_budget" - // ContextCompressReasonRetry indicates compression during context-error retry handling. - ContextCompressReasonRetry ContextCompressReason = "llm_retry" - // ContextCompressReasonSummarize indicates post-turn async summarization. - ContextCompressReasonSummarize ContextCompressReason = "summarize" -) - -// ContextCompressPayload describes a forced history compression. -type ContextCompressPayload struct { - Reason ContextCompressReason - DroppedMessages int - RemainingMessages int -} - -// SessionSummarizePayload describes a completed async session summarization. -type SessionSummarizePayload struct { - SummarizedMessages int - KeptMessages int - SummaryLen int - OmittedOversized bool -} - -// ToolExecStartPayload describes a tool execution request. -type ToolExecStartPayload struct { - Tool string - Arguments map[string]any -} - -// ToolExecEndPayload describes the outcome of a tool execution. -type ToolExecEndPayload struct { - Tool string - Duration time.Duration - ForLLMLen int - ForUserLen int - IsError bool - Async bool -} - -// ToolExecSkippedPayload describes a skipped tool call. -type ToolExecSkippedPayload struct { - Tool string - Reason string -} - -// SteeringInjectedPayload describes steering messages appended before the next LLM call. -type SteeringInjectedPayload struct { - Count int - TotalContentLen int -} - -// FollowUpQueuedPayload describes an async follow-up queued back into the inbound bus. -type FollowUpQueuedPayload struct { - SourceTool string - ContentLen int -} - -type InterruptKind string - -const ( - InterruptKindSteering InterruptKind = "steering" - InterruptKindGraceful InterruptKind = "graceful" - InterruptKindHard InterruptKind = "hard_abort" -) - -// InterruptReceivedPayload describes accepted turn-control input. -type InterruptReceivedPayload struct { - Kind InterruptKind - Role string - ContentLen int - QueueDepth int - HintLen int -} - -// SubTurnSpawnPayload describes the creation of a child turn. -type SubTurnSpawnPayload struct { - AgentID string - Label string - ParentTurnID string -} - -// SubTurnEndPayload describes the completion of a child turn. -type SubTurnEndPayload struct { - AgentID string - Status string -} - -// SubTurnResultDeliveredPayload describes delivery of a sub-turn result. -type SubTurnResultDeliveredPayload struct { - TargetChannel string - TargetChatID string - ContentLen int -} - -// SubTurnOrphanPayload describes a sub-turn result that could not be delivered. -type SubTurnOrphanPayload struct { - ParentTurnID string - ChildTurnID string - Reason string -} - -// ErrorPayload describes an execution error inside the agent loop. -type ErrorPayload struct { - Stage string - Message string +// EventMeta is the legacy name for hook metadata. +type EventMeta = HookMeta + +// Event is the legacy agent event envelope exposed by SubscribeEvents and a +// handful of tests. Runtime code publishes pkg/events.Event internally. +type Event struct { + Kind EventKind + Time time.Time + Meta EventMeta + Context *TurnContext + Payload any } diff --git a/pkg/agent/events_runtime.go b/pkg/agent/events_runtime.go new file mode 100644 index 000000000..2284665e6 --- /dev/null +++ b/pkg/agent/events_runtime.go @@ -0,0 +1,88 @@ +package agent + +import runtimeevents "github.com/sipeed/picoclaw/pkg/events" + +func (al *AgentLoop) publishRuntimeEvent(evt runtimeevents.Event) { + if al == nil || al.runtimeEvents == nil { + return + } + + al.runtimeEvents.PublishNonBlocking(evt) +} + +func runtimeScopeFromHookMeta(meta HookMeta, eventCtx *TurnContext) runtimeevents.Scope { + scope := runtimeevents.Scope{ + AgentID: meta.AgentID, + SessionKey: meta.SessionKey, + TurnID: meta.TurnID, + } + + if eventCtx == nil || eventCtx.Inbound == nil { + return scope + } + + inbound := eventCtx.Inbound + scope.Channel = inbound.Channel + scope.Account = inbound.Account + scope.ChatID = inbound.ChatID + scope.TopicID = inbound.TopicID + scope.SpaceID = inbound.SpaceID + scope.SpaceType = inbound.SpaceType + scope.ChatType = inbound.ChatType + scope.SenderID = inbound.SenderID + scope.MessageID = inbound.MessageID + return scope +} + +func runtimeCorrelationFromHookMeta(meta HookMeta) runtimeevents.Correlation { + return runtimeevents.Correlation{ + TraceID: meta.TracePath, + ParentTurnID: meta.ParentTurnID, + } +} + +func runtimeSeverityForAgentEvent(kind runtimeevents.Kind, payload any) runtimeevents.Severity { + switch kind { + case runtimeevents.KindAgentError, runtimeevents.KindAgentSubTurnOrphan: + return runtimeevents.SeverityError + case runtimeevents.KindAgentLLMRetry, + runtimeevents.KindAgentContextCompress, + runtimeevents.KindAgentToolExecSkipped: + return runtimeevents.SeverityWarn + case runtimeevents.KindAgentTurnEnd: + payload, ok := payload.(TurnEndPayload) + if !ok { + return runtimeevents.SeverityInfo + } + switch payload.Status { + case TurnEndStatusError: + return runtimeevents.SeverityError + case TurnEndStatusAborted: + return runtimeevents.SeverityWarn + default: + return runtimeevents.SeverityInfo + } + case runtimeevents.KindAgentToolExecEnd: + payload, ok := payload.(ToolExecEndPayload) + if ok && payload.IsError { + return runtimeevents.SeverityWarn + } + return runtimeevents.SeverityInfo + default: + return runtimeevents.SeverityInfo + } +} + +func runtimeAttrsFromHookMeta(meta HookMeta) map[string]any { + attrs := make(map[string]any, 2) + if meta.Source != "" { + attrs["agent_source"] = meta.Source + } + if meta.Iteration != 0 { + attrs["iteration"] = meta.Iteration + } + if len(attrs) == 0 { + return nil + } + return attrs +} diff --git a/pkg/agent/evolution_bridge.go b/pkg/agent/evolution_bridge.go new file mode 100644 index 000000000..2e54c8690 --- /dev/null +++ b/pkg/agent/evolution_bridge.go @@ -0,0 +1,444 @@ +package agent + +import ( + "context" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" + "github.com/sipeed/picoclaw/pkg/evolution" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/providers" +) + +type evolutionBridge struct { + cfg config.EvolutionConfig + registry *AgentRegistry + runtime *evolution.Runtime + coldPathRunner *evolution.ColdPathRunner + runtimeSub runtimeevents.Subscription + bgCtx context.Context + cancel context.CancelFunc + closeMu sync.Mutex + closed bool + wg sync.WaitGroup + isCurrent func(*evolutionBridge) bool + + scheduledMu sync.Mutex + scheduledWorkspaces map[string]struct{} +} + +const evolutionDirectDeliveryAttr = "evolution_direct_delivery" + +func newEvolutionBridge( + registry *AgentRegistry, + cfg *config.Config, + provider providers.LLMProvider, +) (*evolutionBridge, error) { + if cfg == nil { + return nil, nil + } + + modelID := resolvedEvolutionModelID(cfg, provider) + runtime, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: cfg.Evolution, + PatternClusterer: evolution.NewLLMPatternClusterer( + provider, + modelID, + evolution.NewHeuristicPatternClusterer(cfg.Evolution.EffectiveMinTaskCount(), nil), + cfg.Evolution.EffectiveMinTaskCount(), + nil, + ), + GeneratorFactory: func(workspace string) evolution.DraftGenerator { + return evolution.NewDraftGeneratorForWorkspace(workspace, provider, modelID) + }, + SuccessJudgeFactory: func(workspace string) evolution.SuccessJudge { + return evolution.NewLLMTaskSuccessJudge(provider, modelID, &evolution.HeuristicSuccessJudge{}) + }, + ApplierFactory: func(workspace string) *evolution.Applier { + return evolution.NewApplier(evolution.NewPaths(workspace, cfg.Evolution.StateDir), nil) + }, + }) + if err != nil { + return nil, err + } + bgCtx, cancel := context.WithCancel(context.Background()) + + bridge := &evolutionBridge{ + cfg: cfg.Evolution, + registry: registry, + runtime: runtime, + bgCtx: bgCtx, + cancel: cancel, + } + if cfg.Evolution.RunsColdPathAutomatically() { + bridge.coldPathRunner = evolution.NewColdPathRunnerWithErrorHandler(runtime, func(err error) { + logger.WarnCF("agent", "Cold path run failed", map[string]any{ + "error": err.Error(), + }) + }) + } + if cfg.Evolution.RunsColdPathScheduled() { + bridge.startScheduledColdPath(cfg.Agents.Defaults.Workspace, cfg.Evolution.EffectiveColdPathTimes()) + bridge.rememberScheduledColdPathWorkspaces(registryWorkspaces(registry)) + } + + return bridge, nil +} + +func resolvedEvolutionModelID(cfg *config.Config, provider providers.LLMProvider) string { + if cfg != nil { + if modelID := cfg.Agents.Defaults.GetModelName(); modelID != "" { + return modelID + } + } + if provider != nil { + return provider.GetDefaultModel() + } + return "" +} + +func (b *evolutionBridge) Close() error { + if b == nil { + return nil + } + + if b.runtimeSub != nil { + if err := b.runtimeSub.Close(); err != nil { + logger.WarnCF("agent", "Failed to close evolution runtime subscription", map[string]any{ + "error": err.Error(), + }) + } + <-b.runtimeSub.Done() + } + + b.closeMu.Lock() + alreadyClosed := b.closed + b.closed = true + b.closeMu.Unlock() + if alreadyClosed { + return nil + } + if b.cancel != nil { + b.cancel() + } + var closeErr error + if b.coldPathRunner != nil { + closeErr = b.coldPathRunner.Close() + } + b.wg.Wait() + return closeErr +} + +func (b *evolutionBridge) OnEvent(_ context.Context, evt Event) error { + if b == nil || !b.cfg.Enabled || b.runtime == nil { + return nil + } + + switch evt.Kind { + case EventKindTurnEnd: + payload, ok := evt.Payload.(TurnEndPayload) + if !ok { + return nil + } + b.handleTurnEndAsync(evt.Meta, payload) + return nil + } + + return nil +} + +func (b *evolutionBridge) OnRuntimeEvent(_ context.Context, evt runtimeevents.Event) error { + if b == nil || !b.cfg.Enabled || b.runtime == nil || evt.Kind != runtimeevents.KindAgentTurnEnd { + return nil + } + if b.isCurrent != nil && !b.isCurrent(b) { + return nil + } + if deliveredDirectly, _ := evt.Attrs[evolutionDirectDeliveryAttr].(bool); deliveredDirectly { + return nil + } + payload, ok := evt.Payload.(TurnEndPayload) + if !ok { + return nil + } + b.handleTurnEndAsync(hookMetaFromRuntimeEvent(evt), payload) + return nil +} + +func (b *evolutionBridge) handleRuntimeTurnEnd(evt runtimeevents.Event) bool { + if b == nil || !b.cfg.Enabled || b.runtime == nil || evt.Kind != runtimeevents.KindAgentTurnEnd { + return false + } + payload, ok := evt.Payload.(TurnEndPayload) + if !ok { + return false + } + return b.handleTurnEndAsync(hookMetaFromRuntimeEvent(evt), payload) +} + +func (b *evolutionBridge) handleTurnEndAsync(meta EventMeta, payload TurnEndPayload) bool { + if b == nil || b.runtime == nil { + return false + } + + input := evolution.TurnCaseInput{ + Workspace: payload.Workspace, + WorkspaceID: payload.Workspace, + TurnID: meta.TurnID, + SessionKey: meta.SessionKey, + AgentID: meta.AgentID, + Status: string(payload.Status), + UserMessage: payload.UserMessage, + FinalContent: payload.FinalContent, + ToolKinds: append([]string(nil), payload.ToolKinds...), + ToolExecutions: toEvolutionToolExecutions(payload.ToolExecutions), + ActiveSkillNames: append([]string(nil), payload.ActiveSkills...), + AttemptedSkillNames: append([]string(nil), payload.AttemptedSkills...), + FinalSuccessfulPath: append([]string(nil), payload.FinalSuccessfulPath...), + SkillContextSnapshots: toEvolutionSkillContextSnapshots(payload.SkillContextSnapshots), + } + b.rememberScheduledColdPathWorkspace(input.Workspace) + + b.closeMu.Lock() + if b.closed { + b.closeMu.Unlock() + return false + } + b.wg.Add(1) + b.closeMu.Unlock() + go func() { + defer b.wg.Done() + if err := b.runtime.FinalizeTurn(b.bgCtx, input); err != nil { + logger.WarnCF("agent", "Evolution finalize turn failed", map[string]any{ + "error": err.Error(), + "turn_id": input.TurnID, + "workspace": input.Workspace, + }) + return + } + if b.coldPathRunner != nil && b.cfg.RunsColdPathAfterTurn() { + b.coldPathRunner.Trigger(input.Workspace) + } + }() + return true +} + +func (b *evolutionBridge) subscribeRuntimeEvents(ch runtimeevents.EventChannel) error { + if b == nil || ch == nil { + return nil + } + sub, err := ch.Source("agent").OfKind(runtimeevents.KindAgentTurnEnd).Subscribe( + b.bgCtx, + runtimeevents.SubscribeOptions{ + Name: "evolution-bridge", + Buffer: hookObserverBufferSize, + Backpressure: runtimeevents.Block, + Concurrency: runtimeevents.Locked, + }, + func(ctx context.Context, evt runtimeevents.Event) error { + return b.OnRuntimeEvent(ctx, evt) + }, + ) + if err != nil { + return err + } + b.runtimeSub = sub + return nil +} + +func (b *evolutionBridge) setCurrentCheck(check func(*evolutionBridge) bool) { + if b == nil { + return + } + b.closeMu.Lock() + defer b.closeMu.Unlock() + b.isCurrent = check +} + +func (b *evolutionBridge) startScheduledColdPath(workspace string, times []string) { + if b == nil || b.coldPathRunner == nil || len(times) == 0 { + return + } + b.rememberScheduledColdPathWorkspace(workspace) + schedule := parseColdPathSchedule(times) + if len(schedule) == 0 { + logger.WarnCF("agent", "No valid evolution cold path schedule times configured", map[string]any{ + "times": times, + }) + return + } + + b.wg.Add(1) + go func() { + defer b.wg.Done() + for { + now := time.Now() + next := nextColdPathScheduledTime(now, schedule) + timer := time.NewTimer(time.Until(next)) + select { + case <-timer.C: + for _, workspace := range b.scheduledColdPathWorkspaces() { + b.coldPathRunner.Trigger(workspace) + } + case <-b.bgCtx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return + } + } + }() +} + +func (b *evolutionBridge) rememberScheduledColdPathWorkspace(workspace string) { + if b == nil || !b.cfg.RunsColdPathScheduled() { + return + } + workspace = strings.TrimSpace(workspace) + if workspace == "" { + return + } + b.scheduledMu.Lock() + defer b.scheduledMu.Unlock() + if b.scheduledWorkspaces == nil { + b.scheduledWorkspaces = make(map[string]struct{}) + } + b.scheduledWorkspaces[workspace] = struct{}{} +} + +func (b *evolutionBridge) rememberScheduledColdPathWorkspaces(workspaces []string) { + for _, workspace := range workspaces { + b.rememberScheduledColdPathWorkspace(workspace) + } +} + +func (b *evolutionBridge) scheduledColdPathWorkspaces() []string { + if b == nil { + return nil + } + b.scheduledMu.Lock() + defer b.scheduledMu.Unlock() + out := make([]string, 0, len(b.scheduledWorkspaces)) + for workspace := range b.scheduledWorkspaces { + out = append(out, workspace) + } + sort.Strings(out) + return out +} + +func registryWorkspaces(registry *AgentRegistry) []string { + if registry == nil { + return nil + } + registry.mu.RLock() + defer registry.mu.RUnlock() + + out := make([]string, 0, len(registry.agents)) + seen := make(map[string]struct{}, len(registry.agents)) + for _, agent := range registry.agents { + if agent == nil { + continue + } + workspace := strings.TrimSpace(agent.Workspace) + if workspace == "" { + continue + } + if _, ok := seen[workspace]; ok { + continue + } + seen[workspace] = struct{}{} + out = append(out, workspace) + } + sort.Strings(out) + return out +} + +type coldPathScheduleTime struct { + hour int + minute int +} + +func parseColdPathSchedule(values []string) []coldPathScheduleTime { + out := make([]coldPathScheduleTime, 0, len(values)) + seen := make(map[coldPathScheduleTime]struct{}, len(values)) + for _, value := range values { + parts := strings.Split(strings.TrimSpace(value), ":") + if len(parts) != 2 { + continue + } + hour, err := strconv.Atoi(parts[0]) + if err != nil || hour < 0 || hour > 23 { + continue + } + minute, err := strconv.Atoi(parts[1]) + if err != nil || minute < 0 || minute > 59 { + continue + } + item := coldPathScheduleTime{hour: hour, minute: minute} + if _, ok := seen[item]; ok { + continue + } + seen[item] = struct{}{} + out = append(out, item) + } + sort.Slice(out, func(i, j int) bool { + if out[i].hour != out[j].hour { + return out[i].hour < out[j].hour + } + return out[i].minute < out[j].minute + }) + return out +} + +func nextColdPathScheduledTime(now time.Time, schedule []coldPathScheduleTime) time.Time { + for _, item := range schedule { + candidate := time.Date(now.Year(), now.Month(), now.Day(), item.hour, item.minute, 0, 0, now.Location()) + if candidate.After(now) { + return candidate + } + } + first := schedule[0] + tomorrow := now.AddDate(0, 0, 1) + return time.Date(tomorrow.Year(), tomorrow.Month(), tomorrow.Day(), first.hour, first.minute, 0, 0, now.Location()) +} + +func toEvolutionSkillContextSnapshots(input []SkillContextSnapshot) []evolution.SkillContextSnapshot { + if len(input) == 0 { + return nil + } + + out := make([]evolution.SkillContextSnapshot, 0, len(input)) + for _, snapshot := range input { + out = append(out, evolution.SkillContextSnapshot{ + Sequence: snapshot.Sequence, + Trigger: snapshot.Trigger, + SkillNames: append([]string(nil), snapshot.SkillNames...), + }) + } + return out +} + +func toEvolutionToolExecutions(input []ToolExecutionRecord) []evolution.ToolExecutionRecord { + if len(input) == 0 { + return nil + } + + out := make([]evolution.ToolExecutionRecord, 0, len(input)) + for _, record := range input { + out = append(out, evolution.ToolExecutionRecord{ + Name: record.Name, + Success: record.Success, + ErrorSummary: record.ErrorSummary, + SkillNames: append([]string(nil), record.SkillNames...), + }) + } + return out +} diff --git a/pkg/agent/evolution_bridge_test.go b/pkg/agent/evolution_bridge_test.go new file mode 100644 index 000000000..8469acd80 --- /dev/null +++ b/pkg/agent/evolution_bridge_test.go @@ -0,0 +1,1344 @@ +package agent + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" + "github.com/sipeed/picoclaw/pkg/evolution" + "github.com/sipeed/picoclaw/pkg/providers" +) + +func TestEvolutionBridge_DisabledWritesNothing(t *testing.T) { + tmpDir := t.TempDir() + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: false, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + resp, err := al.ProcessDirectWithChannel(context.Background(), "hello", "session-disabled", "cli", "direct") + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "ok" { + t.Fatalf("response = %q, want %q", resp, "ok") + } + + assertNotExists(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + assertNotExists(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json")) +} + +func TestEvolutionBridge_ObserveWritesCaseRecord(t *testing.T) { + tmpDir := t.TempDir() + provider := &toolCallRespProvider{ + toolName: "echo_text", + toolArgs: map[string]any{"text": "bridge"}, + response: "done", + } + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, provider) + defer al.Close() + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + defaultAgent.SkillsFilter = []string{"observe-skill"} + al.RegisterTool(&echoTextTool{}) + + resp, err := al.ProcessDirectWithChannel(context.Background(), "hello", "session-observe", "cli", "direct") + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "done" { + t.Fatalf("response = %q, want %q", resp, "done") + } + + record := waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + + if got := record["kind"]; got != string(evolution.RecordKindCase) { + t.Fatalf("kind = %v, want %q", got, evolution.RecordKindCase) + } + if got := record["workspace_id"]; got != tmpDir { + t.Fatalf("workspace_id = %v, want %q", got, tmpDir) + } + if got := record["status"]; got != "new" { + t.Fatalf("status = %v, want %q", got, "new") + } + + for _, field := range []string{"tool_kinds", "tool_executions", "initial_skill_names", "active_skill_names", "attempt_trail", "source"} { + if _, exists := record[field]; exists { + t.Fatalf("%s should not be persisted in slim task record: %#v", field, record[field]) + } + } +} + +func TestEvolutionBridge_TurnEndBypassesHookObserverBackpressure(t *testing.T) { + tmpDir := t.TempDir() + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + blocker := &blockingRuntimeObserver{ + started: make(chan struct{}), + release: make(chan struct{}), + } + defer close(blocker.release) + al.hooks.ConfigureTimeouts(5*time.Second, 0, 0) + if err := al.MountHook(NamedHook("aaa-block-runtime-events", blocker)); err != nil { + t.Fatalf("MountHook: %v", err) + } + + al.publishRuntimeEvent(runtimeevents.Event{ + Kind: runtimeevents.KindAgentTurnStart, + Source: runtimeevents.Source{Component: "agent", Name: "main"}, + }) + select { + case <-blocker.started: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for blocking runtime observer") + } + + for i := 0; i < hookObserverBufferSize+10; i++ { + al.publishRuntimeEvent(runtimeevents.Event{ + Kind: runtimeevents.KindAgentLLMDelta, + Source: runtimeevents.Source{Component: "agent", Name: "main"}, + }) + } + + al.emitEvent(runtimeevents.KindAgentTurnEnd, EventMeta{ + AgentID: "main", + TurnID: "turn-backpressure", + SessionKey: "session-backpressure", + }, TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: tmpDir, + UserMessage: "hello", + FinalContent: "ok", + }) + + record := waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + if got := record["session_key"]; got != "session-backpressure" { + t.Fatalf("session_key = %v, want session-backpressure", got) + } + if got := record["summary"]; got != "hello" { + t.Fatalf("summary = %v, want hello", got) + } +} + +func TestEvolutionBridge_RuntimeBusTurnEndWritesCaseRecord(t *testing.T) { + tmpDir := t.TempDir() + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + result := al.RuntimeEventBus().Publish(context.Background(), runtimeevents.Event{ + Kind: runtimeevents.KindAgentTurnEnd, + Source: runtimeevents.Source{Component: "agent", Name: "main"}, + Scope: runtimeevents.Scope{ + AgentID: "main", + TurnID: "turn-runtime-bus", + SessionKey: "session-runtime-bus", + }, + Payload: TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: tmpDir, + UserMessage: "runtime bus task", + FinalContent: "ok", + }, + }) + if result.Delivered == 0 { + t.Fatalf("runtime bus publish delivered = %d, want > 0", result.Delivered) + } + + record := waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + if got := record["session_key"]; got != "session-runtime-bus" { + t.Fatalf("session_key = %v, want session-runtime-bus", got) + } + if got := record["summary"]; got != "runtime bus task" { + t.Fatalf("summary = %v, want runtime bus task", got) + } +} + +func TestEvolutionBridge_RuntimeBusOnlyCurrentBridgeConsumesTurnEnd(t *testing.T) { + tmpDir := t.TempDir() + cfg := &config.Config{ + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, + } + eventBus := runtimeevents.NewBus() + defer eventBus.Close() + + oldBridge, err := newEvolutionBridge(nil, cfg, nil) + if err != nil { + t.Fatalf("newEvolutionBridge(old): %v", err) + } + defer oldBridge.Close() + newBridge, err := newEvolutionBridge(nil, cfg, nil) + if err != nil { + t.Fatalf("newEvolutionBridge(new): %v", err) + } + defer newBridge.Close() + + current := newBridge + oldBridge.setCurrentCheck(func(bridge *evolutionBridge) bool { + return current == bridge + }) + newBridge.setCurrentCheck(func(bridge *evolutionBridge) bool { + return current == bridge + }) + if err := oldBridge.subscribeRuntimeEvents(eventBus.Channel()); err != nil { + t.Fatalf("old subscribeRuntimeEvents: %v", err) + } + if err := newBridge.subscribeRuntimeEvents(eventBus.Channel()); err != nil { + t.Fatalf("new subscribeRuntimeEvents: %v", err) + } + + eventBus.Publish(context.Background(), runtimeevents.Event{ + Kind: runtimeevents.KindAgentTurnEnd, + Source: runtimeevents.Source{Component: "agent", Name: "main"}, + Scope: runtimeevents.Scope{ + AgentID: "main", + TurnID: "turn-current-bridge", + SessionKey: "session-current-bridge", + }, + Payload: TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: tmpDir, + UserMessage: "current bridge task", + FinalContent: "ok", + }, + }) + + recordsPath := filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl") + waitForEvolutionRecord(t, recordsPath) + time.Sleep(100 * time.Millisecond) + if got := countEvolutionTaskRecords(t, recordsPath); got != 1 { + t.Fatalf("task record count = %d, want 1", got) + } +} + +func TestEvolutionBridge_DirectDeliveryFailureFallsBackToCurrentRuntimeBridge(t *testing.T) { + tmpDir := t.TempDir() + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + oldBridge := al.evolution + if oldBridge == nil { + t.Fatal("expected initial evolution bridge") + } + defer oldBridge.Close() + + newBridge, err := newEvolutionBridge(al.registry, al.cfg, &simpleMockProvider{response: "ok"}) + if err != nil { + t.Fatalf("newEvolutionBridge: %v", err) + } + newBridge.setCurrentCheck(al.isCurrentEvolutionBridge) + if err := newBridge.subscribeRuntimeEvents(al.RuntimeEventBus().Channel()); err != nil { + t.Fatalf("subscribeRuntimeEvents: %v", err) + } + + oldBridge.closeMu.Lock() + done := make(chan struct{}) + go func() { + defer close(done) + al.emitEvent(runtimeevents.KindAgentTurnEnd, EventMeta{ + AgentID: "main", + TurnID: "turn-direct-fallback", + SessionKey: "session-direct-fallback", + }, TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: tmpDir, + UserMessage: "direct fallback task", + FinalContent: "ok", + }) + }() + + time.Sleep(20 * time.Millisecond) + al.mu.Lock() + al.evolution = newBridge + al.mu.Unlock() + oldBridge.closed = true + oldBridge.closeMu.Unlock() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for emitEvent") + } + + recordsPath := filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl") + record := waitForEvolutionRecord(t, recordsPath) + if got := record["session_key"]; got != "session-direct-fallback" { + t.Fatalf("session_key = %v, want session-direct-fallback", got) + } + if got := countEvolutionTaskRecords(t, recordsPath); got != 1 { + t.Fatalf("task record count = %d, want 1", got) + } +} + +func TestEvolutionBridge_CloseCancelsPendingTurnEndRecord(t *testing.T) { + tmpDir := t.TempDir() + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + + al.emitEvent(runtimeevents.KindAgentTurnEnd, EventMeta{ + AgentID: "main", + TurnID: "turn-close-flush", + SessionKey: "session-close-flush", + }, TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: tmpDir, + UserMessage: "close flush task", + FinalContent: "ok", + }) + + done := make(chan struct{}) + go func() { + al.Close() + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Close timed out") + } +} + +func TestEvolutionBridge_ObserveTurnEndPayloadIncludesResolvedAttemptTrail(t *testing.T) { + tmpDir := t.TempDir() + skillDir := filepath.Join(tmpDir, "skills", "observe-skill") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile( + filepath.Join(skillDir, "SKILL.md"), + []byte("---\nname: observe-skill\ndescription: observe test skill\n---\n# Observe Skill\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + defaultAgent.SkillsFilter = []string{"missing-skill", "observe-skill", "observe-skill"} + + sub := al.SubscribeEvents(16) + defer al.UnsubscribeEvents(sub.ID) + + resp, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-observe-attempt-trail", + "cli", + "direct", + ) + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "ok" { + t.Fatalf("response = %q, want %q", resp, "ok") + } + + turnEndEvt := waitForEvent(t, sub.C, 2*time.Second, func(evt Event) bool { + return evt.Kind == EventKindTurnEnd + }) + turnEndPayload, ok := turnEndEvt.Payload.(TurnEndPayload) + if !ok { + t.Fatalf("expected TurnEndPayload, got %T", turnEndEvt.Payload) + } + if got := turnEndPayload.AttemptedSkills; len(got) != 1 || got[0] != "observe-skill" { + t.Fatalf("AttemptedSkills = %v, want [observe-skill]", got) + } + if got := turnEndPayload.FinalSuccessfulPath; len(got) != 1 || got[0] != "observe-skill" { + t.Fatalf("FinalSuccessfulPath = %v, want [observe-skill]", got) + } + if got := turnEndPayload.SkillContextSnapshots; len(got) != 1 || got[0].Trigger != skillContextTriggerInitialBuild { + t.Fatalf("SkillContextSnapshots = %+v, want single initial_build snapshot", got) + } +} + +func TestEvolutionBridge_ObserveTurnEndUsesLatestSkillSnapshotAfterRetry(t *testing.T) { + tmpDir := t.TempDir() + baseSkillDir := filepath.Join(tmpDir, "skills", "base-skill") + if err := os.MkdirAll(baseSkillDir, 0o755); err != nil { + t.Fatalf("MkdirAll(baseSkillDir): %v", err) + } + if err := os.WriteFile( + filepath.Join(baseSkillDir, "SKILL.md"), + []byte("---\nname: base-skill\ndescription: base test skill\n---\n# Base Skill\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile(base-skill): %v", err) + } + + lateSkillPath := filepath.Join(tmpDir, "skills", "late-skill", "SKILL.md") + provider := &lateSkillOnRetryProvider{lateSkillPath: lateSkillPath} + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, provider) + defer al.Close() + + defaultAgent := al.registry.GetDefaultAgent() + if defaultAgent == nil { + t.Fatal("expected default agent") + } + defaultAgent.SkillsFilter = []string{"base-skill", "late-skill"} + + sub := al.SubscribeEvents(16) + defer al.UnsubscribeEvents(sub.ID) + + resp, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-observe-retry-snapshot", + "cli", + "direct", + ) + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "Recovered after retry" { + t.Fatalf("response = %q, want %q", resp, "Recovered after retry") + } + + turnEndEvt := waitForEvent(t, sub.C, 2*time.Second, func(evt Event) bool { + return evt.Kind == EventKindTurnEnd + }) + turnEndPayload, ok := turnEndEvt.Payload.(TurnEndPayload) + if !ok { + t.Fatalf("expected TurnEndPayload, got %T", turnEndEvt.Payload) + } + if got := turnEndPayload.AttemptedSkills; len(got) != 2 || got[0] != "base-skill" || got[1] != "late-skill" { + t.Fatalf("AttemptedSkills = %v, want [base-skill late-skill]", got) + } + if got := turnEndPayload.FinalSuccessfulPath; len(got) != 2 || got[0] != "base-skill" || got[1] != "late-skill" { + t.Fatalf("FinalSuccessfulPath = %v, want [base-skill late-skill]", got) + } + if got := turnEndPayload.SkillContextSnapshots; len(got) != 2 { + t.Fatalf("len(SkillContextSnapshots) = %d, want 2", len(got)) + } + if turnEndPayload.SkillContextSnapshots[0].Trigger != skillContextTriggerInitialBuild { + t.Fatalf( + "SkillContextSnapshots[0].Trigger = %q, want %q", + turnEndPayload.SkillContextSnapshots[0].Trigger, + skillContextTriggerInitialBuild, + ) + } + if turnEndPayload.SkillContextSnapshots[1].Trigger != skillContextTriggerContextRetryRebuild { + t.Fatalf( + "SkillContextSnapshots[1].Trigger = %q, want %q", + turnEndPayload.SkillContextSnapshots[1].Trigger, + skillContextTriggerContextRetryRebuild, + ) + } + if got := turnEndPayload.SkillContextSnapshots[1].SkillNames; len(got) != 2 || got[0] != "base-skill" || + got[1] != "late-skill" { + t.Fatalf("SkillContextSnapshots[1].SkillNames = %v, want [base-skill late-skill]", got) + } +} + +func TestEvolutionBridge_ObserveDoesNotCreateDraftFile(t *testing.T) { + tmpDir := t.TempDir() + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + resp, err := al.ProcessDirectWithChannel(context.Background(), "hello", "session-observe-no-draft", "cli", "direct") + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "ok" { + t.Fatalf("response = %q, want %q", resp, "ok") + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + assertNotExists(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json")) +} + +func TestEvolutionBridge_DraftModeAutomaticallyRunsColdPathAndCreatesDraftFile(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + resp, err := al.ProcessDirectWithChannel(context.Background(), "hello", "session-auto-cold-path", "cli", "direct") + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "ok" { + t.Fatalf("response = %q, want %q", resp, "ok") + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + waitForDrafts(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json"), 1) +} + +func TestEvolutionBridge_ScheduledModeDoesNotRunColdPathAfterTurn(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + ColdPathTrigger: "scheduled", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + resp, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-scheduled-cold-path", + "cli", + "direct", + ) + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "ok" { + t.Fatalf("response = %q, want %q", resp, "ok") + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + time.Sleep(150 * time.Millisecond) + assertNotExists(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json")) +} + +func TestEvolutionBridge_DraftModeUsesProviderBackedDraftGenerator(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, &simpleMockProvider{ + response: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name path first","body_or_patch":"## Start Here\nUse native-name query first."}`, + }) + defer al.Close() + + resp, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-auto-cold-path-llm", + "cli", + "direct", + ) + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp == "" { + t.Fatal("expected non-empty response") + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + drafts := waitForDrafts(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json"), 1) + if drafts[0].HumanSummary != "Prefer native-name path first" { + t.Fatalf("HumanSummary = %q, want %q", drafts[0].HumanSummary, "Prefer native-name path first") + } +} + +func TestEvolutionBridge_DraftModeUsesProviderDefaultModel(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + provider := &capturingEvolutionDraftProvider{ + defaultModel: "provider-explicit-model", + response: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name path first","body_or_patch":"## Start Here\nUse native-name query first."}`, + } + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + }, + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, + } + + al := NewAgentLoop(cfg, bus.NewMessageBus(), provider) + defer al.Close() + + if _, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-auto-cold-path-model", + "cli", + "direct", + ); err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + waitForDrafts(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json"), 1) + if provider.lastModel != "provider-explicit-model" { + t.Fatalf("lastModel = %q, want provider-explicit-model", provider.lastModel) + } +} + +func TestEvolutionBridge_DraftModePrefersConfigDefaultModelName(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + provider := &capturingEvolutionDraftProvider{ + defaultModel: "provider-default-model", + response: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name path first","body_or_patch":"## Start Here\nUse native-name query first."}`, + } + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + }, + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, + } + cfg.Agents.Defaults.ModelName = "resolved-config-model" + + al := NewAgentLoop(cfg, bus.NewMessageBus(), provider) + defer al.Close() + + if _, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-auto-cold-path-model-config", + "cli", + "direct", + ); err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + waitForDrafts(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json"), 1) + if provider.lastModel != "resolved-config-model" { + t.Fatalf("lastModel = %q, want resolved-config-model", provider.lastModel) + } +} + +func TestEvolutionBridge_DraftModeKeepsCandidateDraft(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, &simpleMockProvider{ + response: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"create","human_summary":"Create weather helper","body_or_patch":"---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n"}`, + }) + defer al.Close() + + if _, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-apply-no-auto-apply", + "cli", + "direct", + ); err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + drafts := waitForDrafts(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json"), 1) + if drafts[0].Status != evolution.DraftStatusCandidate { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusCandidate) + } + + assertNotExists(t, filepath.Join(tmpDir, "skills", "weather", "SKILL.md")) + assertProfileNotExists(t, tmpDir, "weather") +} + +func TestEvolutionBridge_ApplyModeAutomaticallyRunsColdPathAndAppliesMergeDraft(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + skillDir := filepath.Join(tmpDir, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + skillPath := filepath.Join(skillDir, "SKILL.md") + original := "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse city names.\n" + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "apply", + }, &simpleMockProvider{ + response: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"merge","human_summary":"Merge native-name path","body_or_patch":"Prefer native-name query first."}`, + }) + defer al.Close() + + if _, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-apply-merge", + "cli", + "direct", + ); err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + drafts := waitForDrafts(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json"), 1) + if drafts[0].Status != evolution.DraftStatusAccepted { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusAccepted) + } + + merged := waitForSkillBody(t, skillPath) + if !strings.Contains(merged, "Use city names.") { + t.Fatalf("merged skill lost original content:\n%s", merged) + } + if !strings.Contains(merged, "## Merged Knowledge") { + t.Fatalf("merged skill missing merged section:\n%s", merged) + } + if !strings.Contains(merged, "Prefer native-name query first.") { + t.Fatalf("merged skill missing learned knowledge:\n%s", merged) + } + + profile := waitForProfile(t, tmpDir, "weather") + if profile.Status != evolution.SkillStatusActive { + t.Fatalf("profile status = %q, want %q", profile.Status, evolution.SkillStatusActive) + } + if profile.CurrentVersion == "" { + t.Fatal("expected applied profile current version") + } +} + +func TestEvolutionBridge_ObserveModeDoesNotRunColdPathOrCreateDraftFile(t *testing.T) { + tmpDir := t.TempDir() + seedReadyRule(t, tmpDir) + + al := newEvolutionTestLoop(t, tmpDir, config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, &simpleMockProvider{response: "ok"}) + defer al.Close() + + resp, err := al.ProcessDirectWithChannel( + context.Background(), + "hello", + "session-no-auto-cold-path", + "cli", + "direct", + ) + if err != nil { + t.Fatalf("ProcessDirectWithChannel failed: %v", err) + } + if resp != "ok" { + t.Fatalf("response = %q, want %q", resp, "ok") + } + + waitForEvolutionRecord(t, filepath.Join(tmpDir, "state", "evolution", "task-records.jsonl")) + assertNotExists(t, filepath.Join(tmpDir, "state", "evolution", "skill-drafts.json")) +} + +func TestEvolutionBridge_TurnEndUsesPayloadWorkspace(t *testing.T) { + workspace := t.TempDir() + cfg := &config.Config{ + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, + } + + bridge, err := newEvolutionBridge(nil, cfg, nil) + if err != nil { + t.Fatalf("newEvolutionBridge: %v", err) + } + + err = bridge.OnEvent(context.Background(), Event{ + Kind: EventKindTurnEnd, + Meta: EventMeta{ + AgentID: "main", + TurnID: "turn-1", + SessionKey: "session-1", + }, + Payload: TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: workspace, + ActiveSkills: []string{"observe-skill"}, + ToolKinds: []string{"echo_text"}, + }, + }) + if err != nil { + t.Fatalf("OnEvent: %v", err) + } + + record := waitForEvolutionRecord(t, filepath.Join(workspace, "state", "evolution", "task-records.jsonl")) + if got := record["workspace_id"]; got != workspace { + t.Fatalf("workspace_id = %v, want %q", got, workspace) + } +} + +func TestEvolutionBridge_TurnEndUsesExplicitAttemptTrail(t *testing.T) { + workspace := t.TempDir() + cfg := &config.Config{ + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, + } + + bridge, err := newEvolutionBridge(nil, cfg, nil) + if err != nil { + t.Fatalf("newEvolutionBridge: %v", err) + } + + err = bridge.OnEvent(context.Background(), Event{ + Kind: EventKindTurnEnd, + Meta: EventMeta{ + AgentID: "main", + TurnID: "turn-1", + SessionKey: "session-1", + }, + Payload: TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: workspace, + ActiveSkills: []string{"weather"}, + AttemptedSkills: []string{"geocode", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + SkillContextSnapshots: []SkillContextSnapshot{ + {Sequence: 1, Trigger: skillContextTriggerInitialBuild, SkillNames: []string{"weather"}}, + { + Sequence: 2, + Trigger: skillContextTriggerContextRetryRebuild, + SkillNames: []string{"geocode", "weather"}, + }, + }, + ToolKinds: []string{"echo_text"}, + }, + }) + if err != nil { + t.Fatalf("OnEvent: %v", err) + } + + record := waitForEvolutionRecord(t, filepath.Join(workspace, "state", "evolution", "task-records.jsonl")) + usedSkills, ok := record["used_skill_names"].([]any) + if !ok || len(usedSkills) != 2 || usedSkills[0] != "geocode" || usedSkills[1] != "weather" { + t.Fatalf("used_skill_names = %#v, want [geocode weather]", record["used_skill_names"]) + } + for _, field := range []string{"attempt_trail", "initial_skill_names", "added_skill_names"} { + if _, exists := record[field]; exists { + t.Fatalf("%s should not be persisted in slim task record: %#v", field, record[field]) + } + } +} + +func TestEvolutionBridge_CloseStopsColdPathRunnerIdempotently(t *testing.T) { + cfg := &config.Config{ + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, + } + + bridge, err := newEvolutionBridge(nil, cfg, nil) + if err != nil { + t.Fatalf("newEvolutionBridge: %v", err) + } + if bridge.coldPathRunner == nil { + t.Fatal("expected cold path runner") + } + + if err := bridge.Close(); err != nil { + t.Fatalf("first Close() error = %v", err) + } + if err := bridge.Close(); err != nil { + t.Fatalf("second Close() error = %v", err) + } + if bridge.coldPathRunner.Trigger(t.TempDir()) { + t.Fatal("expected closed bridge runner to reject new work") + } +} + +func TestEvolutionBridge_CloseRejectsLateTurnEndEvents(t *testing.T) { + workspace := t.TempDir() + cfg := &config.Config{ + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, + } + + bridge, err := newEvolutionBridge(nil, cfg, nil) + if err != nil { + t.Fatalf("newEvolutionBridge: %v", err) + } + + if closeErr := bridge.Close(); closeErr != nil { + t.Fatalf("Close() error = %v", closeErr) + } + + err = bridge.OnEvent(context.Background(), Event{ + Kind: EventKindTurnEnd, + Meta: EventMeta{ + TurnID: "turn-after-close", + SessionKey: "session-after-close", + AgentID: "agent-after-close", + }, + Payload: TurnEndPayload{ + Status: TurnEndStatusCompleted, + Workspace: workspace, + }, + }) + if err != nil { + t.Fatalf("OnEvent() error = %v", err) + } + + assertNotExists(t, filepath.Join(workspace, "state", "evolution", "task-records.jsonl")) +} + +func TestAgentLoop_ReloadProviderAndConfig_RebuildsEvolutionBridge(t *testing.T) { + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + }, + Evolution: config.EvolutionConfig{ + Enabled: false, + Mode: "observe", + }, + } + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockProvider{}) + defer al.Close() + + oldBridge := al.evolution + if oldBridge == nil { + t.Fatal("expected initial evolution bridge") + } + + reloadCfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + }, + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "apply", + StateDir: filepath.Join(t.TempDir(), "evolution-state"), + }, + } + + if err := al.ReloadProviderAndConfig(context.Background(), &mockProvider{}, reloadCfg); err != nil { + t.Fatalf("ReloadProviderAndConfig failed: %v", err) + } + + if al.evolution == nil { + t.Fatal("expected evolution bridge after reload") + } + if al.evolution == oldBridge { + t.Fatal("expected evolution bridge to be rebuilt on reload") + } + if al.evolution.cfg.Enabled != reloadCfg.Evolution.Enabled { + t.Fatalf("reloaded evolution enabled = %v, want %v", al.evolution.cfg.Enabled, reloadCfg.Evolution.Enabled) + } + if al.evolution.cfg.Mode != reloadCfg.Evolution.Mode { + t.Fatalf("reloaded evolution mode = %q, want %q", al.evolution.cfg.Mode, reloadCfg.Evolution.Mode) + } + if al.evolution.cfg.StateDir != reloadCfg.Evolution.StateDir { + t.Fatalf("reloaded evolution state_dir = %q, want %q", al.evolution.cfg.StateDir, reloadCfg.Evolution.StateDir) + } +} + +func TestEvolutionBridge_ColdPathScheduleParsing(t *testing.T) { + schedule := parseColdPathSchedule([]string{"18:30", "bad", "03:05", "18:30", "24:00", "09:99"}) + if len(schedule) != 2 { + t.Fatalf("len(schedule) = %d, want 2: %+v", len(schedule), schedule) + } + if schedule[0].hour != 3 || schedule[0].minute != 5 { + t.Fatalf("schedule[0] = %+v, want 03:05", schedule[0]) + } + if schedule[1].hour != 18 || schedule[1].minute != 30 { + t.Fatalf("schedule[1] = %+v, want 18:30", schedule[1]) + } + + now := time.Date(2026, 5, 7, 4, 0, 0, 0, time.Local) + next := nextColdPathScheduledTime(now, schedule) + want := time.Date(2026, 5, 7, 18, 30, 0, 0, time.Local) + if !next.Equal(want) { + t.Fatalf("next = %v, want %v", next, want) + } + + now = time.Date(2026, 5, 7, 19, 0, 0, 0, time.Local) + next = nextColdPathScheduledTime(now, schedule) + want = time.Date(2026, 5, 8, 3, 5, 0, 0, time.Local) + if !next.Equal(want) { + t.Fatalf("next after day end = %v, want %v", next, want) + } +} + +func TestEvolutionBridge_ScheduledColdPathTracksObservedWorkspaces(t *testing.T) { + bridge := &evolutionBridge{ + cfg: config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + ColdPathTrigger: "scheduled", + ColdPathTimes: []string{"03:00"}, + }, + } + + bridge.rememberScheduledColdPathWorkspace("/tmp/workspace-b") + bridge.rememberScheduledColdPathWorkspace("/tmp/workspace-a") + bridge.rememberScheduledColdPathWorkspace("/tmp/workspace-b") + bridge.rememberScheduledColdPathWorkspace("") + + got := bridge.scheduledColdPathWorkspaces() + want := []string{"/tmp/workspace-a", "/tmp/workspace-b"} + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("scheduled workspaces = %v, want %v", got, want) + } +} + +func TestEvolutionBridge_ScheduledColdPathSeedsConfiguredAgentWorkspaces(t *testing.T) { + defaultWorkspace := t.TempDir() + workerWorkspace := t.TempDir() + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: defaultWorkspace, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + List: []config.AgentConfig{ + {ID: "main", Default: true}, + {ID: "worker", Workspace: workerWorkspace}, + }, + }, + Evolution: config.EvolutionConfig{ + Enabled: true, + Mode: "draft", + ColdPathTrigger: "scheduled", + ColdPathTimes: []string{"03:00"}, + }, + } + registry := NewAgentRegistry(cfg, &simpleMockProvider{response: "ok"}) + bridge, err := newEvolutionBridge(registry, cfg, &simpleMockProvider{response: "ok"}) + if err != nil { + t.Fatalf("newEvolutionBridge: %v", err) + } + defer bridge.Close() + + got := bridge.scheduledColdPathWorkspaces() + want := []string{defaultWorkspace, workerWorkspace} + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("scheduled workspaces = %v, want %v", got, want) + } +} + +func seedReadyRule(t *testing.T, workspace string) { + t.Helper() + + store := evolution.NewStore(evolution.NewPaths(workspace, "")) + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: workspace, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Label: "weather-native-name-path", + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + TaskRecordIDs: []string{"task-1", "task-2"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } +} + +func newEvolutionTestLoop( + t *testing.T, + workspace string, + evo config.EvolutionConfig, + provider providers.LLMProvider, +) *AgentLoop { + t.Helper() + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + }, + Evolution: evo, + } + + return NewAgentLoop(cfg, bus.NewMessageBus(), provider) +} + +func waitForEvolutionRecord(t *testing.T, path string) map[string]any { + t.Helper() + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil { + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + for i := len(lines) - 1; i >= 0; i-- { + if strings.TrimSpace(lines[i]) == "" { + continue + } + var record map[string]any + if err := json.Unmarshal([]byte(lines[i]), &record); err != nil { + t.Fatalf("json.Unmarshal(%s): %v", path, err) + } + if kind, _ := record["kind"].(string); kind == string(evolution.RecordKindTask) { + return record + } + } + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("timed out waiting for evolution record at %s", path) + return nil +} + +func countEvolutionTaskRecords(t *testing.T, path string) int { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s): %v", path, err) + } + count := 0 + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + var record map[string]any + if err := json.Unmarshal([]byte(line), &record); err != nil { + t.Fatalf("json.Unmarshal(%s): %v", path, err) + } + if kind, _ := record["kind"].(string); kind == string(evolution.RecordKindTask) { + count++ + } + } + return count +} + +func waitForDrafts(t *testing.T, path string, want int) []evolution.SkillDraft { + t.Helper() + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil { + var drafts []evolution.SkillDraft + if err := json.Unmarshal(data, &drafts); err != nil { + t.Fatalf("json.Unmarshal(%s): %v", path, err) + } + if len(drafts) == want { + return drafts + } + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("timed out waiting for %d drafts at %s", want, path) + return nil +} + +func waitForSkillBody(t *testing.T, path string) string { + t.Helper() + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil { + return string(data) + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("timed out waiting for skill file at %s", path) + return "" +} + +func waitForProfile(t *testing.T, workspace, skillName string) evolution.SkillProfile { + t.Helper() + + store := evolution.NewStore(evolution.NewPaths(workspace, "")) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + profile, err := store.LoadProfile(skillName) + if err == nil { + return profile + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("timed out waiting for profile %q in %s", skillName, workspace) + return evolution.SkillProfile{} +} + +func assertProfileNotExists(t *testing.T, workspace, skillName string) { + t.Helper() + + store := evolution.NewStore(evolution.NewPaths(workspace, "")) + if _, loadErr := store.LoadProfile(skillName); !os.IsNotExist(loadErr) { + t.Fatalf("profile %q should not exist, got err = %v", skillName, loadErr) + } +} + +func assertNotExists(t *testing.T, path string) { + t.Helper() + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatalf("%s should not exist, stat err = %v", path, statErr) + } +} + +func waitForEvent(t *testing.T, ch <-chan Event, timeout time.Duration, match func(Event) bool) Event { + t.Helper() + + timer := time.NewTimer(timeout) + defer timer.Stop() + for { + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("event channel closed") + } + if match == nil || match(evt) { + return evt + } + case <-timer.C: + t.Fatal("timed out waiting for event") + } + } +} + +type blockingRuntimeObserver struct { + once sync.Once + started chan struct{} + release chan struct{} +} + +func (o *blockingRuntimeObserver) OnRuntimeEvent(ctx context.Context, _ runtimeevents.Event) error { + o.once.Do(func() { + close(o.started) + }) + select { + case <-o.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +type capturingEvolutionDraftProvider struct { + response string + defaultModel string + lastModel string +} + +type lateSkillOnRetryProvider struct { + calls int + lateSkillPath string +} + +func (p *lateSkillOnRetryProvider) Chat( + _ context.Context, + _ []providers.Message, + _ []providers.ToolDefinition, + _ string, + _ map[string]any, +) (*providers.LLMResponse, error) { + p.calls++ + if p.calls == 1 { + if err := os.MkdirAll(filepath.Dir(p.lateSkillPath), 0o755); err != nil { + return nil, err + } + if err := os.WriteFile( + p.lateSkillPath, + []byte("---\nname: late-skill\ndescription: late test skill\n---\n# Late Skill\n"), + 0o644, + ); err != nil { + return nil, err + } + return nil, errors.New("context_window_exceeded") + } + + return &providers.LLMResponse{Content: "Recovered after retry"}, nil +} + +func (p *lateSkillOnRetryProvider) GetDefaultModel() string { + return "mock-model" +} + +func (p *capturingEvolutionDraftProvider) Chat( + _ context.Context, + _ []providers.Message, + _ []providers.ToolDefinition, + model string, + _ map[string]any, +) (*providers.LLMResponse, error) { + p.lastModel = model + return &providers.LLMResponse{Content: p.response}, nil +} + +func (p *capturingEvolutionDraftProvider) GetDefaultModel() string { + return p.defaultModel +} diff --git a/pkg/agent/hook_mount.go b/pkg/agent/hook_mount.go index c92145f1f..c518feee8 100644 --- a/pkg/agent/hook_mount.go +++ b/pkg/agent/hook_mount.go @@ -8,6 +8,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" ) type hookRuntime struct { @@ -295,10 +296,11 @@ func processHookObserveKindsFromConfig(observe []string) ([]string, bool, error) case "", "*", "all": return nil, true, nil default: - if _, ok := validKinds[kind]; !ok { + normalizedKind, ok := validKinds[kind] + if !ok { return nil, false, fmt.Errorf("unsupported observe event %q", kind) } - normalized = append(normalized, kind) + normalized = append(normalized, normalizedKind) } } @@ -308,10 +310,30 @@ func processHookObserveKindsFromConfig(observe []string) ([]string, bool, error) return normalized, true, nil } -func validHookEventKinds() map[string]struct{} { - kinds := make(map[string]struct{}, int(eventKindCount)) - for kind := EventKind(0); kind < eventKindCount; kind++ { - kinds[kind.String()] = struct{}{} +func validHookEventKinds() map[string]string { + runtimeKinds := runtimeevents.KnownKinds() + kinds := make(map[string]string, len(runtimeKinds)*2) + for _, kind := range runtimeKinds { + kinds[kind.String()] = kind.String() } + kinds["turn_start"] = runtimeevents.KindAgentTurnStart.String() + kinds["turn_end"] = runtimeevents.KindAgentTurnEnd.String() + kinds["llm_request"] = runtimeevents.KindAgentLLMRequest.String() + kinds["llm_delta"] = runtimeevents.KindAgentLLMDelta.String() + kinds["llm_response"] = runtimeevents.KindAgentLLMResponse.String() + kinds["llm_retry"] = runtimeevents.KindAgentLLMRetry.String() + kinds["context_compress"] = runtimeevents.KindAgentContextCompress.String() + kinds["session_summarize"] = runtimeevents.KindAgentSessionSummarize.String() + kinds["tool_exec_start"] = runtimeevents.KindAgentToolExecStart.String() + kinds["tool_exec_end"] = runtimeevents.KindAgentToolExecEnd.String() + kinds["tool_exec_skipped"] = runtimeevents.KindAgentToolExecSkipped.String() + kinds["steering_injected"] = runtimeevents.KindAgentSteeringInjected.String() + kinds["follow_up_queued"] = runtimeevents.KindAgentFollowUpQueued.String() + kinds["interrupt_received"] = runtimeevents.KindAgentInterruptReceived.String() + kinds["subturn_spawn"] = runtimeevents.KindAgentSubTurnSpawn.String() + kinds["subturn_end"] = runtimeevents.KindAgentSubTurnEnd.String() + kinds["subturn_result_delivered"] = runtimeevents.KindAgentSubTurnResultDelivered.String() + kinds["subturn_orphan"] = runtimeevents.KindAgentSubTurnOrphan.String() + kinds["error"] = runtimeevents.KindAgentError.String() return kinds } diff --git a/pkg/agent/hook_mount_test.go b/pkg/agent/hook_mount_test.go index 85d8f5c11..5cd64af7b 100644 --- a/pkg/agent/hook_mount_test.go +++ b/pkg/agent/hook_mount_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "path/filepath" + "slices" "testing" "github.com/sipeed/picoclaw/pkg/bus" @@ -155,7 +156,27 @@ func TestAgentLoop_ProcessDirectWithChannel_AutoMountsProcessHook(t *testing.T) t.Fatalf("expected process model, got %q", lastModel) } - waitForFileContains(t, eventLog, "turn_end") + waitForFileContains(t, eventLog, "agent.turn.end") +} + +func TestProcessHookObserveKindsFromConfigAcceptsRuntimeNames(t *testing.T) { + kinds, enabled, err := processHookObserveKindsFromConfig([]string{ + "tool_exec_start", + "agent.tool.exec_end", + "gateway.ready", + "mcp.server.failed", + }) + if err != nil { + t.Fatalf("processHookObserveKindsFromConfig failed: %v", err) + } + if !enabled { + t.Fatal("expected observe to be enabled") + } + + want := []string{"agent.tool.exec_start", "agent.tool.exec_end", "gateway.ready", "mcp.server.failed"} + if !slices.Equal(kinds, want) { + t.Fatalf("observe kinds = %v, want %v", kinds, want) + } } func TestAgentLoop_ProcessDirectWithChannel_InvalidConfiguredHookFails(t *testing.T) { diff --git a/pkg/agent/hook_process.go b/pkg/agent/hook_process.go index ace95f44d..ce8e932d2 100644 --- a/pkg/agent/hook_process.go +++ b/pkg/agent/hook_process.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/isolation" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/tools" @@ -183,7 +184,7 @@ func (ph *ProcessHook) Close() error { return ph.closeErr } -func (ph *ProcessHook) OnEvent(ctx context.Context, evt Event) error { +func (ph *ProcessHook) OnRuntimeEvent(ctx context.Context, evt runtimeevents.Event) error { if ph == nil || !ph.opts.Observe { return nil } @@ -192,7 +193,7 @@ func (ph *ProcessHook) OnEvent(ctx context.Context, evt Event) error { return nil } } - return ph.notify(ctx, "hook.event", evt) + return ph.notify(ctx, "hook.runtime_event", evt) } func (ph *ProcessHook) BeforeLLM( diff --git a/pkg/agent/hook_process_test.go b/pkg/agent/hook_process_test.go index 9e95d105e..0fd1ec38d 100644 --- a/pkg/agent/hook_process_test.go +++ b/pkg/agent/hook_process_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/isolation" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -66,7 +67,7 @@ func TestAgentLoop_MountProcessHook_LLMAndObserver(t *testing.T) { t.Fatalf("expected process model, got %q", lastModel) } - waitForFileContains(t, eventLog, "turn_end") + waitForFileContains(t, eventLog, "agent.turn.end") } func TestAgentLoop_MountProcessHook_ToolRewrite(t *testing.T) { @@ -146,8 +147,13 @@ func TestAgentLoop_MountProcessHook_ApprovalDeny(t *testing.T) { t.Fatalf("MountProcessHook failed: %v", err) } - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentToolExecSkipped, + ) + defer closeRuntimeEvents() resp, err := al.runAgentLoop(context.Background(), agent, processOptions{ SessionKey: "session-1", @@ -167,8 +173,8 @@ func TestAgentLoop_MountProcessHook_ApprovalDeny(t *testing.T) { t.Fatalf("expected %q, got %q", expected, resp) } - events := collectEventStream(sub.C) - skippedEvt, ok := findEvent(events, EventKindToolExecSkipped) + events := collectRuntimeEventStream(runtimeCh) + skippedEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentToolExecSkipped) if !ok { t.Fatal("expected tool skipped event") } @@ -350,12 +356,11 @@ func runProcessHookHelper() error { } if msg.ID == 0 { - if msg.Method == "hook.event" && eventLog != "" { + if msg.Method == "hook.runtime_event" && eventLog != "" { var evt map[string]any if err := json.Unmarshal(msg.Params, &evt); err == nil { - if rawKind, ok := evt["Kind"].(float64); ok { - kind := EventKind(rawKind) - _ = os.WriteFile(eventLog, []byte(kind.String()+"\n"), 0o644) + if kind, ok := evt["kind"].(string); ok { + _ = os.WriteFile(eventLog, []byte(kind+"\n"), 0o644) } } } diff --git a/pkg/agent/hooks.go b/pkg/agent/hooks.go index 9cc3e6951..a4f0fac82 100644 --- a/pkg/agent/hooks.go +++ b/pkg/agent/hooks.go @@ -9,6 +9,7 @@ import ( "sync" "time" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/tools" @@ -71,8 +72,8 @@ func NamedHook(name string, hook any) HookRegistration { } } -type EventObserver interface { - OnEvent(ctx context.Context, evt Event) error +type RuntimeEventObserver interface { + OnRuntimeEvent(ctx context.Context, evt runtimeevents.Event) error } type LLMInterceptor interface { @@ -90,7 +91,7 @@ type ToolApprover interface { } type LLMHookRequest struct { - Meta EventMeta `json:"meta"` + Meta HookMeta `json:"meta"` Context *TurnContext `json:"context,omitempty"` Model string `json:"model"` Messages []providers.Message `json:"messages,omitempty"` @@ -104,7 +105,7 @@ func (r *LLMHookRequest) Clone() *LLMHookRequest { return nil } cloned := *r - cloned.Meta = cloneEventMeta(r.Meta) + cloned.Meta = cloneHookMeta(r.Meta) cloned.Context = cloneTurnContext(r.Context) cloned.Messages = cloneProviderMessages(r.Messages) cloned.Tools = cloneToolDefinitions(r.Tools) @@ -113,7 +114,7 @@ func (r *LLMHookRequest) Clone() *LLMHookRequest { } type LLMHookResponse struct { - Meta EventMeta `json:"meta"` + Meta HookMeta `json:"meta"` Context *TurnContext `json:"context,omitempty"` Model string `json:"model"` Response *providers.LLMResponse `json:"response,omitempty"` @@ -124,14 +125,14 @@ func (r *LLMHookResponse) Clone() *LLMHookResponse { return nil } cloned := *r - cloned.Meta = cloneEventMeta(r.Meta) + cloned.Meta = cloneHookMeta(r.Meta) cloned.Context = cloneTurnContext(r.Context) cloned.Response = cloneLLMResponse(r.Response) return &cloned } type ToolCallHookRequest struct { - Meta EventMeta `json:"meta"` + Meta HookMeta `json:"meta"` Context *TurnContext `json:"context,omitempty"` Tool string `json:"tool"` Arguments map[string]any `json:"arguments,omitempty"` @@ -145,7 +146,7 @@ func (r *ToolCallHookRequest) Clone() *ToolCallHookRequest { return nil } cloned := *r - cloned.Meta = cloneEventMeta(r.Meta) + cloned.Meta = cloneHookMeta(r.Meta) cloned.Context = cloneTurnContext(r.Context) cloned.Arguments = cloneStringAnyMap(r.Arguments) cloned.HookResult = cloneToolResult(r.HookResult) @@ -153,7 +154,7 @@ func (r *ToolCallHookRequest) Clone() *ToolCallHookRequest { } type ToolApprovalRequest struct { - Meta EventMeta `json:"meta"` + Meta HookMeta `json:"meta"` Context *TurnContext `json:"context,omitempty"` Tool string `json:"tool"` Arguments map[string]any `json:"arguments,omitempty"` @@ -164,14 +165,14 @@ func (r *ToolApprovalRequest) Clone() *ToolApprovalRequest { return nil } cloned := *r - cloned.Meta = cloneEventMeta(r.Meta) + cloned.Meta = cloneHookMeta(r.Meta) cloned.Context = cloneTurnContext(r.Context) cloned.Arguments = cloneStringAnyMap(r.Arguments) return &cloned } type ToolResultHookResponse struct { - Meta EventMeta `json:"meta"` + Meta HookMeta `json:"meta"` Context *TurnContext `json:"context,omitempty"` Tool string `json:"tool"` Arguments map[string]any `json:"arguments,omitempty"` @@ -184,7 +185,7 @@ func (r *ToolResultHookResponse) Clone() *ToolResultHookResponse { return nil } cloned := *r - cloned.Meta = cloneEventMeta(r.Meta) + cloned.Meta = cloneHookMeta(r.Meta) cloned.Context = cloneTurnContext(r.Context) cloned.Arguments = cloneStringAnyMap(r.Arguments) cloned.Result = cloneToolResult(r.Result) @@ -192,7 +193,7 @@ func (r *ToolResultHookResponse) Clone() *ToolResultHookResponse { } type HookManager struct { - eventBus *EventBus + runtimeEvents runtimeevents.EventChannel observerTimeout time.Duration interceptorTimeout time.Duration approvalTimeout time.Duration @@ -201,28 +202,39 @@ type HookManager struct { hooks map[string]HookRegistration ordered []HookRegistration - sub EventSubscription - done chan struct{} - closeOnce sync.Once + runtimeSub runtimeevents.Subscription + runtimeDone chan struct{} + closeOnce sync.Once } -func NewHookManager(eventBus *EventBus) *HookManager { +func NewHookManager(runtimeEvents runtimeevents.EventChannel) *HookManager { hm := &HookManager{ - eventBus: eventBus, + runtimeEvents: runtimeEvents, observerTimeout: defaultHookObserverTimeout, interceptorTimeout: defaultHookInterceptorTimeout, approvalTimeout: defaultHookApprovalTimeout, hooks: make(map[string]HookRegistration), - done: make(chan struct{}), + runtimeDone: make(chan struct{}), } - if eventBus == nil { - close(hm.done) - return hm + if runtimeEvents != nil { + sub, ch, err := runtimeEvents.SubscribeChan(context.Background(), runtimeevents.SubscribeOptions{ + Name: "hook-manager-observer", + Buffer: hookObserverBufferSize, + }) + if err != nil { + logger.WarnCF("hooks", "Failed to subscribe runtime events for hooks", map[string]any{ + "error": err.Error(), + }) + close(hm.runtimeDone) + } else { + hm.runtimeSub = sub + go hm.dispatchRuntimeEvents(ch) + } + } else { + close(hm.runtimeDone) } - hm.sub = eventBus.Subscribe(hookObserverBufferSize) - go hm.dispatchEvents() return hm } @@ -232,10 +244,14 @@ func (hm *HookManager) Close() { } hm.closeOnce.Do(func() { - if hm.eventBus != nil { - hm.eventBus.Unsubscribe(hm.sub.ID) + if hm.runtimeSub != nil { + if err := hm.runtimeSub.Close(); err != nil { + logger.WarnCF("hooks", "Failed to close runtime event hook subscription", map[string]any{ + "error": err.Error(), + }) + } } - <-hm.done + <-hm.runtimeDone hm.closeAllHooks() }) } @@ -292,16 +308,16 @@ func (hm *HookManager) Unmount(name string) { hm.rebuildOrdered() } -func (hm *HookManager) dispatchEvents() { - defer close(hm.done) +func (hm *HookManager) dispatchRuntimeEvents(ch <-chan runtimeevents.Event) { + defer close(hm.runtimeDone) - for evt := range hm.sub.C { + for evt := range ch { for _, reg := range hm.snapshotHooks() { - observer, ok := reg.Hook.(EventObserver) + observer, ok := reg.Hook.(RuntimeEventObserver) if !ok { continue } - hm.runObserver(reg.Name, observer, evt) + hm.runRuntimeObserver(reg.Name, observer, evt) } } } @@ -581,26 +597,30 @@ func (hm *HookManager) closeAllHooks() { hm.ordered = nil } -func (hm *HookManager) runObserver(name string, observer EventObserver, evt Event) { +func (hm *HookManager) runRuntimeObserver( + name string, + observer RuntimeEventObserver, + evt runtimeevents.Event, +) { ctx, cancel := context.WithTimeout(context.Background(), hm.observerTimeout) defer cancel() done := make(chan error, 1) go func() { - done <- observer.OnEvent(ctx, evt) + done <- observer.OnRuntimeEvent(ctx, evt) }() select { case err := <-done: if err != nil { - logger.WarnCF("hooks", "Event observer failed", map[string]any{ + logger.WarnCF("hooks", "Runtime event observer failed", map[string]any{ "hook": name, "event": evt.Kind.String(), "error": err.Error(), }) } case <-ctx.Done(): - logger.WarnCF("hooks", "Event observer timed out", map[string]any{ + logger.WarnCF("hooks", "Runtime event observer timed out", map[string]any{ "hook": name, "event": evt.Kind.String(), "timeout_ms": hm.observerTimeout.Milliseconds(), diff --git a/pkg/agent/hooks_test.go b/pkg/agent/hooks_test.go index aa52bf2d5..4deef38c7 100644 --- a/pkg/agent/hooks_test.go +++ b/pkg/agent/hooks_test.go @@ -12,6 +12,7 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/routing" "github.com/sipeed/picoclaw/pkg/session" @@ -111,14 +112,14 @@ func (p *llmHookTestProvider) GetDefaultModel() string { } type llmObserverHook struct { - eventCh chan Event + eventCh chan runtimeevents.Event lastInbound *bus.InboundContext lastRoute *routing.ResolvedRoute lastScope *session.SessionScope } -func (h *llmObserverHook) OnEvent(ctx context.Context, evt Event) error { - if evt.Kind == EventKindTurnEnd { +func (h *llmObserverHook) OnRuntimeEvent(ctx context.Context, evt runtimeevents.Event) error { + if evt.Kind == runtimeevents.KindAgentTurnEnd { select { case h.eventCh <- evt: default: @@ -150,6 +151,20 @@ func (h *llmObserverHook) AfterLLM( return next, HookDecision{Action: HookActionModify}, nil } +type dualRuntimeObserverHook struct { + runtimeCh chan runtimeevents.Event +} + +func (h *dualRuntimeObserverHook) OnRuntimeEvent(ctx context.Context, evt runtimeevents.Event) error { + if evt.Kind == runtimeevents.KindAgentTurnEnd { + select { + case h.runtimeCh <- evt: + default: + } + } + return nil +} + type llmSystemRewriteHook struct{} func (h *llmSystemRewriteHook) BeforeLLM( @@ -417,7 +432,7 @@ func TestAgentLoop_Hooks_ObserverAndLLMInterceptor(t *testing.T) { al, agent, cleanup := newHookTestLoop(t, provider) defer cleanup() - hook := &llmObserverHook{eventCh: make(chan Event, 1)} + hook := &llmObserverHook{eventCh: make(chan runtimeevents.Event, 1)} if err := al.MountHook(NamedHook("llm-observer", hook)); err != nil { t.Fatalf("MountHook failed: %v", err) } @@ -481,30 +496,80 @@ func TestAgentLoop_Hooks_ObserverAndLLMInterceptor(t *testing.T) { select { case evt := <-hook.eventCh: - if evt.Kind != EventKindTurnEnd { + if evt.Kind != runtimeevents.KindAgentTurnEnd { t.Fatalf("expected turn end event, got %v", evt.Kind) } - if evt.Context == nil || evt.Context.Inbound == nil { - t.Fatal("expected observer event to carry inbound context") - } - if evt.Context.Route == nil || evt.Context.Route.AgentID != "main" { - t.Fatalf("expected observer event to carry route context, got %+v", evt.Context.Route) - } - if evt.Context.Scope == nil || evt.Context.Scope.Values["sender"] != "hook-user" { - t.Fatalf("expected observer event to carry session scope, got %+v", evt.Context.Scope) + if evt.Scope.AgentID != "main" || + evt.Scope.SessionKey != "session-1" || + evt.Scope.Channel != "cli" || + evt.Scope.ChatID != "direct" || + evt.Scope.SenderID != "hook-user" { + t.Fatalf("runtime observer scope = %+v", evt.Scope) } case <-time.After(2 * time.Second): t.Fatal("timed out waiting for hook observer event") } } +func TestAgentLoop_Hooks_RuntimeObserverReceivesEvents(t *testing.T) { + provider := &llmHookTestProvider{} + al, agent, cleanup := newHookTestLoop(t, provider) + defer cleanup() + + hook := &dualRuntimeObserverHook{ + runtimeCh: make(chan runtimeevents.Event, 1), + } + if err := al.MountHook(NamedHook("runtime-observer", hook)); err != nil { + t.Fatalf("MountHook failed: %v", err) + } + + resp, err := al.runAgentLoop(context.Background(), agent, processOptions{ + SessionKey: "session-1", + Channel: "cli", + ChatID: "direct", + UserMessage: "hello", + DefaultResponse: defaultResponse, + EnableSummary: false, + SendResponse: false, + InboundContext: &bus.InboundContext{ + Channel: "cli", + Account: "default", + ChatID: "direct", + ChatType: "direct", + SenderID: "hook-user", + MessageID: "msg-1", + }, + }) + if err != nil { + t.Fatalf("runAgentLoop failed: %v", err) + } + if resp != "provider content" { + t.Fatalf("expected provider content, got %q", resp) + } + + select { + case evt := <-hook.runtimeCh: + if evt.Kind != runtimeevents.KindAgentTurnEnd { + t.Fatalf("runtime observer kind = %q", evt.Kind) + } + if evt.Scope.SessionKey != "session-1" || + evt.Scope.Channel != "cli" || + evt.Scope.ChatID != "direct" || + evt.Scope.MessageID != "msg-1" { + t.Fatalf("runtime observer scope = %+v", evt.Scope) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for runtime observer event") + } +} + func TestAgentLoop_BtwCommand_UsesLLMHooks(t *testing.T) { provider := &llmHookTestProvider{} al, agent, cleanup := newHookTestLoop(t, provider) defer cleanup() useTestSideQuestionProvider(al, provider) - hook := &llmObserverHook{eventCh: make(chan Event, 1)} + hook := &llmObserverHook{eventCh: make(chan runtimeevents.Event, 1)} if err := al.MountHook(NamedHook("llm-observer", hook)); err != nil { t.Fatalf("MountHook failed: %v", err) } @@ -800,8 +865,13 @@ func TestAgentLoop_Hooks_ToolApproverCanDeny(t *testing.T) { t.Fatalf("MountHook failed: %v", err) } - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentToolExecSkipped, + ) + defer closeRuntimeEvents() resp, err := al.runAgentLoop(context.Background(), agent, processOptions{ SessionKey: "session-1", @@ -820,8 +890,8 @@ func TestAgentLoop_Hooks_ToolApproverCanDeny(t *testing.T) { t.Fatalf("expected %q, got %q", expected, resp) } - events := collectEventStream(sub.C) - skippedEvt, ok := findEvent(events, EventKindToolExecSkipped) + events := collectRuntimeEventStream(runtimeCh) + skippedEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentToolExecSkipped) if !ok { t.Fatal("expected tool skipped event") } @@ -876,8 +946,13 @@ func TestAgentLoop_Hooks_ToolRespondAction(t *testing.T) { t.Fatalf("MountHook failed: %v", err) } - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentToolExecEnd, + ) + defer closeRuntimeEvents() resp, err := al.runAgentLoop(context.Background(), agent, processOptions{ SessionKey: "session-1", @@ -899,8 +974,8 @@ func TestAgentLoop_Hooks_ToolRespondAction(t *testing.T) { } // Verify event stream has ToolExecEnd, not actual tool execution - events := collectEventStream(sub.C) - endEvt, ok := findEvent(events, EventKindToolExecEnd) + events := collectRuntimeEventStream(runtimeCh) + endEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentToolExecEnd) if !ok { t.Fatal("expected tool exec end event") } @@ -1065,8 +1140,13 @@ func TestAgentLoop_HookRespond_MediaError(t *testing.T) { sendErr: errors.New("channel unavailable"), }) - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentToolExecEnd, + ) + defer closeRuntimeEvents() _, err := al.runAgentLoop(context.Background(), agent, processOptions{ SessionKey: "session-media-err", @@ -1081,8 +1161,8 @@ func TestAgentLoop_HookRespond_MediaError(t *testing.T) { t.Fatalf("runAgentLoop failed: %v", err) } - events := collectEventStream(sub.C) - endEvt, ok := findEvent(events, EventKindToolExecEnd) + events := collectRuntimeEventStream(runtimeCh) + endEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentToolExecEnd) if !ok { t.Fatal("expected ToolExecEnd event") } @@ -1120,8 +1200,13 @@ func TestAgentLoop_HookRespond_BusFallback(t *testing.T) { t.Fatalf("MountHook failed: %v", err) } - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentToolExecEnd, + ) + defer closeRuntimeEvents() resp, err := al.runAgentLoop(context.Background(), agent, processOptions{ SessionKey: "session-bus-fallback", @@ -1136,8 +1221,8 @@ func TestAgentLoop_HookRespond_BusFallback(t *testing.T) { t.Fatalf("runAgentLoop failed: %v", err) } - events := collectEventStream(sub.C) - endEvt, ok := findEvent(events, EventKindToolExecEnd) + events := collectRuntimeEventStream(runtimeCh) + endEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentToolExecEnd) if !ok { t.Fatal("expected ToolExecEnd event") } @@ -1282,8 +1367,13 @@ func TestAgentLoop_HookRespond_InterruptSkipsRemaining(t *testing.T) { t.Fatalf("MountHook failed: %v", err) } - sub := al.SubscribeEvents(32) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 32, + runtimeevents.KindAgentToolExecSkipped, + ) + defer closeRuntimeEvents() sessionKey := session.BuildMainSessionKey(routing.DefaultAgentID) @@ -1322,9 +1412,9 @@ func TestAgentLoop_HookRespond_InterruptSkipsRemaining(t *testing.T) { t.Fatal("timeout waiting for result") } - events := collectEventStream(sub.C) + events := collectRuntimeEventStream(runtimeCh) - skippedEvts := filterEvents(events, EventKindToolExecSkipped) + skippedEvts := filterRuntimeEvents(events, runtimeevents.KindAgentToolExecSkipped) if len(skippedEvts) < 1 { t.Fatal("expected at least one ToolExecSkipped event after interrupt") } @@ -1362,8 +1452,14 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) { t.Fatalf("MountHook failed: %v", err) } - sub := al.SubscribeEvents(32) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 32, + runtimeevents.KindAgentToolExecEnd, + runtimeevents.KindAgentToolExecSkipped, + ) + defer closeRuntimeEvents() sessionKey := session.BuildMainSessionKey(routing.DefaultAgentID) @@ -1383,14 +1479,14 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) { resultCh <- result{resp: resp, err: err} }() - collectedEvents := make([]Event, 0, 8) + collectedEvents := make([]runtimeevents.Event, 0, 8) steered := false deadline := time.After(3 * time.Second) for !steered { select { - case evt := <-sub.C: + case evt := <-runtimeCh: collectedEvents = append(collectedEvents, evt) - if evt.Kind != EventKindToolExecEnd { + if evt.Kind != runtimeevents.KindAgentToolExecEnd { continue } payload, ok := evt.Payload.(ToolExecEndPayload) @@ -1413,9 +1509,9 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) { t.Fatal("timeout waiting for result") } - events := append(collectedEvents, collectEventStream(sub.C)...) + events := append(collectedEvents, collectRuntimeEventStream(runtimeCh)...) - skippedEvts := filterEvents(events, EventKindToolExecSkipped) + skippedEvts := filterRuntimeEvents(events, runtimeevents.KindAgentToolExecSkipped) if len(skippedEvts) < 1 { t.Fatal("expected at least one ToolExecSkipped event after steering") } @@ -1480,13 +1576,3 @@ func TestCloneStringAnyMap_EmptyMapReturnsNonNil(t *testing.T) { } }) } - -func filterEvents(events []Event, kind EventKind) []Event { - var result []Event - for _, evt := range events { - if evt.Kind == kind { - result = append(result, evt) - } - } - return result -} diff --git a/pkg/agent/instance.go b/pkg/agent/instance.go index d0b25a0a8..4ed713035 100644 --- a/pkg/agent/instance.go +++ b/pkg/agent/instance.go @@ -38,8 +38,10 @@ type AgentInstance struct { Sessions session.SessionStore ContextBuilder *ContextBuilder Tools *tools.ToolRegistry + Definition AgentContextDefinition Subagents *config.SubagentsConfig SkillsFilter []string + MCPServerAllowlist map[string]struct{} Candidates []providers.FallbackCandidate // Router is non-nil when model routing is configured and the light model @@ -74,7 +76,9 @@ func NewAgentInstance( workspace := resolveAgentWorkspace(agentCfg, defaults) os.MkdirAll(workspace, 0o755) - model := resolveAgentModel(agentCfg, defaults) + definition := loadAgentDefinition(workspace) + + model := resolveAgentModel(agentCfg, defaults, definition) fallbacks := resolveAgentFallbacks(agentCfg, defaults) restrict := defaults.RestrictToWorkspace @@ -83,8 +87,11 @@ func NewAgentInstance( // Compile path whitelist patterns from config. allowReadPaths := buildAllowReadPatterns(cfg) allowWritePaths := compilePatterns(cfg.Tools.AllowWritePaths) + agentToolAllowlist := resolveAgentToolAllowlist(definition) + agentMCPServerAllowlist := resolveAgentMCPServerAllowlist(definition) toolsRegistry := tools.NewToolRegistry() + toolsRegistry.SetAllowlist(agentToolAllowlist) if cfg.Tools.IsToolEnabled("read_file") { maxReadFileSize := cfg.Tools.ReadFile.MaxReadFileSize @@ -121,7 +128,7 @@ func NewAgentInstance( sessionsDir := filepath.Join(workspace, "sessions") sessions := initSessionStore(sessionsDir) - mcpDiscoveryActive := cfg.Tools.MCP.Enabled && cfg.Tools.MCP.Discovery.Enabled + mcpDiscoveryActive := agentHasDiscoverableMCPServers(cfg, agentMCPServerAllowlist) contextBuilder := NewContextBuilder(workspace). WithToolDiscovery( mcpDiscoveryActive && cfg.Tools.MCP.Discovery.UseBM25, @@ -137,9 +144,14 @@ func NewAgentInstance( if agentCfg != nil { agentID = routing.NormalizeAgentID(agentCfg.ID) agentName = agentCfg.Name + if definition.Agent != nil && strings.TrimSpace(definition.Agent.Frontmatter.Name) != "" { + agentName = strings.TrimSpace(definition.Agent.Frontmatter.Name) + } subagents = agentCfg.Subagents - skillsFilter = agentCfg.Skills + skillsFilter = resolveAgentSkillsFilter(agentCfg, definition) } + provider = resolvePrimaryProviderForAgent(cfg, workspace, agentID, model, provider) + warnOnUnknownAgentMCPServerDeclarations(agentID, workspace, cfg, definition) maxIter := defaults.MaxToolIterations if maxIter == 0 { @@ -199,8 +211,15 @@ func NewAgentInstance( if len(resolved) > 0 { lightModelCfg, err := resolvedModelConfig(cfg, rc.LightModel, workspace) if err != nil { - logger.WarnCF("agent", "Routing light model config invalid; routing disabled", - map[string]any{"light_model": rc.LightModel, "agent_id": agentID, "error": err.Error()}) + logger.WarnCF( + "agent", + "Routing light model config invalid; routing disabled", + map[string]any{ + "light_model": rc.LightModel, + "agent_id": agentID, + "error": err.Error(), + }, + ) } else { lp, _, err := providers.CreateProviderFromConfig(lightModelCfg) if err != nil { @@ -239,8 +258,10 @@ func NewAgentInstance( Sessions: sessions, ContextBuilder: contextBuilder, Tools: toolsRegistry, + Definition: definition, Subagents: subagents, SkillsFilter: skillsFilter, + MCPServerAllowlist: agentMCPServerAllowlist, Candidates: candidates, Router: router, LightCandidates: lightCandidates, @@ -285,13 +306,55 @@ func populateCandidateProvidersFromNames( } } +// resolvePrimaryProviderForAgent resolves a dedicated provider for the active +// primary model when the model points at a model_list entry. This keeps the +// agent's single-candidate path aligned with the selected model's own +// provider/api_base/api_key instead of inheriting the process default provider. +func resolvePrimaryProviderForAgent( + cfg *config.Config, + workspace string, + agentID string, + model string, + fallback providers.LLMProvider, +) providers.LLMProvider { + model = strings.TrimSpace(model) + if cfg == nil || model == "" { + return fallback + } + + modelCfg := lookupModelConfigByRef(cfg, model) + if modelCfg == nil { + return fallback + } + clone := *modelCfg + if clone.Workspace == "" { + clone.Workspace = workspace + } + + resolvedProvider, _, err := providers.CreateProviderFromConfig(&clone) + if err != nil { + logger.WarnCF("agent", "Primary model provider init failed; using injected provider", + map[string]any{ + "agent_id": agentID, + "model": model, + "error": err.Error(), + }) + return fallback + } + if resolvedProvider == nil { + return fallback + } + return resolvedProvider +} + // resolveAgentWorkspace determines the workspace directory for an agent. func resolveAgentWorkspace(agentCfg *config.AgentConfig, defaults *config.AgentDefaults) string { if agentCfg != nil && strings.TrimSpace(agentCfg.Workspace) != "" { return expandHome(strings.TrimSpace(agentCfg.Workspace)) } // Use the configured default workspace (respects PICOCLAW_HOME) - if agentCfg == nil || agentCfg.Default || agentCfg.ID == "" || routing.NormalizeAgentID(agentCfg.ID) == "main" { + if agentCfg == nil || agentCfg.Default || agentCfg.ID == "" || + routing.NormalizeAgentID(agentCfg.ID) == "main" { return expandHome(defaults.Workspace) } // For named agents without explicit workspace, use default workspace with agent ID suffix @@ -300,7 +363,14 @@ func resolveAgentWorkspace(agentCfg *config.AgentConfig, defaults *config.AgentD } // resolveAgentModel resolves the primary model for an agent. -func resolveAgentModel(agentCfg *config.AgentConfig, defaults *config.AgentDefaults) string { +func resolveAgentModel( + agentCfg *config.AgentConfig, + defaults *config.AgentDefaults, + definition AgentContextDefinition, +) string { + if definition.Agent != nil && strings.TrimSpace(definition.Agent.Frontmatter.Model) != "" { + return strings.TrimSpace(definition.Agent.Frontmatter.Model) + } if agentCfg != nil && agentCfg.Model != nil && strings.TrimSpace(agentCfg.Model.Primary) != "" { return strings.TrimSpace(agentCfg.Model.Primary) } @@ -315,6 +385,27 @@ func resolveAgentFallbacks(agentCfg *config.AgentConfig, defaults *config.AgentD return defaults.ModelFallbacks } +func resolveAgentSkillsFilter( + agentCfg *config.AgentConfig, + definition AgentContextDefinition, +) []string { + if definition.Agent != nil && definition.Agent.Frontmatter.Skills != nil { + return append([]string(nil), definition.Agent.Frontmatter.Skills...) + } + if agentCfg == nil || agentCfg.Skills == nil { + return nil + } + return append([]string(nil), agentCfg.Skills...) +} + +func (a *AgentInstance) AllowsMCPServer(serverName string) bool { + if a == nil || a.MCPServerAllowlist == nil { + return true + } + _, ok := a.MCPServerAllowlist[strings.ToLower(strings.TrimSpace(serverName))] + return ok +} + func compilePatterns(patterns []string) []*regexp.Regexp { compiled := make([]*regexp.Regexp, 0, len(patterns)) for _, p := range patterns { diff --git a/pkg/agent/instance_test.go b/pkg/agent/instance_test.go index 42bb53d86..dff2c0f2f 100644 --- a/pkg/agent/instance_test.go +++ b/pkg/agent/instance_test.go @@ -10,6 +10,7 @@ import ( "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/tools" ) func TestNewAgentInstance_UsesDefaultsTemperatureAndMaxTokens(t *testing.T) { @@ -616,3 +617,285 @@ func TestNewAgentInstance_InvalidExecConfigDoesNotExit(t *testing.T) { t.Fatal("read_file tool should still be registered") } } + +func TestNewAgentInstance_UsesFrontmatterModelAndSkills(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +model: frontmatter-model +skills: [frontmatter-skill] +mcpServers: [GitHub, filesystem] +--- +# Agent + +Use frontmatter identity. +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + ModelName: "default-model", + }, + }, + } + + agent := NewAgentInstance(&config.AgentConfig{ + ID: "research", + Workspace: workspace, + Model: &config.AgentModelConfig{ + Primary: "config-model", + }, + Skills: []string{"config-skill"}, + }, &cfg.Agents.Defaults, cfg, &mockProvider{}) + + if agent.Model != "frontmatter-model" { + t.Fatalf("agent.Model = %q, want frontmatter-model", agent.Model) + } + if len(agent.SkillsFilter) != 1 || agent.SkillsFilter[0] != "frontmatter-skill" { + t.Fatalf("agent.SkillsFilter = %v, want [frontmatter-skill]", agent.SkillsFilter) + } + if !agent.AllowsMCPServer("github") { + t.Fatal("expected github MCP server to be allowed from frontmatter") + } + if !agent.AllowsMCPServer("FILESYSTEM") { + t.Fatal("expected filesystem MCP server matching to be case-insensitive") + } + if agent.AllowsMCPServer("slack") { + t.Fatal("expected slack MCP server to be blocked by frontmatter allowlist") + } +} + +func TestNewAgentInstance_UsesResolvedProviderForFrontmatterPrimaryModel(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +model: claude-frontmatter +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + Provider: "openai", + ModelName: "default-model", + }, + }, + ModelList: []*config.ModelConfig{ + { + ModelName: "claude-frontmatter", + Model: "anthropic/claude-3-7-sonnet", + APIKeys: config.SimpleSecureStrings("test-anthropic-key"), + Workspace: workspace, + }, + }, + } + + defaultProvider := &mockProvider{} + agent := NewAgentInstance(&config.AgentConfig{ + ID: "research", + Workspace: workspace, + }, &cfg.Agents.Defaults, cfg, defaultProvider) + + if agent.Model != "claude-frontmatter" { + t.Fatalf("agent.Model = %q, want %q", agent.Model, "claude-frontmatter") + } + if len(agent.Candidates) != 1 { + t.Fatalf("len(agent.Candidates) = %d, want 1", len(agent.Candidates)) + } + if got := agent.Candidates[0].Provider; got != "anthropic" { + t.Fatalf("primary candidate provider = %q, want %q", got, "anthropic") + } + if got := agent.Candidates[0].Model; got != "claude-3-7-sonnet" { + t.Fatalf("primary candidate model = %q, want %q", got, "claude-3-7-sonnet") + } + if agent.Provider == defaultProvider { + t.Fatal("expected primary provider to be resolved from model_list instead of using injected default provider") + } +} + +func TestNewAgentInstance_SuppressesToolDiscoveryPromptWhenNoMCPServersSelected(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +mcpServers: [] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + ModelName: "default-model", + }, + }, + Tools: config.ToolsConfig{ + MCP: config.MCPConfig{ + ToolConfig: config.ToolConfig{Enabled: true}, + Discovery: config.ToolDiscoveryConfig{ + Enabled: true, + UseBM25: true, + UseRegex: false, + }, + Servers: map[string]config.MCPServerConfig{ + "github": {Enabled: true}, + }, + }, + }, + } + + agent := NewAgentInstance(&config.AgentConfig{ + ID: "research", + Workspace: workspace, + }, &cfg.Agents.Defaults, cfg, &mockProvider{}) + + if agent.AllowsMCPServer("github") { + t.Fatal("expected empty mcpServers allowlist to deny all servers") + } + messages := agent.ContextBuilder.BuildMessagesFromPrompt(PromptBuildRequest{CurrentMessage: "hello"}) + if prompt := messages[0].Content; strings.Contains(prompt, tools.BM25SearchToolName) { + t.Fatalf("expected no tool discovery prompt when no MCP servers are selected, got %q", prompt) + } +} + +func TestNewAgentInstance_IncludesToolDiscoveryPromptWhenDiscoverableMCPServerSelected(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +mcpServers: [github] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + ModelName: "default-model", + }, + }, + Tools: config.ToolsConfig{ + MCP: config.MCPConfig{ + ToolConfig: config.ToolConfig{Enabled: true}, + Discovery: config.ToolDiscoveryConfig{ + Enabled: true, + UseBM25: true, + UseRegex: false, + }, + Servers: map[string]config.MCPServerConfig{ + "github": {Enabled: true}, + }, + }, + }, + } + + agent := NewAgentInstance(&config.AgentConfig{ + ID: "research", + Workspace: workspace, + }, &cfg.Agents.Defaults, cfg, &mockProvider{}) + + messages := agent.ContextBuilder.BuildMessagesFromPrompt(PromptBuildRequest{CurrentMessage: "hello"}) + if prompt := messages[0].Content; !strings.Contains(prompt, tools.BM25SearchToolName) { + t.Fatalf("expected tool discovery prompt when a discoverable MCP server is selected, got %q", prompt) + } +} + +func TestNewAgentInstance_InvalidFrontmatterFailsClosedForToolsAndMCPServers(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +tools: [read_file +mcpServers: [github] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + ModelName: "default-model", + }, + }, + Tools: config.ToolsConfig{ + ReadFile: config.ReadFileToolConfig{Enabled: true}, + }, + } + + agent := NewAgentInstance(&config.AgentConfig{ + ID: "research", + Workspace: workspace, + }, &cfg.Agents.Defaults, cfg, &mockProvider{}) + + if _, ok := agent.Tools.Get("read_file"); ok { + t.Fatal("expected malformed frontmatter to fail closed and block read_file") + } + if agent.AllowsMCPServer("github") { + t.Fatal("expected malformed frontmatter to fail closed for MCP servers") + } +} + +func TestNewAgentInstance_ExplicitEmptyToolsFieldBlocksAllTools(t *testing.T) { + tests := []struct { + name string + toolsSnippet string + }{ + { + name: "empty list", + toolsSnippet: "tools: []", + }, + { + name: "blank field", + toolsSnippet: "tools:", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +` + tt.toolsSnippet + ` +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: workspace, + ModelName: "default-model", + }, + }, + Tools: config.ToolsConfig{ + ReadFile: config.ReadFileToolConfig{Enabled: true}, + ListDir: config.ToolConfig{Enabled: true}, + }, + } + + agent := NewAgentInstance(&config.AgentConfig{ + ID: "research", + Workspace: workspace, + }, &cfg.Agents.Defaults, cfg, &mockProvider{}) + + if got := agent.Tools.List(); len(got) != 0 { + t.Fatalf("agent tools = %v, want no registered tools", got) + } + if _, ok := agent.Tools.Get("read_file"); ok { + t.Fatal("expected read_file to be blocked by explicit empty tools field") + } + if _, ok := agent.Tools.Get("list_dir"); ok { + t.Fatal("expected list_dir to be blocked by explicit empty tools field") + } + }) + } +} diff --git a/pkg/agent/legacy_events.go b/pkg/agent/legacy_events.go new file mode 100644 index 000000000..30761e8e6 --- /dev/null +++ b/pkg/agent/legacy_events.go @@ -0,0 +1,177 @@ +package agent + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/sipeed/picoclaw/pkg/bus" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +const defaultEventSubscriberBuffer = 16 + +// EventSubscription identifies a legacy subscriber channel returned by +// AgentLoop.SubscribeEvents. +type EventSubscription struct { + ID uint64 + C <-chan Event +} + +type legacyEventSubscription struct { + cancel context.CancelFunc + sub runtimeevents.Subscription +} + +var ( + legacyEventSubSeq atomic.Uint64 + legacyEventSubLock sync.Map +) + +// SubscribeEvents exposes the previous in-agent event subscription API on top +// of the runtime event bus for tests and compatibility. +func (al *AgentLoop) SubscribeEvents(buffer int) EventSubscription { + if buffer <= 0 { + buffer = defaultEventSubscriberBuffer + } + + out := make(chan Event, buffer) + if al == nil || al.runtimeEvents == nil { + close(out) + return EventSubscription{C: out} + } + + ctx, cancel := context.WithCancel(context.Background()) + sub, in, err := al.runtimeEvents.Channel(). + Source("agent"). + OfKind(legacyAgentEventKinds()...). + SubscribeChan(ctx, runtimeevents.SubscribeOptions{ + Name: "legacy-agent-events", + Buffer: buffer, + }) + if err != nil { + cancel() + close(out) + return EventSubscription{C: out} + } + + id := legacyEventSubSeq.Add(1) + legacyEventSubLock.Store(id, legacyEventSubscription{cancel: cancel, sub: sub}) + go func() { + defer legacyEventSubLock.LoadAndDelete(id) + defer close(out) + for { + select { + case <-ctx.Done(): + return + case evt, ok := <-in: + if !ok { + return + } + select { + case out <- legacyEventFromRuntimeEvent(evt): + case <-ctx.Done(): + return + } + } + } + }() + + return EventSubscription{ID: id, C: out} +} + +func (al *AgentLoop) UnsubscribeEvents(id uint64) { + if id == 0 { + return + } + value, ok := legacyEventSubLock.LoadAndDelete(id) + if !ok { + return + } + sub := value.(legacyEventSubscription) + sub.cancel() + if sub.sub != nil { + _ = sub.sub.Close() + } +} + +func legacyEventFromRuntimeEvent(evt runtimeevents.Event) Event { + meta := hookMetaFromRuntimeEvent(evt) + return Event{ + Kind: evt.Kind, + Time: evt.Time, + Meta: meta, + Context: turnContextFromRuntimeScope(evt.Scope), + Payload: evt.Payload, + } +} + +func hookMetaFromRuntimeEvent(evt runtimeevents.Event) HookMeta { + meta := HookMeta{ + AgentID: evt.Scope.AgentID, + TurnID: evt.Scope.TurnID, + ParentTurnID: evt.Correlation.ParentTurnID, + SessionKey: evt.Scope.SessionKey, + TracePath: evt.Correlation.TraceID, + } + if evt.Attrs != nil { + if source, ok := evt.Attrs["agent_source"].(string); ok { + meta.Source = source + } + if iteration, ok := evt.Attrs["iteration"].(int); ok { + meta.Iteration = iteration + } + } + return meta +} + +func turnContextFromRuntimeScope(scope runtimeevents.Scope) *TurnContext { + if scope.Channel == "" && + scope.Account == "" && + scope.ChatID == "" && + scope.ChatType == "" && + scope.TopicID == "" && + scope.SpaceID == "" && + scope.SpaceType == "" && + scope.SenderID == "" && + scope.MessageID == "" { + return nil + } + return &TurnContext{ + Inbound: &bus.InboundContext{ + Channel: scope.Channel, + Account: scope.Account, + ChatID: scope.ChatID, + ChatType: scope.ChatType, + TopicID: scope.TopicID, + SpaceID: scope.SpaceID, + SpaceType: scope.SpaceType, + SenderID: scope.SenderID, + MessageID: scope.MessageID, + }, + } +} + +func legacyAgentEventKinds() []runtimeevents.Kind { + return []runtimeevents.Kind{ + EventKindTurnStart, + EventKindTurnEnd, + EventKindLLMRequest, + EventKindLLMDelta, + EventKindLLMResponse, + EventKindLLMRetry, + EventKindContextCompress, + EventKindSessionSummarize, + EventKindToolExecStart, + EventKindToolExecEnd, + EventKindToolExecSkipped, + EventKindSteeringInjected, + EventKindFollowUpQueued, + EventKindInterruptReceived, + EventKindSubTurnSpawn, + EventKindSubTurnEnd, + EventKindSubTurnResultDelivered, + EventKindSubTurnOrphan, + EventKindError, + } +} diff --git a/pkg/agent/legacy_events_test.go b/pkg/agent/legacy_events_test.go new file mode 100644 index 000000000..3aa7782fe --- /dev/null +++ b/pkg/agent/legacy_events_test.go @@ -0,0 +1,76 @@ +package agent + +import ( + "context" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +func TestSubscribeEventsFiltersRuntimeBusToLegacyAgentEvents(t *testing.T) { + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: t.TempDir(), + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 3, + }, + }, + } + al := NewAgentLoop(cfg, bus.NewMessageBus(), &simpleMockProvider{response: "ok"}) + defer al.Close() + + sub := al.SubscribeEvents(4) + defer al.UnsubscribeEvents(sub.ID) + + al.RuntimeEventBus().Publish(context.Background(), runtimeevents.Event{ + Kind: runtimeevents.KindGatewayReady, + Source: runtimeevents.Source{Component: "gateway"}, + }) + select { + case evt := <-sub.C: + t.Fatalf("legacy subscriber received non-agent runtime event: %s", evt.Kind) + case <-time.After(50 * time.Millisecond): + } + + al.RuntimeEventBus().Publish(context.Background(), runtimeevents.Event{ + Kind: runtimeevents.KindAgentTurnStart, + Source: runtimeevents.Source{Component: "agent", Name: "main"}, + Scope: runtimeevents.Scope{ + AgentID: "main", + TurnID: "turn-1", + SessionKey: "session-1", + Channel: "telegram", + Account: "bot-1", + ChatID: "chat-1", + ChatType: "private", + TopicID: "topic-1", + SpaceID: "space-1", + SpaceType: "dm", + SenderID: "sender-1", + MessageID: "message-1", + }, + Payload: TurnStartPayload{UserMessage: "hello"}, + }) + + evt := waitForEvent(t, sub.C, 2*time.Second, nil) + if evt.Kind != EventKindTurnStart { + t.Fatalf("event kind = %q, want %q", evt.Kind, EventKindTurnStart) + } + if evt.Context == nil || evt.Context.Inbound == nil { + t.Fatalf("expected legacy event inbound context, got %#v", evt.Context) + } + if got := evt.Context.Inbound.Channel; got != "telegram" { + t.Fatalf("inbound channel = %q, want telegram", got) + } + if got := evt.Context.Inbound.ChatID; got != "chat-1" { + t.Fatalf("inbound chat_id = %q, want chat-1", got) + } + if got := evt.Context.Inbound.MessageID; got != "message-1" { + t.Fatalf("inbound message_id = %q, want message-1", got) + } +} diff --git a/pkg/agent/llm_media.go b/pkg/agent/llm_media.go index eb1908777..31692174b 100644 --- a/pkg/agent/llm_media.go +++ b/pkg/agent/llm_media.go @@ -56,5 +56,12 @@ func isVisionUnsupportedError(err error) bool { return true } + // DeepSeek and other strict providers reject the image_url field at the + // JSON schema level with an "unknown variant" error rather than a semantic + // "not supported" message. + if strings.Contains(msg, "unknown variant") && strings.Contains(msg, "image_url") { + return true + } + return false } diff --git a/pkg/agent/pipeline_execute.go b/pkg/agent/pipeline_execute.go index f6a8eaad6..567e56d17 100644 --- a/pkg/agent/pipeline_execute.go +++ b/pkg/agent/pipeline_execute.go @@ -6,16 +6,103 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" + "sort" + "strings" "time" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/constants" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/tools" "github.com/sipeed/picoclaw/pkg/utils" ) +func toolErrorSummary(result *tools.ToolResult) string { + if result == nil || !result.IsError { + return "" + } + content := strings.TrimSpace(result.ContentForLLM()) + if content == "" && result.Err != nil { + content = strings.TrimSpace(result.Err.Error()) + } + return utils.Truncate(content, 200) +} + +func inferSkillNamesFromToolCall(ts *turnState, toolName string, toolArgs map[string]any) []string { + if ts == nil || toolName != "read_file" { + return nil + } + + rawPath, ok := toolArgs["path"].(string) + if !ok { + return nil + } + path := strings.TrimSpace(rawPath) + if path == "" { + return nil + } + + cleanPath := filepath.Clean(path) + if !filepath.IsAbs(cleanPath) { + cleanPath = filepath.Join(ts.workspace, cleanPath) + } + if filepath.Base(cleanPath) != "SKILL.md" { + return nil + } + + var roots []string + if ts.agent != nil && ts.agent.ContextBuilder != nil { + roots = ts.agent.ContextBuilder.skillRoots() + } + if len(roots) == 0 && strings.TrimSpace(ts.workspace) != "" { + roots = []string{filepath.Join(ts.workspace, "skills")} + } + + found := make(map[string]struct{}) + for _, root := range roots { + root = strings.TrimSpace(root) + if root == "" { + continue + } + rel, err := filepath.Rel(filepath.Clean(root), cleanPath) + if err != nil { + continue + } + if rel == "." || rel == "" || strings.HasPrefix(rel, "..") { + continue + } + parts := strings.Split(rel, string(filepath.Separator)) + if len(parts) != 2 || parts[1] != "SKILL.md" { + continue + } + + skillName := strings.TrimSpace(parts[0]) + if skillName == "" { + continue + } + if ts.agent != nil && ts.agent.ContextBuilder != nil { + if canonical, ok := ts.agent.ContextBuilder.ResolveSkillName(skillName); ok { + skillName = canonical + } + } + found[skillName] = struct{}{} + } + + if len(found) == 0 { + return nil + } + + names := make([]string, 0, len(found)) + for skillName := range found { + names = append(names, skillName) + } + sort.Strings(names) + return names +} + // ExecuteTools executes the tool loop, handling BeforeTool/ApproveTool/AfterTool hooks, // tool execution with async callbacks, media delivery, and steering injection. // Returns ToolControl indicating what the coordinator should do next: @@ -72,7 +159,7 @@ toolLoop: }) al.emitEvent( - EventKindToolExecStart, + runtimeevents.KindAgentToolExecStart, ts.eventMeta("runTurn", "turn.tool.start"), ToolExecStartPayload{ Tool: toolName, @@ -191,7 +278,7 @@ toolLoop: } al.emitEvent( - EventKindToolExecEnd, + runtimeevents.KindAgentToolExecEnd, ts.eventMeta("runTurn", "turn.tool.end"), ToolExecEndPayload{ Tool: toolName, @@ -202,6 +289,12 @@ toolLoop: Async: hookResult.Async, }, ) + ts.recordToolExecution( + toolName, + !hookResult.IsError, + toolErrorSummary(hookResult), + inferSkillNamesFromToolCall(ts, toolName, toolArgs), + ) messages = append(messages, toolResultMsg) if !ts.opts.NoHistory { @@ -237,7 +330,7 @@ toolLoop: for j := i + 1; j < len(normalizedToolCalls); j++ { skippedTC := normalizedToolCalls[j] al.emitEvent( - EventKindToolExecSkipped, + runtimeevents.KindAgentToolExecSkipped, ts.eventMeta("runTurn", "turn.tool.skipped"), ToolExecSkippedPayload{ Tool: skippedTC.Name, @@ -284,7 +377,7 @@ toolLoop: exec.allResponsesHandled = false denyContent := hookDeniedToolContent("Tool execution denied by hook", decision.Reason) al.emitEvent( - EventKindToolExecSkipped, + runtimeevents.KindAgentToolExecSkipped, ts.eventMeta("runTurn", "turn.tool.skipped"), ToolExecSkippedPayload{ Tool: toolName, @@ -323,7 +416,7 @@ toolLoop: exec.allResponsesHandled = false denyContent := hookDeniedToolContent("Tool execution denied by approval hook", approval.Reason) al.emitEvent( - EventKindToolExecSkipped, + runtimeevents.KindAgentToolExecSkipped, ts.eventMeta("runTurn", "turn.tool.skipped"), ToolExecSkippedPayload{ Tool: toolName, @@ -353,7 +446,7 @@ toolLoop: "iteration": iteration, }) al.emitEvent( - EventKindToolExecStart, + runtimeevents.KindAgentToolExecStart, ts.eventMeta("runTurn", "turn.tool.start"), ToolExecStartPayload{ Tool: toolName, @@ -401,7 +494,7 @@ toolLoop: "channel": ts.channel, }) al.emitEvent( - EventKindFollowUpQueued, + runtimeevents.KindAgentFollowUpQueued, ts.scope.meta(iteration, "runTurn", "turn.follow_up.queued"), FollowUpQueuedPayload{ SourceTool: asyncToolName, @@ -567,7 +660,7 @@ toolLoop: toolResultMsg.Media = append(toolResultMsg.Media, toolResult.Media...) } al.emitEvent( - EventKindToolExecEnd, + runtimeevents.KindAgentToolExecEnd, ts.eventMeta("runTurn", "turn.tool.end"), ToolExecEndPayload{ Tool: toolName, @@ -578,6 +671,12 @@ toolLoop: Async: toolResult.Async, }, ) + ts.recordToolExecution( + toolName, + !toolResult.IsError, + toolErrorSummary(toolResult), + inferSkillNamesFromToolCall(ts, toolName, toolArgs), + ) messages = append(messages, toolResultMsg) if !ts.opts.NoHistory { ts.agent.Sessions.AddFullMessage(ts.sessionKey, toolResultMsg) @@ -612,7 +711,7 @@ toolLoop: for j := i + 1; j < len(normalizedToolCalls); j++ { skippedTC := normalizedToolCalls[j] al.emitEvent( - EventKindToolExecSkipped, + runtimeevents.KindAgentToolExecSkipped, ts.eventMeta("runTurn", "turn.tool.skipped"), ToolExecSkippedPayload{ Tool: skippedTC.Name, diff --git a/pkg/agent/pipeline_execute_test.go b/pkg/agent/pipeline_execute_test.go new file mode 100644 index 000000000..404da320c --- /dev/null +++ b/pkg/agent/pipeline_execute_test.go @@ -0,0 +1,50 @@ +package agent + +import ( + "os" + "path/filepath" + "testing" +) + +func TestInferSkillNamesFromToolCall_ReadFileSkillMarkdown(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "three-one") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile( + filepath.Join(skillDir, "SKILL.md"), + []byte("---\nname: three-one\ndescription: test\n---\n# Three One\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + cb := NewContextBuilder(workspace) + ts := &turnState{ + workspace: workspace, + agent: &AgentInstance{ + Workspace: workspace, + ContextBuilder: cb, + }, + } + + got := inferSkillNamesFromToolCall(ts, "read_file", map[string]any{ + "path": filepath.Join(workspace, "skills", "three-one", "SKILL.md"), + }) + if len(got) != 1 || got[0] != "three-one" { + t.Fatalf("inferSkillNamesFromToolCall = %v, want [three-one]", got) + } +} + +func TestInferSkillNamesFromToolCall_NonSkillFileIgnored(t *testing.T) { + workspace := t.TempDir() + ts := &turnState{workspace: workspace} + + got := inferSkillNamesFromToolCall(ts, "read_file", map[string]any{ + "path": filepath.Join(workspace, "README.md"), + }) + if len(got) != 0 { + t.Fatalf("inferSkillNamesFromToolCall = %v, want empty", got) + } +} diff --git a/pkg/agent/pipeline_finalize.go b/pkg/agent/pipeline_finalize.go index a2be6f65b..1f407825e 100644 --- a/pkg/agent/pipeline_finalize.go +++ b/pkg/agent/pipeline_finalize.go @@ -6,6 +6,7 @@ import ( "context" "github.com/sipeed/picoclaw/pkg/bus" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -50,7 +51,7 @@ func (p *Pipeline) Finalize( ts.ingestMessage(turnCtx, al, finalMsg) if err := ts.agent.Sessions.Save(ts.sessionKey); err != nil { al.emitEvent( - EventKindError, + runtimeevents.KindAgentError, ts.eventMeta("runTurn", "turn.error"), ErrorPayload{ Stage: "session_save", diff --git a/pkg/agent/pipeline_llm.go b/pkg/agent/pipeline_llm.go index 6bf55fa39..3a3c496f6 100644 --- a/pkg/agent/pipeline_llm.go +++ b/pkg/agent/pipeline_llm.go @@ -11,6 +11,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/constants" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -113,7 +114,7 @@ func (p *Pipeline) CallLLM( } al.emitEvent( - EventKindLLMRequest, + runtimeevents.KindAgentLLMRequest, ts.eventMeta("runTurn", "turn.llm.request"), LLMRequestPayload{ Model: exec.llmModel, @@ -184,7 +185,14 @@ func (p *Pipeline) CallLLM( // Retry loop var err error - maxRetries := 2 + maxRetries := p.Cfg.Agents.Defaults.MaxLLMRetries + if maxRetries <= 0 { + maxRetries = 2 + } + backoffSecs := p.Cfg.Agents.Defaults.LLMRetryBackoffSecs + if backoffSecs <= 0 { + backoffSecs = 2 + } for retry := 0; retry <= maxRetries; retry++ { exec.response, err = callLLM(exec.callMessages, exec.providerToolDefs) if err == nil { @@ -199,7 +207,7 @@ func (p *Pipeline) CallLLM( // Retry without media if vision is unsupported if hasMediaRefs(exec.callMessages) && isVisionUnsupportedError(err) && retry < maxRetries { al.emitEvent( - EventKindLLMRetry, + runtimeevents.KindAgentLLMRetry, ts.eventMeta("runTurn", "turn.llm.retry"), LLMRetryPayload{ Attempt: retry + 1, @@ -232,6 +240,15 @@ func (p *Pipeline) CallLLM( strings.Contains(errMsg, "timed out") || strings.Contains(errMsg, "timeout exceeded") + isNetworkError := !isTimeoutError && (strings.Contains(errMsg, "connection reset") || + strings.Contains(errMsg, "connection refused") || + strings.Contains(errMsg, "broken pipe") || + strings.Contains(errMsg, "no such host") || + strings.Contains(errMsg, "network is unreachable") || + strings.Contains(errMsg, "read tcp") || + strings.Contains(errMsg, "write tcp") || + strings.Contains(errMsg, "eof")) + isContextError := !isTimeoutError && (strings.Contains(errMsg, "context_length_exceeded") || strings.Contains(errMsg, "context window") || strings.Contains(errMsg, "context_window") || @@ -244,9 +261,9 @@ func (p *Pipeline) CallLLM( strings.Contains(errMsg, "request too large")) if isTimeoutError && retry < maxRetries { - backoff := time.Duration(retry+1) * 5 * time.Second + backoff := time.Duration(retry+1) * time.Duration(backoffSecs) * time.Second al.emitEvent( - EventKindLLMRetry, + runtimeevents.KindAgentLLMRetry, ts.eventMeta("runTurn", "turn.llm.retry"), LLMRetryPayload{ Attempt: retry + 1, @@ -272,9 +289,38 @@ func (p *Pipeline) CallLLM( continue } + if isNetworkError && retry < maxRetries { + backoff := time.Duration(retry+1) * time.Duration(backoffSecs) * time.Second + al.emitEvent( + runtimeevents.KindAgentLLMRetry, + ts.eventMeta("runTurn", "turn.llm.retry"), + LLMRetryPayload{ + Attempt: retry + 1, + MaxRetries: maxRetries, + Reason: "network", + Error: err.Error(), + Backoff: backoff, + }, + ) + logger.WarnCF("agent", "Network error, retrying after backoff", map[string]any{ + "error": err.Error(), + "retry": retry, + "backoff": backoff.String(), + }) + if sleepErr := sleepWithContext(turnCtx, backoff); sleepErr != nil { + if ts.hardAbortRequested() { + _ = ts.requestHardAbort() + return ControlBreak, nil + } + err = sleepErr + break + } + continue + } + if isContextError && retry < maxRetries && !ts.opts.NoHistory { al.emitEvent( - EventKindLLMRetry, + runtimeevents.KindAgentLLMRetry, ts.eventMeta("runTurn", "turn.llm.retry"), LLMRetryPayload{ Attempt: retry + 1, @@ -318,9 +364,14 @@ func (p *Pipeline) CallLLM( exec.history = asmResp.History exec.summary = asmResp.Summary } - exec.messages = ts.agent.ContextBuilder.BuildMessagesFromPrompt( - promptBuildRequestForTurn(ts, exec.history, exec.summary, "", nil), - ) + contextualSkills := ts.activeSkills + if ts.agent.ContextBuilder != nil { + contextualSkills = ts.agent.ContextBuilder.ResolveActiveSkillsForContext(ts.activeSkills) + } + ts.recordSkillContextSnapshot(skillContextTriggerContextRetryRebuild, contextualSkills) + rebuildPromptReq := promptBuildRequestForTurn(ts, exec.history, exec.summary, "", nil) + rebuildPromptReq.ActiveSkills = append([]string(nil), contextualSkills...) + exec.messages = ts.agent.ContextBuilder.BuildMessagesFromPrompt(rebuildPromptReq) exec.callMessages = exec.messages if exec.gracefulTerminal { msgs := append([]providers.Message(nil), exec.messages...) @@ -333,7 +384,7 @@ func (p *Pipeline) CallLLM( if err != nil { al.emitEvent( - EventKindError, + runtimeevents.KindAgentError, ts.eventMeta("runTurn", "turn.error"), ErrorPayload{ Stage: "llm", @@ -397,7 +448,7 @@ func (p *Pipeline) CallLLM( ) } al.emitEvent( - EventKindLLMResponse, + runtimeevents.KindAgentLLMResponse, ts.eventMeta("runTurn", "turn.llm.response"), LLMResponsePayload{ ContentLen: len(exec.response.Content), diff --git a/pkg/agent/pipeline_setup.go b/pkg/agent/pipeline_setup.go index 219e4e5de..f6fed09de 100644 --- a/pkg/agent/pipeline_setup.go +++ b/pkg/agent/pipeline_setup.go @@ -31,9 +31,14 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution } ts.captureRestorePoint(history, summary) - messages := ts.agent.ContextBuilder.BuildMessagesFromPrompt( - promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media), - ) + contextualSkills := ts.activeSkills + if ts.agent.ContextBuilder != nil { + contextualSkills = ts.agent.ContextBuilder.ResolveActiveSkillsForContext(ts.activeSkills) + } + ts.recordSkillContextSnapshot(skillContextTriggerInitialBuild, contextualSkills) + initialPromptReq := promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media) + initialPromptReq.ActiveSkills = append([]string(nil), contextualSkills...) + messages := ts.agent.ContextBuilder.BuildMessagesFromPrompt(initialPromptReq) messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize) @@ -61,9 +66,9 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution history = resp.History summary = resp.Summary } - messages = ts.agent.ContextBuilder.BuildMessagesFromPrompt( - promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media), - ) + rebuildPromptReq := promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media) + rebuildPromptReq.ActiveSkills = append([]string(nil), contextualSkills...) + messages = ts.agent.ContextBuilder.BuildMessagesFromPrompt(rebuildPromptReq) messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize) } } diff --git a/pkg/agent/prompt.go b/pkg/agent/prompt.go index be5ccddf2..02c850360 100644 --- a/pkg/agent/prompt.go +++ b/pkg/agent/prompt.go @@ -52,6 +52,7 @@ const ( PromptSourceMemory PromptSourceID = "memory:workspace" PromptSourceSkillCatalog PromptSourceID = "skill:index" PromptSourceActiveSkills PromptSourceID = "skill:active" + PromptSourceAgentDiscovery PromptSourceID = "agent:discovery" PromptSourceToolRegistry PromptSourceID = "tool_registry:native" PromptSourceToolDiscovery PromptSourceID = "tool_registry:discovery" PromptSourceOutputPolicy PromptSourceID = "runtime.output" @@ -195,6 +196,13 @@ func builtinPromptSources() []PromptSourceDescriptor { Allowed: []PromptPlacement{{Layer: PromptLayerCapability, Slot: PromptSlotActiveSkill}}, StableByDefault: false, }, + { + ID: PromptSourceAgentDiscovery, + Owner: "agent", + Description: "Structured multi-agent discovery registry", + Allowed: []PromptPlacement{{Layer: PromptLayerCapability, Slot: PromptSlotTooling}}, + StableByDefault: false, + }, { ID: PromptSourceMemory, Owner: "memory", diff --git a/pkg/agent/prompt_contributors.go b/pkg/agent/prompt_contributors.go index 960572e03..d6a2c09ec 100644 --- a/pkg/agent/prompt_contributors.go +++ b/pkg/agent/prompt_contributors.go @@ -93,6 +93,47 @@ func (c mcpServerPromptContributor) ContributePrompt( }, nil } +type agentDiscoveryPromptContributor struct { + agentID string + discover func(agentID string) []AgentDescriptor +} + +func (c agentDiscoveryPromptContributor) PromptSource() PromptSourceDescriptor { + return PromptSourceDescriptor{ + ID: PromptSourceAgentDiscovery, + Owner: "agent", + Description: "Structured multi-agent discovery registry", + Allowed: []PromptPlacement{{Layer: PromptLayerCapability, Slot: PromptSlotTooling}}, + StableByDefault: false, + } +} + +func (c agentDiscoveryPromptContributor) ContributePrompt( + _ context.Context, + _ PromptBuildRequest, +) ([]PromptPart, error) { + if c.discover == nil { + return nil, nil + } + content := formatAgentDiscoverySection(c.discover(c.agentID)) + if strings.TrimSpace(content) == "" { + return nil, nil + } + + return []PromptPart{ + { + ID: "capability.agent_discovery", + Layer: PromptLayerCapability, + Slot: PromptSlotTooling, + Source: PromptSource{ID: PromptSourceAgentDiscovery, Name: "agent:discovery"}, + Title: "agent discovery", + Content: content, + Stable: false, + Cache: PromptCacheNone, + }, + }, nil +} + func mcpPromptSourceID(serverName string) PromptSourceID { return PromptSourceID("mcp:" + promptSourceComponent(serverName)) } diff --git a/pkg/agent/registry.go b/pkg/agent/registry.go index 8aa11e37b..821ad4187 100644 --- a/pkg/agent/registry.go +++ b/pkg/agent/registry.go @@ -13,6 +13,7 @@ import ( // AgentRegistry manages multiple agent instances and routes messages to them. type AgentRegistry struct { + cfg *config.Config agents map[string]*AgentInstance resolver *routing.RouteResolver mu sync.RWMutex @@ -24,6 +25,7 @@ func NewAgentRegistry( provider providers.LLMProvider, ) *AgentRegistry { registry := &AgentRegistry{ + cfg: cfg, agents: make(map[string]*AgentInstance), resolver: routing.NewRouteResolver(cfg), } @@ -53,6 +55,12 @@ func NewAgentRegistry( } } + for _, instance := range registry.agents { + if instance.ContextBuilder != nil { + instance.ContextBuilder.WithAgentDiscovery(instance.ID, registry.ListSpawnableAgents) + } + } + return registry } @@ -81,16 +89,43 @@ func (r *AgentRegistry) ListAgentIDs() []string { return ids } +func (r *AgentRegistry) allowedMCPServers() map[string]struct{} { + r.mu.RLock() + defer r.mu.RUnlock() + + if len(r.agents) == 0 { + return nil + } + + union := make(map[string]struct{}) + for _, agent := range r.agents { + if agent == nil { + continue + } + if agent.MCPServerAllowlist == nil { + return nil + } + for serverName := range agent.MCPServerAllowlist { + union[serverName] = struct{}{} + } + } + + return union +} + // CanSpawnSubagent checks if parentAgentID is allowed to spawn targetAgentID. func (r *AgentRegistry) CanSpawnSubagent(parentAgentID, targetAgentID string) bool { parent, ok := r.GetAgent(parentAgentID) if !ok { return false } - if parent.Subagents == nil || parent.Subagents.AllowAgents == nil { + return agentAllowsSubagent(parent, routing.NormalizeAgentID(targetAgentID)) +} + +func agentAllowsSubagent(parent *AgentInstance, targetNorm string) bool { + if parent == nil || parent.Subagents == nil || parent.Subagents.AllowAgents == nil { return false } - targetNorm := routing.NormalizeAgentID(targetAgentID) for _, allowed := range parent.Subagents.AllowAgents { if allowed == "*" { return true @@ -102,6 +137,14 @@ func (r *AgentRegistry) CanSpawnSubagent(parentAgentID, targetAgentID string) bo return false } +func agentHasSpawnTool(agent *AgentInstance) bool { + if agent == nil || agent.Tools == nil { + return false + } + _, ok := agent.Tools.Get("spawn") + return ok +} + // ForEachTool calls fn for every tool registered under the given name // across all agents. This is useful for propagating dependencies (e.g. // MediaStore) to tools after registry construction. @@ -131,11 +174,13 @@ func (r *AgentRegistry) Close() { func (r *AgentRegistry) GetDefaultAgent() *AgentInstance { r.mu.RLock() defer r.mu.RUnlock() - if agent, ok := r.agents["main"]; ok { - return agent + if id := r.defaultAgentIDLocked(); id != "" { + if agent, ok := r.agents[id]; ok { + return agent + } } - for _, agent := range r.agents { - return agent + for id := range r.agents { + return r.agents[id] } return nil } diff --git a/pkg/agent/registry_test.go b/pkg/agent/registry_test.go index b173ef967..62b2ea6eb 100644 --- a/pkg/agent/registry_test.go +++ b/pkg/agent/registry_test.go @@ -2,8 +2,10 @@ package agent import ( "context" + "slices" "testing" + "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -200,6 +202,112 @@ func TestAgentInstance_FallbackExplicitEmpty(t *testing.T) { agent, _ := registry.GetAgent("no-fallback") if len(agent.Fallbacks) != 0 { - t.Errorf("expected 0 fallbacks (explicit empty), got %d: %v", len(agent.Fallbacks), agent.Fallbacks) + t.Errorf( + "expected 0 fallbacks (explicit empty), got %d: %v", + len(agent.Fallbacks), + agent.Fallbacks, + ) + } +} + +func TestNewAgentLoop_AgentToolAllowlistFiltersRuntimeTools(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": "# Agent\nMain agent.\n", + }) + defer cleanupWorkspace(t, mainWorkspace) + + researchWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +tools: [read_file, write_file, web_search, web_fetch, message] +skills: [deep-research] +--- +# Agent + +Research agent. +`, + }) + defer cleanupWorkspace(t, researchWorkspace) + + cfg := testCfg([]config.AgentConfig{ + {ID: "main", Default: true, Workspace: mainWorkspace}, + { + ID: "research", + Workspace: researchWorkspace, + }, + }) + cfg.Agents.Defaults.Workspace = mainWorkspace + cfg.Tools.ReadFile.Enabled = true + cfg.Tools.WriteFile.Enabled = true + cfg.Tools.ListDir.Enabled = true + cfg.Tools.Exec.Enabled = true + cfg.Tools.Message.Enabled = true + cfg.Tools.Web.Enabled = true + cfg.Tools.Web.DuckDuckGo.Enabled = true + cfg.Tools.WebFetch.Enabled = true + cfg.Tools.Spawn.Enabled = true + cfg.Tools.Subagent.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + research, ok := al.GetRegistry().GetAgent("research") + if !ok || research == nil { + t.Fatal("expected research agent") + } + + got := research.Tools.List() + want := []string{"message", "read_file", "web_fetch", "web_search", "write_file"} + if !slices.Equal(got, want) { + t.Fatalf("research tools = %v, want %v", got, want) + } + + for _, blocked := range []string{"exec", "list_dir", "spawn", "subagent"} { + if _, ok := research.Tools.Get(blocked); ok { + t.Fatalf("expected %q to be blocked by allowlist", blocked) + } + } +} + +func TestNewAgentLoop_AgentToolAllowlistRequiresExactRuntimeToolNames(t *testing.T) { + mainWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": "# Agent\nMain agent.\n", + }) + defer cleanupWorkspace(t, mainWorkspace) + + researchWorkspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +tools: [web] +--- +# Agent + +Research agent. +`, + }) + defer cleanupWorkspace(t, researchWorkspace) + + cfg := testCfg([]config.AgentConfig{ + {ID: "main", Default: true, Workspace: mainWorkspace}, + { + ID: "research", + Workspace: researchWorkspace, + }, + }) + cfg.Agents.Defaults.Workspace = mainWorkspace + cfg.Tools.Web.Enabled = true + cfg.Tools.Web.DuckDuckGo.Enabled = true + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockRegistryProvider{}) + defer al.Close() + + research, ok := al.GetRegistry().GetAgent("research") + if !ok || research == nil { + t.Fatal("expected research agent") + } + + if _, ok := research.Tools.Get("web_search"); ok { + t.Fatal("web_search should not be registered when allowlist contains only web") + } + if slices.Contains(research.Tools.List(), "web_search") { + t.Fatalf("research tools = %v, expected web_search to be absent", research.Tools.List()) } } diff --git a/pkg/agent/runtime_event_logger.go b/pkg/agent/runtime_event_logger.go new file mode 100644 index 000000000..1035ffe35 --- /dev/null +++ b/pkg/agent/runtime_event_logger.go @@ -0,0 +1,408 @@ +package agent + +import ( + "context" + "fmt" + "path" + "strings" + "sync" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" + "github.com/sipeed/picoclaw/pkg/logger" +) + +const ( + runtimeEventLoggerBuffer = 256 + runtimeEventLoggerDrainTimeout = 2 * time.Second +) + +type runtimeEventLogger struct { + mu sync.RWMutex + cfg config.EventLoggingConfig +} + +func (al *AgentLoop) refreshRuntimeEventLogger(cfg *config.Config) { + if al == nil { + return + } + logCfg := config.EffectiveEventLoggingConfig(cfg) + + al.runtimeEventLogMu.Lock() + if !logCfg.Enabled { + oldSub := al.runtimeEventLogSub + al.runtimeEventLogger = nil + al.runtimeEventLogSub = nil + al.runtimeEventLogMu.Unlock() + closeRuntimeEventLoggerSubscription(oldSub) + return + } + + if al.runtimeEventLogger != nil && al.runtimeEventLogSub != nil { + al.runtimeEventLogger.updateConfig(logCfg) + al.runtimeEventLogMu.Unlock() + return + } + al.runtimeEventLogMu.Unlock() + + eventLogger := newRuntimeEventLoggerFromConfig(logCfg) + sub, err := eventLogger.subscribe(context.Background(), al.runtimeEvents) + if err != nil { + logger.WarnCF("events", "Failed to subscribe runtime event logger", map[string]any{"error": err.Error()}) + return + } + + al.runtimeEventLogMu.Lock() + oldSub := al.runtimeEventLogSub + al.runtimeEventLogger = eventLogger + al.runtimeEventLogSub = sub + al.runtimeEventLogMu.Unlock() + closeRuntimeEventLoggerSubscription(oldSub) +} + +func (al *AgentLoop) closeRuntimeEventLogger() { + if al == nil { + return + } + al.runtimeEventLogMu.Lock() + oldSub := al.runtimeEventLogSub + al.runtimeEventLogger = nil + al.runtimeEventLogSub = nil + al.runtimeEventLogMu.Unlock() + closeRuntimeEventLoggerSubscription(oldSub) +} + +func closeRuntimeEventLoggerSubscription(sub runtimeevents.Subscription) { + if sub == nil { + return + } + if err := sub.Close(); err != nil { + logger.WarnCF("events", "Failed to close runtime event logger subscription", map[string]any{ + "error": err.Error(), + }) + } + + timer := time.NewTimer(runtimeEventLoggerDrainTimeout) + defer timer.Stop() + select { + case <-sub.Done(): + case <-timer.C: + logger.WarnCF("events", "Timed out waiting for runtime event logger to drain", map[string]any{ + "timeout": runtimeEventLoggerDrainTimeout.String(), + }) + } +} + +func newRuntimeEventLogger(cfg *config.Config) *runtimeEventLogger { + logCfg := config.EffectiveEventLoggingConfig(cfg) + if !logCfg.Enabled { + return nil + } + return newRuntimeEventLoggerFromConfig(logCfg) +} + +func newRuntimeEventLoggerFromConfig(logCfg config.EventLoggingConfig) *runtimeEventLogger { + return &runtimeEventLogger{cfg: logCfg} +} + +func (l *runtimeEventLogger) updateConfig(cfg config.EventLoggingConfig) { + if l == nil { + return + } + l.mu.Lock() + l.cfg = cfg + l.mu.Unlock() +} + +func (l *runtimeEventLogger) configSnapshot() config.EventLoggingConfig { + if l == nil { + return config.EventLoggingConfig{} + } + l.mu.RLock() + defer l.mu.RUnlock() + return l.cfg +} + +func (l *runtimeEventLogger) subscribe( + ctx context.Context, + eventBus runtimeevents.Bus, +) (runtimeevents.Subscription, error) { + if l == nil || eventBus == nil { + return nil, nil + } + return eventBus.Channel().Subscribe(ctx, runtimeevents.SubscribeOptions{ + Name: "runtime-event-logger", + Buffer: runtimeEventLoggerBuffer, + Concurrency: runtimeevents.Locked, + Backpressure: runtimeevents.DropNewest, + PanicPolicy: runtimeevents.RecoverAndLog, + }, l.handle) +} + +func (l *runtimeEventLogger) handle(_ context.Context, evt runtimeevents.Event) error { + if l == nil || !l.shouldLog(evt) { + return nil + } + + fields := runtimeEventLogFields(evt) + if l.configSnapshot().IncludePayload && evt.Payload != nil { + fields["payload"] = evt.Payload + } + + logRuntimeEvent(evt, fields) + return nil +} + +func (l *runtimeEventLogger) shouldLog(evt runtimeevents.Event) bool { + if l == nil { + return false + } + cfg := l.configSnapshot() + if !cfg.Enabled { + return false + } + if runtimeEventSeverityRank(evt.Severity) < runtimeEventSeverityRank(parseRuntimeEventSeverity(cfg.MinSeverity)) { + return false + } + + kind := evt.Kind.String() + if !matchAnyRuntimeEventPattern(cfg.Include, kind, true) { + return false + } + return !matchAnyRuntimeEventPattern(cfg.Exclude, kind, false) +} + +func logRuntimeEvent(evt runtimeevents.Event, fields map[string]any) { + message := fmt.Sprintf("Runtime event: %s", evt.Kind.String()) + switch normalizeRuntimeEventSeverity(evt.Severity) { + case runtimeevents.SeverityDebug: + logger.DebugCF("events", message, fields) + case runtimeevents.SeverityWarn: + logger.WarnCF("events", message, fields) + case runtimeevents.SeverityError: + logger.ErrorCF("events", message, fields) + default: + logger.InfoCF("events", message, fields) + } +} + +func runtimeEventLogFields(evt runtimeevents.Event) map[string]any { + fields := map[string]any{ + "event_id": evt.ID, + "event_kind": evt.Kind.String(), + "severity": string(normalizeRuntimeEventSeverity(evt.Severity)), + } + if !evt.Time.IsZero() { + fields["event_time"] = evt.Time.Format(time.RFC3339Nano) + } + appendRuntimeEventSourceFields(fields, evt.Source) + appendRuntimeEventScopeFields(fields, evt.Scope) + appendRuntimeEventCorrelationFields(fields, evt.Correlation) + appendRuntimeEventAttrs(fields, evt.Attrs) + appendRuntimeEventPayloadSummary(fields, evt.Payload) + return fields +} + +func appendRuntimeEventSourceFields(fields map[string]any, source runtimeevents.Source) { + if source.Component != "" { + fields["source_component"] = source.Component + } + if source.Name != "" { + fields["source_name"] = source.Name + } +} + +func appendRuntimeEventScopeFields(fields map[string]any, scope runtimeevents.Scope) { + setStringField(fields, "runtime_id", scope.RuntimeID) + setStringField(fields, "agent_id", scope.AgentID) + setStringField(fields, "session_key", scope.SessionKey) + setStringField(fields, "turn_id", scope.TurnID) + setStringField(fields, "channel", scope.Channel) + setStringField(fields, "account", scope.Account) + setStringField(fields, "chat_id", scope.ChatID) + setStringField(fields, "topic_id", scope.TopicID) + setStringField(fields, "space_id", scope.SpaceID) + setStringField(fields, "space_type", scope.SpaceType) + setStringField(fields, "chat_type", scope.ChatType) + setStringField(fields, "sender_id", scope.SenderID) + setStringField(fields, "message_id", scope.MessageID) +} + +func appendRuntimeEventCorrelationFields(fields map[string]any, correlation runtimeevents.Correlation) { + setStringField(fields, "trace_id", correlation.TraceID) + setStringField(fields, "parent_turn_id", correlation.ParentTurnID) + setStringField(fields, "request_id", correlation.RequestID) + setStringField(fields, "reply_to_id", correlation.ReplyToID) +} + +func appendRuntimeEventAttrs(fields map[string]any, attrs map[string]any) { + for key, value := range attrs { + if key == "" || value == nil { + continue + } + if _, exists := fields[key]; exists { + fields["attr_"+key] = value + continue + } + fields[key] = value + } +} + +func appendRuntimeEventPayloadSummary(fields map[string]any, payload any) { + switch payload := payload.(type) { + case TurnStartPayload: + fields["user_len"] = len(payload.UserMessage) + fields["media_count"] = payload.MediaCount + case TurnEndPayload: + fields["status"] = payload.Status + fields["iterations_total"] = payload.Iterations + fields["duration_ms"] = payload.Duration.Milliseconds() + fields["final_len"] = payload.FinalContentLen + case LLMRequestPayload: + fields["model"] = payload.Model + fields["messages"] = payload.MessagesCount + fields["tools"] = payload.ToolsCount + fields["max_tokens"] = payload.MaxTokens + case LLMDeltaPayload: + fields["content_delta_len"] = payload.ContentDeltaLen + fields["reasoning_delta_len"] = payload.ReasoningDeltaLen + case LLMResponsePayload: + fields["content_len"] = payload.ContentLen + fields["tool_calls"] = payload.ToolCalls + fields["has_reasoning"] = payload.HasReasoning + case LLMRetryPayload: + fields["attempt"] = payload.Attempt + fields["max_retries"] = payload.MaxRetries + fields["reason"] = payload.Reason + fields["error"] = payload.Error + fields["backoff_ms"] = payload.Backoff.Milliseconds() + case ContextCompressPayload: + fields["reason"] = payload.Reason + fields["dropped_messages"] = payload.DroppedMessages + fields["remaining_messages"] = payload.RemainingMessages + case SessionSummarizePayload: + fields["summarized_messages"] = payload.SummarizedMessages + fields["kept_messages"] = payload.KeptMessages + fields["summary_len"] = payload.SummaryLen + fields["omitted_oversized"] = payload.OmittedOversized + case ToolExecStartPayload: + fields["tool"] = payload.Tool + fields["args_count"] = len(payload.Arguments) + case ToolExecEndPayload: + fields["tool"] = payload.Tool + fields["duration_ms"] = payload.Duration.Milliseconds() + fields["for_llm_len"] = payload.ForLLMLen + fields["for_user_len"] = payload.ForUserLen + fields["is_error"] = payload.IsError + fields["async"] = payload.Async + case ToolExecSkippedPayload: + fields["tool"] = payload.Tool + fields["reason"] = payload.Reason + case SteeringInjectedPayload: + fields["count"] = payload.Count + fields["total_content_len"] = payload.TotalContentLen + case FollowUpQueuedPayload: + fields["source_tool"] = payload.SourceTool + fields["content_len"] = payload.ContentLen + case InterruptReceivedPayload: + fields["interrupt_kind"] = payload.Kind + fields["role"] = payload.Role + fields["content_len"] = payload.ContentLen + fields["queue_depth"] = payload.QueueDepth + fields["hint_len"] = payload.HintLen + case SubTurnSpawnPayload: + fields["child_agent_id"] = payload.AgentID + fields["label"] = payload.Label + case SubTurnEndPayload: + fields["child_agent_id"] = payload.AgentID + fields["status"] = payload.Status + case SubTurnResultDeliveredPayload: + fields["target_channel"] = payload.TargetChannel + fields["target_chat_id"] = payload.TargetChatID + fields["content_len"] = payload.ContentLen + case SubTurnOrphanPayload: + fields["parent_turn_id"] = payload.ParentTurnID + fields["child_turn_id"] = payload.ChildTurnID + fields["reason"] = payload.Reason + case ErrorPayload: + fields["stage"] = payload.Stage + fields["error"] = payload.Message + } +} + +func setStringField(fields map[string]any, key, value string) { + if value != "" { + fields[key] = value + } +} + +func matchAnyRuntimeEventPattern(patterns []string, kind string, emptyMatches bool) bool { + if len(patterns) == 0 { + return emptyMatches + } + for _, pattern := range patterns { + if matchRuntimeEventPattern(pattern, kind) { + return true + } + } + return false +} + +func matchRuntimeEventPattern(pattern, kind string) bool { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + return false + } + if pattern == "*" { + return true + } + if strings.HasSuffix(pattern, ".*") { + return strings.HasPrefix(kind, strings.TrimSuffix(pattern, "*")) + } + matched, err := path.Match(pattern, kind) + if err == nil { + return matched + } + return pattern == kind +} + +func parseRuntimeEventSeverity(severity string) runtimeevents.Severity { + switch strings.ToLower(strings.TrimSpace(severity)) { + case "debug": + return runtimeevents.SeverityDebug + case "warn", "warning": + return runtimeevents.SeverityWarn + case "error": + return runtimeevents.SeverityError + default: + return runtimeevents.SeverityInfo + } +} + +func normalizeRuntimeEventSeverity(severity runtimeevents.Severity) runtimeevents.Severity { + switch severity { + case runtimeevents.SeverityDebug, + runtimeevents.SeverityInfo, + runtimeevents.SeverityWarn, + runtimeevents.SeverityError: + return severity + default: + return runtimeevents.SeverityInfo + } +} + +func runtimeEventSeverityRank(severity runtimeevents.Severity) int { + switch normalizeRuntimeEventSeverity(severity) { + case runtimeevents.SeverityDebug: + return 0 + case runtimeevents.SeverityInfo: + return 1 + case runtimeevents.SeverityWarn: + return 2 + case runtimeevents.SeverityError: + return 3 + default: + return 1 + } +} diff --git a/pkg/agent/runtime_event_logger_test.go b/pkg/agent/runtime_event_logger_test.go new file mode 100644 index 000000000..1c95b365c --- /dev/null +++ b/pkg/agent/runtime_event_logger_test.go @@ -0,0 +1,259 @@ +package agent + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +func TestRuntimeEventLoggerFiltering(t *testing.T) { + cfg := config.DefaultConfig() + eventLogger := newRuntimeEventLogger(cfg) + if eventLogger == nil { + t.Fatal("default runtime event logger is nil") + } + + if !eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindAgentTurnStart, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("default config should log agent events") + } + if eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindChannelLifecycleStarted, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("default config should not log non-agent events") + } + + cfg.Events.Logging.Include = []string{"*"} + cfg.Events.Logging.Exclude = []string{"mcp.*"} + eventLogger = newRuntimeEventLogger(cfg) + if !eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindGatewayReady, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("include * should log gateway events") + } + if eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindMCPServerConnected, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("exclude mcp.* should suppress MCP events") + } + + cfg.Events.Logging.Exclude = nil + cfg.Events.Logging.MinSeverity = "warn" + eventLogger = newRuntimeEventLogger(cfg) + if eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindGatewayReady, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("min severity warn should suppress info events") + } + if !eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindGatewayReloadFailed, + Severity: runtimeevents.SeverityError, + }) { + t.Fatal("min severity warn should allow error events") + } + + cfg.Events.Logging.Enabled = false + if newRuntimeEventLogger(cfg) != nil { + t.Fatal("disabled config should not create runtime event logger") + } +} + +func TestRuntimeEventLogFieldsSummarizeAgentPayload(t *testing.T) { + fields := runtimeEventLogFields(runtimeevents.Event{ + ID: "evt-test", + Kind: runtimeevents.KindAgentToolExecStart, + Severity: runtimeevents.SeverityInfo, + Source: runtimeevents.Source{ + Component: "agent", + Name: "main", + }, + Scope: runtimeevents.Scope{ + AgentID: "main", + SessionKey: "session-1", + TurnID: "turn-1", + }, + Payload: ToolExecStartPayload{ + Tool: "exec", + Arguments: map[string]any{ + "secret": "should-not-be-logged-by-default", + }, + }, + }) + + if fields["event_id"] != "evt-test" || fields["source_component"] != "agent" { + t.Fatalf("missing common event fields: %#v", fields) + } + if fields["tool"] != "exec" || fields["args_count"] != 1 { + t.Fatalf("missing safe agent payload summary fields: %#v", fields) + } + if _, ok := fields["payload"]; ok { + t.Fatalf("raw payload should not be included by runtimeEventLogFields: %#v", fields) + } +} + +func TestRuntimeEventLogFieldsIncludeSafeAttrs(t *testing.T) { + fields := runtimeEventLogFields(runtimeevents.Event{ + ID: "evt-gateway", + Kind: runtimeevents.KindGatewayReady, + Severity: runtimeevents.SeverityInfo, + Attrs: map[string]any{ + "duration_ms": 42, + "error": "startup failed", + "event_kind": "conflict", + }, + }) + + if fields["duration_ms"] != 42 || fields["error"] != "startup failed" { + t.Fatalf("missing safe attrs: %#v", fields) + } + if fields["event_kind"] != runtimeevents.KindGatewayReady.String() { + t.Fatalf("event_kind overwritten by attrs: %#v", fields) + } + if fields["attr_event_kind"] != "conflict" { + t.Fatalf("conflicting attr not preserved with prefix: %#v", fields) + } + if _, ok := fields["payload"]; ok { + t.Fatalf("raw payload should not be included by runtimeEventLogFields: %#v", fields) + } +} + +func runtimeEventLoggerStateForTest( + al *AgentLoop, +) (*runtimeEventLogger, runtimeevents.Subscription) { + al.runtimeEventLogMu.RLock() + defer al.runtimeEventLogMu.RUnlock() + return al.runtimeEventLogger, al.runtimeEventLogSub +} + +func TestReloadProviderAndConfigRefreshesRuntimeEventLogger(t *testing.T) { + cfg := config.DefaultConfig() + cfg.Agents.Defaults.Workspace = t.TempDir() + cfg.Events.Logging.Include = []string{"agent.*"} + + al := NewAgentLoop(cfg, bus.NewMessageBus(), &mockProvider{}) + defer al.Close() + + eventLogger, logSub := runtimeEventLoggerStateForTest(al) + if eventLogger == nil || logSub == nil { + t.Fatal("expected initial runtime event logger subscription") + } + if eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindGatewayReloadCompleted, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("initial agent-only logging should not log gateway reload events") + } + + reloaded := config.DefaultConfig() + reloaded.Agents.Defaults.Workspace = cfg.Agents.Defaults.Workspace + reloaded.Events.Logging.Include = []string{"gateway.*"} + if err := al.ReloadProviderAndConfig(context.Background(), &mockProvider{}, reloaded); err != nil { + t.Fatalf("ReloadProviderAndConfig() error = %v", err) + } + + eventLogger, logSub = runtimeEventLoggerStateForTest(al) + if eventLogger == nil || logSub == nil { + t.Fatal("expected runtime event logger subscription after reload") + } + if !eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindGatewayReloadCompleted, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("reloaded gateway logging should log gateway reload events") + } + if eventLogger.shouldLog(runtimeevents.Event{ + Kind: runtimeevents.KindAgentTurnStart, + Severity: runtimeevents.SeverityInfo, + }) { + t.Fatal("reloaded gateway-only logging should not log agent events") + } + + disabled := config.DefaultConfig() + disabled.Agents.Defaults.Workspace = cfg.Agents.Defaults.Workspace + disabled.Events.Logging.Enabled = false + if err := al.ReloadProviderAndConfig(context.Background(), &mockProvider{}, disabled); err != nil { + t.Fatalf("ReloadProviderAndConfig() with disabled logging error = %v", err) + } + eventLogger, logSub = runtimeEventLoggerStateForTest(al) + if eventLogger != nil || logSub != nil { + t.Fatal("expected runtime event logger to be disabled after reload") + } +} + +func TestCloseRuntimeEventLoggerSubscriptionWaitsForDrain(t *testing.T) { + eventBus := runtimeevents.NewBus() + defer func() { + if err := eventBus.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + }() + + var handled atomic.Uint64 + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + sub, err := eventBus.Channel().Subscribe( + context.Background(), + runtimeevents.SubscribeOptions{ + Name: "runtime-event-logger", + Buffer: 2, + Concurrency: runtimeevents.Locked, + }, + func(context.Context, runtimeevents.Event) error { + if handled.Add(1) == 1 { + close(firstStarted) + <-releaseFirst + } + return nil + }, + ) + if err != nil { + t.Fatalf("Subscribe failed: %v", err) + } + + first := eventBus.Publish(context.Background(), runtimeevents.Event{Kind: runtimeevents.Kind("test.first")}) + if first.Delivered != 1 { + t.Fatalf("first Publish = %+v, want one delivered event", first) + } + select { + case <-firstStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for first handler to start") + } + second := eventBus.Publish(context.Background(), runtimeevents.Event{Kind: runtimeevents.Kind("test.second")}) + if second.Delivered != 1 { + t.Fatalf("second Publish = %+v, want one delivered event", second) + } + + closeReturned := make(chan struct{}) + go func() { + closeRuntimeEventLoggerSubscription(sub) + close(closeReturned) + }() + + select { + case <-closeReturned: + t.Fatal("runtime event logger close returned before buffered events drained") + case <-time.After(50 * time.Millisecond): + } + + close(releaseFirst) + select { + case <-closeReturned: + case <-time.After(time.Second): + t.Fatal("timed out waiting for runtime event logger close to return") + } + if got := handled.Load(); got != 2 { + t.Fatalf("handled = %d, want 2", got) + } +} diff --git a/pkg/agent/runtime_event_test.go b/pkg/agent/runtime_event_test.go new file mode 100644 index 000000000..162ccf424 --- /dev/null +++ b/pkg/agent/runtime_event_test.go @@ -0,0 +1,103 @@ +package agent + +import ( + "testing" + "time" + + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +func subscribeRuntimeEventsForTest( + t *testing.T, + al *AgentLoop, + buffer int, + kinds ...runtimeevents.Kind, +) (<-chan runtimeevents.Event, func()) { + t.Helper() + + if al == nil { + t.Fatal("agent loop is nil") + } + channel := al.RuntimeEvents() + if channel == nil { + t.Fatal("runtime event channel is nil") + } + if len(kinds) > 0 { + channel = channel.OfKind(kinds...) + } + sub, ch, err := channel.SubscribeChan( + t.Context(), + runtimeevents.SubscribeOptions{Name: "agent-runtime-test", Buffer: buffer}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + return ch, func() { + if err := sub.Close(); err != nil { + t.Errorf("runtime subscription close failed: %v", err) + } + } +} + +func waitForRuntimeEvent( + t *testing.T, + ch <-chan runtimeevents.Event, + timeout time.Duration, + match func(runtimeevents.Event) bool, +) runtimeevents.Event { + t.Helper() + + timer := time.NewTimer(timeout) + defer timer.Stop() + + for { + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("runtime event stream closed before expected event arrived") + } + if match(evt) { + return evt + } + case <-timer.C: + t.Fatal("timed out waiting for expected runtime event") + } + } +} + +func collectRuntimeEventStream(ch <-chan runtimeevents.Event) []runtimeevents.Event { + var events []runtimeevents.Event + for { + select { + case evt, ok := <-ch: + if !ok { + return events + } + events = append(events, evt) + default: + return events + } + } +} + +func findRuntimeEvent( + events []runtimeevents.Event, + kind runtimeevents.Kind, +) (runtimeevents.Event, bool) { + for _, evt := range events { + if evt.Kind == kind { + return evt, true + } + } + return runtimeevents.Event{}, false +} + +func filterRuntimeEvents(events []runtimeevents.Event, kind runtimeevents.Kind) []runtimeevents.Event { + var filtered []runtimeevents.Event + for _, evt := range events { + if evt.Kind == kind { + filtered = append(filtered, evt) + } + } + return filtered +} diff --git a/pkg/agent/steering.go b/pkg/agent/steering.go index 2efa7bbf4..7bddbfc31 100644 --- a/pkg/agent/steering.go +++ b/pkg/agent/steering.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/sipeed/picoclaw/pkg/bus" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/session" @@ -155,6 +156,18 @@ func (sq *steeringQueue) lenScope(scope string) int { return len(sq.queues[normalizeSteeringScope(scope)]) } +func (sq *steeringQueue) clearScope(scope string) int { + sq.mu.Lock() + defer sq.mu.Unlock() + + scope = normalizeSteeringScope(scope) + count := len(sq.queues[scope]) + if count > 0 { + delete(sq.queues, scope) + } + return count +} + // setMode updates the steering mode. func (sq *steeringQueue) setMode(mode SteeringMode) { sq.mu.Lock() @@ -206,7 +219,7 @@ func (al *AgentLoop) enqueueSteeringMessage(scope, agentID string, msg providers "scope": normalizeSteeringScope(scope), }) - meta := EventMeta{ + meta := HookMeta{ Source: "Steer", TracePath: "turn.interrupt.received", } @@ -230,7 +243,7 @@ func (al *AgentLoop) enqueueSteeringMessage(scope, agentID string, msg providers } al.emitEvent( - EventKindInterruptReceived, + runtimeevents.KindAgentInterruptReceived, meta, InterruptReceivedPayload{ Kind: InterruptKindSteering, @@ -289,6 +302,13 @@ func (al *AgentLoop) pendingSteeringCountForScope(scope string) int { return al.steering.lenScope(scope) } +func (al *AgentLoop) clearSteeringMessagesForScope(scope string) int { + if al.steering == nil { + return 0 + } + return al.steering.clearScope(scope) +} + func (al *AgentLoop) continueWithSteeringMessages( ctx context.Context, agent *AgentInstance, @@ -410,7 +430,7 @@ func (al *AgentLoop) InterruptGraceful(hint string) error { } al.emitEvent( - EventKindInterruptReceived, + runtimeevents.KindAgentInterruptReceived, ts.eventMeta("InterruptGraceful", "turn.interrupt.received"), InterruptReceivedPayload{ Kind: InterruptKindGraceful, @@ -438,7 +458,7 @@ func (al *AgentLoop) InterruptHard() error { } al.emitEvent( - EventKindInterruptReceived, + runtimeevents.KindAgentInterruptReceived, ts.eventMeta("InterruptHard", "turn.interrupt.received"), InterruptReceivedPayload{ Kind: InterruptKindHard, @@ -510,6 +530,10 @@ func (al *AgentLoop) HardAbort(sessionKey string) error { "initial_history_length": ts.initialHistoryLength, }) + // Cancel the active provider/tool turn contexts immediately so long-running + // execution stops as soon as possible on the root turn. + _ = ts.requestHardAbort() + // IMPORTANT: Trigger cascading cancellation FIRST to stop all child SubTurns // from adding more messages to the session. This prevents race conditions // where rollback happens while children are still writing. diff --git a/pkg/agent/steering_test.go b/pkg/agent/steering_test.go index 1ff699976..23d34840e 100644 --- a/pkg/agent/steering_test.go +++ b/pkg/agent/steering_test.go @@ -12,8 +12,10 @@ import ( "testing" "time" + "github.com/sipeed/picoclaw/pkg/audio/asr" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/routing" @@ -476,6 +478,16 @@ func (p *lateSteeringProvider) GetDefaultModel() string { return "late-steering-mock" } +type fixedTranscriber struct { + text string +} + +func (f *fixedTranscriber) Name() string { return "fixed" } + +func (f *fixedTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*asr.TranscriptionResponse, error) { + return &asr.TranscriptionResponse{Text: f.text}, nil +} + type blockingDirectProvider struct { mu sync.Mutex calls int @@ -839,6 +851,307 @@ func TestAgentLoop_Run_AutoContinuesLateSteeringMessage(t *testing.T) { } } +func TestAgentLoop_Run_QueuedVoiceMessageIsTranscribedBeforeSteering(t *testing.T) { + tmpDir := t.TempDir() + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + }, + }, + } + + msgBus := bus.NewMessageBus() + provider := &lateSteeringProvider{ + firstCallStarted: make(chan struct{}), + releaseFirstCall: make(chan struct{}), + } + al := NewAgentLoop(cfg, msgBus, provider) + + store := media.NewFileMediaStore() + audioPath := filepath.Join(tmpDir, "voice.ogg") + if err := os.WriteFile(audioPath, []byte("fake audio"), 0o644); err != nil { + t.Fatalf("write audio fixture: %v", err) + } + ref, err := store.Store(audioPath, media.MediaMeta{ + Filename: "voice.ogg", + ContentType: "audio/ogg", + CleanupPolicy: media.CleanupPolicyForgetOnly, + }, "scope-voice") + if err != nil { + t.Fatalf("store audio fixture: %v", err) + } + al.SetMediaStore(store) + al.SetTranscriber(&fixedTranscriber{text: "and also two pieces of bread"}) + + runCtx, cancelRun := context.WithCancel(context.Background()) + defer cancelRun() + + runErrCh := make(chan error, 1) + go func() { + runErrCh <- al.Run(runCtx) + }() + + first := bus.InboundMessage{ + Context: bus.InboundContext{ + Channel: "test", + ChatID: "chat1", + ChatType: "direct", + SenderID: "user1", + }, + Content: "first meal", + } + late := bus.InboundMessage{ + Context: bus.InboundContext{ + Channel: "test", + ChatID: "chat1", + ChatType: "direct", + SenderID: "user1", + }, + Content: "[voice]", + Media: []string{ref}, + } + + pubCtx, pubCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer pubCancel() + if err := msgBus.PublishInbound(pubCtx, first); err != nil { + t.Fatalf("publish first inbound: %v", err) + } + + select { + case <-provider.firstCallStarted: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for first provider call to start") + } + + if err := msgBus.PublishInbound(pubCtx, late); err != nil { + t.Fatalf("publish late voice inbound: %v", err) + } + + close(provider.releaseFirstCall) + + subCtx, subCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer subCancel() + select { + case <-msgBus.OutboundChan(): + case <-subCtx.Done(): + t.Fatal("expected outbound response") + } + + cancelRun() + select { + case err := <-runErrCh: + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for Run to stop") + } + + provider.mu.Lock() + secondMessages := append([]providers.Message(nil), provider.secondCallMessages...) + provider.mu.Unlock() + + foundTranscribedVoice := false + for _, msg := range secondMessages { + if msg.Role == "user" && strings.Contains(msg.Content, "[voice: and also two pieces of bread]") { + foundTranscribedVoice = true + break + } + } + if !foundTranscribedVoice { + t.Fatalf("expected queued voice message to be transcribed before steering injection, got %#v", secondMessages) + } +} + +func TestAgentLoop_Run_PendingStopStillContinuesQueuedFollowUp(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "agent-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + MaxParallelTurns: 1, + }, + }, + } + + msgBus := bus.NewMessageBus() + provider := &lateSteeringProvider{ + firstCallStarted: make(chan struct{}), + releaseFirstCall: make(chan struct{}), + } + al := NewAgentLoop(cfg, msgBus, provider) + + runCtx, cancelRun := context.WithCancel(context.Background()) + defer cancelRun() + + runErrCh := make(chan error, 1) + go func() { + runErrCh <- al.Run(runCtx) + }() + defer func() { + cancelRun() + select { + case err := <-runErrCh: + if err != nil { + t.Fatalf("Run() error = %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for Run to stop") + } + }() + + blockerSessionKey := session.BuildOpaqueSessionKey("agent:main:test:blocker") + targetSessionKey := session.BuildOpaqueSessionKey("agent:main:test:target") + blockerCtx := bus.InboundContext{ + Channel: "test", + ChatID: "blocker-chat", + ChatType: "direct", + SenderID: "user1", + } + targetCtx := bus.InboundContext{ + Channel: "test", + ChatID: "target-chat", + ChatType: "direct", + SenderID: "user1", + } + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: blockerCtx, + Content: "block worker pool", + SessionKey: blockerSessionKey, + }); err != nil { + t.Fatalf("PublishInbound(blocker) error = %v", err) + } + + select { + case <-provider.firstCallStarted: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for blocker turn to start") + } + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: targetCtx, + Content: "skip this turn", + SessionKey: targetSessionKey, + }); err != nil { + t.Fatalf("PublishInbound(target start) error = %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for { + ts := al.getActiveTurnState(targetSessionKey) + if ts != nil && strings.HasPrefix(ts.turnID, pendingTurnPrefix) { + break + } + if time.Now().After(deadline) { + t.Fatal("timeout waiting for pending placeholder") + } + time.Sleep(10 * time.Millisecond) + } + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: targetCtx, + Content: "/stop", + SessionKey: targetSessionKey, + }); err != nil { + t.Fatalf("PublishInbound(/stop) error = %v", err) + } + + deadline = time.Now().Add(2 * time.Second) + stopSeen := false + for !stopSeen { + select { + case outbound := <-msgBus.OutboundChan(): + if outbound.ChatID == "target-chat" && outbound.Content == "Task stopped. Current task was canceled." { + stopSeen = true + } + case <-time.After(10 * time.Millisecond): + if time.Now().After(deadline) { + t.Fatal("timeout waiting for /stop reply") + } + } + } + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: targetCtx, + Content: "run this instead", + SessionKey: targetSessionKey, + }); err != nil { + t.Fatalf("PublishInbound(follow-up) error = %v", err) + } + + deadline = time.Now().Add(2 * time.Second) + for al.pendingSteeringCountForScope(targetSessionKey) == 0 { + if time.Now().After(deadline) { + t.Fatal("timeout waiting for follow-up to enter scoped steering queue") + } + time.Sleep(10 * time.Millisecond) + } + + close(provider.releaseFirstCall) + + deadline = time.Now().Add(5 * time.Second) + followUpSeen := false + for !followUpSeen { + select { + case outbound := <-msgBus.OutboundChan(): + if outbound.ChatID == "target-chat" && outbound.Content == "continued response" { + followUpSeen = true + } + case <-time.After(10 * time.Millisecond): + if time.Now().After(deadline) { + t.Fatal("timeout waiting for queued follow-up continuation") + } + } + } + + deadline = time.Now().Add(2 * time.Second) + for { + if al.GetActiveTurnBySession(targetSessionKey) == nil && + al.pendingSteeringCountForScope(targetSessionKey) == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("timeout waiting for target session to go idle") + } + time.Sleep(10 * time.Millisecond) + } + + provider.mu.Lock() + calls := provider.calls + secondMessages := append([]providers.Message(nil), provider.secondCallMessages...) + provider.mu.Unlock() + + if calls != 2 { + t.Fatalf("expected 2 provider calls (blocker + continuation), got %d", calls) + } + + foundFollowUp := false + for _, msg := range secondMessages { + if msg.Role == "user" && msg.Content == "run this instead" { + foundFollowUp = true + } + if msg.Role == "user" && msg.Content == "skip this turn" { + t.Fatalf("unexpected canceled message in continuation context: %q", msg.Content) + } + } + if !foundFollowUp { + t.Fatal("expected queued follow-up to be processed after pending stop") + } +} + func TestAgentLoop_Steering_DirectResponseContinuesWithQueuedMessage(t *testing.T) { tmpDir, err := os.MkdirTemp("", "agent-test-*") if err != nil { @@ -1134,8 +1447,14 @@ func TestAgentLoop_InterruptGraceful_UsesTerminalNoToolCall(t *testing.T) { al.RegisterTool(tool2) sessionKey := session.BuildMainSessionKey(routing.DefaultAgentID) - sub := al.SubscribeEvents(32) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 32, + runtimeevents.KindAgentInterruptReceived, + runtimeevents.KindAgentTurnEnd, + ) + defer closeRuntimeEvents() type result struct { resp string @@ -1222,8 +1541,8 @@ func TestAgentLoop_InterruptGraceful_UsesTerminalNoToolCall(t *testing.T) { t.Fatal("expected remaining tool to be marked as skipped after graceful interrupt") } - events := collectEventStream(sub.C) - interruptEvt, ok := findEvent(events, EventKindInterruptReceived) + events := collectRuntimeEventStream(runtimeCh) + interruptEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentInterruptReceived) if !ok { t.Fatal("expected interrupt received event") } @@ -1235,7 +1554,7 @@ func TestAgentLoop_InterruptGraceful_UsesTerminalNoToolCall(t *testing.T) { t.Fatalf("expected graceful interrupt payload, got %q", interruptPayload.Kind) } - turnEndEvt, ok := findEvent(events, EventKindTurnEnd) + turnEndEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentTurnEnd) if !ok { t.Fatal("expected turn end event") } @@ -1299,8 +1618,14 @@ func TestAgentLoop_InterruptHard_RestoresSession(t *testing.T) { } defaultAgent.Sessions.SetHistory(sessionKey, originalHistory) - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentInterruptReceived, + runtimeevents.KindAgentTurnEnd, + ) + defer closeRuntimeEvents() type result struct { resp string @@ -1353,8 +1678,8 @@ func TestAgentLoop_InterruptHard_RestoresSession(t *testing.T) { t.Fatalf("expected history rollback after hard abort, got %#v", finalHistory) } - events := collectEventStream(sub.C) - interruptEvt, ok := findEvent(events, EventKindInterruptReceived) + events := collectRuntimeEventStream(runtimeCh) + interruptEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentInterruptReceived) if !ok { t.Fatal("expected interrupt received event") } @@ -1366,7 +1691,7 @@ func TestAgentLoop_InterruptHard_RestoresSession(t *testing.T) { t.Fatalf("expected hard interrupt payload, got %q", interruptPayload.Kind) } - turnEndEvt, ok := findEvent(events, EventKindTurnEnd) + turnEndEvt, ok := findRuntimeEvent(events, runtimeevents.KindAgentTurnEnd) if !ok { t.Fatal("expected turn end event") } @@ -1379,6 +1704,149 @@ func TestAgentLoop_InterruptHard_RestoresSession(t *testing.T) { } } +func TestAgentLoop_StopCommand_AbortsActiveTurnAndClearsQueuedSteering(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "agent-test-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + }, + }, + } + + msgBus := bus.NewMessageBus() + provider := &toolCallProvider{ + toolCalls: []providers.ToolCall{ + { + ID: "call_1", + Type: "function", + Name: "cancel_tool", + Function: &providers.FunctionCall{ + Name: "cancel_tool", + Arguments: "{}", + }, + Arguments: map[string]any{}, + }, + }, + finalResp: "should not continue", + } + + al := NewAgentLoop(cfg, msgBus, provider) + started := make(chan struct{}) + al.RegisterTool(&interruptibleTool{name: "cancel_tool", started: started}) + sessionKey := session.BuildMainSessionKey(routing.DefaultAgentID) + + runCtx, cancelRun := context.WithCancel(context.Background()) + defer cancelRun() + + runErrCh := make(chan error, 1) + go func() { + runErrCh <- al.Run(runCtx) + }() + defer func() { + cancelRun() + select { + case err := <-runErrCh: + if err != nil { + t.Fatalf("Run() error = %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for Run to stop") + } + }() + + baseMsg := testInboundMessage(bus.InboundMessage{ + Context: bus.InboundContext{ + Channel: "test", + ChatID: "chat1", + ChatType: "direct", + SenderID: "user1", + }, + SessionKey: sessionKey, + }) + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: baseMsg.Context, + Content: "do work", + SessionKey: sessionKey, + }); err != nil { + t.Fatalf("PublishInbound(start) error = %v", err) + } + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for interruptible tool to start") + } + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: baseMsg.Context, + Content: "follow up after cancel", + SessionKey: sessionKey, + }); err != nil { + t.Fatalf("PublishInbound(follow-up) error = %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for al.pendingSteeringCountForScope(sessionKey) == 0 { + if time.Now().After(deadline) { + t.Fatal("timeout waiting for follow-up message to enter steering queue") + } + time.Sleep(10 * time.Millisecond) + } + + if err := msgBus.PublishInbound(context.Background(), bus.InboundMessage{ + Context: baseMsg.Context, + Content: "/stop", + SessionKey: sessionKey, + }); err != nil { + t.Fatalf("PublishInbound(/stop) error = %v", err) + } + + select { + case outbound := <-msgBus.OutboundChan(): + want := "Task stopped. \"do work\" was canceled." + if outbound.Content != want { + t.Fatalf("stop reply = %q, want %q", outbound.Content, want) + } + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for /stop reply") + } + + deadline = time.Now().Add(5 * time.Second) + for al.GetActiveTurnBySession(sessionKey) != nil { + if time.Now().After(deadline) { + t.Fatal("timeout waiting for active turn to stop") + } + time.Sleep(10 * time.Millisecond) + } + + if got := al.pendingSteeringCountForScope(sessionKey); got != 0 { + t.Fatalf("expected cleared steering queue, got %d pending message(s)", got) + } + + select { + case outbound := <-msgBus.OutboundChan(): + t.Fatalf("unexpected outbound after stop: %q", outbound.Content) + case <-time.After(300 * time.Millisecond): + } + + provider.mu.Lock() + calls := provider.calls + provider.mu.Unlock() + if calls != 1 { + t.Fatalf("expected provider to stop before follow-up turn, got %d calls", calls) + } +} + // capturingMockProvider captures messages sent to Chat for inspection. type capturingMockProvider struct { response string diff --git a/pkg/agent/subturn.go b/pkg/agent/subturn.go index 4d824bd3a..86617d02f 100644 --- a/pkg/agent/subturn.go +++ b/pkg/agent/subturn.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/providers/messageutil" @@ -173,7 +174,10 @@ type SubTurnConfig struct { // Used by team tool to enforce token limits across all team members. InitialTokenBudget *atomic.Int64 - // Can be extended with temperature, topP, etc. + // TargetAgentID, when set, runs the sub-turn as the specified agent. + // The target agent's workspace, model, tools, and system prompt are used + // instead of the caller's. If empty, the sub-turn runs as the parent agent. + TargetAgentID string } // ====================== Context Keys ====================== @@ -231,6 +235,7 @@ func (s *AgentLoopSpawner) SpawnSubTurn( Critical: cfg.Critical, Timeout: cfg.Timeout, MaxContextRunes: cfg.MaxContextRunes, + TargetAgentID: cfg.TargetAgentID, } return spawnSubTurn(ctx, s.al, parentTS, agentCfg) @@ -313,8 +318,9 @@ func spawnSubTurn( return nil, ErrDepthLimitExceeded } - // 2. Config validation - if cfg.Model == "" { + // 2. Config validation: Model is required unless TargetAgentID is set + // (the target agent provides its own model). + if cfg.Model == "" && cfg.TargetAgentID == "" { return nil, ErrInvalidSubTurnConfig } @@ -332,12 +338,22 @@ func spawnSubTurn( childID := al.generateSubTurnID() - // Get the agent instance from parent, falling back to the default agent. - // Wrap it in a shallow copy that uses an ephemeral (in-memory only) session store - // so that child turns never pollute or persist to the parent's session history. - baseAgent := parentTS.agent - if baseAgent == nil { - baseAgent = al.registry.GetDefaultAgent() + // Resolve the agent instance for the child turn. + // When TargetAgentID is set, look up that agent from the registry so the + // child runs with the target's workspace, model, tools, and system prompt. + // Otherwise fall back to the parent's agent (existing behavior). + var baseAgent *AgentInstance + if cfg.TargetAgentID != "" { + var ok bool + baseAgent, ok = al.registry.GetAgent(cfg.TargetAgentID) + if !ok { + return nil, fmt.Errorf("target agent %q not found in registry", cfg.TargetAgentID) + } + } else { + baseAgent = parentTS.agent + if baseAgent == nil { + baseAgent = al.registry.GetDefaultAgent() + } } if baseAgent == nil { return nil, errors.New("parent turnState has no agent instance") @@ -422,7 +438,7 @@ func spawnSubTurn( parentTS.mu.Unlock() // 6. Emit Spawn event - al.emitEvent(EventKindSubTurnSpawn, + al.emitEvent(runtimeevents.KindAgentSubTurnSpawn, childTS.eventMeta("spawnSubTurn", "subturn.spawn"), SubTurnSpawnPayload{ AgentID: childTS.agentID, @@ -453,7 +469,7 @@ func spawnSubTurn( if err != nil { status = "error" } - al.emitEvent(EventKindSubTurnEnd, + al.emitEvent(runtimeevents.KindAgentSubTurnEnd, childTS.eventMeta("spawnSubTurn", "subturn.end"), SubTurnEndPayload{ AgentID: childTS.agentID, @@ -504,16 +520,16 @@ func spawnSubTurn( // // Delivery behavior: // - If parent turn is still running: attempts to deliver to pendingResults channel -// - If channel is full: emits SubTurnOrphanResultEvent (result is lost from channel but tracked) -// - If parent turn has finished: emits SubTurnOrphanResultEvent (late arrival) +// - If channel is full: emits agent.subturn.orphan (result is lost from channel but tracked) +// - If parent turn has finished: emits agent.subturn.orphan (late arrival) // // Thread safety: // - Reads parent state under lock, then releases lock before channel send // - Small race window exists but is acceptable (worst case: result becomes orphan) // // Event emissions: -// - SubTurnResultDeliveredEvent: successful delivery to channel -// - SubTurnOrphanResultEvent: delivery failed (parent finished or channel full) +// - agent.subturn.result_delivered: successful delivery to channel +// - agent.subturn.orphan: delivery failed (parent finished or channel full) func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, result *tools.ToolResult) { // Let GC clean up the pendingResults channel; parent Finish will no longer close it. // We use defer/recover to catch any unlikely channel panics if it were ever closed. @@ -526,7 +542,7 @@ func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, re "recover": r, }) if result != nil && al != nil { - al.emitEvent(EventKindSubTurnOrphan, + al.emitEvent(runtimeevents.KindAgentSubTurnOrphan, parentTS.eventMeta("deliverSubTurnResult", "subturn.orphan"), SubTurnOrphanPayload{ParentTurnID: parentTS.turnID, ChildTurnID: childID, Reason: "panic"}, ) @@ -541,7 +557,7 @@ func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, re // If parent turn has already finished, treat this as an orphan result if isFinished || resultChan == nil { if result != nil && al != nil { - al.emitEvent(EventKindSubTurnOrphan, + al.emitEvent(runtimeevents.KindAgentSubTurnOrphan, parentTS.eventMeta("deliverSubTurnResult", "subturn.orphan"), SubTurnOrphanPayload{ParentTurnID: parentTS.turnID, ChildTurnID: childID, Reason: "parent_finished"}, ) @@ -557,7 +573,7 @@ func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, re case resultChan <- result: // Successfully delivered if al != nil { - al.emitEvent(EventKindSubTurnResultDelivered, + al.emitEvent(runtimeevents.KindAgentSubTurnResultDelivered, parentTS.eventMeta("deliverSubTurnResult", "subturn.result_delivered"), SubTurnResultDeliveredPayload{ContentLen: len(result.ForLLM)}, ) @@ -571,7 +587,7 @@ func deliverSubTurnResult(al *AgentLoop, parentTS *turnState, childID string, re }) if result != nil && al != nil { al.emitEvent( - EventKindSubTurnOrphan, + runtimeevents.KindAgentSubTurnOrphan, parentTS.eventMeta("deliverSubTurnResult", "subturn.orphan"), SubTurnOrphanPayload{ ParentTurnID: parentTS.turnID, diff --git a/pkg/agent/subturn_test.go b/pkg/agent/subturn_test.go index 040063249..e9f557c82 100644 --- a/pkg/agent/subturn_test.go +++ b/pkg/agent/subturn_test.go @@ -4,12 +4,16 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" + "strings" "sync" "testing" "time" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/providers" "github.com/sipeed/picoclaw/pkg/tools" ) @@ -22,30 +26,38 @@ const ( // ====================== Test Helper: Event Collector ====================== type eventCollector struct { mu sync.Mutex - events []Event + events []runtimeevents.Event } func newEventCollector(t *testing.T, al *AgentLoop) (*eventCollector, func()) { t.Helper() c := &eventCollector{} - sub := al.SubscribeEvents(16) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentSubTurnSpawn, + runtimeevents.KindAgentSubTurnEnd, + runtimeevents.KindAgentSubTurnResultDelivered, + runtimeevents.KindAgentSubTurnOrphan, + ) done := make(chan struct{}) go func() { defer close(done) - for evt := range sub.C { + for evt := range runtimeCh { c.mu.Lock() c.events = append(c.events, evt) c.mu.Unlock() } }() cleanup := func() { - al.UnsubscribeEvents(sub.ID) + closeRuntimeEvents() <-done } return c, cleanup } -func (c *eventCollector) hasEventOfKind(kind EventKind) bool { +func (c *eventCollector) hasEventOfKind(kind runtimeevents.Kind) bool { c.mu.Lock() defer c.mu.Unlock() for _, e := range c.events { @@ -131,7 +143,7 @@ func TestSpawnSubTurn(t *testing.T) { agent: al.registry.GetDefaultAgent(), } - // Subscribe to real EventBus to capture events + // Subscribe to runtime events to capture sub-turn lifecycle. collector, collectCleanup := newEventCollector(t, al) defer collectCleanup() @@ -158,12 +170,12 @@ func TestSpawnSubTurn(t *testing.T) { // Verify event emission time.Sleep(10 * time.Millisecond) // let event goroutine flush if tt.wantSpawn { - if !collector.hasEventOfKind(EventKindSubTurnSpawn) { + if !collector.hasEventOfKind(runtimeevents.KindAgentSubTurnSpawn) { t.Error("SubTurnSpawnEvent not emitted") } } if tt.wantEnd { - if !collector.hasEventOfKind(EventKindSubTurnEnd) { + if !collector.hasEventOfKind(runtimeevents.KindAgentSubTurnEnd) { t.Error("SubTurnEndEvent not emitted") } } @@ -316,8 +328,8 @@ func TestSpawnSubTurn_OrphanResultRouting(t *testing.T) { time.Sleep(10 * time.Millisecond) // let event goroutine flush // Verify Orphan event is emitted - if !collector.hasEventOfKind(EventKindSubTurnOrphan) { - t.Error("SubTurnOrphanResultEvent not emitted for finished parent") + if !collector.hasEventOfKind(runtimeevents.KindAgentSubTurnOrphan) { + t.Error("agent.subturn.orphan not emitted for finished parent") } // Verify history is NOT polluted @@ -591,12 +603,16 @@ func TestNestedSubTurnHierarchy(t *testing.T) { var spawnedTurns []turnInfo var mu sync.Mutex - // Subscribe to real EventBus to capture spawn events - sub := al.SubscribeEvents(16) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 16, + runtimeevents.KindAgentSubTurnSpawn, + ) + defer closeRuntimeEvents() go func() { - for evt := range sub.C { - if evt.Kind == EventKindSubTurnSpawn { + for evt := range runtimeCh { + if evt.Kind == runtimeevents.KindAgentSubTurnSpawn { p, _ := evt.Payload.(SubTurnSpawnPayload) mu.Lock() spawnedTurns = append(spawnedTurns, turnInfo{ @@ -879,7 +895,7 @@ func TestSpawnSubTurn_PanicRecovery(t *testing.T) { time.Sleep(10 * time.Millisecond) // let event goroutine flush // SubTurnEndEvent should still be emitted - if !collector.hasEventOfKind(EventKindSubTurnEnd) { + if !collector.hasEventOfKind(runtimeevents.KindAgentSubTurnEnd) { t.Error("SubTurnEndEvent not emitted after panic") } @@ -1229,18 +1245,23 @@ func TestDeliverSubTurnResult_RaceWithFinish(t *testing.T) { al, _, _, _, cleanup := newTestAgentLoop(t) //nolint:dogsled defer cleanup() - // Collect events via real EventBus var mu sync.Mutex var deliveredCount, orphanCount int - sub := al.SubscribeEvents(64) - defer al.UnsubscribeEvents(sub.ID) + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 64, + runtimeevents.KindAgentSubTurnResultDelivered, + runtimeevents.KindAgentSubTurnOrphan, + ) + defer closeRuntimeEvents() go func() { - for evt := range sub.C { + for evt := range runtimeCh { mu.Lock() switch evt.Kind { - case EventKindSubTurnResultDelivered: + case runtimeevents.KindAgentSubTurnResultDelivered: deliveredCount++ - case EventKindSubTurnOrphan: + case runtimeevents.KindAgentSubTurnOrphan: orphanCount++ } mu.Unlock() @@ -1795,13 +1816,20 @@ func TestAsyncSubTurn_ParentFinishesEarly(t *testing.T) { provider := &slowMockProvider{delay: 5 * time.Second} // SubTurn takes 5 seconds al := NewAgentLoop(cfg, msgBus, provider) - // Capture events via real EventBus var mu sync.Mutex - var events []Event - sub := al.SubscribeEvents(32) - defer al.UnsubscribeEvents(sub.ID) + var events []runtimeevents.Event + runtimeCh, closeRuntimeEvents := subscribeRuntimeEventsForTest( + t, + al, + 32, + runtimeevents.KindAgentSubTurnSpawn, + runtimeevents.KindAgentSubTurnEnd, + runtimeevents.KindAgentSubTurnResultDelivered, + runtimeevents.KindAgentSubTurnOrphan, + ) + defer closeRuntimeEvents() go func() { - for evt := range sub.C { + for evt := range runtimeCh { mu.Lock() events = append(events, evt) mu.Unlock() @@ -2097,3 +2125,206 @@ func TestSubTurn_IndependentContext(t *testing.T) { t.Log("✓ SubTurn completed successfully (independent context)") } } + +// ====================== TargetAgentID Tests ====================== + +// modelRecordingProvider captures the model passed to Chat for test assertions. +type modelRecordingProvider struct { + mu sync.Mutex + lastModel string +} + +func (rp *modelRecordingProvider) Chat( + _ context.Context, + _ []providers.Message, + _ []providers.ToolDefinition, + model string, + _ map[string]any, +) (*providers.LLMResponse, error) { + rp.mu.Lock() + rp.lastModel = model + rp.mu.Unlock() + return &providers.LLMResponse{Content: "Mock response"}, nil +} + +func (rp *modelRecordingProvider) GetDefaultModel() string { return "mock-model" } + +func (rp *modelRecordingProvider) getLastModel() string { + rp.mu.Lock() + defer rp.mu.Unlock() + return rp.lastModel +} + +// newMultiAgentLoop creates an AgentLoop with two named agents for testing +// cross-agent delegation via TargetAgentID. +func newMultiAgentLoop(t *testing.T, provider providers.LLMProvider) (*AgentLoop, func()) { + t.Helper() + tmpDir, err := os.MkdirTemp("", "multiagent-test-*") + if err != nil { + t.Fatalf("create temp dir: %v", err) + } + + alphaDir := filepath.Join(tmpDir, "alpha") + betaDir := filepath.Join(tmpDir, "beta") + os.MkdirAll(alphaDir, 0o755) + os.MkdirAll(betaDir, 0o755) + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "default-model", + MaxTokens: 4096, + MaxToolIterations: 10, + }, + List: []config.AgentConfig{ + { + ID: "alpha", + Workspace: alphaDir, + Model: &config.AgentModelConfig{Primary: "model-alpha"}, + }, + { + ID: "beta", + Workspace: betaDir, + Model: &config.AgentModelConfig{Primary: "model-beta"}, + }, + }, + }, + } + + msgBus := bus.NewMessageBus() + al := NewAgentLoop(cfg, msgBus, provider) + + return al, func() { os.RemoveAll(tmpDir) } +} + +func TestSpawnSubTurn_TargetAgentID_UsesTargetAgent(t *testing.T) { + rp := &modelRecordingProvider{} + al, cleanup := newMultiAgentLoop(t, rp) + defer cleanup() + + alphaAgent, ok := al.registry.GetAgent("alpha") + if !ok { + t.Fatal("alpha agent not in registry") + } + + // Parent is alpha, target is beta + parent := &turnState{ + ctx: context.Background(), + turnID: "parent-alpha", + depth: 0, + childTurnIDs: []string{}, + pendingResults: make(chan *tools.ToolResult, 4), + concurrencySem: make(chan struct{}, testMaxConcurrentSubTurns), + session: &ephemeralSessionStore{}, + agent: alphaAgent, + } + + result, err := spawnSubTurn(context.Background(), al, parent, SubTurnConfig{ + TargetAgentID: "beta", + SystemPrompt: "task for beta", + }) + if err != nil { + t.Fatalf("spawnSubTurn failed: %v", err) + } + if result == nil { + t.Fatal("expected non-nil result") + } + + // The recording provider captures the model passed to Chat(). + // If TargetAgentID works correctly, the child turn should have + // used beta's model, not alpha's. + if got := rp.getLastModel(); got != "model-beta" { + t.Errorf("child turn used model %q, want %q", got, "model-beta") + } +} + +func TestSpawnSubTurn_TargetAgentID_NotFound(t *testing.T) { + al, cleanup := newMultiAgentLoop(t, &mockProvider{}) + defer cleanup() + + alphaAgent, _ := al.registry.GetAgent("alpha") + parent := &turnState{ + ctx: context.Background(), + turnID: "parent-alpha", + depth: 0, + childTurnIDs: []string{}, + pendingResults: make(chan *tools.ToolResult, 4), + concurrencySem: make(chan struct{}, testMaxConcurrentSubTurns), + session: &ephemeralSessionStore{}, + agent: alphaAgent, + } + + _, err := spawnSubTurn(context.Background(), al, parent, SubTurnConfig{ + TargetAgentID: "nonexistent", + SystemPrompt: "task", + }) + + if err == nil { + t.Fatal("expected error for nonexistent agent") + } + if !strings.Contains(err.Error(), "not found") { + t.Errorf("error should mention 'not found', got: %v", err) + } +} + +func TestSpawnSubTurn_TargetAgentID_EmptyModelAccepted(t *testing.T) { + al, cleanup := newMultiAgentLoop(t, &mockProvider{}) + defer cleanup() + + alphaAgent, _ := al.registry.GetAgent("alpha") + parent := &turnState{ + ctx: context.Background(), + turnID: "parent-alpha", + depth: 0, + childTurnIDs: []string{}, + pendingResults: make(chan *tools.ToolResult, 4), + concurrencySem: make(chan struct{}, testMaxConcurrentSubTurns), + session: &ephemeralSessionStore{}, + agent: alphaAgent, + } + + // Model is empty but TargetAgentID is set — should NOT fail validation + result, err := spawnSubTurn(context.Background(), al, parent, SubTurnConfig{ + Model: "", // intentionally empty + TargetAgentID: "beta", + SystemPrompt: "task for beta", + }) + if err != nil { + t.Fatalf("should accept empty Model when TargetAgentID is set, got: %v", err) + } + if result == nil { + t.Fatal("expected non-nil result") + } +} + +func TestDelegateToolNotRegistered_SingleAgent(t *testing.T) { + // Single-agent setup: delegate should not be registered + al, _, _, provider, cleanup := newTestAgentLoop(t) + _ = provider + defer cleanup() + + agent := al.registry.GetDefaultAgent() + if agent == nil { + t.Fatal("default agent should exist") + } + if _, has := agent.Tools.Get("delegate"); has { + t.Error("delegate tool should not be registered in single-agent setup") + } +} + +func TestDelegateToolRegistered_MultiAgent(t *testing.T) { + al, cleanup := newMultiAgentLoop(t, &mockProvider{}) + defer cleanup() + + // Both agents should have the delegate tool + for _, id := range []string{"alpha", "beta"} { + agent, ok := al.registry.GetAgent(id) + if !ok { + t.Fatalf("agent %q not found", id) + } + if _, has := agent.Tools.Get("delegate"); !has { + t.Errorf("agent %q should have delegate tool in multi-agent setup", id) + } + } +} diff --git a/pkg/agent/tool_allowlist.go b/pkg/agent/tool_allowlist.go new file mode 100644 index 000000000..962f7ec05 --- /dev/null +++ b/pkg/agent/tool_allowlist.go @@ -0,0 +1,203 @@ +package agent + +import ( + "sort" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/tools" +) + +const dynamicMCPToolPrefix = "mcp_" + +func normalizeMCPServerName(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +func normalizedMCPServerNameSet( + servers map[string]config.MCPServerConfig, +) map[string]struct{} { + normalized := make(map[string]struct{}, len(servers)) + for serverName := range servers { + name := normalizeMCPServerName(serverName) + if name == "" { + continue + } + normalized[name] = struct{}{} + } + return normalized +} + +func warnOnUnknownAgentToolDeclarations( + agentID, workspace string, + definition AgentContextDefinition, + registry *tools.ToolRegistry, +) { + if registry == nil || frontmatterParseFailed(definition) { + return + } + + if unknownTools := unknownAgentToolNames(registry, definition); len(unknownTools) > 0 { + logger.WarnCF("agent", "AGENT.md declares unregistered tool names", + map[string]any{ + "agent_id": agentID, + "workspace": workspace, + "tools": unknownTools, + }) + } +} + +func warnOnUnknownAgentMCPServerDeclarations( + agentID, workspace string, + cfg *config.Config, + definition AgentContextDefinition, +) { + if cfg == nil || frontmatterParseFailed(definition) { + return + } + + if unknownServers := unknownAgentMCPServerNames(cfg, definition); len(unknownServers) > 0 { + logger.WarnCF("agent", "AGENT.md declares unknown MCP server names", + map[string]any{ + "agent_id": agentID, + "workspace": workspace, + "mcp_servers": unknownServers, + }) + } +} + +func unknownAgentToolNames( + registry *tools.ToolRegistry, + definition AgentContextDefinition, +) []string { + if definition.Agent == nil || definition.Agent.Frontmatter.Tools == nil { + return nil + } + + known := registeredRuntimeToolNames(registry) + unknown := make(map[string]struct{}) + for _, raw := range definition.Agent.Frontmatter.Tools { + name := strings.ToLower(strings.TrimSpace(raw)) + if name == "" || strings.HasPrefix(name, dynamicMCPToolPrefix) { + continue + } + if _, ok := known[name]; ok { + continue + } + unknown[name] = struct{}{} + } + + return sortedKeys(unknown) +} + +func registeredRuntimeToolNames(registry *tools.ToolRegistry) map[string]struct{} { + known := make(map[string]struct{}) + if registry == nil { + return known + } + for _, raw := range registry.List() { + name := strings.ToLower(strings.TrimSpace(raw)) + if name == "" { + continue + } + known[name] = struct{}{} + } + return known +} + +func unknownAgentMCPServerNames(cfg *config.Config, definition AgentContextDefinition) []string { + if cfg == nil || definition.Agent == nil || definition.Agent.Frontmatter.MCPServers == nil { + return nil + } + + knownServers := normalizedMCPServerNameSet(cfg.Tools.MCP.Servers) + unknown := make(map[string]struct{}) + for _, raw := range definition.Agent.Frontmatter.MCPServers { + name := normalizeMCPServerName(raw) + if name == "" { + continue + } + if _, ok := knownServers[name]; ok { + continue + } + unknown[name] = struct{}{} + } + + return sortedKeys(unknown) +} + +func sortedKeys(values map[string]struct{}) []string { + if len(values) == 0 { + return nil + } + + result := make([]string, 0, len(values)) + for value := range values { + result = append(result, value) + } + sort.Strings(result) + return result +} + +func resolveAgentToolAllowlist(definition AgentContextDefinition) []string { + if frontmatterParseFailed(definition) { + return []string{} + } + if definition.Agent == nil || !frontmatterDeclaresField(definition, "tools") { + return nil + } + + allowlist := make(map[string]struct{}, len(definition.Agent.Frontmatter.Tools)) + for _, raw := range definition.Agent.Frontmatter.Tools { + trimmed := strings.ToLower(strings.TrimSpace(raw)) + if trimmed == "" { + continue + } + allowlist[trimmed] = struct{}{} + } + + if len(allowlist) == 0 { + return []string{} + } + + return sortedKeys(allowlist) +} + +func resolveAgentMCPServerAllowlist(definition AgentContextDefinition) map[string]struct{} { + if frontmatterParseFailed(definition) { + return map[string]struct{}{} + } + if definition.Agent == nil || !frontmatterDeclaresField(definition, "mcpServers") { + return nil + } + + allowlist := make(map[string]struct{}, len(definition.Agent.Frontmatter.MCPServers)) + for _, raw := range definition.Agent.Frontmatter.MCPServers { + trimmed := strings.ToLower(strings.TrimSpace(raw)) + if trimmed == "" { + continue + } + allowlist[trimmed] = struct{}{} + } + + return allowlist +} + +func frontmatterDeclaresField(definition AgentContextDefinition, field string) bool { + if definition.Agent == nil || definition.Agent.Frontmatter.Fields == nil { + return false + } + _, ok := definition.Agent.Frontmatter.Fields[field] + return ok +} + +func frontmatterParseFailed(definition AgentContextDefinition) bool { + if definition.Agent == nil { + return false + } + if strings.TrimSpace(definition.Agent.RawFrontmatter) == "" { + return false + } + return strings.TrimSpace(definition.Agent.FrontmatterErr) != "" +} diff --git a/pkg/agent/tool_allowlist_test.go b/pkg/agent/tool_allowlist_test.go new file mode 100644 index 000000000..5ed35d4c6 --- /dev/null +++ b/pkg/agent/tool_allowlist_test.go @@ -0,0 +1,184 @@ +package agent + +import ( + "context" + "testing" + + "github.com/sipeed/picoclaw/pkg/config" + agenttools "github.com/sipeed/picoclaw/pkg/tools" +) + +type allowlistTestTool struct { + name string +} + +func (t *allowlistTestTool) Name() string { return t.name } + +func (t *allowlistTestTool) Description() string { return "test tool" } + +func (t *allowlistTestTool) Parameters() map[string]any { + return map[string]any{"type": "object"} +} + +func (t *allowlistTestTool) Execute( + _ context.Context, + _ map[string]any, +) *agenttools.ToolResult { + return agenttools.NewToolResult("ok") +} + +func TestUnknownAgentToolNames(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +tools: [read_file, web_serach, mcp_github_search] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + registry := agenttools.NewToolRegistry() + registry.Register(&allowlistTestTool{name: "read_file"}) + registry.Register(&allowlistTestTool{name: "web_search"}) + + unknown := unknownAgentToolNames(registry, loadAgentDefinition(workspace)) + if len(unknown) != 1 || unknown[0] != "web_serach" { + t.Fatalf("unknownAgentToolNames() = %v, want [web_serach]", unknown) + } +} + +func TestUnknownAgentToolNamesUsesRegisteredRuntimeTools(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +tools: [serial, reaction, send_tts, load_image, delegate, made_up] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + registry := agenttools.NewToolRegistry() + for _, name := range []string{"serial", "reaction", "send_tts", "load_image", "delegate"} { + registry.Register(&allowlistTestTool{name: name}) + } + + unknown := unknownAgentToolNames(registry, loadAgentDefinition(workspace)) + if len(unknown) != 1 || unknown[0] != "made_up" { + t.Fatalf("unknownAgentToolNames() = %v, want [made_up]", unknown) + } +} + +func TestResolveAgentToolAllowlistDistinguishesMissingAndEmptyToolsField(t *testing.T) { + tests := []struct { + name string + agentMD string + wantNil bool + wantEmpty bool + }{ + { + name: "missing tools field allows all tools", + agentMD: `--- +name: pico +--- +# Agent +`, + wantNil: true, + }, + { + name: "explicit empty tools list blocks all tools", + agentMD: `--- +tools: [] +--- +# Agent +`, + wantEmpty: true, + }, + { + name: "blank tools field blocks all tools", + agentMD: `--- +tools: +--- +# Agent +`, + wantEmpty: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": tt.agentMD, + }) + defer cleanupWorkspace(t, workspace) + + allowlist := resolveAgentToolAllowlist(loadAgentDefinition(workspace)) + + if tt.wantNil { + if allowlist != nil { + t.Fatalf("resolveAgentToolAllowlist() = %v, want nil", allowlist) + } + return + } + + if allowlist == nil { + t.Fatal("resolveAgentToolAllowlist() = nil, want explicit empty allowlist") + } + if len(allowlist) != 0 { + t.Fatalf("resolveAgentToolAllowlist() = %v, want empty allowlist", allowlist) + } + }) + } +} + +func TestUnknownAgentMCPServerNames(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +mcpServers: [github, githb] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Tools: config.ToolsConfig{ + MCP: config.MCPConfig{ + Servers: map[string]config.MCPServerConfig{ + "github": {Enabled: true}, + }, + }, + }, + } + + unknown := unknownAgentMCPServerNames(cfg, loadAgentDefinition(workspace)) + if len(unknown) != 1 || unknown[0] != "githb" { + t.Fatalf("unknownAgentMCPServerNames() = %v, want [githb]", unknown) + } +} + +func TestUnknownAgentMCPServerNamesMatchesConfigCaseInsensitively(t *testing.T) { + workspace := setupWorkspace(t, map[string]string{ + "AGENT.md": `--- +mcpServers: [github, FileSystem, slak] +--- +# Agent +`, + }) + defer cleanupWorkspace(t, workspace) + + cfg := &config.Config{ + Tools: config.ToolsConfig{ + MCP: config.MCPConfig{ + Servers: map[string]config.MCPServerConfig{ + "GitHub": {Enabled: true}, + "filesystem": {Enabled: true}, + }, + }, + }, + } + + unknown := unknownAgentMCPServerNames(cfg, loadAgentDefinition(workspace)) + if len(unknown) != 1 || unknown[0] != "slak" { + t.Fatalf("unknownAgentMCPServerNames() = %v, want [slak]", unknown) + } +} diff --git a/pkg/agent/turn_context.go b/pkg/agent/turn_context.go index 8913993aa..c675590ce 100644 --- a/pkg/agent/turn_context.go +++ b/pkg/agent/turn_context.go @@ -61,7 +61,7 @@ func cloneStringMap(src map[string]string) map[string]string { return cloned } -func cloneEventMeta(meta EventMeta) EventMeta { +func cloneHookMeta(meta HookMeta) HookMeta { meta.turnContext = cloneTurnContext(meta.turnContext) return meta } diff --git a/pkg/agent/turn_coord.go b/pkg/agent/turn_coord.go index ade2b7c21..060346339 100644 --- a/pkg/agent/turn_coord.go +++ b/pkg/agent/turn_coord.go @@ -9,6 +9,7 @@ import ( "time" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" ) @@ -25,22 +26,50 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel al.registerActiveTurn(ts) defer al.clearActiveTurn(ts) + if al.takePendingStop(ts.sessionKey) { + _ = ts.requestHardAbort() + } + turnStatus := TurnEndStatusCompleted defer func() { + attemptedSkills := ts.attemptedSkillsSnapshot() + skillContextSnapshots := ts.skillContextSnapshotsSnapshot() + finalSuccessfulPath := []string(nil) + if turnStatus == TurnEndStatusCompleted { + if latest := ts.latestSkillContextSnapshot(); len(latest) > 0 { + finalSuccessfulPath = latest + } else { + finalSuccessfulPath = append([]string(nil), attemptedSkills...) + } + } al.emitEvent( - EventKindTurnEnd, + runtimeevents.KindAgentTurnEnd, ts.eventMeta("runTurn", "turn.end"), TurnEndPayload{ - Status: turnStatus, - Iterations: ts.currentIteration(), - Duration: time.Since(ts.startedAt), - FinalContentLen: ts.finalContentLen(), + Status: turnStatus, + Workspace: ts.workspace, + Iterations: ts.currentIteration(), + Duration: time.Since(ts.startedAt), + FinalContentLen: ts.finalContentLen(), + UserMessage: ts.userMessage, + FinalContent: ts.finalContentSnapshot(), + ActiveSkills: append([]string(nil), ts.activeSkills...), + AttemptedSkills: attemptedSkills, + FinalSuccessfulPath: finalSuccessfulPath, + SkillContextSnapshots: skillContextSnapshots, + ToolKinds: ts.toolKindsSnapshot(), + ToolExecutions: ts.toolExecutionsSnapshot(), }, ) }() + if ts.hardAbortRequested() { + turnStatus = TurnEndStatusAborted + return al.abortTurn(ts) + } + al.emitEvent( - EventKindTurnStart, + runtimeevents.KindAgentTurnStart, ts.eventMeta("runTurn", "turn.start"), TurnStartPayload{ UserMessage: ts.userMessage, @@ -140,7 +169,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel }) } al.emitEvent( - EventKindSteeringInjected, + runtimeevents.KindAgentSteeringInjected, ts.eventMeta("runTurn", "turn.steering.injected"), SteeringInjectedPayload{ Count: len(pendingMessages), @@ -190,7 +219,11 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel if finalContent == "" { finalContent = ts.opts.DefaultResponse } - return pipeline.Finalize(ctx, turnCtx, ts, exec, turnStatus, finalContent) + result, finalizeErr := pipeline.Finalize(ctx, turnCtx, ts, exec, turnStatus, finalContent) + if finalizeErr != nil { + turnStatus = TurnEndStatusError + } + return result, finalizeErr case ControlToolLoop: // Execute tools via Pipeline toolCtrl := pipeline.ExecuteTools(ctx, turnCtx, ts, exec, iteration) @@ -217,7 +250,11 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel if exec.allResponsesHandled { finalContent = "" } - return pipeline.Finalize(ctx, turnCtx, ts, exec, turnStatus, finalContent) + result, finalizeErr := pipeline.Finalize(ctx, turnCtx, ts, exec, turnStatus, finalContent) + if finalizeErr != nil { + turnStatus = TurnEndStatusError + } + return result, finalizeErr } } } @@ -241,7 +278,11 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel return al.abortTurn(ts) } - return pipeline.Finalize(ctx, turnCtx, ts, exec, turnStatus, finalContent) + result, err := pipeline.Finalize(ctx, turnCtx, ts, exec, turnStatus, finalContent) + if err != nil { + turnStatus = TurnEndStatusError + } + return result, err } func (al *AgentLoop) abortTurn(ts *turnState) (turnResult, error) { @@ -249,7 +290,7 @@ func (al *AgentLoop) abortTurn(ts *turnState) (turnResult, error) { if !ts.opts.NoHistory { if err := ts.restoreSession(ts.agent); err != nil { al.emitEvent( - EventKindError, + runtimeevents.KindAgentError, ts.eventMeta("abortTurn", "turn.error"), ErrorPayload{ Stage: "session_restore", @@ -414,7 +455,7 @@ func (al *AgentLoop) askSideQuestion( llmModel := activeModel if al.hooks != nil { llmReq, decision := al.hooks.BeforeLLM(ctx, &LLMHookRequest{ - Meta: EventMeta{ + Meta: HookMeta{ Source: "askSideQuestion", TracePath: "turn.llm.request", turnContext: cloneTurnContext(turnCtx), @@ -494,8 +535,8 @@ func (al *AgentLoop) askSideQuestion( resp, err = callSideLLM(messages) if err != nil && hasMediaRefs(messages) && isVisionUnsupportedError(err) { al.emitEvent( - EventKindLLMRetry, - EventMeta{ + runtimeevents.KindAgentLLMRetry, + HookMeta{ Source: "askSideQuestion", TracePath: "turn.llm.retry", turnContext: cloneTurnContext(turnCtx), @@ -521,7 +562,7 @@ func (al *AgentLoop) askSideQuestion( // Apply after_llm hooks if al.hooks != nil { llmResp, decision := al.hooks.AfterLLM(ctx, &LLMHookResponse{ - Meta: EventMeta{ + Meta: HookMeta{ Source: "askSideQuestion", TracePath: "turn.llm.response", turnContext: cloneTurnContext(turnCtx), diff --git a/pkg/agent/turn_coord_test.go b/pkg/agent/turn_coord_test.go index c059d0a39..c7cdd8a32 100644 --- a/pkg/agent/turn_coord_test.go +++ b/pkg/agent/turn_coord_test.go @@ -10,6 +10,7 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/session" ) // ============================================================================= @@ -135,6 +136,16 @@ func (p *errorProvider) Chat( return nil, errors.New("context_length_exceeded") case "vision": return nil, errors.New("vision_unsupported") + case "connection_reset": + return nil, errors.New("connection reset by peer") + case "broken_pipe": + return nil, errors.New("broken pipe") + case "read_tcp": + return nil, errors.New("read tcp 127.0.0.1:8080: connection reset") + case "eof": + return nil, errors.New("EOF") + case "connection_refused": + return nil, errors.New("connection refused") default: return nil, errors.New("unknown error") } @@ -188,6 +199,15 @@ func makeTestProcessOpts(sessionKey string) processOptions { } } +type saveFailingSessionStore struct { + session.SessionStore + err error +} + +func (s *saveFailingSessionStore) Save(_ string) error { + return s.err +} + // ============================================================================= // Pipeline Method Tests: SetupTurn // ============================================================================= @@ -251,6 +271,44 @@ func TestPipeline_CallLLM_SimpleResponse(t *testing.T) { } } +func TestRunTurn_FinalizeSaveErrorEmitsErrorTurnEnd(t *testing.T) { + al, agent, cleanup := newTurnCoordTestLoop(t, &simpleConvProvider{}) + defer cleanup() + + saveErr := errors.New("session save failed") + agent.Sessions = &saveFailingSessionStore{ + SessionStore: session.NewSessionManager(""), + err: saveErr, + } + + sub := al.SubscribeEvents(8) + defer al.UnsubscribeEvents(sub.ID) + + if _, err := al.ProcessDirect(context.Background(), "hello", "session-save-fail"); err == nil { + t.Fatal("expected ProcessDirect to fail") + } + + deadline := time.After(2 * time.Second) + for { + select { + case evt := <-sub.C: + if evt.Kind != EventKindTurnEnd { + continue + } + payload, ok := evt.Payload.(TurnEndPayload) + if !ok { + t.Fatalf("TurnEnd payload type = %T", evt.Payload) + } + if payload.Status != TurnEndStatusError { + t.Fatalf("TurnEnd status = %q, want %q", payload.Status, TurnEndStatusError) + } + return + case <-deadline: + t.Fatal("timed out waiting for turn_end event") + } + } +} + func TestPipeline_CallLLM_WithToolCall(t *testing.T) { provider := &toolCallRespProvider{ toolName: "web_search", @@ -366,6 +424,163 @@ func TestPipeline_CallLLM_ContextLengthError(t *testing.T) { t.Logf("CallLLM result after context error: err=%v", err) } +func TestPipeline_CallLLM_NetworkErrorRetry(t *testing.T) { + testCases := []struct { + name string + errType string + }{ + {"connection_reset", "connection_reset"}, + {"broken_pipe", "broken_pipe"}, + {"read_tcp", "read_tcp"}, + {"eof", "eof"}, + {"connection_refused", "connection_refused"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + errorPrv := &errorProvider{errType: tc.errType} + al, agent, cleanup := newTurnCoordTestLoop(t, errorPrv) + defer cleanup() + + pipeline := NewPipeline(al) + ts := newTurnState(agent, makeTestProcessOpts("test-session"), turnEventScope{ + turnID: "turn-1", + context: newTurnContext(nil, nil, nil), + }) + + exec, err := pipeline.SetupTurn(context.Background(), ts) + if err != nil { + t.Fatalf("SetupTurn failed: %v", err) + } + + _, err = pipeline.CallLLM(context.Background(), context.Background(), ts, exec, 1) + if err == nil { + t.Error("expected error after network error retries") + } + }) + } +} + +func TestPipeline_CallLLM_RetryConfigRespected(t *testing.T) { + tmpDir := t.TempDir() + + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + MaxLLMRetries: 3, + LLMRetryBackoffSecs: 1, + }, + }, + } + + msgBus := bus.NewMessageBus() + provider := &errorProvider{errType: "connection_reset"} + al := NewAgentLoop(cfg, msgBus, provider) + defer al.Close() + agent := al.registry.GetDefaultAgent() + if agent == nil { + t.Fatal("expected default agent") + } + + pipeline := NewPipeline(al) + ts := newTurnState(agent, makeTestProcessOpts("test-session"), turnEventScope{ + turnID: "turn-1", + context: newTurnContext(nil, nil, nil), + }) + + exec, err := pipeline.SetupTurn(context.Background(), ts) + if err != nil { + t.Fatalf("SetupTurn failed: %v", err) + } + + start := time.Now() + _, err = pipeline.CallLLM(context.Background(), context.Background(), ts, exec, 1) + elapsed := time.Since(start) + + if err == nil { + t.Error("expected error after retries") + } + + expectedMinTime := 3 * time.Second + if elapsed < expectedMinTime { + t.Errorf("expected at least %v of backoff, got %v", expectedMinTime, elapsed) + } +} + +func TestPipeline_CallLLM_RetryCountLimit(t *testing.T) { + tmpDir := t.TempDir() + + counterPrv := &countingErrorProvider{errType: "connection_reset", targetCalls: 5} + cfg := &config.Config{ + Agents: config.AgentsConfig{ + Defaults: config.AgentDefaults{ + Workspace: tmpDir, + ModelName: "test-model", + MaxTokens: 4096, + MaxToolIterations: 10, + MaxLLMRetries: 2, + LLMRetryBackoffSecs: 0, + }, + }, + } + + msgBus := bus.NewMessageBus() + al := NewAgentLoop(cfg, msgBus, counterPrv) + defer al.Close() + agent := al.registry.GetDefaultAgent() + if agent == nil { + t.Fatal("expected default agent") + } + + pipeline := NewPipeline(al) + ts := newTurnState(agent, makeTestProcessOpts("test-session"), turnEventScope{ + turnID: "turn-1", + context: newTurnContext(nil, nil, nil), + }) + + exec, err := pipeline.SetupTurn(context.Background(), ts) + if err != nil { + t.Fatalf("SetupTurn failed: %v", err) + } + + _, err = pipeline.CallLLM(context.Background(), context.Background(), ts, exec, 1) + if err == nil { + t.Error("expected error after retries") + } + + if counterPrv.callCount != 3 { + t.Errorf("expected exactly 3 calls (1 initial + 2 retries), got %d", counterPrv.callCount) + } +} + +type countingErrorProvider struct { + errType string + targetCalls int + callCount int + mu sync.Mutex +} + +func (p *countingErrorProvider) Chat( + ctx context.Context, + messages []providers.Message, + tools []providers.ToolDefinition, + model string, + opts map[string]any, +) (*providers.LLMResponse, error) { + p.mu.Lock() + p.callCount++ + p.mu.Unlock() + return nil, errors.New("connection reset by peer") +} + +func (p *countingErrorProvider) GetDefaultModel() string { + return "counting-error-model" +} + // ============================================================================= // Pipeline Method Tests: ExecuteTools // ============================================================================= @@ -613,3 +828,30 @@ func TestTurnState_HardAbortRequested(t *testing.T) { t.Error("expected hard abort to be requested") } } + +func TestTurnState_SkillContextSnapshotsTrackLatestSuccessfulPath(t *testing.T) { + ts := &turnState{} + + ts.recordSkillContextSnapshot(skillContextTriggerInitialBuild, []string{"skill-a"}) + ts.recordSkillContextSnapshot(skillContextTriggerContextRetryRebuild, []string{"skill-b", "skill-c"}) + + if got := ts.attemptedSkillsSnapshot(); len(got) != 3 || got[0] != "skill-a" || got[1] != "skill-b" || + got[2] != "skill-c" { + t.Fatalf("attemptedSkillsSnapshot = %v, want [skill-a skill-b skill-c]", got) + } + + if got := ts.latestSkillContextSnapshot(); len(got) != 2 || got[0] != "skill-b" || got[1] != "skill-c" { + t.Fatalf("latestSkillContextSnapshot = %v, want [skill-b skill-c]", got) + } + + snapshots := ts.skillContextSnapshotsSnapshot() + if len(snapshots) != 2 { + t.Fatalf("len(skillContextSnapshotsSnapshot()) = %d, want 2", len(snapshots)) + } + if snapshots[0].Sequence != 1 || snapshots[0].Trigger != skillContextTriggerInitialBuild { + t.Fatalf("snapshots[0] = %+v, want sequence=1 trigger=%q", snapshots[0], skillContextTriggerInitialBuild) + } + if snapshots[1].Sequence != 2 || snapshots[1].Trigger != skillContextTriggerContextRetryRebuild { + t.Fatalf("snapshots[1] = %+v, want sequence=2 trigger=%q", snapshots[1], skillContextTriggerContextRetryRebuild) + } +} diff --git a/pkg/agent/turn_state.go b/pkg/agent/turn_state.go index 360c3b7d5..ae058e49d 100644 --- a/pkg/agent/turn_state.go +++ b/pkg/agent/turn_state.go @@ -5,6 +5,7 @@ package agent import ( "context" "reflect" + "strings" "sync" "sync/atomic" "time" @@ -176,13 +177,19 @@ type turnState struct { opts processOptions scope turnEventScope - turnID string - agentID string - sessionKey string - turnCtx *TurnContext + turnID string + agentID string + sessionKey string + activeSkills []string + attemptedSkills []string + skillContextTrace []SkillContextSnapshot + toolKinds []string + toolExecutions []ToolExecutionRecord + turnCtx *TurnContext channel string chatID string + workspace string userMessage string media []string @@ -238,25 +245,30 @@ type turnState struct { func newTurnState(agent *AgentInstance, opts processOptions, scope turnEventScope) *turnState { ts := &turnState{ - agent: agent, - opts: opts, - scope: scope, - turnID: scope.turnID, - agentID: agent.ID, - sessionKey: opts.Dispatch.SessionKey, - turnCtx: cloneTurnContext(scope.context), - channel: opts.Dispatch.Channel(), - chatID: opts.Dispatch.ChatID(), - userMessage: opts.Dispatch.UserMessage, - media: append([]string(nil), opts.Dispatch.Media...), - phase: TurnPhaseSetup, - startedAt: time.Now(), + agent: agent, + opts: opts, + scope: scope, + turnID: scope.turnID, + agentID: agent.ID, + sessionKey: opts.Dispatch.SessionKey, + activeSkills: activeSkillNames(agent, opts), + turnCtx: cloneTurnContext(scope.context), + channel: opts.Dispatch.Channel(), + chatID: opts.Dispatch.ChatID(), + workspace: agent.Workspace, + userMessage: opts.Dispatch.UserMessage, + media: append([]string(nil), opts.Dispatch.Media...), + phase: TurnPhaseSetup, + startedAt: time.Now(), } // Bind session store and capture initial history length for rollback logic if agent != nil && agent.Sessions != nil { ts.session = agent.Sessions - ts.initialHistoryLength = len(agent.Sessions.GetHistory(opts.Dispatch.SessionKey)) + history := agent.Sessions.GetHistory(opts.Dispatch.SessionKey) + ts.initialHistoryLength = len(history) + ts.restorePointHistory = append([]providers.Message(nil), history...) + ts.restorePointSummary = agent.Sessions.GetSummary(opts.Dispatch.SessionKey) } return ts @@ -375,6 +387,160 @@ func (ts *turnState) finalContentLen() int { return len(ts.finalContent) } +func (ts *turnState) finalContentSnapshot() string { + ts.mu.RLock() + defer ts.mu.RUnlock() + return ts.finalContent +} + +func (ts *turnState) recordToolKind(tool string) { + tool = strings.TrimSpace(tool) + if tool == "" { + return + } + + ts.mu.Lock() + defer ts.mu.Unlock() + + for _, existing := range ts.toolKinds { + if existing == tool { + return + } + } + ts.toolKinds = append(ts.toolKinds, tool) +} + +func (ts *turnState) toolKindsSnapshot() []string { + ts.mu.RLock() + defer ts.mu.RUnlock() + return append([]string(nil), ts.toolKinds...) +} + +func (ts *turnState) recordToolExecution(tool string, success bool, errorSummary string, skillNames []string) { + tool = strings.TrimSpace(tool) + if tool == "" { + return + } + + ts.recordToolKind(tool) + + ts.mu.Lock() + defer ts.mu.Unlock() + ts.toolExecutions = append(ts.toolExecutions, ToolExecutionRecord{ + Name: tool, + Success: success, + ErrorSummary: strings.TrimSpace(errorSummary), + SkillNames: append([]string(nil), skillNames...), + }) +} + +func (ts *turnState) toolExecutionsSnapshot() []ToolExecutionRecord { + ts.mu.RLock() + defer ts.mu.RUnlock() + if len(ts.toolExecutions) == 0 { + return nil + } + + out := make([]ToolExecutionRecord, 0, len(ts.toolExecutions)) + for _, exec := range ts.toolExecutions { + out = append(out, ToolExecutionRecord{ + Name: exec.Name, + Success: exec.Success, + ErrorSummary: exec.ErrorSummary, + SkillNames: append([]string(nil), exec.SkillNames...), + }) + } + return out +} + +func (ts *turnState) recordAttemptedSkills(skillNames []string) { + if len(skillNames) == 0 { + return + } + + ts.mu.Lock() + defer ts.mu.Unlock() + + for _, skillName := range skillNames { + skillName = strings.TrimSpace(skillName) + if skillName == "" { + continue + } + seen := false + for _, existing := range ts.attemptedSkills { + if existing == skillName { + seen = true + break + } + } + if seen { + continue + } + ts.attemptedSkills = append(ts.attemptedSkills, skillName) + } +} + +func (ts *turnState) attemptedSkillsSnapshot() []string { + ts.mu.RLock() + defer ts.mu.RUnlock() + return append([]string(nil), ts.attemptedSkills...) +} + +func (ts *turnState) recordSkillContextSnapshot(trigger string, skillNames []string) { + if len(skillNames) == 0 { + return + } + + filtered := make([]string, 0, len(skillNames)) + for _, skillName := range skillNames { + skillName = strings.TrimSpace(skillName) + if skillName == "" { + continue + } + filtered = append(filtered, skillName) + } + if len(filtered) == 0 { + return + } + + ts.recordAttemptedSkills(filtered) + + ts.mu.Lock() + defer ts.mu.Unlock() + ts.skillContextTrace = append(ts.skillContextTrace, SkillContextSnapshot{ + Sequence: len(ts.skillContextTrace) + 1, + Trigger: trigger, + SkillNames: append([]string(nil), filtered...), + }) +} + +func (ts *turnState) latestSkillContextSnapshot() []string { + ts.mu.RLock() + defer ts.mu.RUnlock() + if len(ts.skillContextTrace) == 0 { + return nil + } + return append([]string(nil), ts.skillContextTrace[len(ts.skillContextTrace)-1].SkillNames...) +} + +func (ts *turnState) skillContextSnapshotsSnapshot() []SkillContextSnapshot { + ts.mu.RLock() + defer ts.mu.RUnlock() + if len(ts.skillContextTrace) == 0 { + return nil + } + + snapshots := make([]SkillContextSnapshot, 0, len(ts.skillContextTrace)) + for _, snapshot := range ts.skillContextTrace { + snapshots = append(snapshots, SkillContextSnapshot{ + Sequence: snapshot.Sequence, + Trigger: snapshot.Trigger, + SkillNames: append([]string(nil), snapshot.SkillNames...), + }) + } + return snapshots +} + func (ts *turnState) setTurnCancel(cancel context.CancelFunc) { ts.mu.Lock() defer ts.mu.Unlock() @@ -442,9 +608,9 @@ func (ts *turnState) hardAbortRequested() bool { return ts.hardAbort } -func (ts *turnState) eventMeta(source, tracePath string) EventMeta { +func (ts *turnState) eventMeta(source, tracePath string) HookMeta { snap := ts.snapshot() - return EventMeta{ + return HookMeta{ AgentID: snap.AgentID, TurnID: snap.TurnID, SessionKey: snap.SessionKey, diff --git a/pkg/audio/asr/README.md b/pkg/audio/asr/README.md index 0477276dd..99d2a8c90 100644 --- a/pkg/audio/asr/README.md +++ b/pkg/audio/asr/README.md @@ -82,7 +82,8 @@ Notes: "model_list": [ { "model_name": "elevenlabs-asr", - "model": "elevenlabs/scribe_v1" + "provider": "elevenlabs", + "model": "scribe_v1" } ] } @@ -130,7 +131,7 @@ PicoClaw currently supports three main ASR routes: | Route | Example models | Behavior | | --- | --- | --- | -| ElevenLabs ASR | `elevenlabs/scribe_v1` | Uses the ElevenLabs transcription API. | +| ElevenLabs ASR | `provider: elevenlabs`, `model: scribe_v1` | Uses the ElevenLabs transcription API. | | Whisper endpoint models | `openai/whisper-1`, `groq/whisper-large-v3` | Uses an OpenAI-compatible `/audio/transcriptions` endpoint. | | Audio-capable chat models **(Under construction)** | `openai/gpt-4o-audio-preview`, `gemini/gemini-2.5-flash` | Sends audio to a multimodal chat model and asks it to transcribe. | @@ -142,7 +143,7 @@ If you are unsure which one to pick, choose Groq Whisper or ElevenLabs first. 1. **Preferred path**: resolve `voice.model_name` against `model_list`. 2. If that resolved model is: - - `elevenlabs/...`, PicoClaw uses the ElevenLabs transcriber. + - an `elevenlabs` provider model, PicoClaw uses the ElevenLabs transcriber. - an OpenAI-compatible Whisper model, PicoClaw uses the Whisper transcriber. - an audio-capable chat model, PicoClaw uses `AudioModelTranscriber`. 3. **Fallback path**: if `voice.model_name` is not set, PicoClaw performs a compatibility scan through `model_list` for legacy auto-detected ASR entries. diff --git a/pkg/audio/asr/README.zh.md b/pkg/audio/asr/README.zh.md index 104116080..670698cb8 100644 --- a/pkg/audio/asr/README.zh.md +++ b/pkg/audio/asr/README.zh.md @@ -82,7 +82,8 @@ model_list: "model_list": [ { "model_name": "elevenlabs-asr", - "model": "elevenlabs/scribe_v1" + "provider": "elevenlabs", + "model": "scribe_v1" } ] } @@ -130,7 +131,7 @@ PicoClaw 目前主要支持三种 ASR 路径： | 路径 | 示例模型 | 行为说明 | | --- | --- | --- | -| ElevenLabs ASR | `elevenlabs/scribe_v1` | 使用 ElevenLabs 的语音转录接口。 | +| ElevenLabs ASR | `provider: elevenlabs`，`model: scribe_v1` | 使用 ElevenLabs 的语音转录接口。 | | Whisper 接口模型 | `openai/whisper-1`、`groq/whisper-large-v3` | 使用 OpenAI 兼容的 `/audio/transcriptions` 接口。 | | 支持音频的聊天模型 **（重构中）** | `openai/gpt-4o-audio-preview`、`gemini/gemini-2.5-flash` | 把音频发给多模态聊天模型，并要求它返回转录结果。 | @@ -142,7 +143,7 @@ PicoClaw 目前主要支持三种 ASR 路径： 1. **首选路径**：根据 `voice.model_name` 在 `model_list` 中找到对应模型。 2. 如果找到的模型属于以下类型： - - `elevenlabs/...`，则使用 ElevenLabs transcriber。 + - `provider=elevenlabs` 的模型，则使用 ElevenLabs transcriber。 - OpenAI 兼容的 Whisper 模型，则使用 Whisper transcriber。 - 支持音频输入的聊天模型，则使用 `AudioModelTranscriber`。 3. **回退路径**：如果没有设置 `voice.model_name`，PicoClaw 会为了兼容旧配置，扫描 `model_list` 中可自动识别的 ASR 条目。 diff --git a/pkg/audio/asr/asr.go b/pkg/audio/asr/asr.go index 1482f40bb..a7c93e578 100644 --- a/pkg/audio/asr/asr.go +++ b/pkg/audio/asr/asr.go @@ -8,6 +8,12 @@ import ( "github.com/sipeed/picoclaw/pkg/providers" ) +const elevenLabsSupportedModelID = "scribe_v1" + +func ElevenLabsSupportedModelID() string { + return elevenLabsSupportedModelID +} + type Transcriber interface { Name() string Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) @@ -72,14 +78,23 @@ func whisperModelID(modelCfg *config.ModelConfig) string { return "" } +func isElevenLabsTranscriptionModel(modelCfg *config.ModelConfig) bool { + if modelCfg == nil || modelCfg.APIKey() == "" { + return false + } + + protocol, _ := providers.ExtractProtocol(modelCfg) + return protocol == "elevenlabs" +} + func transcriberFromModelConfig(modelCfg *config.ModelConfig) Transcriber { if modelCfg == nil { return nil } - protocol, _ := providers.ExtractProtocol(modelCfg) - if protocol == "elevenlabs" && modelCfg.APIKey() != "" { - return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase) + if isElevenLabsTranscriptionModel(modelCfg) { + _, modelID := providers.ExtractProtocol(modelCfg) + return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase, modelID) } if modelID := whisperModelID(modelCfg); modelID != "" { return NewWhisperTranscriber(modelCfg) @@ -95,9 +110,9 @@ func fallbackTranscriberFromModelConfig(modelCfg *config.ModelConfig) Transcribe return nil } - protocol, _ := providers.ExtractProtocol(modelCfg) - if protocol == "elevenlabs" && modelCfg.APIKey() != "" { - return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase) + if isElevenLabsTranscriptionModel(modelCfg) { + _, modelID := providers.ExtractProtocol(modelCfg) + return NewElevenLabsTranscriber(modelCfg.APIKey(), modelCfg.APIBase, modelID) } if modelID := whisperModelID(modelCfg); modelID != "" { return NewWhisperTranscriber(modelCfg) diff --git a/pkg/audio/asr/asr_test.go b/pkg/audio/asr/asr_test.go index 0970d69f4..f877b1198 100644 --- a/pkg/audio/asr/asr_test.go +++ b/pkg/audio/asr/asr_test.go @@ -46,6 +46,21 @@ func TestDetectTranscriber(t *testing.T) { }, wantName: "elevenlabs", }, + { + name: "explicit elevenlabs provider selects elevenlabs transcriber", + cfg: &config.Config{ + Voice: config.VoiceConfig{ModelName: "my-asr-model"}, + ModelList: []*config.ModelConfig{ + { + ModelName: "my-asr-model", + Provider: "elevenlabs", + Model: "scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }, + }, + }, + wantName: "elevenlabs", + }, { name: "voice model name alias selects whisper transcriber for groq", cfg: &config.Config{ diff --git a/pkg/audio/asr/elevenlabs_transcriber.go b/pkg/audio/asr/elevenlabs_transcriber.go index 452b9512d..a89d62848 100644 --- a/pkg/audio/asr/elevenlabs_transcriber.go +++ b/pkg/audio/asr/elevenlabs_transcriber.go @@ -20,19 +20,24 @@ import ( type ElevenLabsTranscriber struct { apiKey string apiBase string + modelID string httpClient *http.Client } -func NewElevenLabsTranscriber(apiKey, apiBase string) *ElevenLabsTranscriber { +func NewElevenLabsTranscriber(apiKey, apiBase, modelID string) *ElevenLabsTranscriber { logger.DebugCF("voice", "Creating ElevenLabs transcriber", map[string]any{"has_api_key": apiKey != ""}) if apiBase == "" { apiBase = "https://api.elevenlabs.io" } + if modelID == "" || modelID != ElevenLabsSupportedModelID() { + modelID = ElevenLabsSupportedModelID() + } return &ElevenLabsTranscriber{ apiKey: apiKey, apiBase: apiBase, + modelID: modelID, httpClient: &http.Client{ Timeout: 120 * time.Second, }, @@ -74,7 +79,7 @@ func (t *ElevenLabsTranscriber) Transcribe(ctx context.Context, audioFilePath st return nil, fmt.Errorf("failed to copy file content: %w", err) } - if err = writer.WriteField("model_id", "scribe_v1"); err != nil { + if err = writer.WriteField("model_id", t.modelID); err != nil { return nil, fmt.Errorf("failed to write model_id field: %w", err) } diff --git a/pkg/audio/asr/elevenlabs_transcriber_test.go b/pkg/audio/asr/elevenlabs_transcriber_test.go index fa80110be..bbc827578 100644 --- a/pkg/audio/asr/elevenlabs_transcriber_test.go +++ b/pkg/audio/asr/elevenlabs_transcriber_test.go @@ -3,10 +3,14 @@ package asr import ( "context" "encoding/json" + "io" + "mime" + "mime/multipart" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "testing" ) @@ -14,7 +18,7 @@ import ( var _ Transcriber = (*ElevenLabsTranscriber)(nil) func TestElevenLabsTranscriberName(t *testing.T) { - tr := NewElevenLabsTranscriber("sk_test", "") + tr := NewElevenLabsTranscriber("sk_test", "", "scribe_v1") if got := tr.Name(); got != "elevenlabs" { t.Errorf("Name() = %q, want %q", got, "elevenlabs") } @@ -35,6 +39,35 @@ func TestElevenLabsTranscribe(t *testing.T) { if r.Header.Get("Xi-Api-Key") != "sk_test" { t.Errorf("unexpected xi-api-key header: %s", r.Header.Get("Xi-Api-Key")) } + mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + t.Fatalf("ParseMediaType() error = %v", err) + } + if mediaType != "multipart/form-data" { + t.Fatalf("content-type = %q, want multipart/form-data", mediaType) + } + reader := multipart.NewReader(r.Body, params["boundary"]) + var gotModelID string + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart() error = %v", err) + } + if part.FormName() != "model_id" { + continue + } + body, err := io.ReadAll(part) + if err != nil { + t.Fatalf("ReadAll(part) error = %v", err) + } + gotModelID = strings.TrimSpace(string(body)) + } + if gotModelID != "scribe_v1" { + t.Fatalf("model_id = %q, want %q", gotModelID, "scribe_v1") + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(TranscriptionResponse{ Text: "hello from elevenlabs", @@ -43,7 +76,7 @@ func TestElevenLabsTranscribe(t *testing.T) { })) defer srv.Close() - tr := NewElevenLabsTranscriber("sk_test", "") + tr := NewElevenLabsTranscriber("sk_test", "", "scribe_v1") tr.apiBase = srv.URL resp, err := tr.Transcribe(context.Background(), audioPath) @@ -64,7 +97,7 @@ func TestElevenLabsTranscribe(t *testing.T) { })) defer srv.Close() - tr := NewElevenLabsTranscriber("sk_bad", "") + tr := NewElevenLabsTranscriber("sk_bad", "", "scribe_v1") tr.apiBase = srv.URL _, err := tr.Transcribe(context.Background(), audioPath) @@ -74,10 +107,54 @@ func TestElevenLabsTranscribe(t *testing.T) { }) t.Run("missing file", func(t *testing.T) { - tr := NewElevenLabsTranscriber("sk_test", "") + tr := NewElevenLabsTranscriber("sk_test", "", "scribe_v1") _, err := tr.Transcribe(context.Background(), filepath.Join(tmpDir, "nonexistent.ogg")) if err == nil { t.Fatal("expected error for missing file, got nil") } }) + + t.Run("unsupported model falls back to scribe_v1", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + t.Fatalf("ParseMediaType() error = %v", err) + } + if mediaType != "multipart/form-data" { + t.Fatalf("content-type = %q, want multipart/form-data", mediaType) + } + reader := multipart.NewReader(r.Body, params["boundary"]) + var gotModelID string + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart() error = %v", err) + } + if part.FormName() != "model_id" { + continue + } + body, err := io.ReadAll(part) + if err != nil { + t.Fatalf("ReadAll(part) error = %v", err) + } + gotModelID = strings.TrimSpace(string(body)) + } + if gotModelID != "scribe_v1" { + t.Fatalf("model_id = %q, want runtime fallback to %q", gotModelID, "scribe_v1") + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(TranscriptionResponse{Text: "ok"}) + })) + defer srv.Close() + + tr := NewElevenLabsTranscriber("sk_test", "", "unsupported-model") + tr.apiBase = srv.URL + + if _, err := tr.Transcribe(context.Background(), audioPath); err != nil { + t.Fatalf("Transcribe() error: %v", err) + } + }) } diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 9a05d4f95..dee67d87c 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -6,6 +6,7 @@ import ( "sync" "sync/atomic" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -48,6 +49,13 @@ type MessageBus struct { closed atomic.Bool wg sync.WaitGroup streamDelegate atomic.Value // stores StreamDelegate + eventPublisher atomic.Value // stores EventPublisher +} + +// EventPublisher is the minimal runtime event publisher used by MessageBus. +type EventPublisher interface { + Publish(ctx context.Context, evt runtimeevents.Event) runtimeevents.PublishResult + PublishNonBlocking(evt runtimeevents.Event) runtimeevents.PublishResult } func NewMessageBus() *MessageBus { @@ -92,9 +100,14 @@ func publish[T any](ctx context.Context, mb *MessageBus, ch chan T, msg T) error func (mb *MessageBus) PublishInbound(ctx context.Context, msg InboundMessage) error { msg = NormalizeInboundMessage(msg) if msg.Context.isZero() { + mb.publishFailure("inbound", runtimeScopeFromInboundContext(msg.Context), ErrMissingInboundContext) return ErrMissingInboundContext } - return publish(ctx, mb, mb.inbound, msg) + if err := publish(ctx, mb, mb.inbound, msg); err != nil { + mb.publishFailure("inbound", runtimeScopeFromInboundContext(msg.Context), err) + return err + } + return nil } func (mb *MessageBus) InboundChan() <-chan InboundMessage { @@ -104,9 +117,14 @@ func (mb *MessageBus) InboundChan() <-chan InboundMessage { func (mb *MessageBus) PublishOutbound(ctx context.Context, msg OutboundMessage) error { msg = NormalizeOutboundMessage(msg) if msg.Context.isZero() { + mb.publishFailure("outbound", runtimeScopeFromInboundContext(msg.Context), ErrMissingOutboundContext) return ErrMissingOutboundContext } - return publish(ctx, mb, mb.outbound, msg) + if err := publish(ctx, mb, mb.outbound, msg); err != nil { + mb.publishFailure("outbound", runtimeScopeFromInboundContext(msg.Context), err) + return err + } + return nil } func (mb *MessageBus) OutboundChan() <-chan OutboundMessage { @@ -116,9 +134,14 @@ func (mb *MessageBus) OutboundChan() <-chan OutboundMessage { func (mb *MessageBus) PublishOutboundMedia(ctx context.Context, msg OutboundMediaMessage) error { msg = NormalizeOutboundMediaMessage(msg) if msg.Context.isZero() { + mb.publishFailure("outbound_media", runtimeScopeFromInboundContext(msg.Context), ErrMissingOutboundMediaContext) return ErrMissingOutboundMediaContext } - return publish(ctx, mb, mb.outboundMedia, msg) + if err := publish(ctx, mb, mb.outboundMedia, msg); err != nil { + mb.publishFailure("outbound_media", runtimeScopeFromInboundContext(msg.Context), err) + return err + } + return nil } func (mb *MessageBus) OutboundMediaChan() <-chan OutboundMediaMessage { @@ -126,7 +149,11 @@ func (mb *MessageBus) OutboundMediaChan() <-chan OutboundMediaMessage { } func (mb *MessageBus) PublishAudioChunk(ctx context.Context, chunk AudioChunk) error { - return publish(ctx, mb, mb.audioChunks, chunk) + if err := publish(ctx, mb, mb.audioChunks, chunk); err != nil { + mb.publishFailure("audio_chunk", runtimeScopeFromAudioChunk(chunk), err) + return err + } + return nil } func (mb *MessageBus) AudioChunksChan() <-chan AudioChunk { @@ -134,7 +161,11 @@ func (mb *MessageBus) AudioChunksChan() <-chan AudioChunk { } func (mb *MessageBus) PublishVoiceControl(ctx context.Context, ctrl VoiceControl) error { - return publish(ctx, mb, mb.voiceControls, ctrl) + if err := publish(ctx, mb, mb.voiceControls, ctrl); err != nil { + mb.publishFailure("voice_control", runtimeScopeFromVoiceControl(ctrl), err) + return err + } + return nil } func (mb *MessageBus) VoiceControlsChan() <-chan VoiceControl { @@ -146,6 +177,11 @@ func (mb *MessageBus) SetStreamDelegate(d StreamDelegate) { mb.streamDelegate.Store(d) } +// SetEventPublisher registers a runtime event publisher for bus errors and lifecycle events. +func (mb *MessageBus) SetEventPublisher(p EventPublisher) { + mb.eventPublisher.Store(p) +} + // GetStreamer returns a Streamer for the given channel+chatID via the delegate. func (mb *MessageBus) GetStreamer(ctx context.Context, channel, chatID string) (Streamer, bool) { if d, ok := mb.streamDelegate.Load().(StreamDelegate); ok && d != nil { @@ -156,6 +192,7 @@ func (mb *MessageBus) GetStreamer(ctx context.Context, channel, chatID string) ( func (mb *MessageBus) Close() { mb.closeOnce.Do(func() { + mb.publishCloseEvent(runtimeevents.KindBusCloseStarted, 0) // notify all blocked publishers to exit close(mb.done) @@ -195,6 +232,8 @@ func (mb *MessageBus) Close() { logger.DebugCF("bus", "Drained buffered messages during close", map[string]any{ "count": drained, }) + mb.publishCloseEvent(runtimeevents.KindBusCloseDrained, drained) } + mb.publishCloseEvent(runtimeevents.KindBusCloseCompleted, drained) }) } diff --git a/pkg/bus/bus_test.go b/pkg/bus/bus_test.go index 5145d4759..a0a9e1e14 100644 --- a/pkg/bus/bus_test.go +++ b/pkg/bus/bus_test.go @@ -5,6 +5,8 @@ import ( "sync" "testing" "time" + + runtimeevents "github.com/sipeed/picoclaw/pkg/events" ) func TestPublishConsume(t *testing.T) { @@ -171,6 +173,86 @@ func TestPublishInbound_BackfillsContextFromLegacyFields(t *testing.T) { } } +func TestMessageBusPublishesRuntimeFailureAndCloseEvents(t *testing.T) { + eventBus := runtimeevents.NewBus() + defer func() { + if err := eventBus.Close(); err != nil { + t.Errorf("event bus close failed: %v", err) + } + }() + + _, eventsCh, err := eventBus.Channel().OfKind( + runtimeevents.KindBusPublishFailed, + runtimeevents.KindBusCloseStarted, + runtimeevents.KindBusCloseDrained, + runtimeevents.KindBusCloseCompleted, + ).SubscribeChan(t.Context(), runtimeevents.SubscribeOptions{Name: "bus-events", Buffer: 4}) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + mb := NewMessageBus() + mb.SetEventPublisher(eventBus) + + if err := mb.PublishInbound(context.Background(), InboundMessage{}); err == nil { + t.Fatal("expected PublishInbound to fail") + } + failed := receiveBusRuntimeEvent(t, eventsCh) + if failed.Kind != runtimeevents.KindBusPublishFailed || + failed.Source.Name != "inbound" || + failed.Severity != runtimeevents.SeverityError { + t.Fatalf("publish failed event = %+v", failed) + } + if failed.Attrs["stream"] != "inbound" || failed.Attrs["error"] == "" { + t.Fatalf("publish failed attrs = %#v, want stream and error", failed.Attrs) + } + + if err := mb.PublishOutbound(context.Background(), OutboundMessage{ + Context: NewOutboundContext("telegram", "chat-1", ""), + Content: "queued", + }); err != nil { + t.Fatalf("PublishOutbound failed: %v", err) + } + mb.Close() + + seen := map[runtimeevents.Kind]bool{} + var drainedAttrs map[string]any + for range 3 { + evt := receiveBusRuntimeEvent(t, eventsCh) + seen[evt.Kind] = true + if evt.Kind == runtimeevents.KindBusCloseDrained { + drainedAttrs = evt.Attrs + } + } + for _, kind := range []runtimeevents.Kind{ + runtimeevents.KindBusCloseStarted, + runtimeevents.KindBusCloseDrained, + runtimeevents.KindBusCloseCompleted, + } { + if !seen[kind] { + t.Fatalf("missing %s event, seen=%v", kind, seen) + } + } + if drainedAttrs["drained"] != 1 { + t.Fatalf("bus close drained attrs = %#v, want drained count", drainedAttrs) + } +} + +func receiveBusRuntimeEvent(t *testing.T, ch <-chan runtimeevents.Event) runtimeevents.Event { + t.Helper() + + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("runtime event channel closed before expected event") + } + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for runtime event") + return runtimeevents.Event{} + } +} + func TestPublishOutboundSubscribe(t *testing.T) { mb := NewMessageBus() defer mb.Close() diff --git a/pkg/bus/events.go b/pkg/bus/events.go new file mode 100644 index 000000000..4640ed1fc --- /dev/null +++ b/pkg/bus/events.go @@ -0,0 +1,88 @@ +package bus + +import ( + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +type busPublishFailedPayload struct { + Stream string `json:"stream"` + Error string `json:"error"` +} + +type busClosePayload struct { + Drained int `json:"drained,omitempty"` +} + +func (mb *MessageBus) publishFailure(stream string, scope runtimeevents.Scope, err error) { + if mb == nil || err == nil { + return + } + publisher, ok := mb.eventPublisher.Load().(EventPublisher) + if !ok || publisher == nil { + return + } + + publisher.PublishNonBlocking(runtimeevents.Event{ + Kind: runtimeevents.KindBusPublishFailed, + Source: runtimeevents.Source{Component: "bus", Name: stream}, + Scope: scope, + Severity: runtimeevents.SeverityError, + Payload: busPublishFailedPayload{ + Stream: stream, + Error: err.Error(), + }, + Attrs: map[string]any{ + "stream": stream, + "error": err.Error(), + }, + }) +} + +func (mb *MessageBus) publishCloseEvent(kind runtimeevents.Kind, drained int) { + if mb == nil { + return + } + publisher, ok := mb.eventPublisher.Load().(EventPublisher) + if !ok || publisher == nil { + return + } + + attrs := map[string]any{} + if drained > 0 { + attrs["drained"] = drained + } + publisher.PublishNonBlocking(runtimeevents.Event{ + Kind: kind, + Source: runtimeevents.Source{Component: "bus"}, + Severity: runtimeevents.SeverityInfo, + Payload: busClosePayload{Drained: drained}, + Attrs: attrs, + }) +} + +func runtimeScopeFromInboundContext(ctx InboundContext) runtimeevents.Scope { + return runtimeevents.Scope{ + Channel: ctx.Channel, + Account: ctx.Account, + ChatID: ctx.ChatID, + TopicID: ctx.TopicID, + SpaceID: ctx.SpaceID, + SpaceType: ctx.SpaceType, + ChatType: ctx.ChatType, + SenderID: ctx.SenderID, + MessageID: ctx.MessageID, + } +} + +func runtimeScopeFromAudioChunk(chunk AudioChunk) runtimeevents.Scope { + return runtimeevents.Scope{ + Channel: chunk.Channel, + ChatID: chunk.ChatID, + } +} + +func runtimeScopeFromVoiceControl(ctrl VoiceControl) runtimeevents.Scope { + return runtimeevents.Scope{ + ChatID: ctrl.ChatID, + } +} diff --git a/pkg/channels/README.md b/pkg/channels/README.md index 1cab1a4a6..c3decd242 100644 --- a/pkg/channels/README.md +++ b/pkg/channels/README.md @@ -1310,6 +1310,7 @@ make test # Full test suite | `pkg/channels/whatsapp/` | `"whatsapp"` | — (Bridge mode) | | `pkg/channels/whatsapp_native/` | `"whatsapp_native"` | — (Native whatsmeow mode) | | `pkg/channels/maixcam/` | `"maixcam"` | — | +| `pkg/channels/mqtt/` | `"mqtt"` | — | | `pkg/channels/pico/` | `"pico"` | TypingCapable, PlaceholderCapable, MessageEditor, WebhookHandler | ### A.3 Interface Quick Reference diff --git a/pkg/channels/README.zh.md b/pkg/channels/README.zh.md index c44859c20..d71c30104 100644 --- a/pkg/channels/README.zh.md +++ b/pkg/channels/README.zh.md @@ -1308,6 +1308,7 @@ make test # 全量测试 | `pkg/channels/whatsapp/` | `"whatsapp"` | — (Bridge 模式) | | `pkg/channels/whatsapp_native/` | `"whatsapp_native"` | — (原生 whatsmeow 模式) | | `pkg/channels/maixcam/` | `"maixcam"` | — | +| `pkg/channels/mqtt/` | `"mqtt"` | — | | `pkg/channels/pico/` | `"pico"` | TypingCapable, PlaceholderCapable, MessageEditor, WebhookHandler | ### A.3 接口速查表 diff --git a/pkg/channels/events.go b/pkg/channels/events.go new file mode 100644 index 000000000..60e5640f0 --- /dev/null +++ b/pkg/channels/events.go @@ -0,0 +1,197 @@ +package channels + +import ( + "github.com/sipeed/picoclaw/pkg/bus" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +func channelTypeForEvent(m *Manager, channelName string) string { + if m == nil || m.config == nil { + return channelName + } + if bc := m.config.Channels.Get(channelName); bc != nil && bc.Type != "" { + return bc.Type + } + return channelName +} + +func (m *Manager) publishChannelEvent( + kind runtimeevents.Kind, + channelName string, + scope runtimeevents.Scope, + severity runtimeevents.Severity, + payload any, +) { + if m == nil || m.runtimeEvents == nil { + return + } + if scope.Channel == "" { + scope.Channel = channelName + } + m.runtimeEvents.PublishNonBlocking(runtimeevents.Event{ + Kind: kind, + Source: runtimeevents.Source{Component: "channel", Name: channelName}, + Scope: scope, + Severity: severity, + Payload: payload, + Attrs: channelEventAttrs(payload), + }) +} + +func channelEventAttrs(payload any) map[string]any { + switch payload := payload.(type) { + case ChannelLifecyclePayload: + attrs := map[string]any{} + setAttrString(attrs, "type", payload.Type) + setAttrString(attrs, "error", payload.Error) + return attrs + case ChannelOutboundPayload: + attrs := map[string]any{} + if payload.Media { + attrs["media"] = payload.Media + } + if payload.ContentLen > 0 { + attrs["content_len"] = payload.ContentLen + } + if len(payload.MessageIDs) > 0 { + attrs["message_ids_count"] = len(payload.MessageIDs) + } + setAttrString(attrs, "reply_to_message_id", payload.ReplyToMessageID) + setAttrString(attrs, "error", payload.Error) + if payload.Retries > 0 { + attrs["retries"] = payload.Retries + } + return attrs + default: + return nil + } +} + +func setAttrString(attrs map[string]any, key, value string) { + if value != "" { + attrs[key] = value + } +} + +func (m *Manager) publishOutboundSent( + channelName string, + msg bus.OutboundMessage, + messageIDs []string, +) { + m.publishChannelEvent( + runtimeevents.KindChannelMessageOutboundSent, + channelName, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityInfo, + ChannelOutboundPayload{ + ContentLen: len([]rune(msg.Content)), + MessageIDs: append([]string(nil), messageIDs...), + ReplyToMessageID: msg.ReplyToMessageID, + }, + ) +} + +func (m *Manager) publishOutboundQueued( + channelName string, + msg bus.OutboundMessage, +) { + m.publishChannelEvent( + runtimeevents.KindChannelMessageOutboundQueued, + channelName, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityInfo, + ChannelOutboundPayload{ + ContentLen: len([]rune(msg.Content)), + ReplyToMessageID: msg.ReplyToMessageID, + }, + ) +} + +func (m *Manager) publishOutboundFailed( + channelName string, + msg bus.OutboundMessage, + err error, + media bool, +) { + payload := ChannelOutboundPayload{ + Media: media, + ContentLen: len([]rune(msg.Content)), + ReplyToMessageID: msg.ReplyToMessageID, + Retries: maxRetries, + } + if err != nil { + payload.Error = err.Error() + } + m.publishChannelEvent( + runtimeevents.KindChannelMessageOutboundFailed, + channelName, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityError, + payload, + ) +} + +func (m *Manager) publishOutboundMediaSent( + channelName string, + msg bus.OutboundMediaMessage, + messageIDs []string, +) { + m.publishChannelEvent( + runtimeevents.KindChannelMessageOutboundSent, + channelName, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityInfo, + ChannelOutboundPayload{ + Media: true, + MessageIDs: append([]string(nil), messageIDs...), + }, + ) +} + +func (m *Manager) publishOutboundMediaQueued( + channelName string, + msg bus.OutboundMediaMessage, +) { + m.publishChannelEvent( + runtimeevents.KindChannelMessageOutboundQueued, + channelName, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityInfo, + ChannelOutboundPayload{Media: true}, + ) +} + +func (m *Manager) publishOutboundMediaFailed( + channelName string, + msg bus.OutboundMediaMessage, + err error, +) { + payload := ChannelOutboundPayload{ + Media: true, + Retries: maxRetries, + } + if err != nil { + payload.Error = err.Error() + } + m.publishChannelEvent( + runtimeevents.KindChannelMessageOutboundFailed, + channelName, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityError, + payload, + ) +} + +func scopeFromOutboundContext(ctx bus.InboundContext) runtimeevents.Scope { + return runtimeevents.Scope{ + Channel: ctx.Channel, + Account: ctx.Account, + ChatID: ctx.ChatID, + TopicID: ctx.TopicID, + SpaceID: ctx.SpaceID, + SpaceType: ctx.SpaceType, + ChatType: ctx.ChatType, + SenderID: ctx.SenderID, + MessageID: ctx.MessageID, + } +} diff --git a/pkg/channels/line/line.go b/pkg/channels/line/line.go index 760506a31..d4d34211d 100644 --- a/pkg/channels/line/line.go +++ b/pkg/channels/line/line.go @@ -1,19 +1,17 @@ package line import ( - "bytes" "context" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/json" + "errors" "fmt" - "io" "net/http" "strings" "sync" "time" + "github.com/line/line-bot-sdk-go/v8/linebot/messaging_api" + "github.com/line/line-bot-sdk-go/v8/linebot/webhook" + "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" @@ -24,13 +22,7 @@ import ( ) const ( - lineAPIBase = "https://api.line.me/v2/bot" - lineDataAPIBase = "https://api-data.line.me/v2/bot" - lineReplyEndpoint = lineAPIBase + "/message/reply" - linePushEndpoint = lineAPIBase + "/message/push" - lineContentEndpoint = lineDataAPIBase + "/message/%s/content" - lineBotInfoEndpoint = lineAPIBase + "/info" - lineLoadingEndpoint = lineAPIBase + "/chat/loading/start" + lineContentEndpoint = "https://api-data.line.me/v2/bot/message/%s/content" lineReplyTokenMaxAge = 25 * time.Second // Limit request body to prevent memory exhaustion (DoS). @@ -45,17 +37,16 @@ type replyTokenEntry struct { // LINEChannel implements the Channel interface for LINE Official Account // using the LINE Messaging API with HTTP webhook for receiving messages -// and REST API for sending messages. +// and the official LINE Bot SDK for sending messages. type LINEChannel struct { *channels.BaseChannel config *config.LINESettings - infoClient *http.Client // for bot info lookups (short timeout) - apiClient *http.Client // for messaging API calls - botUserID string // Bot's user ID - botBasicID string // Bot's basic ID (e.g. @216ru...) - botDisplayName string // Bot's display name for text-based mention detection - replyTokens sync.Map // chatID -> replyTokenEntry - quoteTokens sync.Map // chatID -> quoteToken (string) + client *messaging_api.MessagingApiAPI + botUserID string // Bot's user ID + botBasicID string // Bot's basic ID (e.g. @216ru...) + botDisplayName string // Bot's display name for text-based mention detection + replyTokens sync.Map // chatID -> replyTokenEntry + quoteTokens sync.Map // chatID -> quoteToken (string) ctx context.Context cancel context.CancelFunc } @@ -70,6 +61,14 @@ func NewLINEChannel( return nil, fmt.Errorf("line channel_secret and channel_access_token are required") } + client, err := messaging_api.NewMessagingApiAPI( + cfg.ChannelAccessToken.String(), + messaging_api.WithHTTPClient(&http.Client{Timeout: 30 * time.Second}), + ) + if err != nil { + return nil, fmt.Errorf("failed to create LINE messaging client: %w", err) + } + base := channels.NewBaseChannel("line", cfg, messageBus, bc.AllowFrom, channels.WithMaxMessageLength(5000), channels.WithGroupTrigger(bc.GroupTrigger), @@ -79,8 +78,7 @@ func NewLINEChannel( return &LINEChannel{ BaseChannel: base, config: cfg, - infoClient: &http.Client{Timeout: 10 * time.Second}, - apiClient: &http.Client{Timeout: 30 * time.Second}, + client: client, }, nil } @@ -91,11 +89,15 @@ func (c *LINEChannel) Start(ctx context.Context) error { c.ctx, c.cancel = context.WithCancel(ctx) // Fetch bot profile to get bot's userId for mention detection - if err := c.fetchBotInfo(); err != nil { + info, err := c.client.WithContext(ctx).GetBotInfo() + if err != nil { logger.WarnCF("line", "Failed to fetch bot info (mention detection disabled)", map[string]any{ "error": err.Error(), }) } else { + c.botUserID = info.UserId + c.botBasicID = info.BasicId + c.botDisplayName = info.DisplayName logger.InfoCF("line", "Bot info fetched", map[string]any{ "bot_user_id": c.botUserID, "basic_id": c.botBasicID, @@ -108,39 +110,6 @@ func (c *LINEChannel) Start(ctx context.Context) error { return nil } -// fetchBotInfo retrieves the bot's userId, basicId, and displayName from the LINE API. -func (c *LINEChannel) fetchBotInfo() error { - req, err := http.NewRequest(http.MethodGet, lineBotInfoEndpoint, nil) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+c.config.ChannelAccessToken.String()) - - resp, err := c.infoClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bot info API returned status %d", resp.StatusCode) - } - - var info struct { - UserID string `json:"userId"` - BasicID string `json:"basicId"` - DisplayName string `json:"displayName"` - } - if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { - return err - } - - c.botUserID = info.UserID - c.botBasicID = info.BasicID - c.botDisplayName = info.DisplayName - return nil -} - // Stop gracefully stops the LINE channel. func (c *LINEChannel) Stop(ctx context.Context) error { logger.InfoC("line", "Stopping LINE channel") @@ -174,140 +143,70 @@ func (c *LINEChannel) webhookHandler(w http.ResponseWriter, r *http.Request) { return } - body, err := io.ReadAll(io.LimitReader(r.Body, maxWebhookBodySize+1)) + // Limit body size to prevent memory exhaustion (DoS). + // ParseRequest reads r.Body internally via io.ReadAll; wrapping with + // MaxBytesReader ensures oversized payloads are rejected before full + // allocation. + r.Body = http.MaxBytesReader(w, r.Body, maxWebhookBodySize) + + cb, err := webhook.ParseRequest(c.config.ChannelSecret.String(), r) if err != nil { - logger.ErrorCF("line", "Failed to read request body", map[string]any{ - "error": err.Error(), - }) - http.Error(w, "Bad request", http.StatusBadRequest) - return - } - if int64(len(body)) > maxWebhookBodySize { - logger.WarnC("line", "Webhook request body too large, rejected") - http.Error(w, "Request entity too large", http.StatusRequestEntityTooLarge) - return - } - - signature := r.Header.Get("X-Line-Signature") - if !c.verifySignature(body, signature) { - logger.WarnC("line", "Invalid webhook signature") - http.Error(w, "Forbidden", http.StatusForbidden) - return - } - - var payload struct { - Events []lineEvent `json:"events"` - } - if err := json.Unmarshal(body, &payload); err != nil { - logger.ErrorCF("line", "Failed to parse webhook payload", map[string]any{ - "error": err.Error(), - }) - http.Error(w, "Bad request", http.StatusBadRequest) + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + logger.WarnC("line", "Webhook request body too large, rejected") + http.Error(w, "Request entity too large", http.StatusRequestEntityTooLarge) + } else if errors.Is(err, webhook.ErrInvalidSignature) { + logger.WarnC("line", "Invalid webhook signature") + http.Error(w, "Forbidden", http.StatusForbidden) + } else { + logger.ErrorCF("line", "Failed to parse webhook request", map[string]any{ + "error": err.Error(), + }) + http.Error(w, "Bad request", http.StatusBadRequest) + } return } // Return 200 immediately, process events asynchronously w.WriteHeader(http.StatusOK) - for _, event := range payload.Events { + for _, event := range cb.Events { go c.processEvent(event) } } -// verifySignature validates the X-Line-Signature using HMAC-SHA256. -func (c *LINEChannel) verifySignature(body []byte, signature string) bool { - if signature == "" { - return false - } - - mac := hmac.New(sha256.New, []byte(c.config.ChannelSecret.String())) - mac.Write(body) - expected := base64.StdEncoding.EncodeToString(mac.Sum(nil)) - - return hmac.Equal([]byte(expected), []byte(signature)) -} - -// LINE webhook event types -type lineEvent struct { - Type string `json:"type"` - ReplyToken string `json:"replyToken"` - Source lineSource `json:"source"` - Message json.RawMessage `json:"message"` - Timestamp int64 `json:"timestamp"` -} - -type lineSource struct { - Type string `json:"type"` // "user", "group", "room" - UserID string `json:"userId"` - GroupID string `json:"groupId"` - RoomID string `json:"roomId"` -} - -type lineMessage struct { - ID string `json:"id"` - Type string `json:"type"` // "text", "image", "video", "audio", "file", "sticker" - Text string `json:"text"` - QuoteToken string `json:"quoteToken"` - Mention *struct { - Mentionees []lineMentionee `json:"mentionees"` - } `json:"mention"` - ContentProvider struct { - Type string `json:"type"` - } `json:"contentProvider"` -} - -type lineMentionee struct { - Index int `json:"index"` - Length int `json:"length"` - Type string `json:"type"` // "user", "all" - UserID string `json:"userId"` -} - -func (c *LINEChannel) processEvent(event lineEvent) { - if event.Type != "message" { +func (c *LINEChannel) processEvent(event webhook.EventInterface) { + msgEvent, ok := event.(webhook.MessageEvent) + if !ok { logger.DebugCF("line", "Ignoring non-message event", map[string]any{ - "type": event.Type, + "type": event.GetType(), }) return } - senderID := event.Source.UserID - chatID := c.resolveChatID(event.Source) - isGroup := event.Source.Type == "group" || event.Source.Type == "room" - - var msg lineMessage - if err := json.Unmarshal(event.Message, &msg); err != nil { - logger.ErrorCF("line", "Failed to parse message", map[string]any{ - "error": err.Error(), - }) - return - } + senderID, chatID, sourceType := c.resolveSource(msgEvent.Source) + isGroup := sourceType == "group" || sourceType == "room" // Store reply token for later use - if event.ReplyToken != "" { + if msgEvent.ReplyToken != "" { c.replyTokens.Store(chatID, replyTokenEntry{ - token: event.ReplyToken, + token: msgEvent.ReplyToken, timestamp: time.Now(), }) } - // Store quote token for quoting the original message in reply - if msg.QuoteToken != "" { - c.quoteTokens.Store(chatID, msg.QuoteToken) - } - var content string var mediaPaths []string - - scope := channels.BuildMediaScope("line", chatID, msg.ID) + var messageID string + var quoteToken string + var isMentioned bool // Helper to register a local file with the media store - storeMedia := func(localPath, filename string) string { + storeMedia := func(localPath, filename, scope string) string { if store := c.GetMediaStore(); store != nil { ref, err := store.Store(localPath, media.MediaMeta{ - Filename: filename, - Source: "line", - CleanupPolicy: media.CleanupPolicyDeleteOnCleanup, + Filename: filename, + Source: "line", }, scope) if err == nil { return ref @@ -316,37 +215,70 @@ func (c *LINEChannel) processEvent(event lineEvent) { return localPath // fallback } - switch msg.Type { - case "text": + switch msg := msgEvent.Message.(type) { + case webhook.TextMessageContent: + messageID = msg.Id content = msg.Text + isMentioned = c.isBotMentioned(msg) + // Store quote token for quoting the original message in reply + if msg.QuoteToken != "" { + quoteToken = msg.QuoteToken + c.quoteTokens.Store(chatID, msg.QuoteToken) + } // Strip bot mention from text in group chats if isGroup { content = c.stripBotMention(content, msg) } - case "image": - localPath := c.downloadContent(msg.ID, "image.jpg") - if localPath != "" { - mediaPaths = append(mediaPaths, storeMedia(localPath, "image.jpg")) + case webhook.ImageMessageContent: + messageID = msg.Id + if msg.QuoteToken != "" { + quoteToken = msg.QuoteToken + c.quoteTokens.Store(chatID, msg.QuoteToken) + } + if localPath := c.downloadContent(msg.Id, "image.jpg"); localPath != "" { + scope := channels.BuildMediaScope("line", chatID, msg.Id) + mediaPaths = append(mediaPaths, storeMedia(localPath, "image.jpg", scope)) content = "[image]" } - case "audio": - localPath := c.downloadContent(msg.ID, "audio.m4a") - if localPath != "" { - mediaPaths = append(mediaPaths, storeMedia(localPath, "audio.m4a")) + case webhook.AudioMessageContent: + messageID = msg.Id + if localPath := c.downloadContent(msg.Id, "audio.m4a"); localPath != "" { + scope := channels.BuildMediaScope("line", chatID, msg.Id) + mediaPaths = append(mediaPaths, storeMedia(localPath, "audio.m4a", scope)) content = "[audio]" } - case "video": - localPath := c.downloadContent(msg.ID, "video.mp4") - if localPath != "" { - mediaPaths = append(mediaPaths, storeMedia(localPath, "video.mp4")) + case webhook.VideoMessageContent: + messageID = msg.Id + if msg.QuoteToken != "" { + quoteToken = msg.QuoteToken + c.quoteTokens.Store(chatID, msg.QuoteToken) + } + if localPath := c.downloadContent(msg.Id, "video.mp4"); localPath != "" { + scope := channels.BuildMediaScope("line", chatID, msg.Id) + mediaPaths = append(mediaPaths, storeMedia(localPath, "video.mp4", scope)) content = "[video]" } - case "file": + case webhook.FileMessageContent: + messageID = msg.Id content = "[file]" - case "sticker": + case webhook.LocationMessageContent: + messageID = msg.Id + content = "[location]" + if msg.Title != "" { + content = fmt.Sprintf("[location: %s]", msg.Title) + } + case webhook.StickerMessageContent: + messageID = msg.Id + if msg.QuoteToken != "" { + quoteToken = msg.QuoteToken + c.quoteTokens.Store(chatID, msg.QuoteToken) + } content = "[sticker]" default: - content = fmt.Sprintf("[%s]", msg.Type) + logger.DebugCF("line", "Ignoring unsupported message type", map[string]any{ + "type": msgEvent.Message.GetType(), + }) + return } if strings.TrimSpace(content) == "" { @@ -354,9 +286,7 @@ func (c *LINEChannel) processEvent(event lineEvent) { } // In group chats, apply unified group trigger filtering - isMentioned := false if isGroup { - isMentioned = c.isBotMentioned(msg) respond, cleaned := c.ShouldRespondInGroup(isMentioned, content) if !respond { logger.DebugCF("line", "Ignoring group message by group trigger", map[string]any{ @@ -369,13 +299,13 @@ func (c *LINEChannel) processEvent(event lineEvent) { metadata := map[string]string{ "platform": "line", - "source_type": event.Source.Type, + "source_type": sourceType, } logger.DebugCF("line", "Received message", map[string]any{ "sender_id": senderID, "chat_id": chatID, - "message_type": msg.Type, + "message_type": msgEvent.Message.GetType(), "is_group": isGroup, "preview": utils.Truncate(content, 50), }) @@ -395,16 +325,16 @@ func (c *LINEChannel) processEvent(event lineEvent) { ChatID: chatID, ChatType: map[bool]string{true: "group", false: "direct"}[isGroup], SenderID: senderID, - MessageID: msg.ID, + MessageID: messageID, Mentioned: isMentioned, Raw: metadata, } - if event.ReplyToken != "" { + if msgEvent.ReplyToken != "" { inboundCtx.ReplyHandles = map[string]string{ - "reply_token": event.ReplyToken, + "reply_token": msgEvent.ReplyToken, } - if msg.QuoteToken != "" { - inboundCtx.ReplyHandles["quote_token"] = msg.QuoteToken + if quoteToken != "" { + inboundCtx.ReplyHandles["quote_token"] = quoteToken } } @@ -412,30 +342,28 @@ func (c *LINEChannel) processEvent(event lineEvent) { } // isBotMentioned checks if the bot is mentioned in the message. -// It first checks the mention metadata (userId match), then falls back +// It first checks the mention metadata (userId match or IsSelf), then falls back // to text-based detection using the bot's display name, since LINE may // not include userId in mentionees for Official Accounts. -func (c *LINEChannel) isBotMentioned(msg lineMessage) bool { - // Check mention metadata +func (c *LINEChannel) isBotMentioned(msg webhook.TextMessageContent) bool { if msg.Mention != nil { for _, m := range msg.Mention.Mentionees { - if m.Type == "all" { + switch mentionee := m.(type) { + case webhook.AllMentionee: return true - } - if c.botUserID != "" && m.UserID == c.botUserID { - return true - } - } - // Mention metadata exists with mentionees but bot not matched by userId. - // The bot IS likely mentioned (LINE includes mention struct when bot is @-ed), - // so check if any mentionee overlaps with bot display name in text. - if c.botDisplayName != "" { - for _, m := range msg.Mention.Mentionees { - if m.Index >= 0 && m.Length > 0 { + case webhook.UserMentionee: + if mentionee.IsSelf { + return true + } + if c.botUserID != "" && mentionee.UserId == c.botUserID { + return true + } + // Check if mentionee text overlaps with bot display name + if c.botDisplayName != "" && mentionee.Index >= 0 && mentionee.Length > 0 { runes := []rune(msg.Text) - end := m.Index + m.Length + end := int(mentionee.Index) + int(mentionee.Length) if end <= len(runes) { - mentionText := string(runes[m.Index:end]) + mentionText := string(runes[mentionee.Index:end]) if strings.Contains(mentionText, c.botDisplayName) { return true } @@ -454,30 +382,43 @@ func (c *LINEChannel) isBotMentioned(msg lineMessage) bool { } // stripBotMention removes the @BotName mention text from the message. -func (c *LINEChannel) stripBotMention(text string, msg lineMessage) string { +func (c *LINEChannel) stripBotMention(text string, msg webhook.TextMessageContent) string { stripped := false - // Try to strip using mention metadata indices if msg.Mention != nil { runes := []rune(text) for i := len(msg.Mention.Mentionees) - 1; i >= 0; i-- { m := msg.Mention.Mentionees[i] - // Strip if userId matches OR if the mention text contains the bot display name shouldStrip := false - if c.botUserID != "" && m.UserID == c.botUserID { - shouldStrip = true - } else if c.botDisplayName != "" && m.Index >= 0 && m.Length > 0 { - end := m.Index + m.Length - if end <= len(runes) { - mentionText := string(runes[m.Index:end]) - if strings.Contains(mentionText, c.botDisplayName) { - shouldStrip = true + var index, length int32 + + switch mentionee := m.(type) { + case webhook.UserMentionee: + index = mentionee.Index + length = mentionee.Length + if mentionee.IsSelf { + shouldStrip = true + } else if c.botUserID != "" && mentionee.UserId == c.botUserID { + shouldStrip = true + } else if c.botDisplayName != "" && index >= 0 && length > 0 { + end := int(index) + int(length) + if end <= len(runes) { + mentionText := string(runes[index:end]) + if strings.Contains(mentionText, c.botDisplayName) { + shouldStrip = true + } } } + case webhook.AllMentionee: + // Don't strip @All mentions + continue + default: + continue } + if shouldStrip { - start := m.Index - end := m.Index + m.Length + start := int(index) + end := int(index) + int(length) if start >= 0 && end <= len(runes) { runes = append(runes[:start], runes[end:]...) stripped = true @@ -497,16 +438,20 @@ func (c *LINEChannel) stripBotMention(text string, msg lineMessage) string { return strings.TrimSpace(text) } -// resolveChatID determines the chat ID from the event source. -// For group/room messages, use the group/room ID; for 1:1, use the user ID. -func (c *LINEChannel) resolveChatID(source lineSource) string { - switch source.Type { - case "group": - return source.GroupID - case "room": - return source.RoomID +// resolveSource extracts senderID, chatID, and source type from the event source. +func (c *LINEChannel) resolveSource(source webhook.SourceInterface) (senderID, chatID, sourceType string) { + switch src := source.(type) { + case webhook.GroupSource: + return src.UserId, src.GroupId, "group" + case webhook.RoomSource: + return src.UserId, src.RoomId, "room" + case webhook.UserSource: + return src.UserId, src.UserId, "user" default: - return source.UserID + logger.WarnCF("line", "Unknown source type", map[string]any{ + "type": fmt.Sprintf("%T", source), + }) + return "", "", "unknown" } } @@ -523,23 +468,41 @@ func (c *LINEChannel) Send(ctx context.Context, msg bus.OutboundMessage) ([]stri quoteToken = qt.(string) } + textMsg := messaging_api.TextMessage{ + Text: msg.Content, + QuoteToken: quoteToken, + } + // Try reply token first (free, valid for ~25 seconds) if entry, ok := c.replyTokens.LoadAndDelete(msg.ChatID); ok { tokenEntry := entry.(replyTokenEntry) if time.Since(tokenEntry.timestamp) < lineReplyTokenMaxAge { - if err := c.sendReply(ctx, tokenEntry.token, msg.Content, quoteToken); err == nil { + resp, _, err := c.client.WithContext(ctx).ReplyMessageWithHttpInfo(&messaging_api.ReplyMessageRequest{ + ReplyToken: tokenEntry.token, + Messages: []messaging_api.MessageInterface{&textMsg}, + }) + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + if err == nil { logger.DebugCF("line", "Message sent via Reply API", map[string]any{ "chat_id": msg.ChatID, "quoted": quoteToken != "", }) return nil, nil } - logger.DebugC("line", "Reply API failed, falling back to Push API") + logger.DebugCF("line", "Reply API failed, falling back to Push API", map[string]any{ + "error": err.Error(), + }) } } // Fall back to Push API - return nil, c.sendPush(ctx, msg.ChatID, msg.Content, quoteToken) + resp, _, err := c.client.WithContext(ctx).PushMessageWithHttpInfo(&messaging_api.PushMessageRequest{ + To: msg.ChatID, + Messages: []messaging_api.MessageInterface{&textMsg}, + }, "") + return nil, classifySDKError(resp, err) } // SendMedia implements the channels.MediaSender interface. @@ -564,46 +527,19 @@ func (c *LINEChannel) SendMedia(ctx context.Context, msg bus.OutboundMediaMessag caption = fmt.Sprintf("[%s: %s]", part.Type, part.Filename) } - if err := c.sendPush(ctx, msg.ChatID, caption, ""); err != nil { - return nil, err + textMsg := messaging_api.TextMessage{Text: caption} + resp, _, err := c.client.WithContext(ctx).PushMessageWithHttpInfo(&messaging_api.PushMessageRequest{ + To: msg.ChatID, + Messages: []messaging_api.MessageInterface{&textMsg}, + }, "") + if sdkErr := classifySDKError(resp, err); sdkErr != nil { + return nil, sdkErr } } return nil, nil } -// buildTextMessage creates a text message object, optionally with quoteToken. -func buildTextMessage(content, quoteToken string) map[string]string { - msg := map[string]string{ - "type": "text", - "text": content, - } - if quoteToken != "" { - msg["quoteToken"] = quoteToken - } - return msg -} - -// sendReply sends a message using the LINE Reply API. -func (c *LINEChannel) sendReply(ctx context.Context, replyToken, content, quoteToken string) error { - payload := map[string]any{ - "replyToken": replyToken, - "messages": []map[string]string{buildTextMessage(content, quoteToken)}, - } - - return c.callAPI(ctx, lineReplyEndpoint, payload) -} - -// sendPush sends a message using the LINE Push API. -func (c *LINEChannel) sendPush(ctx context.Context, to, content, quoteToken string) error { - payload := map[string]any{ - "to": to, - "messages": []map[string]string{buildTextMessage(content, quoteToken)}, - } - - return c.callAPI(ctx, linePushEndpoint, payload) -} - // StartTyping implements channels.TypingCapable using LINE's loading animation. // // NOTE: The LINE loading animation API only works for 1:1 chats. @@ -649,48 +585,31 @@ func (c *LINEChannel) StartTyping(ctx context.Context, chatID string) (func(), e return stop, nil } +// classifySDKError maps an SDK HTTP response to the project's sentinel errors. +func classifySDKError(resp *http.Response, err error) error { + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + if err == nil { + return nil + } + if resp != nil { + return channels.ClassifySendError(resp.StatusCode, err) + } + return channels.ClassifyNetError(err) +} + // sendLoading sends a loading animation indicator to the chat. func (c *LINEChannel) sendLoading(ctx context.Context, chatID string) error { - payload := map[string]any{ - "chatId": chatID, - "loadingSeconds": 60, + req := &messaging_api.ShowLoadingAnimationRequest{ + ChatId: chatID, + LoadingSeconds: 60, } - return c.callAPI(ctx, lineLoadingEndpoint, payload) + resp, _, err := c.client.WithContext(ctx).ShowLoadingAnimationWithHttpInfo(req) + return classifySDKError(resp, err) } -// callAPI makes an authenticated POST request to the LINE API. -func (c *LINEChannel) callAPI(ctx context.Context, endpoint string, payload any) error { - body, err := json.Marshal(payload) - if err != nil { - return fmt.Errorf("failed to marshal payload: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+c.config.ChannelAccessToken.String()) - - resp, err := c.apiClient.Do(req) - if err != nil { - return channels.ClassifyNetError(err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return channels.ClassifySendError(resp.StatusCode, fmt.Errorf("reading LINE API error response: %w", err)) - } - return channels.ClassifySendError(resp.StatusCode, fmt.Errorf("LINE API error: %s", string(respBody))) - } - - return nil -} - -// downloadContent downloads media content from the LINE API. +// downloadContent downloads media content from the LINE content API. func (c *LINEChannel) downloadContent(messageID, filename string) string { url := fmt.Sprintf(lineContentEndpoint, messageID) return utils.DownloadFile(url, filename, utils.DownloadOptions{ diff --git a/pkg/channels/line/line_test.go b/pkg/channels/line/line_test.go index c5f4e9be2..83af04a0d 100644 --- a/pkg/channels/line/line_test.go +++ b/pkg/channels/line/line_test.go @@ -11,7 +11,7 @@ import ( ) func TestWebhookRejectsOversizedBody(t *testing.T) { - ch := &LINEChannel{} + ch := &LINEChannel{config: &config.LINESettings{}} oversized := bytes.Repeat([]byte("A"), maxWebhookBodySize+1) req := httptest.NewRequest(http.MethodPost, "/webhook", bytes.NewReader(oversized)) @@ -25,7 +25,7 @@ func TestWebhookRejectsOversizedBody(t *testing.T) { } func TestWebhookAcceptsMaxBodySize(t *testing.T) { - ch := &LINEChannel{} + ch := &LINEChannel{config: &config.LINESettings{}} body := bytes.Repeat([]byte("A"), maxWebhookBodySize) req := httptest.NewRequest(http.MethodPost, "/webhook", bytes.NewReader(body)) @@ -40,7 +40,7 @@ func TestWebhookAcceptsMaxBodySize(t *testing.T) { } func TestWebhookRejectsOversizedBodyBeforeSignatureCheck(t *testing.T) { - ch := &LINEChannel{} + ch := &LINEChannel{config: &config.LINESettings{}} oversized := bytes.Repeat([]byte("A"), maxWebhookBodySize+1) req := httptest.NewRequest(http.MethodPost, "/webhook", bytes.NewReader(oversized)) @@ -55,7 +55,7 @@ func TestWebhookRejectsOversizedBodyBeforeSignatureCheck(t *testing.T) { } func TestWebhookRejectsNonPostMethod(t *testing.T) { - ch := &LINEChannel{} + ch := &LINEChannel{config: &config.LINESettings{}} req := httptest.NewRequest(http.MethodGet, "/webhook", nil) rec := httptest.NewRecorder() diff --git a/pkg/channels/manager.go b/pkg/channels/manager.go index c6dcfebe3..d345a5d0b 100644 --- a/pkg/channels/manager.go +++ b/pkg/channels/manager.go @@ -23,6 +23,7 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/constants" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/health" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" @@ -84,6 +85,7 @@ type Manager struct { channels map[string]Channel workers map[string]*channelWorker bus *bus.MessageBus + runtimeEvents runtimeevents.Bus config *config.Config mediaStore media.MediaStore dispatchTask *asyncTask @@ -98,6 +100,36 @@ type Manager struct { channelHashes map[string]string // channel name → config hash } +type mediaStoreSetter interface { + SetMediaStore(s media.MediaStore) +} + +// ManagerOption configures a channel Manager. +type ManagerOption func(*Manager) + +// WithRuntimeEvents injects the runtime event bus used for channel observations. +func WithRuntimeEvents(eventBus runtimeevents.Bus) ManagerOption { + return func(m *Manager) { + m.runtimeEvents = eventBus + } +} + +// ChannelLifecyclePayload describes channel lifecycle runtime events. +type ChannelLifecyclePayload struct { + Type string `json:"type,omitempty"` + Error string `json:"error,omitempty"` +} + +// ChannelOutboundPayload describes channel outbound message runtime events. +type ChannelOutboundPayload struct { + Media bool `json:"media,omitempty"` + ContentLen int `json:"content_len,omitempty"` + MessageIDs []string `json:"message_ids,omitempty"` + ReplyToMessageID string `json:"reply_to_message_id,omitempty"` + Error string `json:"error,omitempty"` + Retries int `json:"retries,omitempty"` +} + type toolFeedbackMessageTracker interface { RecordToolFeedbackMessage(chatID, messageID, content string) ClearToolFeedbackMessage(chatID string) @@ -424,7 +456,12 @@ func (m *Manager) preSendMedia(ctx context.Context, name string, msg bus.Outboun } } -func NewManager(cfg *config.Config, messageBus *bus.MessageBus, store media.MediaStore) (*Manager, error) { +func NewManager( + cfg *config.Config, + messageBus *bus.MessageBus, + store media.MediaStore, + opts ...ManagerOption, +) (*Manager, error) { m := &Manager{ channels: make(map[string]Channel), workers: make(map[string]*channelWorker), @@ -433,6 +470,11 @@ func NewManager(cfg *config.Config, messageBus *bus.MessageBus, store media.Medi mediaStore: store, channelHashes: make(map[string]string), } + for _, opt := range opts { + if opt != nil { + opt(m) + } + } // Register as streaming delegate so the agent loop can obtain streamers messageBus.SetStreamDelegate(m) @@ -447,6 +489,22 @@ func NewManager(cfg *config.Config, messageBus *bus.MessageBus, store media.Medi return m, nil } +// SetMediaStore updates the store used by the manager and every channel that +// accepts media store injection. Gateway reload creates a fresh store, so +// keeping existing channels on the same store as the agent is required for +// inbound media refs to remain resolvable after reload. +func (m *Manager) SetMediaStore(store media.MediaStore) { + m.mu.Lock() + defer m.mu.Unlock() + + m.mediaStore = store + for _, ch := range m.channels { + if setter, ok := ch.(mediaStoreSetter); ok { + setter.SetMediaStore(store) + } + } +} + // GetStreamer implements bus.StreamDelegate. // It checks if the named channel supports streaming and returns a Streamer. func (m *Manager) GetStreamer(ctx context.Context, channelName, chatID string) (bus.Streamer, bool) { @@ -544,7 +602,7 @@ func (m *Manager) initChannel(typeName, channelName string) { } else { // Inject MediaStore if channel supports it if m.mediaStore != nil { - if setter, ok := ch.(interface{ SetMediaStore(s media.MediaStore) }); ok { + if setter, ok := ch.(mediaStoreSetter); ok { setter.SetMediaStore(m.mediaStore) } } @@ -557,6 +615,13 @@ func (m *Manager) initChannel(typeName, channelName string) { setter.SetOwner(ch) } m.channels[channelName] = ch + m.publishChannelEvent( + runtimeevents.KindChannelLifecycleInitialized, + channelName, + runtimeevents.Scope{Channel: channelName}, + runtimeevents.SeverityInfo, + ChannelLifecyclePayload{Type: typeName}, + ) logger.InfoCF("channels", "Channel enabled successfully", map[string]any{ "channel": channelName, "type": typeName, @@ -623,10 +688,14 @@ func (m *Manager) getChannelConfigAndEnabled(channelName string) (*config.Channe return bc, true case *config.TeamsWebhookSettings: return bc, true + case *config.SlackWebhookSettings: + return bc, true case *config.DiscordSettings: return bc, settings.Token.String() != "" case *config.VKSettings: return bc, settings.GroupID != 0 && settings.Token.String() != "" + case *config.MQTTSettings: + return bc, settings.Broker != "" && settings.AgentID != "" } return bc, bc.Enabled @@ -702,6 +771,13 @@ func (m *Manager) registerHTTPHandlersLocked() { func (m *Manager) registerChannelHTTPHandler(name string, ch Channel) { if wh, ok := ch.(WebhookHandler); ok { m.mux.Handle(wh.WebhookPath(), wh) + m.publishChannelEvent( + runtimeevents.KindChannelWebhookRegistered, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityInfo, + ChannelLifecyclePayload{Type: channelTypeForEvent(m, name)}, + ) logger.InfoCF("channels", "Webhook handler registered", map[string]any{ "channel": name, "path": wh.WebhookPath(), @@ -721,6 +797,13 @@ func (m *Manager) registerChannelHTTPHandler(name string, ch Channel) { func (m *Manager) unregisterChannelHTTPHandler(name string, ch Channel) { if wh, ok := ch.(WebhookHandler); ok { m.mux.Unhandle(wh.WebhookPath()) + m.publishChannelEvent( + runtimeevents.KindChannelWebhookUnregistered, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityInfo, + ChannelLifecyclePayload{Type: channelTypeForEvent(m, name)}, + ) logger.InfoCF("channels", "Webhook handler unregistered", map[string]any{ "channel": name, "path": wh.WebhookPath(), @@ -759,6 +842,13 @@ func (m *Manager) StartAll(ctx context.Context) error { "channel": name, "error": err.Error(), }) + m.publishChannelEvent( + runtimeevents.KindChannelLifecycleStartFailed, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityError, + ChannelLifecyclePayload{Type: channelTypeForEvent(m, name), Error: err.Error()}, + ) failedStarts = append(failedStarts, fmt.Errorf("channel %s: %w", name, err)) failedNames = append(failedNames, name) continue @@ -774,6 +864,13 @@ func (m *Manager) StartAll(ctx context.Context) error { m.workers[name] = w go m.runWorker(dispatchCtx, name, w) go m.runMediaWorker(dispatchCtx, name, w) + m.publishChannelEvent( + runtimeevents.KindChannelLifecycleStarted, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityInfo, + ChannelLifecyclePayload{Type: channelType}, + ) } if len(m.channels) > 0 && len(m.workers) == 0 { @@ -910,7 +1007,15 @@ func (m *Manager) StopAll(ctx context.Context) error { "channel": name, "error": err.Error(), }) + continue } + m.publishChannelEvent( + runtimeevents.KindChannelLifecycleStopped, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityInfo, + ChannelLifecyclePayload{Type: channelTypeForEvent(m, name)}, + ) } logger.InfoC("channels", "All channels stopped") @@ -1020,11 +1125,23 @@ func (m *Manager) sendWithRetry( // Rate limit: wait for token if err := w.limiter.Wait(ctx); err != nil { // ctx canceled, shutting down + m.publishChannelEvent( + runtimeevents.KindChannelRateLimited, + name, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityWarn, + ChannelOutboundPayload{ + ContentLen: len([]rune(msg.Content)), + ReplyToMessageID: msg.ReplyToMessageID, + Error: err.Error(), + }, + ) return nil, false } // Pre-send: stop typing and try to edit placeholder if msgIDs, handled := m.preSend(ctx, name, msg, w.ch); handled { + m.publishOutboundSent(name, msg, msgIDs) return msgIDs, true } @@ -1033,6 +1150,7 @@ func (m *Manager) sendWithRetry( for attempt := 0; attempt <= maxRetries; attempt++ { msgIDs, lastErr = w.ch.Send(ctx, msg) if lastErr == nil { + m.publishOutboundSent(name, msg, msgIDs) return msgIDs, true } @@ -1072,6 +1190,7 @@ func (m *Manager) sendWithRetry( "error": lastErr.Error(), "retries": maxRetries, }) + m.publishOutboundFailed(name, msg, lastErr, false) return nil, false } @@ -1134,6 +1253,7 @@ func (m *Manager) dispatchOutbound(ctx context.Context) { func(ctx context.Context, w *channelWorker, msg bus.OutboundMessage) bool { select { case w.queue <- msg: + m.publishOutboundQueued(outboundMessageChannel(msg), msg) return true case <-ctx.Done(): return false @@ -1154,6 +1274,7 @@ func (m *Manager) dispatchOutboundMedia(ctx context.Context) { func(ctx context.Context, w *channelWorker, msg bus.OutboundMediaMessage) bool { select { case w.mediaQueue <- msg: + m.publishOutboundMediaQueued(outboundMediaChannel(msg), msg) return true case <-ctx.Done(): return false @@ -1203,6 +1324,16 @@ func (m *Manager) sendMediaWithRetry( // Rate limit: wait for token if err := w.limiter.Wait(ctx); err != nil { + m.publishChannelEvent( + runtimeevents.KindChannelRateLimited, + name, + scopeFromOutboundContext(msg.Context), + runtimeevents.SeverityWarn, + ChannelOutboundPayload{ + Media: true, + Error: err.Error(), + }, + ) return nil, err } @@ -1214,6 +1345,7 @@ func (m *Manager) sendMediaWithRetry( for attempt := 0; attempt <= maxRetries; attempt++ { msgIDs, lastErr = ms.SendMedia(ctx, msg) if lastErr == nil { + m.publishOutboundMediaSent(name, msg, msgIDs) return msgIDs, nil } @@ -1253,6 +1385,7 @@ func (m *Manager) sendMediaWithRetry( "error": lastErr.Error(), "retries": maxRetries, }) + m.publishOutboundMediaFailed(name, msg, lastErr) return nil, lastErr } @@ -1390,6 +1523,13 @@ func (m *Manager) Reload(ctx context.Context, cfg *config.Config) error { "channel": name, "error": err.Error(), }) + m.publishChannelEvent( + runtimeevents.KindChannelLifecycleStartFailed, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityError, + ChannelLifecyclePayload{Type: channelTypeForEvent(m, name), Error: err.Error()}, + ) continue } // Lazily create worker only after channel starts successfully @@ -1403,6 +1543,13 @@ func (m *Manager) Reload(ctx context.Context, cfg *config.Config) error { m.workers[name] = w go m.runWorker(dispatchCtx, name, w) go m.runMediaWorker(dispatchCtx, name, w) + m.publishChannelEvent( + runtimeevents.KindChannelLifecycleStarted, + name, + runtimeevents.Scope{Channel: name}, + runtimeevents.SeverityInfo, + ChannelLifecyclePayload{Type: channelType}, + ) deferFuncs = append(deferFuncs, func() { m.RegisterChannel(name, channel) }) @@ -1525,6 +1672,7 @@ func (m *Manager) SendToChannel(ctx context.Context, channelName, chatID, conten if wExists && w != nil { select { case w.queue <- msg: + m.publishOutboundQueued(channelName, msg) return nil case <-ctx.Done(): return ctx.Err() diff --git a/pkg/channels/manager_channel.go b/pkg/channels/manager_channel.go index 1f5978e7d..9dbd35cab 100644 --- a/pkg/channels/manager_channel.go +++ b/pkg/channels/manager_channel.go @@ -102,6 +102,24 @@ func hiddenValues(key string, value map[string]any, ch *config.Channel) { } } value["webhooks"] = webhooks + case "mqtt": + if settings, ok := v.(*config.MQTTSettings); ok { + value["username"] = settings.Username.String() + value["password"] = settings.Password.String() + } + case "slack_webhook": + // Expose webhook URLs for hash computation (they contain secrets) + if settings, ok := v.(*config.SlackWebhookSettings); ok { + webhooks := make(map[string]any) + for name, target := range settings.Webhooks { + webhooks[name] = map[string]any{ + "webhook_url": target.WebhookURL.String(), + "username": target.Username, + "icon_emoji": target.IconEmoji, + } + } + value["webhooks"] = webhooks + } } } diff --git a/pkg/channels/manager_test.go b/pkg/channels/manager_test.go index 6c518780d..8c2f6ecf8 100644 --- a/pkg/channels/manager_test.go +++ b/pkg/channels/manager_test.go @@ -14,6 +14,8 @@ import ( "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" + "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -148,6 +150,26 @@ func newTestManager() *Manager { } } +func TestSetMediaStorePropagatesToExistingChannels(t *testing.T) { + oldStore := media.NewFileMediaStore() + newStore := media.NewFileMediaStore() + ch := &mockChannel{} + ch.SetMediaStore(oldStore) + + m := newTestManager() + m.mediaStore = oldStore + m.channels["telegram"] = ch + + m.SetMediaStore(newStore) + + if m.mediaStore != newStore { + t.Fatal("manager media store was not updated") + } + if got := ch.GetMediaStore(); got != newStore { + t.Fatalf("channel media store = %p, want %p", got, newStore) + } +} + func TestStartAll_AllChannelsFail_ReturnsJoinedError(t *testing.T) { m := newTestManager() errA := errors.New("channel-a start failed") @@ -242,6 +264,57 @@ func TestStartAll_PartialFailure_StartsSuccessfulWorkers(t *testing.T) { } } +func TestStartAllPublishesLifecycleRuntimeEvents(t *testing.T) { + eventBus := runtimeevents.NewBus() + defer func() { + if err := eventBus.Close(); err != nil { + t.Errorf("event bus close failed: %v", err) + } + }() + + _, eventsCh, err := eventBus.Channel().SubscribeChan( + t.Context(), + runtimeevents.SubscribeOptions{Name: "channel-lifecycle", Buffer: 4}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + m := newTestManager() + m.runtimeEvents = eventBus + m.config = &config.Config{Channels: config.ChannelsConfig{}} + m.channels["good"] = &mockChannel{} + m.channels["bad"] = &mockChannel{ + startFn: func(_ context.Context) error { return errors.New("bad start") }, + } + + if err := m.StartAll(t.Context()); err != nil { + t.Fatalf("StartAll() error = %v", err) + } + t.Cleanup(func() { + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := m.StopAll(stopCtx); err != nil { + t.Errorf("StopAll() error = %v", err) + } + }) + + events := []runtimeevents.Event{ + receiveChannelRuntimeEvent(t, eventsCh), + receiveChannelRuntimeEvent(t, eventsCh), + } + seen := map[runtimeevents.Kind]runtimeevents.Event{} + for _, evt := range events { + seen[evt.Kind] = evt + } + if evt, ok := seen[runtimeevents.KindChannelLifecycleStarted]; !ok || evt.Scope.Channel != "good" { + t.Fatalf("missing started event for good channel: %+v", events) + } + if evt, ok := seen[runtimeevents.KindChannelLifecycleStartFailed]; !ok || evt.Scope.Channel != "bad" { + t.Fatalf("missing failed event for bad channel: %+v", events) + } +} + func testOutboundMessage(msg bus.OutboundMessage) bus.OutboundMessage { if msg.Context.Channel == "" && msg.Context.ChatID == "" { msg.Context = bus.NewOutboundContext(msg.Channel, msg.ChatID, msg.ReplyToMessageID) @@ -256,6 +329,21 @@ func testOutboundMediaMessage(msg bus.OutboundMediaMessage) bus.OutboundMediaMes return bus.NormalizeOutboundMediaMessage(msg) } +func receiveChannelRuntimeEvent(t *testing.T, ch <-chan runtimeevents.Event) runtimeevents.Event { + t.Helper() + + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("runtime event channel closed before expected event") + } + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for runtime event") + return runtimeevents.Event{} + } +} + func TestSendWithRetry_Success(t *testing.T) { m := newTestManager() var callCount int @@ -280,6 +368,69 @@ func TestSendWithRetry_Success(t *testing.T) { } } +func TestSendWithRetryPublishesOutboundRuntimeEvents(t *testing.T) { + eventBus := runtimeevents.NewBus() + defer func() { + if err := eventBus.Close(); err != nil { + t.Errorf("event bus close failed: %v", err) + } + }() + + _, eventsCh, err := eventBus.Channel().OfKind( + runtimeevents.KindChannelMessageOutboundSent, + runtimeevents.KindChannelMessageOutboundFailed, + ).SubscribeChan(t.Context(), runtimeevents.SubscribeOptions{Name: "channel-outbound", Buffer: 2}) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + m := newTestManager() + m.runtimeEvents = eventBus + + successWorker := &channelWorker{ + ch: &mockChannel{}, + limiter: rate.NewLimiter(rate.Inf, 1), + } + m.sendWithRetry( + context.Background(), + "test", + successWorker, + testOutboundMessage(bus.OutboundMessage{Channel: "test", ChatID: "chat-1", Content: "hello"}), + ) + sent := receiveChannelRuntimeEvent(t, eventsCh) + if sent.Kind != runtimeevents.KindChannelMessageOutboundSent || sent.Scope.ChatID != "chat-1" { + t.Fatalf("sent event = %+v", sent) + } + if sent.Attrs["content_len"] != 5 { + t.Fatalf("sent attrs = %#v, want content_len", sent.Attrs) + } + + failWorker := &channelWorker{ + ch: &mockChannel{ + sendFn: func(context.Context, bus.OutboundMessage) error { + return fmt.Errorf("send failed: %w", ErrSendFailed) + }, + }, + limiter: rate.NewLimiter(rate.Inf, 1), + } + m.sendWithRetry( + context.Background(), + "test", + failWorker, + testOutboundMessage(bus.OutboundMessage{Channel: "test", ChatID: "chat-2", Content: "hello"}), + ) + failed := receiveChannelRuntimeEvent(t, eventsCh) + if failed.Kind != runtimeevents.KindChannelMessageOutboundFailed || failed.Scope.ChatID != "chat-2" { + t.Fatalf("failed event = %+v", failed) + } + if failed.Severity != runtimeevents.SeverityError { + t.Fatalf("failed severity = %q", failed.Severity) + } + if failed.Attrs["error"] == "" || failed.Attrs["retries"] != maxRetries { + t.Fatalf("failed attrs = %#v, want error and retries", failed.Attrs) + } +} + func TestSendWithRetry_TemporaryThenSuccess(t *testing.T) { m := newTestManager() var callCount int diff --git a/pkg/channels/mqtt/init.go b/pkg/channels/mqtt/init.go new file mode 100644 index 000000000..c9cec7e83 --- /dev/null +++ b/pkg/channels/mqtt/init.go @@ -0,0 +1,16 @@ +package mqtt + +import ( + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/channels" + "github.com/sipeed/picoclaw/pkg/config" +) + +func init() { + channels.RegisterSafeFactory( + config.ChannelMQTT, + func(bc *config.Channel, cfg *config.MQTTSettings, b *bus.MessageBus) (channels.Channel, error) { + return NewMQTTChannel(bc, cfg, b) + }, + ) +} diff --git a/pkg/channels/mqtt/mqtt.go b/pkg/channels/mqtt/mqtt.go new file mode 100644 index 000000000..c34bc79bf --- /dev/null +++ b/pkg/channels/mqtt/mqtt.go @@ -0,0 +1,255 @@ +package mqtt + +import ( + "context" + "crypto/rand" + "crypto/tls" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + pahomqtt "github.com/eclipse/paho.mqtt.golang" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/channels" + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" +) + +// mqttPayload is the JSON payload for both inbound and outbound messages. +type mqttPayload struct { + Text string `json:"text"` +} + +// MQTTChannel implements the Channel interface for MQTT-based communication. +type MQTTChannel struct { + *channels.BaseChannel + bc *config.Channel + cfg *config.MQTTSettings + client pahomqtt.Client + qos byte + clientID string +} + +// NewMQTTChannel creates a new MQTT channel instance. +func NewMQTTChannel(bc *config.Channel, cfg *config.MQTTSettings, b *bus.MessageBus) (*MQTTChannel, error) { + if cfg.Broker == "" { + return nil, fmt.Errorf("mqtt broker is required") + } + if cfg.AgentID == "" { + return nil, fmt.Errorf("mqtt agent_id is required") + } + + base := channels.NewBaseChannel("mqtt", cfg, b, bc.AllowFrom, + channels.WithGroupTrigger(bc.GroupTrigger), + channels.WithReasoningChannelID(bc.ReasoningChannelID), + ) + + mqttClientID := cfg.ClientID + if mqttClientID == "" { + var suffix [4]byte + _, _ = rand.Read(suffix[:]) + mqttClientID = fmt.Sprintf("picoclaw-mqtt-%s-%s", cfg.AgentID, hex.EncodeToString(suffix[:])) + } + + return &MQTTChannel{ + BaseChannel: base, + bc: bc, + cfg: cfg, + qos: byte(cfg.QoS), + clientID: mqttClientID, + }, nil +} + +// Start connects to the MQTT broker and begins listening for inbound messages. +func (c *MQTTChannel) Start(ctx context.Context) error { + logger.InfoC("mqtt", "Starting MQTT channel") + + keepAlive := c.cfg.KeepAlive + if keepAlive <= 0 { + keepAlive = 60 + } + + opts := pahomqtt.NewClientOptions() + opts.AddBroker(c.cfg.Broker) + opts.SetClientID(c.clientID) + opts.SetKeepAlive(time.Duration(keepAlive) * time.Second) + opts.SetAutoReconnect(true) + opts.SetConnectRetry(true) + opts.SetConnectRetryInterval(5 * time.Second) + opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true}) //nolint:gosec + + if c.cfg.Username.String() != "" { + opts.SetUsername(c.cfg.Username.String()) + opts.SetPassword(c.cfg.Password.String()) + } + + firstSubscribe := make(chan error, 1) + var once sync.Once + + opts.SetOnConnectHandler(func(client pahomqtt.Client) { + logger.InfoC("mqtt", "MQTT connected, subscribing to inbound topic") + err := c.subscribe(client) + once.Do(func() { firstSubscribe <- err }) + }) + + opts.SetConnectionLostHandler(func(_ pahomqtt.Client, err error) { + logger.WarnCF("mqtt", "MQTT connection lost", map[string]any{"error": err.Error()}) + }) + + client := pahomqtt.NewClient(opts) + token := client.Connect() + if !token.WaitTimeout(10 * time.Second) { + client.Disconnect(250) + return fmt.Errorf("mqtt connect timed out after 10s (broker: %s)", c.cfg.Broker) + } + if err := token.Error(); err != nil { + client.Disconnect(250) + return fmt.Errorf("mqtt connect failed: %w", err) + } + + if err := <-firstSubscribe; err != nil { + client.Disconnect(250) + return fmt.Errorf("mqtt subscribe failed: %w", err) + } + + c.client = client + c.SetRunning(true) + + logger.InfoCF("mqtt", "MQTT channel started", map[string]any{ + "broker": c.cfg.Broker, + "agent_id": c.cfg.AgentID, + }) + return nil +} + +// topicPrefix returns the configured topic prefix, normalizing slashes. +// Trailing slashes are stripped; the result may or may not have a leading slash +// depending on what the user configured. +func (c *MQTTChannel) topicPrefix() string { + p := strings.TrimRight(c.cfg.TopicPrefix, "/") + if p == "" { + return "/picoclaw" + } + return p +} + +// clientIDFromTopic extracts the client_id segment from a received topic. +// Topic structure: {prefix}/{agent_id}/{client_id}/request +func (c *MQTTChannel) clientIDFromTopic(topic string) (string, bool) { + prefix := c.topicPrefix() + // Build the expected fixed portion: {prefix}/{agent_id}/ + fixed := prefix + "/" + c.cfg.AgentID + "/" + after, ok := strings.CutPrefix(topic, fixed) + if !ok { + return "", false + } + // after = "{client_id}/request" + slash := strings.IndexByte(after, '/') + if slash < 0 { + return "", false + } + return after[:slash], true +} + +// subscribe subscribes to the inbound topic for this agent. +func (c *MQTTChannel) subscribe(client pahomqtt.Client) error { + topic := fmt.Sprintf("%s/%s/+/request", c.topicPrefix(), c.cfg.AgentID) + token := client.Subscribe(topic, c.qos, func(_ pahomqtt.Client, msg pahomqtt.Message) { + c.handleInbound(msg) + }) + token.Wait() + if err := token.Error(); err != nil { + logger.ErrorCF("mqtt", "Failed to subscribe", map[string]any{ + "topic": topic, + "error": err.Error(), + }) + return err + } + logger.InfoCF("mqtt", "Subscribed to inbound topic", map[string]any{"topic": topic}) + return nil +} + +// handleInbound processes an inbound MQTT message. +func (c *MQTTChannel) handleInbound(msg pahomqtt.Message) { + topic := msg.Topic() + + clientID, ok := c.clientIDFromTopic(topic) + if !ok { + logger.WarnCF("mqtt", "Unexpected topic format", map[string]any{"topic": topic}) + return + } + chatID := "mqtt:" + clientID + + var payload mqttPayload + if err := json.Unmarshal(msg.Payload(), &payload); err != nil { + logger.WarnCF("mqtt", "Failed to parse inbound payload", map[string]any{ + "topic": topic, + "error": err.Error(), + }) + return + } + + if payload.Text == "" { + logger.WarnCF("mqtt", "Inbound payload missing text", map[string]any{"topic": topic}) + return + } + + inboundCtx := bus.InboundContext{ + Channel: "mqtt", + ChatID: chatID, + ChatType: "direct", + SenderID: clientID, + } + + c.HandleInboundContext(context.Background(), chatID, payload.Text, nil, inboundCtx) +} + +// Stop disconnects from the MQTT broker. +func (c *MQTTChannel) Stop(_ context.Context) error { + logger.InfoC("mqtt", "Stopping MQTT channel") + c.SetRunning(false) + + if c.client != nil { + c.client.Disconnect(500) + } + + logger.InfoC("mqtt", "MQTT channel stopped") + return nil +} + +// Send publishes a response to the client via MQTT. +func (c *MQTTChannel) Send(_ context.Context, msg bus.OutboundMessage) ([]string, error) { + if !c.IsRunning() { + return nil, channels.ErrNotRunning + } + + if strings.TrimSpace(msg.Content) == "" { + return nil, nil + } + + clientID := strings.TrimPrefix(msg.ChatID, "mqtt:") + if clientID == msg.ChatID { + logger.WarnCF("mqtt", "Send called with unexpected chatID format", map[string]any{"chat_id": msg.ChatID}) + return nil, nil + } + + topic := fmt.Sprintf("%s/%s/%s/response", c.topicPrefix(), c.cfg.AgentID, clientID) + + data, err := json.Marshal(mqttPayload{Text: msg.Content}) + if err != nil { + return nil, fmt.Errorf("mqtt: failed to marshal outbound payload: %w", err) + } + + token := c.client.Publish(topic, c.qos, false, data) + token.Wait() + if err := token.Error(); err != nil { + return nil, fmt.Errorf("mqtt: publish failed: %w", err) + } + + logger.DebugCF("mqtt", "Published response", map[string]any{"topic": topic}) + return nil, nil +} diff --git a/pkg/channels/slack_webhook/convert.go b/pkg/channels/slack_webhook/convert.go new file mode 100644 index 000000000..6ee2be2f8 --- /dev/null +++ b/pkg/channels/slack_webhook/convert.go @@ -0,0 +1,263 @@ +package slackwebhook + +import ( + "fmt" + "regexp" + "strings" +) + +const maxTableRowWidth = 60 + +var ( + boldRe = regexp.MustCompile(`\*\*([^*]+)\*\*`) + strikeRe = regexp.MustCompile(`~~([^~]+)~~`) + linkRe = regexp.MustCompile(`\[([^\]]+)\]\(([^)]+)\)`) + headerRe = regexp.MustCompile(`(?m)^#{1,6}\s+(.+)$`) + bulletRe = regexp.MustCompile(`(?m)^- (.+)$`) + markdownTableRe = regexp.MustCompile(`(?m)^(\|[^\n]+\|)\n(\|[-:\|\s]+\|)\n((?:\|[^\n]+\|\n?)+)`) + codeBlockRe = regexp.MustCompile("(?s)```.*?```") + inlineCodeRe = regexp.MustCompile("`[^`]+`") + italicRe = regexp.MustCompile(`(?:^|[^*])\*([^*]+)\*(?:[^*]|$)`) +) + +type contentSegment struct { + content string + isTable bool +} + +func convertMarkdownToMrkdwn(text string) string { + // Protect code blocks from conversion + var codeBlocks []string + text = codeBlockRe.ReplaceAllStringFunc(text, func(match string) string { + codeBlocks = append(codeBlocks, match) + return "\x00CODEBLOCK\x00" + }) + + // Protect inline code + var inlineCode []string + text = inlineCodeRe.ReplaceAllStringFunc(text, func(match string) string { + inlineCode = append(inlineCode, match) + return "\x00INLINE\x00" + }) + + // Convert italic *text* → _text_ BEFORE bold conversion + text = italicRe.ReplaceAllStringFunc(text, func(match string) string { + // Find the asterisk positions + firstAsterisk := strings.Index(match, "*") + lastAsterisk := strings.LastIndex(match, "*") + if firstAsterisk == lastAsterisk { + return match // Only one asterisk, not italic + } + + // Extract content between asterisks + content := match[firstAsterisk+1 : lastAsterisk] + + // Replace with underscores, preserving any prefix/suffix + return match[:firstAsterisk] + "_" + content + "_" + match[lastAsterisk+1:] + }) + + // Convert bold **text** → *text* + text = boldRe.ReplaceAllString(text, "*$1*") + + // Convert strikethrough ~~text~~ → ~text~ + text = strikeRe.ReplaceAllString(text, "~$1~") + + // Convert links [text](url) → + text = linkRe.ReplaceAllString(text, "<$2|$1>") + + // Convert headers # text → *text* + text = headerRe.ReplaceAllString(text, "*$1*") + + // Convert bullet lists - item → • item + text = bulletRe.ReplaceAllString(text, "• $1") + + // Restore inline code + for _, code := range inlineCode { + text = strings.Replace(text, "\x00INLINE\x00", code, 1) + } + + // Restore code blocks + for _, block := range codeBlocks { + text = strings.Replace(text, "\x00CODEBLOCK\x00", block, 1) + } + + return text +} + +func splitContentWithTables(content string) []contentSegment { + var segments []contentSegment + + // Protect code blocks from table detection using unique placeholders + var codeBlocks []string + blockIdx := 0 + protected := codeBlockRe.ReplaceAllStringFunc(content, func(match string) string { + codeBlocks = append(codeBlocks, match) + placeholder := fmt.Sprintf("\x00CODEBLOCK_%d\x00", blockIdx) + blockIdx++ + return placeholder + }) + + matches := markdownTableRe.FindAllStringSubmatchIndex(protected, -1) + if len(matches) == 0 { + return []contentSegment{{content: content, isTable: false}} + } + + // Restore code blocks using indexed placeholders + restoreCodeBlocks := func(s string) string { + result := s + for i, block := range codeBlocks { + placeholder := fmt.Sprintf("\x00CODEBLOCK_%d\x00", i) + result = strings.Replace(result, placeholder, block, 1) + } + return result + } + + lastEnd := 0 + for _, match := range matches { + if match[0] > lastEnd { + segments = append(segments, contentSegment{ + content: restoreCodeBlocks(protected[lastEnd:match[0]]), + isTable: false, + }) + } + segments = append(segments, contentSegment{ + content: restoreCodeBlocks(protected[match[0]:match[1]]), + isTable: true, + }) + lastEnd = match[1] + } + + if lastEnd < len(protected) { + segments = append(segments, contentSegment{ + content: restoreCodeBlocks(protected[lastEnd:]), + isTable: false, + }) + } + + return segments +} + +func renderTable(tableStr string) string { + lines := strings.Split(strings.TrimSpace(tableStr), "\n") + if len(lines) < 2 { + return "```\n" + tableStr + "\n```" + } + + // Parse all rows to get column widths + var allRows [][]string + maxCols := 0 + for i, line := range lines { + if i == 1 && isSeparatorRow(line) { + continue + } + cells := parseTableRow(line) + if len(cells) > 0 { + allRows = append(allRows, cells) + if len(cells) > maxCols { + maxCols = len(cells) + } + } + } + + if len(allRows) == 0 { + return "```\n" + tableStr + "\n```" + } + + // Calculate max width for each column using rune count + colWidths := make([]int, maxCols) + for _, row := range allRows { + for i, cell := range row { + runeLen := len([]rune(cell)) + if runeLen > colWidths[i] { + colWidths[i] = runeLen + } + } + } + + // Check if table is narrow enough for mrkdwn format + totalWidth := 0 + for _, w := range colWidths { + totalWidth += w + } + if len(colWidths) > 1 { + totalWidth += 3 * (len(colWidths) - 1) // " | " separators between columns + } + if totalWidth <= maxTableRowWidth { + // Render as formatted text with bold headers + var result strings.Builder + for i, row := range allRows { + if i == 0 { + var boldCells []string + for _, cell := range row { + boldCells = append(boldCells, "*"+cell+"*") + } + result.WriteString(strings.Join(boldCells, " | ")) + } else { + result.WriteString(strings.Join(row, " | ")) + } + result.WriteString("\n") + } + return strings.TrimSuffix(result.String(), "\n") + } + + // Render as aligned code block + var result strings.Builder + result.WriteString("```\n") + for i, row := range allRows { + var paddedCells []string + for j, cell := range row { + if j < len(colWidths) { + paddedCells = append(paddedCells, padRight(cell, colWidths[j])) + } else { + paddedCells = append(paddedCells, cell) + } + } + result.WriteString("| ") + result.WriteString(strings.Join(paddedCells, " | ")) + result.WriteString(" |\n") + + // Add separator after header + if i == 0 { + var sepParts []string + for _, w := range colWidths { + sepParts = append(sepParts, strings.Repeat("-", w)) + } + result.WriteString("|-") + result.WriteString(strings.Join(sepParts, "-|-")) + result.WriteString("-|\n") + } + } + result.WriteString("```") + return result.String() +} + +func padRight(s string, width int) string { + runeLen := len([]rune(s)) + if runeLen >= width { + return s + } + return s + strings.Repeat(" ", width-runeLen) +} + +func isSeparatorRow(line string) bool { + cleaned := strings.ReplaceAll(line, "|", "") + cleaned = strings.ReplaceAll(cleaned, " ", "") + cleaned = strings.ReplaceAll(cleaned, "-", "") + cleaned = strings.ReplaceAll(cleaned, ":", "") + return cleaned == "" +} + +func parseTableRow(line string) []string { + line = strings.TrimSpace(line) + line = strings.TrimPrefix(line, "|") + line = strings.TrimSuffix(line, "|") + if line == "" { + return nil + } + parts := strings.Split(line, "|") + var cells []string + for _, p := range parts { + cells = append(cells, strings.TrimSpace(p)) + } + return cells +} diff --git a/pkg/channels/slack_webhook/convert_test.go b/pkg/channels/slack_webhook/convert_test.go new file mode 100644 index 000000000..39ff4ac04 --- /dev/null +++ b/pkg/channels/slack_webhook/convert_test.go @@ -0,0 +1,187 @@ +package slackwebhook + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConvertMarkdownToMrkdwn(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "bold double asterisk", + input: "This is **bold** text", + expected: "This is *bold* text", + }, + { + name: "italic single asterisk", + input: "This is *italic* text", + expected: "This is _italic_ text", + }, + { + name: "italic underscore", + input: "This is _italic_ text", + expected: "This is _italic_ text", + }, + { + name: "strikethrough", + input: "This is ~~struck~~ text", + expected: "This is ~struck~ text", + }, + { + name: "inline code unchanged", + input: "Use `code` here", + expected: "Use `code` here", + }, + { + name: "link conversion", + input: "Click [here](https://example.com) now", + expected: "Click now", + }, + { + name: "header to bold", + input: "# Header One", + expected: "*Header One*", + }, + { + name: "header level 2", + input: "## Header Two", + expected: "*Header Two*", + }, + { + name: "bullet list", + input: "- item one\n- item two", + expected: "• item one\n• item two", + }, + { + name: "mixed formatting", + input: "**bold** and *italic* and [link](http://x.com)", + expected: "*bold* and _italic_ and ", + }, + { + name: "code block unchanged", + input: "```\ncode here\n```", + expected: "```\ncode here\n```", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := convertMarkdownToMrkdwn(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestSplitContentWithTables(t *testing.T) { + tests := []struct { + name string + input string + expectedCount int + expectedTables int + }{ + { + name: "no table", + input: "Just some text", + expectedCount: 1, + expectedTables: 0, + }, + { + name: "simple table", + input: "| A | B |\n|---|---|\n| 1 | 2 |", + expectedCount: 1, + expectedTables: 1, + }, + { + name: "text before table", + input: "Intro text\n\n| A | B |\n|---|---|\n| 1 | 2 |", + expectedCount: 2, + expectedTables: 1, + }, + { + name: "text before and after table", + input: "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter", + expectedCount: 3, + expectedTables: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + segments := splitContentWithTables(tt.input) + assert.Equal(t, tt.expectedCount, len(segments)) + tableCount := 0 + for _, seg := range segments { + if seg.isTable { + tableCount++ + } + } + assert.Equal(t, tt.expectedTables, tableCount) + }) + } +} + +func TestRenderTable(t *testing.T) { + tests := []struct { + name string + input string + expectCode bool + }{ + { + name: "narrow table renders as text", + input: "| A | B |\n|---|---|\n| 1 | 2 |", + expectCode: false, + }, + { + name: "wide table renders as code block", + input: "| This is a very long column header | Another extremely long column header here |\n|---|---|\n| Some long value content here | More long value content |", + expectCode: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := renderTable(tt.input) + if tt.expectCode { + assert.Contains(t, result, "```") + } else { + assert.NotContains(t, result, "```") + assert.Contains(t, result, "*") // Bold headers + } + }) + } +} + +func TestRenderTable_Alignment(t *testing.T) { + input := "| Name | Status | Count |\n|---|---|---|\n| foo | OK | 1 |\n| barbaz | PENDING | 123 |" + result := renderTable(input) + + // Should be mrkdwn (narrow table) + assert.NotContains(t, result, "```") + assert.Contains(t, result, "*Name*") + + // Test wide table alignment + wideInput := "| This is a very long column header | Another extremely long column header here |\n|---|---|\n| Short | Longer value here |" + wideResult := renderTable(wideInput) + + assert.Contains(t, wideResult, "```") + // Check that columns are padded - header and value should have same column width + lines := strings.Split(wideResult, "\n") + // Find the header line and a data line + var headerLine, dataLine string + for _, line := range lines { + if strings.Contains(line, "This is a very long") { + headerLine = line + } + if strings.Contains(line, "Short") { + dataLine = line + } + } + // Both lines should have same length (aligned columns) + assert.Equal(t, len(headerLine), len(dataLine), "columns should be aligned") +} diff --git a/pkg/channels/slack_webhook/init.go b/pkg/channels/slack_webhook/init.go new file mode 100644 index 000000000..eed3ae083 --- /dev/null +++ b/pkg/channels/slack_webhook/init.go @@ -0,0 +1,32 @@ +package slackwebhook + +import ( + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/channels" + "github.com/sipeed/picoclaw/pkg/config" +) + +func init() { + channels.RegisterFactory( + config.ChannelSlackWebHook, + func(channelName, channelType string, cfg *config.Config, b *bus.MessageBus) (channels.Channel, error) { + bc := cfg.Channels[channelName] + decoded, err := bc.GetDecoded() + if err != nil { + return nil, err + } + c, ok := decoded.(*config.SlackWebhookSettings) + if !ok { + return nil, channels.ErrSendFailed + } + ch, err := NewSlackWebhookChannel(bc, c, b) + if err != nil { + return nil, err + } + if channelName != config.ChannelSlackWebHook { + ch.SetName(channelName) + } + return ch, nil + }, + ) +} diff --git a/pkg/channels/slack_webhook/slack_webhook.go b/pkg/channels/slack_webhook/slack_webhook.go new file mode 100644 index 000000000..95951de66 --- /dev/null +++ b/pkg/channels/slack_webhook/slack_webhook.go @@ -0,0 +1,316 @@ +package slackwebhook + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/channels" + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" +) + +const maxTextBlockLength = 3000 + +// SlackWebhookChannel is an output-only channel that sends messages +// to Slack via Incoming Webhooks using Block Kit formatting. +type SlackWebhookChannel struct { + *channels.BaseChannel + bc *config.Channel + config *config.SlackWebhookSettings + client *http.Client +} + +// NewSlackWebhookChannel creates a new Slack webhook channel. +func NewSlackWebhookChannel( + bc *config.Channel, + cfg *config.SlackWebhookSettings, + bus *bus.MessageBus, +) (*SlackWebhookChannel, error) { + if len(cfg.Webhooks) == 0 { + return nil, fmt.Errorf("slack_webhook: at least one webhook target is required") + } + + if _, hasDefault := cfg.Webhooks["default"]; !hasDefault { + return nil, fmt.Errorf("slack_webhook: a 'default' webhook target is required") + } + + for name, target := range cfg.Webhooks { + webhookURL := target.WebhookURL.String() + if webhookURL == "" { + return nil, fmt.Errorf("slack_webhook: webhook %q has empty webhook_url", name) + } + parsed, err := url.Parse(webhookURL) + if err != nil { + return nil, fmt.Errorf("slack_webhook: webhook %q has invalid URL format: %w", name, err) + } + if !strings.EqualFold(parsed.Scheme, "https") { + return nil, fmt.Errorf("slack_webhook: webhook %q must use HTTPS (got %q)", name, parsed.Scheme) + } + } + + base := channels.NewBaseChannel( + "slack_webhook", + cfg, + bus, + []string{"*"}, + channels.WithMaxMessageLength(40000), + ) + + return &SlackWebhookChannel{ + BaseChannel: base, + bc: bc, + config: cfg, + client: &http.Client{ + Timeout: 30 * time.Second, + }, + }, nil +} + +// Start initializes the channel. For output-only channels, this is a no-op. +func (c *SlackWebhookChannel) Start(ctx context.Context) error { + targets := make([]string, 0, len(c.config.Webhooks)) + for name := range c.config.Webhooks { + targets = append(targets, name) + } + sort.Strings(targets) + logger.InfoCF("slack_webhook", "Starting Slack webhook channel (output-only)", map[string]any{ + "targets": targets, + }) + c.SetRunning(true) + return nil +} + +// Stop shuts down the channel. +func (c *SlackWebhookChannel) Stop(ctx context.Context) error { + logger.InfoC("slack_webhook", "Stopping Slack webhook channel") + c.SetRunning(false) + return nil +} + +// Send delivers a message to the specified Slack webhook target. +func (c *SlackWebhookChannel) Send(ctx context.Context, msg bus.OutboundMessage) ([]string, error) { + if !c.IsRunning() { + return nil, channels.ErrNotRunning + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + targetName := msg.ChatID + if targetName == "" { + targetName = "default" + } + + target, ok := c.config.Webhooks[targetName] + if !ok { + logger.WarnCF("slack_webhook", "Unknown target, falling back to default", map[string]any{ + "requested": msg.ChatID, + "using": "default", + }) + target = c.config.Webhooks["default"] + targetName = "default" + } + + payload := c.buildPayload(msg, target) + + jsonData, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("slack_webhook: failed to marshal payload: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target.WebhookURL.String(), bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("slack_webhook: failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.client.Do(req) + if err != nil { + logger.ErrorCF("slack_webhook", "Failed to send message", map[string]any{ + "target": targetName, + }) + // Don't expose raw error - it may contain webhook URL secrets + return nil, fmt.Errorf("slack_webhook: network error: %w", channels.ErrTemporary) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + respText := strings.TrimSpace(string(respBody)) + if respText == "" { + respText = http.StatusText(resp.StatusCode) + if respText == "" { + respText = "unknown error" + } + } + logger.ErrorCF("slack_webhook", "Slack API error", map[string]any{ + "target": targetName, + "status": resp.StatusCode, + "response": respText, + }) + sendErr := fmt.Errorf("status %d: %s", resp.StatusCode, respText) + return nil, fmt.Errorf("slack_webhook: %w", channels.ClassifySendError(resp.StatusCode, sendErr)) + } + + logger.DebugCF("slack_webhook", "Message sent successfully", map[string]any{ + "target": targetName, + }) + + return nil, nil +} + +func (c *SlackWebhookChannel) buildPayload(msg bus.OutboundMessage, target config.SlackWebhookTarget) map[string]any { + payload := make(map[string]any) + + if target.Username != "" { + payload["username"] = target.Username + } + if target.IconEmoji != "" { + payload["icon_emoji"] = target.IconEmoji + } + + content := msg.Content + if content == "" { + content = "(empty message)" + } + + blocks := c.buildBlocks(content) + payload["blocks"] = blocks + + return payload +} + +func (c *SlackWebhookChannel) buildBlocks(content string) []map[string]any { + var blocks []map[string]any + + segments := splitContentWithTables(content) + + for _, seg := range segments { + if seg.isTable { + tableText := renderTable(seg.content) + for _, chunk := range splitText(tableText, maxTextBlockLength) { + blocks = append(blocks, c.textSection(chunk)) + } + } else { + text := strings.TrimSpace(seg.content) + if text == "" { + continue + } + converted := convertMarkdownToMrkdwn(text) + for _, chunk := range splitText(converted, maxTextBlockLength) { + blocks = append(blocks, c.textSection(chunk)) + } + } + } + + if len(blocks) == 0 { + blocks = append(blocks, c.textSection("(empty message)")) + } + + return blocks +} + +func (c *SlackWebhookChannel) textSection(text string) map[string]any { + return map[string]any{ + "type": "section", + "text": map[string]any{ + "type": "mrkdwn", + "text": text, + }, + } +} + +func splitText(text string, maxLen int) []string { + runes := []rune(text) + if len(runes) <= maxLen { + return []string{text} + } + + const fencePrefix = "```\n" + const fenceSuffix = "\n```" + fencePrefixLen := len([]rune(fencePrefix)) + fenceSuffixLen := len([]rune(fenceSuffix)) + + var chunks []string + inFence := false + + for len(runes) > 0 { + // Calculate content budget reserving space for fence markers + prefixLen := 0 + if inFence { + prefixLen = fencePrefixLen + } + contentBudget := maxLen - prefixLen - fenceSuffixLen + if contentBudget <= 0 { + contentBudget = maxLen + } + + splitAt := len(runes) + if splitAt > contentBudget { + splitAt = findSplitPoint(runes, contentBudget) + if splitAt <= 0 || splitAt > contentBudget { + splitAt = contentBudget + } + } + + chunkBody := string(runes[:splitAt]) + chunkEndsInFence := endsInsideFence(chunkBody, inFence) + chunk := wrapFenceChunk(chunkBody, inFence, chunkEndsInFence) + + chunks = append(chunks, chunk) + inFence = chunkEndsInFence + runes = runes[splitAt:] + } + + return chunks +} + +func wrapFenceChunk(text string, wasInFence bool, endsInFence bool) string { + if wasInFence && !strings.HasPrefix(strings.TrimSpace(text), "```") { + text = "```\n" + text + } + if endsInFence { + text = strings.TrimSuffix(text, "\n") + "\n```" + } + return text +} + +func findSplitPoint(runes []rune, maxLen int) int { + if len(runes) <= maxLen { + return len(runes) + } + window := string(runes[:maxLen]) + + // Try splitting on newline + if idx := strings.LastIndex(window, "\n"); idx > 0 { + return len([]rune(window[:idx])) + 1 + } + + // Try splitting on space + if idx := strings.LastIndex(window, " "); idx > 0 { + return len([]rune(window[:idx])) + 1 + } + + // Try to split before a fence marker + if idx := strings.LastIndex(window, "```"); idx > 0 { + return len([]rune(window[:idx])) + } + + return maxLen +} + +func endsInsideFence(text string, wasInFence bool) bool { + return wasInFence != (strings.Count(text, "```")%2 == 1) +} diff --git a/pkg/channels/slack_webhook/slack_webhook_test.go b/pkg/channels/slack_webhook/slack_webhook_test.go new file mode 100644 index 000000000..83a4c0522 --- /dev/null +++ b/pkg/channels/slack_webhook/slack_webhook_test.go @@ -0,0 +1,281 @@ +package slackwebhook + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sipeed/picoclaw/pkg/bus" + "github.com/sipeed/picoclaw/pkg/channels" + "github.com/sipeed/picoclaw/pkg/config" +) + +func TestNewSlackWebhookChannel_Validation(t *testing.T) { + tests := []struct { + name string + webhooks map[string]config.SlackWebhookTarget + expectErr string + }{ + { + name: "empty webhooks", + webhooks: map[string]config.SlackWebhookTarget{}, + expectErr: "at least one webhook target is required", + }, + { + name: "missing default", + webhooks: map[string]config.SlackWebhookTarget{ + "alerts": { + WebhookURL: *config.NewSecureString("https://hooks.slack.com/services/T/B/x"), + }, + }, + expectErr: "a 'default' webhook target is required", + }, + { + name: "empty webhook URL", + webhooks: map[string]config.SlackWebhookTarget{ + "default": {WebhookURL: *config.NewSecureString("")}, + }, + expectErr: "has empty webhook_url", + }, + { + name: "non-HTTPS URL", + webhooks: map[string]config.SlackWebhookTarget{ + "default": { + WebhookURL: *config.NewSecureString("http://hooks.slack.com/services/T/B/x"), + }, + }, + expectErr: "must use HTTPS", + }, + { + name: "valid config", + webhooks: map[string]config.SlackWebhookTarget{ + "default": { + WebhookURL: *config.NewSecureString("https://hooks.slack.com/services/T/B/x"), + Username: "TestBot", + IconEmoji: ":robot_face:", + }, + }, + expectErr: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &config.SlackWebhookSettings{Webhooks: tt.webhooks} + bc := &config.Channel{Enabled: true} + mb := bus.NewMessageBus() + + ch, err := NewSlackWebhookChannel(bc, cfg, mb) + if tt.expectErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectErr) + } else { + require.NoError(t, err) + assert.NotNil(t, ch) + } + }) + } +} + +func TestSlackWebhookChannel_Send(t *testing.T) { + payloadCh := make(chan map[string]any, 1) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var payload map[string]any + json.Unmarshal(body, &payload) + payloadCh <- payload + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + cfg := &config.SlackWebhookSettings{ + Webhooks: map[string]config.SlackWebhookTarget{ + "default": { + WebhookURL: *config.NewSecureString(server.URL), + Username: "TestBot", + IconEmoji: ":test:", + }, + }, + } + bc := &config.Channel{Enabled: true} + mb := bus.NewMessageBus() + + ch, err := NewSlackWebhookChannel(bc, cfg, mb) + require.NoError(t, err) + + // Use the test server's client to skip TLS verification + ch.client = server.Client() + + err = ch.Start(context.Background()) + require.NoError(t, err) + + _, err = ch.Send(context.Background(), bus.OutboundMessage{ + Content: "Hello **world**", + ChatID: "default", + }) + require.NoError(t, err) + + // Verify payload structure + receivedPayload := <-payloadCh + assert.Equal(t, "TestBot", receivedPayload["username"]) + assert.Equal(t, ":test:", receivedPayload["icon_emoji"]) + blocks, ok := receivedPayload["blocks"].([]any) + require.True(t, ok) + require.Len(t, blocks, 1) +} + +func TestSlackWebhookChannel_FallbackToDefault(t *testing.T) { + var requestCount atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + cfg := &config.SlackWebhookSettings{ + Webhooks: map[string]config.SlackWebhookTarget{ + "default": {WebhookURL: *config.NewSecureString(server.URL)}, + }, + } + bc := &config.Channel{Enabled: true} + mb := bus.NewMessageBus() + + ch, err := NewSlackWebhookChannel(bc, cfg, mb) + require.NoError(t, err) + ch.client = server.Client() + err = ch.Start(context.Background()) + require.NoError(t, err) + + // Send to unknown target - should fall back to default + _, err = ch.Send(context.Background(), bus.OutboundMessage{ + Content: "Test", + ChatID: "unknown_target", + }) + require.NoError(t, err) + assert.Equal(t, int32(1), requestCount.Load()) +} + +func TestSlackWebhookChannel_ErrorClassification(t *testing.T) { + tests := []struct { + name string + statusCode int + expectTemp bool + }{ + {"400 Bad Request", 400, false}, + {"401 Unauthorized", 401, false}, + {"403 Forbidden", 403, false}, + {"404 Not Found", 404, false}, + {"500 Internal Error", 500, true}, + {"502 Bad Gateway", 502, true}, + {"503 Service Unavailable", 503, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewTLSServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.statusCode) + }), + ) + defer server.Close() + + cfg := &config.SlackWebhookSettings{ + Webhooks: map[string]config.SlackWebhookTarget{ + "default": {WebhookURL: *config.NewSecureString(server.URL)}, + }, + } + bc := &config.Channel{Enabled: true} + mb := bus.NewMessageBus() + + ch, err := NewSlackWebhookChannel(bc, cfg, mb) + require.NoError(t, err) + ch.client = server.Client() + err = ch.Start(context.Background()) + require.NoError(t, err) + + _, err = ch.Send(context.Background(), bus.OutboundMessage{Content: "Test"}) + require.Error(t, err) + + if tt.expectTemp { + assert.True( + t, + errors.Is(err, channels.ErrTemporary), + "expected temporary error for %d", + tt.statusCode, + ) + } else { + assert.True(t, errors.Is(err, channels.ErrSendFailed), "expected permanent error for %d", tt.statusCode) + } + }) + } +} + +func TestSplitText_ChunkSizeLimit(t *testing.T) { + tests := []struct { + name string + input string + maxLen int + }{ + { + name: "plain text", + input: strings.Repeat("a", 5000), + maxLen: 3000, + }, + { + name: "text with code block", + input: "```\n" + strings.Repeat("x", 5000) + "\n```", + maxLen: 3000, + }, + { + name: "multiple code blocks", + input: "text\n```\n" + strings.Repeat( + "code ", + 800, + ) + "\n```\nmore text\n```\n" + strings.Repeat( + "more ", + 800, + ) + "\n```", + maxLen: 3000, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + chunks := splitText(tt.input, tt.maxLen) + for i, chunk := range chunks { + runeLen := len([]rune(chunk)) + assert.LessOrEqual(t, runeLen, tt.maxLen, + "chunk %d has %d runes, exceeds max %d", i, runeLen, tt.maxLen) + } + }) + } +} + +func TestSplitText_FenceIntegrity(t *testing.T) { + input := "```\n" + strings.Repeat("line of code\n", 300) + "```" + + chunks := splitText(input, 3000) + require.Greater(t, len(chunks), 1, "expected multiple chunks") + + for i, chunk := range chunks { + openCount := strings.Count(chunk, "```") + assert.Equal(t, 0, openCount%2, + "chunk %d has unbalanced fence markers (count=%d)", i, openCount) + } +} + +func TestSplitText_ShortText(t *testing.T) { + input := "short text" + chunks := splitText(input, 3000) + require.Len(t, chunks, 1) + assert.Equal(t, input, chunks[0]) +} diff --git a/pkg/channels/telegram/telegram.go b/pkg/channels/telegram/telegram.go index cebebfed6..0965bcedc 100644 --- a/pkg/channels/telegram/telegram.go +++ b/pkg/channels/telegram/telegram.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "regexp" + "slices" "strconv" "strings" "sync" @@ -43,20 +44,38 @@ var ( reInlineCode = regexp.MustCompile("`([^`]+)`") ) +const defaultMediaGroupDelay = 500 * time.Millisecond + type TelegramChannel struct { *channels.BaseChannel - bot *telego.Bot - bh *th.BotHandler - bc *config.Channel - chatIDs map[string]int64 - ctx context.Context - cancel context.CancelFunc - tgCfg *config.TelegramSettings - progress *channels.ToolFeedbackAnimator + bot *telego.Bot + bh *th.BotHandler + bc *config.Channel + chatIDsMu sync.Mutex + chatIDs map[string]int64 + ctx context.Context + cancel context.CancelFunc + tgCfg *config.TelegramSettings + progress *channels.ToolFeedbackAnimator registerFunc func(context.Context, []commands.Definition) error commandRegDelayFn func(int) time.Duration commandRegCancel context.CancelFunc + + mediaGroupMu sync.Mutex + mediaGroups map[string]*telegramMediaGroup + mediaGroupDelay time.Duration +} + +type telegramMediaGroup struct { + messages []*telego.Message + timer *time.Timer + generation uint64 +} + +type telegramMessageParts struct { + content []string + mediaPaths []string } func NewTelegramChannel( @@ -112,11 +131,21 @@ func NewTelegramChannel( bc: bc, chatIDs: make(map[string]int64), tgCfg: telegramCfg, + + mediaGroups: make(map[string]*telegramMediaGroup), + mediaGroupDelay: telegramMediaGroupDelay(telegramCfg), } ch.progress = channels.NewToolFeedbackAnimator(ch.EditMessage) return ch, nil } +func telegramMediaGroupDelay(telegramCfg *config.TelegramSettings) time.Duration { + if telegramCfg != nil && telegramCfg.MediaGroupDelayMS > 0 { + return time.Duration(telegramCfg.MediaGroupDelayMS) * time.Millisecond + } + return defaultMediaGroupDelay +} + func (c *TelegramChannel) Start(ctx context.Context) error { logger.InfoC("telegram", "Starting Telegram bot (polling mode)...") @@ -167,6 +196,7 @@ func (c *TelegramChannel) Stop(ctx context.Context) error { if c.bh != nil { _ = c.bh.StopWithContext(ctx) } + c.flushPendingMediaGroups(ctx) // Cancel our context (stops long polling) if c.cancel != nil { @@ -713,6 +743,131 @@ func (c *TelegramChannel) SendMedia(ctx context.Context, msg bus.OutboundMediaMe } func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Message) error { + if message != nil && strings.TrimSpace(message.MediaGroupID) != "" { + return c.bufferMediaGroupMessage(ctx, message) + } + return c.handleMessages(ctx, []*telego.Message{message}) +} + +func (c *TelegramChannel) bufferMediaGroupMessage(ctx context.Context, message *telego.Message) error { + if message == nil { + return fmt.Errorf("message is nil") + } + groupID := strings.TrimSpace(message.MediaGroupID) + if groupID == "" { + return c.handleMessages(ctx, []*telego.Message{message}) + } + + msgCopy := *message + msgCopy.Photo = append([]telego.PhotoSize(nil), message.Photo...) + key := fmt.Sprintf("%d:%s", message.Chat.ID, groupID) + + c.mediaGroupMu.Lock() + if c.mediaGroups == nil { + c.mediaGroups = make(map[string]*telegramMediaGroup) + } + group := c.mediaGroups[key] + if group == nil { + group = &telegramMediaGroup{} + c.mediaGroups[key] = group + } + group.messages = append(group.messages, &msgCopy) + group.generation++ + generation := group.generation + if group.timer != nil { + group.timer.Stop() + } + delay := c.mediaGroupDelay + if delay <= 0 { + delay = defaultMediaGroupDelay + } + group.timer = time.AfterFunc(delay, func() { + c.flushMediaGroup(c.ctx, key, generation) + }) + c.mediaGroupMu.Unlock() + + logger.DebugCF("telegram", "Buffered media group message", map[string]any{ + "chat_id": message.Chat.ID, + "media_group_id": groupID, + "message_id": message.MessageID, + }) + return nil +} + +func (c *TelegramChannel) flushPendingMediaGroups(ctx context.Context) { + c.mediaGroupMu.Lock() + keys := make([]string, 0, len(c.mediaGroups)) + for key, group := range c.mediaGroups { + if group.timer != nil { + group.timer.Stop() + } + keys = append(keys, key) + } + c.mediaGroupMu.Unlock() + + for _, key := range keys { + c.flushMediaGroup(ctx, key, 0) + } +} + +func (c *TelegramChannel) flushMediaGroup(ctx context.Context, key string, generation uint64) { + c.mediaGroupMu.Lock() + group := c.mediaGroups[key] + if group == nil { + c.mediaGroupMu.Unlock() + return + } + if generation != 0 && group.generation != generation { + c.mediaGroupMu.Unlock() + return + } + delete(c.mediaGroups, key) + if group.timer != nil { + group.timer.Stop() + } + messages := append([]*telego.Message(nil), group.messages...) + c.mediaGroupMu.Unlock() + + if len(messages) == 0 { + return + } + slices.SortFunc(messages, func(a, b *telego.Message) int { + switch { + case a == nil && b == nil: + return 0 + case a == nil: + return -1 + case b == nil: + return 1 + default: + return a.MessageID - b.MessageID + } + }) + if ctx == nil { + ctx = context.Background() + } + if err := c.handleMessages(ctx, messages); err != nil { + logger.ErrorCF("telegram", "Failed to handle media group", map[string]any{ + "key": key, + "error": err.Error(), + }) + } +} + +func (c *TelegramChannel) handleMessages(ctx context.Context, messages []*telego.Message) error { + if len(messages) == 0 { + return nil + } + message := messages[0] + for _, candidate := range messages { + if candidate == nil { + continue + } + if strings.TrimSpace(candidate.Text) != "" || strings.TrimSpace(candidate.Caption) != "" { + message = candidate + break + } + } if message == nil { return fmt.Errorf("message is nil") } @@ -740,7 +895,9 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes } chatID := message.Chat.ID + c.chatIDsMu.Lock() c.chatIDs[platformID] = chatID + c.chatIDsMu.Unlock() content := "" mediaPaths := []string{} @@ -764,61 +921,18 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes return localPath // fallback: use raw path } - if message.Text != "" { - content += message.Text - } - - if message.Caption != "" { - if content != "" { - content += "\n" + for i, msg := range messages { + if msg == nil { + continue } - content += message.Caption - } - - if len(message.Photo) > 0 { - photo := message.Photo[len(message.Photo)-1] - photoPath := c.downloadPhoto(ctx, photo.FileID) - if photoPath != "" { - mediaPaths = append(mediaPaths, storeMedia(photoPath, "photo.jpg")) + parts := c.collectTelegramMessageParts(ctx, msg, i, len(messages), storeMedia) + for _, part := range parts.content { if content != "" { content += "\n" } - content += "[image: photo]" - } - } - - if message.Voice != nil { - voicePath := c.downloadFile(ctx, message.Voice.FileID, ".ogg") - if voicePath != "" { - mediaPaths = append(mediaPaths, storeMedia(voicePath, "voice.ogg")) - - if content != "" { - content += "\n" - } - content += "[voice]" - } - } - - if message.Audio != nil { - audioPath := c.downloadFile(ctx, message.Audio.FileID, ".mp3") - if audioPath != "" { - mediaPaths = append(mediaPaths, storeMedia(audioPath, "audio.mp3")) - if content != "" { - content += "\n" - } - content += "[audio]" - } - } - - if message.Document != nil { - docPath := c.downloadFile(ctx, message.Document.FileID, "") - if docPath != "" { - mediaPaths = append(mediaPaths, storeMedia(docPath, "document")) - if content != "" { - content += "\n" - } - content += "[file]" + content += part } + mediaPaths = append(mediaPaths, parts.mediaPaths...) } if content == "" && len(mediaPaths) == 0 { @@ -917,6 +1031,74 @@ func (c *TelegramChannel) handleMessage(ctx context.Context, message *telego.Mes return nil } +func (c *TelegramChannel) collectTelegramMessageParts( + ctx context.Context, + msg *telego.Message, + index int, + total int, + storeMedia func(localPath, filename string) string, +) telegramMessageParts { + parts := telegramMessageParts{} + if msg == nil { + return parts + } + if text := strings.TrimSpace(msg.Text); text != "" { + parts.content = append(parts.content, text) + } + if caption := strings.TrimSpace(msg.Caption); caption != "" { + parts.content = append(parts.content, caption) + } + if len(msg.Photo) > 0 { + photo := msg.Photo[len(msg.Photo)-1] + photoPath := c.downloadPhoto(ctx, photo.FileID) + if photoPath != "" { + photoNumber := index + 1 + parts.mediaPaths = append(parts.mediaPaths, storeMedia(photoPath, fmt.Sprintf("photo-%d.jpg", photoNumber))) + parts.content = append(parts.content, fmt.Sprintf("[image: photo %d]", photoNumber)) + } + } + if msg.Voice != nil { + voicePath := c.downloadFile(ctx, msg.Voice.FileID, ".ogg") + if voicePath != "" { + parts.mediaPaths = append( + parts.mediaPaths, + storeMedia(voicePath, indexedMediaFilename("voice", ".ogg", index, total)), + ) + parts.content = append(parts.content, "[voice]") + } + } + if msg.Audio != nil { + audioPath := c.downloadFile(ctx, msg.Audio.FileID, ".mp3") + if audioPath != "" { + filename := msg.Audio.FileName + if strings.TrimSpace(filename) == "" { + filename = indexedMediaFilename("audio", ".mp3", index, total) + } + parts.mediaPaths = append(parts.mediaPaths, storeMedia(audioPath, filename)) + parts.content = append(parts.content, "[audio]") + } + } + if msg.Document != nil { + docPath := c.downloadFile(ctx, msg.Document.FileID, "") + if docPath != "" { + filename := msg.Document.FileName + if strings.TrimSpace(filename) == "" { + filename = indexedMediaFilename("document", "", index, total) + } + parts.mediaPaths = append(parts.mediaPaths, storeMedia(docPath, filename)) + parts.content = append(parts.content, "[file]") + } + } + return parts +} + +func indexedMediaFilename(prefix, ext string, index int, total int) string { + if total <= 1 { + return prefix + ext + } + return fmt.Sprintf("%s-%d%s", prefix, index+1, ext) +} + func (c *TelegramChannel) prependTelegramQuotedReply(content string, reply *telego.Message) string { quoted := strings.TrimSpace(telegramQuotedContent(reply)) if quoted == "" { diff --git a/pkg/channels/telegram/telegram_test.go b/pkg/channels/telegram/telegram_test.go index 69c76b430..14d025064 100644 --- a/pkg/channels/telegram/telegram_test.go +++ b/pkg/channels/telegram/telegram_test.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/mymmrac/telego" ta "github.com/mymmrac/telego/telegoapi" @@ -1100,3 +1101,190 @@ func TestHandleMessage_EmptyContent_Ignored(t *testing.T) { default: } } + +func TestHandleMessage_MediaGroupCombinesCaptionMessages(t *testing.T) { + messageBus, ch := newMediaGroupTestChannel(10 * time.Millisecond) + base := testMediaGroupMessage("album-1") + first := base + first.MessageID = 1 + second := base + second.MessageID = 2 + second.Caption = "meal caption" + + require.NoError(t, ch.handleMessage(context.Background(), &first)) + require.NoError(t, ch.handleMessage(context.Background(), &second)) + + select { + case inbound := <-messageBus.InboundChan(): + assert.Equal(t, "2", inbound.Context.MessageID) + assert.Equal(t, "meal caption", inbound.Content) + case <-time.After(time.Second): + t.Fatal("timed out waiting for combined media group message") + } +} + +func TestHandleMessage_MediaGroupWaitsForStaggeredMessages(t *testing.T) { + messageBus, ch := newMediaGroupTestChannel(100 * time.Millisecond) + base := testMediaGroupMessage("album-staggered") + first := base + first.MessageID = 1 + first.Caption = "first caption" + second := base + second.MessageID = 2 + second.Caption = "second caption" + + require.NoError(t, ch.handleMessage(context.Background(), &first)) + time.Sleep(50 * time.Millisecond) + require.NoError(t, ch.handleMessage(context.Background(), &second)) + + select { + case inbound := <-messageBus.InboundChan(): + t.Fatalf("media group flushed before idle delay reset: %#v", inbound) + case <-time.After(75 * time.Millisecond): + } + + select { + case inbound := <-messageBus.InboundChan(): + assert.Equal(t, "1", inbound.Context.MessageID) + assert.Equal(t, "first caption\nsecond caption", inbound.Content) + case <-time.After(time.Second): + t.Fatal("timed out waiting for staggered media group message") + } +} + +func TestFlushMediaGroupIgnoresStaleTimerGeneration(t *testing.T) { + messageBus, ch := newMediaGroupTestChannel(time.Hour) + base := testMediaGroupMessage("album-generation") + first := base + first.MessageID = 1 + first.Caption = "first" + second := base + second.MessageID = 2 + second.Caption = "second" + key := "456:album-generation" + + ch.mediaGroupMu.Lock() + ch.mediaGroups[key] = &telegramMediaGroup{ + messages: []*telego.Message{&first, &second}, + generation: 2, + } + ch.mediaGroupMu.Unlock() + + ch.flushMediaGroup(context.Background(), key, 1) + + select { + case inbound := <-messageBus.InboundChan(): + t.Fatalf("stale media group generation flushed unexpectedly: %#v", inbound) + default: + } + + ch.mediaGroupMu.Lock() + _, stillPending := ch.mediaGroups[key] + ch.mediaGroupMu.Unlock() + require.True(t, stillPending, "stale flush should leave the current batch pending") + + ch.flushMediaGroup(context.Background(), key, 2) + + select { + case inbound := <-messageBus.InboundChan(): + assert.Equal(t, "1", inbound.Context.MessageID) + assert.Equal(t, "first\nsecond", inbound.Content) + case <-time.After(time.Second): + t.Fatal("timed out waiting for current generation media group flush") + } +} + +func TestHandleMessage_MediaGroupAfterDelayStartsNewBatch(t *testing.T) { + messageBus, ch := newMediaGroupTestChannel(10 * time.Millisecond) + base := testMediaGroupMessage("album-split") + first := base + first.MessageID = 1 + first.Caption = "first" + second := base + second.MessageID = 2 + second.Caption = "second" + + require.NoError(t, ch.handleMessage(context.Background(), &first)) + select { + case inbound := <-messageBus.InboundChan(): + assert.Equal(t, "1", inbound.Context.MessageID) + assert.Equal(t, "first", inbound.Content) + case <-time.After(time.Second): + t.Fatal("timed out waiting for first media group batch") + } + + require.NoError(t, ch.handleMessage(context.Background(), &second)) + select { + case inbound := <-messageBus.InboundChan(): + assert.Equal(t, "2", inbound.Context.MessageID) + assert.Equal(t, "second", inbound.Content) + case <-time.After(time.Second): + t.Fatal("timed out waiting for second media group batch") + } +} + +func TestStopFlushesPendingMediaGroups(t *testing.T) { + messageBus, ch := newMediaGroupTestChannel(time.Hour) + base := testMediaGroupMessage("album-stop") + msg := base + msg.MessageID = 1 + msg.Caption = "caption before stop" + + require.NoError(t, ch.handleMessage(context.Background(), &msg)) + require.NoError(t, ch.Stop(context.Background())) + + select { + case inbound := <-messageBus.InboundChan(): + assert.Equal(t, "1", inbound.Context.MessageID) + assert.Equal(t, "caption before stop", inbound.Content) + case <-time.After(time.Second): + t.Fatal("timed out waiting for pending media group flush on stop") + } +} + +func TestNewTelegramChannelUsesConfiguredMediaGroupDelay(t *testing.T) { + ch, err := NewTelegramChannel( + &config.Channel{Type: config.ChannelTelegram, Enabled: true}, + &config.TelegramSettings{ + Token: *config.NewSecureString(testToken), + MediaGroupDelayMS: 750, + }, + bus.NewMessageBus(), + ) + require.NoError(t, err) + assert.Equal(t, 750*time.Millisecond, ch.mediaGroupDelay) + + ch, err = NewTelegramChannel( + &config.Channel{Type: config.ChannelTelegram, Enabled: true}, + &config.TelegramSettings{Token: *config.NewSecureString(testToken)}, + bus.NewMessageBus(), + ) + require.NoError(t, err) + assert.Equal(t, defaultMediaGroupDelay, ch.mediaGroupDelay) +} + +func newMediaGroupTestChannel(delay time.Duration) (*bus.MessageBus, *TelegramChannel) { + messageBus := bus.NewMessageBus() + ch := &TelegramChannel{ + BaseChannel: channels.NewBaseChannel("telegram", nil, messageBus, nil), + chatIDs: make(map[string]int64), + ctx: context.Background(), + mediaGroups: make(map[string]*telegramMediaGroup), + mediaGroupDelay: delay, + } + return messageBus, ch +} + +func testMediaGroupMessage(mediaGroupID string) telego.Message { + return telego.Message{ + Chat: telego.Chat{ + ID: 456, + Type: "private", + }, + From: &telego.User{ + ID: 789, + FirstName: "User", + }, + MediaGroupID: mediaGroupID, + } +} diff --git a/pkg/commands/builtin.go b/pkg/commands/builtin.go index a7e401bb8..e268812a0 100644 --- a/pkg/commands/builtin.go +++ b/pkg/commands/builtin.go @@ -8,6 +8,7 @@ func BuiltinDefinitions() []Definition { return []Definition{ startCommand(), helpCommand(), + stopCommand(), showCommand(), listCommand(), useCommand(), diff --git a/pkg/commands/builtin_test.go b/pkg/commands/builtin_test.go index efd27fa00..bb9abe360 100644 --- a/pkg/commands/builtin_test.go +++ b/pkg/commands/builtin_test.go @@ -42,6 +42,9 @@ func TestBuiltinHelpHandler_ReturnsFormattedMessage(t *testing.T) { if !strings.Contains(reply, "/list [models|channels|agents|skills|mcp]") { t.Fatalf("/help reply missing /list usage, got %q", reply) } + if !strings.Contains(reply, "/stop") { + t.Fatalf("/help reply missing /stop usage, got %q", reply) + } if !strings.Contains(reply, "/use ") { if !strings.Contains(reply, "/use [message]") { t.Fatalf("/help reply missing /use usage, got %q", reply) @@ -49,6 +52,59 @@ func TestBuiltinHelpHandler_ReturnsFormattedMessage(t *testing.T) { } } +func TestBuiltinStop_UsesRuntimeStopper(t *testing.T) { + rt := &Runtime{ + StopActiveTurn: func() (StopResult, error) { + return StopResult{ + Stopped: true, + TaskName: "sync the long running job", + }, nil + }, + } + defs := BuiltinDefinitions() + ex := NewExecutor(NewRegistry(defs), rt) + + var reply string + res := ex.Execute(context.Background(), Request{ + Text: "/stop", + Reply: func(text string) error { + reply = text + return nil + }, + }) + if res.Outcome != OutcomeHandled { + t.Fatalf("/stop: outcome=%v, want=%v", res.Outcome, OutcomeHandled) + } + if reply != "Task stopped. \"sync the long running job\" was canceled." { + t.Fatalf("/stop reply=%q", reply) + } +} + +func TestBuiltinStop_NoActiveTask(t *testing.T) { + rt := &Runtime{ + StopActiveTurn: func() (StopResult, error) { + return StopResult{}, nil + }, + } + defs := BuiltinDefinitions() + ex := NewExecutor(NewRegistry(defs), rt) + + var reply string + res := ex.Execute(context.Background(), Request{ + Text: "/stop", + Reply: func(text string) error { + reply = text + return nil + }, + }) + if res.Outcome != OutcomeHandled { + t.Fatalf("/stop: outcome=%v, want=%v", res.Outcome, OutcomeHandled) + } + if reply != "No active task to stop." { + t.Fatalf("/stop reply=%q, want no-active message", reply) + } +} + func TestBuiltinShowChannel_PreservesUserVisibleBehavior(t *testing.T) { defs := BuiltinDefinitions() ex := NewExecutor(NewRegistry(defs), nil) diff --git a/pkg/commands/cmd_stop.go b/pkg/commands/cmd_stop.go new file mode 100644 index 000000000..147688bdc --- /dev/null +++ b/pkg/commands/cmd_stop.go @@ -0,0 +1,52 @@ +package commands + +import ( + "context" + "fmt" + "strings" +) + +func stopCommand() Definition { + return Definition{ + Name: "stop", + Description: "Stop the current task", + Usage: "/stop", + Handler: func(_ context.Context, req Request, rt *Runtime) error { + if rt == nil || rt.StopActiveTurn == nil { + return req.Reply(unavailableMsg) + } + + result, err := rt.StopActiveTurn() + if err != nil { + return req.Reply("Failed to stop task: " + err.Error()) + } + + return req.Reply(FormatStopReply(result)) + }, + } +} + +// FormatStopReply renders a user-facing reply for a stop request. +func FormatStopReply(result StopResult) string { + if !result.Stopped { + return "No active task to stop." + } + + taskName := compactStopTaskName(result.TaskName) + if taskName == "" { + return "Task stopped. Current task was canceled." + } + + return fmt.Sprintf("Task stopped. %q was canceled.", taskName) +} + +func compactStopTaskName(taskName string) string { + taskName = strings.Join(strings.Fields(strings.TrimSpace(taskName)), " ") + if taskName == "" { + return "" + } + if len(taskName) > 80 { + return taskName[:77] + "..." + } + return taskName +} diff --git a/pkg/commands/runtime.go b/pkg/commands/runtime.go index c17b7cf1c..b0327c863 100644 --- a/pkg/commands/runtime.go +++ b/pkg/commands/runtime.go @@ -36,6 +36,12 @@ type ContextStats struct { MessageCount int } +// StopResult describes the outcome of a stop request for the current session. +type StopResult struct { + Stopped bool + TaskName string +} + // Runtime provides runtime dependencies to command handlers. It is constructed // per-request by the agent loop so that per-request state (like session scope) // can coexist with long-lived callbacks (like GetModelInfo). @@ -55,4 +61,5 @@ type Runtime struct { SwitchChannel func(value string) error ClearHistory func() error ReloadConfig func() error + StopActiveTurn func() (StopResult, error) } diff --git a/pkg/config/config.go b/pkg/config/config.go index d0b3f9207..79dfab4f1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/sipeed/picoclaw/pkg" "github.com/sipeed/picoclaw/pkg/fileutil" "github.com/sipeed/picoclaw/pkg/logger" + providercommon "github.com/sipeed/picoclaw/pkg/providers/common" ) // rrCounter is a global counter for round-robin load balancing across models. @@ -36,9 +37,11 @@ type Config struct { Isolation IsolationConfig `json:"isolation,omitempty" yaml:"-"` Agents AgentsConfig `json:"agents" yaml:"-"` Session SessionConfig `json:"session,omitempty" yaml:"-"` + Evolution EvolutionConfig `json:"evolution,omitempty" yaml:"-"` Channels ChannelsConfig `json:"channel_list" yaml:"channel_list"` ModelList SecureModelList `json:"model_list" yaml:"model_list"` // New model-centric provider configuration Gateway GatewayConfig `json:"gateway" yaml:"-"` + Events EventsConfig `json:"events,omitempty" yaml:"-"` Hooks HooksConfig `json:"hooks,omitempty" yaml:"-"` Tools ToolsConfig `json:"tools" yaml:",inline"` Heartbeat HeartbeatConfig `json:"heartbeat" yaml:"-"` @@ -51,6 +54,126 @@ type Config struct { sensitiveCache *SensitiveDataCache } +type EvolutionConfig struct { + Enabled bool `json:"enabled,omitempty"` + Mode string `json:"mode,omitempty"` + StateDir string `json:"state_dir,omitempty"` + MinTaskCount int `json:"min_task_count,omitempty"` + MinSuccessRatio float64 `json:"min_success_ratio,omitempty"` + ColdPathTrigger string `json:"cold_path_trigger,omitempty"` + ColdPathTimes []string `json:"cold_path_times,omitempty"` + // Deprecated: use MinTaskCount. + MinCaseCount int `json:"min_case_count,omitempty"` + // Deprecated: use MinSuccessRatio. + MinSuccessRate float64 `json:"min_success_rate,omitempty"` +} + +func (c EvolutionConfig) MarshalJSON() ([]byte, error) { + out := struct { + Enabled bool `json:"enabled,omitempty"` + Mode string `json:"mode,omitempty"` + StateDir string `json:"state_dir,omitempty"` + MinTaskCount int `json:"min_task_count,omitempty"` + MinSuccessRatio float64 `json:"min_success_ratio,omitempty"` + ColdPathTrigger string `json:"cold_path_trigger,omitempty"` + ColdPathTimes []string `json:"cold_path_times,omitempty"` + }{ + Enabled: c.Enabled, + Mode: c.Mode, + StateDir: c.StateDir, + MinTaskCount: c.EffectiveMinTaskCount(), + MinSuccessRatio: c.EffectiveMinSuccessRatio(), + ColdPathTrigger: strings.TrimSpace(c.ColdPathTrigger), + ColdPathTimes: c.EffectiveColdPathTimes(), + } + if !out.Enabled { + out.Mode = "" + out.ColdPathTrigger = "" + out.ColdPathTimes = nil + } + return json.Marshal(out) +} + +func (c EvolutionConfig) EffectiveMode() string { + if !c.Enabled { + return "" + } + switch strings.ToLower(strings.TrimSpace(c.Mode)) { + case "draft": + return "draft" + case "apply": + return "apply" + case "", "observe": + return "observe" + default: + return "observe" + } +} + +func (c EvolutionConfig) RunsColdPathAutomatically() bool { + return c.RunsColdPathAfterTurn() || c.RunsColdPathScheduled() +} + +func (c EvolutionConfig) ColdPathTriggerMode() string { + if c.EffectiveMode() != "draft" && c.EffectiveMode() != "apply" { + return "" + } + switch strings.ToLower(strings.TrimSpace(c.ColdPathTrigger)) { + case "", "after_turn": + return "after_turn" + case "scheduled": + return "scheduled" + case "manual", "none", "off": + return "manual" + default: + return "after_turn" + } +} + +func (c EvolutionConfig) RunsColdPathAfterTurn() bool { + return c.ColdPathTriggerMode() == "after_turn" +} + +func (c EvolutionConfig) RunsColdPathScheduled() bool { + return c.ColdPathTriggerMode() == "scheduled" +} + +func (c EvolutionConfig) EffectiveMinTaskCount() int { + if c.MinTaskCount > 0 { + return c.MinTaskCount + } + if c.MinCaseCount > 0 { + return c.MinCaseCount + } + return 2 +} + +func (c EvolutionConfig) EffectiveMinSuccessRatio() float64 { + if c.MinSuccessRatio > 0 { + return c.MinSuccessRatio + } + if c.MinSuccessRate > 0 { + return c.MinSuccessRate + } + return 0.7 +} + +func (c EvolutionConfig) EffectiveColdPathTimes() []string { + out := make([]string, 0, len(c.ColdPathTimes)) + for _, value := range c.ColdPathTimes { + value = strings.TrimSpace(value) + if value == "" { + continue + } + out = append(out, value) + } + return out +} + +func (c EvolutionConfig) AutoAppliesDrafts() bool { + return c.EffectiveMode() == "apply" +} + // IsolationConfig controls subprocess isolation for commands started by PicoClaw. // It is applied by the isolation package rather than by sandboxing the main process. type IsolationConfig struct { @@ -277,6 +400,8 @@ type AgentDefaults struct { SplitOnMarker bool `json:"split_on_marker" env:"PICOCLAW_AGENTS_DEFAULTS_SPLIT_ON_MARKER"` // split messages on <|[SPLIT]|> marker ContextManager string `json:"context_manager,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_CONTEXT_MANAGER"` ContextManagerConfig json.RawMessage `json:"context_manager_config,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_CONTEXT_MANAGER_CONFIG"` + MaxLLMRetries int `json:"max_llm_retries,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_MAX_LLM_RETRIES"` + LLMRetryBackoffSecs int `json:"llm_retry_backoff_secs,omitempty" env:"PICOCLAW_AGENTS_DEFAULTS_LLM_RETRY_BACKOFF_SECS"` } const DefaultMaxMediaSize = 20 * 1024 * 1024 // 20 MB @@ -356,11 +481,12 @@ type WhatsAppSettings struct { } type TelegramSettings struct { - Token SecureString `json:"token,omitzero" yaml:"token,omitempty" env:"PICOCLAW_CHANNELS_TELEGRAM_TOKEN"` - BaseURL string `json:"base_url" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_BASE_URL"` - Proxy string `json:"proxy" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_PROXY"` - Streaming StreamingConfig `json:"streaming,omitempty" yaml:"-"` - UseMarkdownV2 bool `json:"use_markdown_v2" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_USE_MARKDOWN_V2"` + Token SecureString `json:"token,omitzero" yaml:"token,omitempty" env:"PICOCLAW_CHANNELS_TELEGRAM_TOKEN"` + BaseURL string `json:"base_url" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_BASE_URL"` + Proxy string `json:"proxy" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_PROXY"` + Streaming StreamingConfig `json:"streaming,omitempty" yaml:"-"` + UseMarkdownV2 bool `json:"use_markdown_v2" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_USE_MARKDOWN_V2"` + MediaGroupDelayMS int `json:"media_group_delay_ms" yaml:"-" env:"PICOCLAW_CHANNELS_TELEGRAM_MEDIA_GROUP_DELAY_MS"` } type FeishuSettings struct { @@ -513,6 +639,29 @@ type TeamsWebhookTarget struct { Title string `json:"title,omitempty" yaml:"-"` } +type MQTTSettings struct { + Broker string `json:"broker" yaml:"-" env:"PICOCLAW_CHANNELS_MQTT_BROKER"` + AgentID string `json:"agent_id" yaml:"-" env:"PICOCLAW_CHANNELS_MQTT_AGENT_ID"` + TopicPrefix string `json:"topic_prefix,omitempty" yaml:"-" env:"PICOCLAW_CHANNELS_MQTT_TOPIC_PREFIX"` + Username SecureString `json:"username,omitzero" yaml:"username,omitempty" env:"PICOCLAW_CHANNELS_MQTT_USERNAME"` + Password SecureString `json:"password,omitzero" yaml:"password,omitempty" env:"PICOCLAW_CHANNELS_MQTT_PASSWORD"` + ClientID string `json:"client_id,omitempty" yaml:"-" env:"PICOCLAW_CHANNELS_MQTT_CLIENT_ID"` + KeepAlive int `json:"keep_alive,omitempty" yaml:"-" env:"PICOCLAW_CHANNELS_MQTT_KEEP_ALIVE"` + QoS int `json:"qos,omitempty" yaml:"-" env:"PICOCLAW_CHANNELS_MQTT_QOS"` +} + +// SlackWebhookSettings configures the output-only Slack webhook channel. +type SlackWebhookSettings struct { + Webhooks map[string]SlackWebhookTarget `json:"webhooks" yaml:"webhooks,omitempty"` +} + +// SlackWebhookTarget represents a single Slack Incoming Webhook destination. +type SlackWebhookTarget struct { + WebhookURL SecureString `json:"webhook_url,omitzero" yaml:"webhook_url,omitempty"` + Username string `json:"username,omitempty" yaml:"-"` + IconEmoji string `json:"icon_emoji,omitempty" yaml:"-"` +} + type HeartbeatConfig struct { Enabled bool `json:"enabled" env:"PICOCLAW_HEARTBEAT_ENABLED"` Interval int `json:"interval" env:"PICOCLAW_HEARTBEAT_INTERVAL"` // minutes, min 5 @@ -554,12 +703,13 @@ type ModelConfig struct { Workspace string `json:"workspace,omitempty"` // Workspace path for CLI-based providers // Optional optimizations - RPM int `json:"rpm,omitempty"` // Requests per minute limit - MaxTokensField string `json:"max_tokens_field,omitempty"` // Field name for max tokens (e.g., "max_completion_tokens") - RequestTimeout int `json:"request_timeout,omitempty"` - ThinkingLevel string `json:"thinking_level,omitempty"` // Extended thinking: off|low|medium|high|xhigh|adaptive - ExtraBody map[string]any `json:"extra_body,omitempty"` // Additional fields to inject into request body - CustomHeaders map[string]string `json:"custom_headers,omitempty"` // Additional headers to inject into every HTTP request + RPM int `json:"rpm,omitempty"` // Requests per minute limit + MaxTokensField string `json:"max_tokens_field,omitempty"` // Field name for max tokens (e.g., "max_completion_tokens") + RequestTimeout int `json:"request_timeout,omitempty"` + ThinkingLevel string `json:"thinking_level,omitempty"` // Extended thinking: off|low|medium|high|xhigh|adaptive + ToolSchemaTransform string `json:"tool_schema_transform,omitempty"` // Optional tool schema compatibility transform (e.g. "simple") + ExtraBody map[string]any `json:"extra_body,omitempty"` // Additional fields to inject into request body + CustomHeaders map[string]string `json:"custom_headers,omitempty"` // Additional headers to inject into every HTTP request APIKeys SecureStrings `json:"api_keys,omitzero" yaml:"api_keys,omitempty"` // API authentication keys (multiple keys for failover) @@ -596,6 +746,24 @@ func (c *ModelConfig) Validate() error { if c.Model == "" { return fmt.Errorf("model is required") } + if _, err := providercommon.NormalizeToolSchemaTransform(c.ToolSchemaTransform); err != nil { + return err + } + + // Reject whitespace in model identifier + if strings.ContainsAny(c.Model, " \t\n\r") { + return fmt.Errorf("model identifier contains whitespace") + } + + // Reject leading slash + if strings.HasPrefix(c.Model, "/") { + return fmt.Errorf("model identifier must not start with /") + } + + // Reject consecutive slashes + if strings.Contains(c.Model, "//") { + return fmt.Errorf("model identifier must not contain //") + } return nil } @@ -680,6 +848,13 @@ type SogouConfig struct { MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_SOGOU_MAX_RESULTS"` } +type GeminiSearchConfig struct { + Enabled bool `json:"enabled" yaml:"-" env:"PICOCLAW_TOOLS_WEB_GEMINI_ENABLED"` + APIKey SecureString `json:"api_key,omitzero" yaml:"api_key,omitempty" env:"PICOCLAW_TOOLS_WEB_GEMINI_API_KEY"` + Model string `json:"model" yaml:"-" env:"PICOCLAW_TOOLS_WEB_GEMINI_MODEL"` + MaxResults int `json:"max_results" yaml:"-" env:"PICOCLAW_TOOLS_WEB_GEMINI_MAX_RESULTS"` +} + type PerplexityConfig struct { Enabled bool `json:"enabled" yaml:"-" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_ENABLED"` APIKeys SecureStrings `json:"api_keys,omitzero" yaml:"api_keys,omitempty" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_API_KEYS"` @@ -723,16 +898,17 @@ type BaiduSearchConfig struct { } type WebToolsConfig struct { - ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_WEB_"` - Brave BraveConfig `yaml:"brave,omitempty" json:"brave"` - Tavily TavilyConfig `yaml:"tavily,omitempty" json:"tavily"` - Sogou SogouConfig `yaml:"-" json:"sogou"` - DuckDuckGo DuckDuckGoConfig `yaml:"-" json:"duckduckgo"` - Perplexity PerplexityConfig `yaml:"perplexity,omitempty" json:"perplexity"` - SearXNG SearXNGConfig `yaml:"-" json:"searxng"` - GLMSearch GLMSearchConfig `yaml:"glm_search,omitempty" json:"glm_search"` - BaiduSearch BaiduSearchConfig `yaml:"baidu_search,omitempty" json:"baidu_search"` - Provider string `yaml:"-" json:"provider,omitempty" env:"PICOCLAW_TOOLS_WEB_PROVIDER"` + ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_WEB_"` + Brave BraveConfig `yaml:"brave,omitempty" json:"brave"` + Tavily TavilyConfig `yaml:"tavily,omitempty" json:"tavily"` + Sogou SogouConfig `yaml:"-" json:"sogou"` + DuckDuckGo DuckDuckGoConfig `yaml:"-" json:"duckduckgo"` + Gemini GeminiSearchConfig `yaml:"gemini,omitempty" json:"gemini"` + Perplexity PerplexityConfig `yaml:"perplexity,omitempty" json:"perplexity"` + SearXNG SearXNGConfig `yaml:"-" json:"searxng"` + GLMSearch GLMSearchConfig `yaml:"glm_search,omitempty" json:"glm_search"` + BaiduSearch BaiduSearchConfig `yaml:"baidu_search,omitempty" json:"baidu_search"` + Provider string `yaml:"-" json:"provider,omitempty" env:"PICOCLAW_TOOLS_WEB_PROVIDER"` // PreferNative controls whether to use provider-native web search when // the active LLM supports it (e.g. OpenAI web_search_preview). When true, // the client-side web_search tool is hidden to avoid duplicate search surfaces, @@ -1467,23 +1643,24 @@ func expandMultiKeyModels(models []*ModelConfig) []*ModelConfig { // Create a copy for the additional key additionalEntry := &ModelConfig{ - ModelName: expandedName, - Provider: m.Provider, - Model: m.Model, - APIBase: m.APIBase, - APIKeys: SimpleSecureStrings(keys[i]), - Proxy: m.Proxy, - AuthMethod: m.AuthMethod, - ConnectMode: m.ConnectMode, - Workspace: m.Workspace, - RPM: m.RPM, - MaxTokensField: m.MaxTokensField, - RequestTimeout: m.RequestTimeout, - ThinkingLevel: m.ThinkingLevel, - ExtraBody: m.ExtraBody, - CustomHeaders: m.CustomHeaders, - UserAgent: m.UserAgent, - isVirtual: true, + ModelName: expandedName, + Provider: m.Provider, + Model: m.Model, + APIBase: m.APIBase, + APIKeys: SimpleSecureStrings(keys[i]), + Proxy: m.Proxy, + AuthMethod: m.AuthMethod, + ConnectMode: m.ConnectMode, + Workspace: m.Workspace, + RPM: m.RPM, + MaxTokensField: m.MaxTokensField, + RequestTimeout: m.RequestTimeout, + ThinkingLevel: m.ThinkingLevel, + ToolSchemaTransform: m.ToolSchemaTransform, + ExtraBody: m.ExtraBody, + CustomHeaders: m.CustomHeaders, + UserAgent: m.UserAgent, + isVirtual: true, } expanded = append(expanded, additionalEntry) fallbackNames = append(fallbackNames, expandedName) @@ -1491,22 +1668,23 @@ func expandMultiKeyModels(models []*ModelConfig) []*ModelConfig { // Create the primary entry with first key and fallbacks primaryEntry := &ModelConfig{ - ModelName: originalName, - Provider: m.Provider, - Model: m.Model, - APIBase: m.APIBase, - Proxy: m.Proxy, - AuthMethod: m.AuthMethod, - ConnectMode: m.ConnectMode, - Workspace: m.Workspace, - RPM: m.RPM, - MaxTokensField: m.MaxTokensField, - RequestTimeout: m.RequestTimeout, - ThinkingLevel: m.ThinkingLevel, - ExtraBody: m.ExtraBody, - CustomHeaders: m.CustomHeaders, - UserAgent: m.UserAgent, - APIKeys: SimpleSecureStrings(keys[0]), + ModelName: originalName, + Provider: m.Provider, + Model: m.Model, + APIBase: m.APIBase, + Proxy: m.Proxy, + AuthMethod: m.AuthMethod, + ConnectMode: m.ConnectMode, + Workspace: m.Workspace, + RPM: m.RPM, + MaxTokensField: m.MaxTokensField, + RequestTimeout: m.RequestTimeout, + ThinkingLevel: m.ThinkingLevel, + ToolSchemaTransform: m.ToolSchemaTransform, + ExtraBody: m.ExtraBody, + CustomHeaders: m.CustomHeaders, + UserAgent: m.UserAgent, + APIKeys: SimpleSecureStrings(keys[0]), } // Prepend new fallbacks to existing ones diff --git a/pkg/config/config_channel.go b/pkg/config/config_channel.go index 4e87fcc3e..52d4b4934 100644 --- a/pkg/config/config_channel.go +++ b/pkg/config/config_channel.go @@ -33,6 +33,8 @@ const ( ChannelWhatsApp = "whatsapp" ChannelWhatsAppNative = "whatsapp_native" ChannelTeamsWebHook = "teams_webhook" + ChannelMQTT = "mqtt" + ChannelSlackWebHook = "slack_webhook" ) func initChannel() { @@ -640,6 +642,8 @@ var channelSettingsFactory = map[string]any{ ChannelWhatsApp: (WhatsAppSettings{}), ChannelWhatsAppNative: (WhatsAppSettings{}), ChannelTeamsWebHook: (TeamsWebhookSettings{}), + ChannelMQTT: (MQTTSettings{}), + ChannelSlackWebHook: (SlackWebhookSettings{}), } // newChannelSettings creates a fresh zero-value pointer for the given channel type. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d455572eb..d744e15dc 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -96,17 +96,17 @@ func TestAgentConfig_FullParse(t *testing.T) { "name": "Sales Bot", "model": "gpt-4" }, - { - "id": "support", - "name": "Support Bot", - "model": { - "primary": "claude-opus", - "fallbacks": ["haiku"] - }, - "subagents": { - "allow_agents": ["sales"] - } + { + "id": "support", + "name": "Support Bot", + "model": { + "primary": "claude-opus", + "fallbacks": ["haiku"] + }, + "subagents": { + "allow_agents": ["sales"] } + } ] }, "session": { @@ -171,6 +171,317 @@ func TestDefaultConfig_MCPMaxInlineTextChars(t *testing.T) { } } +func TestDefaultConfig_EvolutionDefaults(t *testing.T) { + cfg := DefaultConfig() + + assert.False(t, cfg.Evolution.Enabled) + assert.Equal(t, "observe", cfg.Evolution.Mode) + assert.Equal(t, "", cfg.Evolution.StateDir) + assert.Equal(t, 2, cfg.Evolution.MinTaskCount) + assert.Equal(t, 0.7, cfg.Evolution.MinSuccessRatio) + assert.Equal(t, "after_turn", cfg.Evolution.ColdPathTrigger) + assert.Equal(t, 2, cfg.Evolution.EffectiveMinTaskCount()) + assert.Equal(t, 0.7, cfg.Evolution.EffectiveMinSuccessRatio()) + assert.False(t, cfg.Evolution.RunsColdPathAutomatically()) + assert.False(t, cfg.Evolution.AutoAppliesDrafts()) +} + +func TestEvolutionConfig_EffectiveMode(t *testing.T) { + tests := []struct { + name string + cfg EvolutionConfig + want string + }{ + { + name: "disabled returns empty", + cfg: EvolutionConfig{ + Enabled: false, + Mode: "apply", + }, + want: "", + }, + { + name: "enabled empty mode defaults to observe", + cfg: EvolutionConfig{ + Enabled: true, + }, + want: "observe", + }, + { + name: "enabled whitespace mode defaults to observe", + cfg: EvolutionConfig{ + Enabled: true, + Mode: " \t\n ", + }, + want: "observe", + }, + { + name: "enabled returns configured mode", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, + want: "draft", + }, + { + name: "enabled trims and normalizes mode", + cfg: EvolutionConfig{ + Enabled: true, + Mode: " Draft ", + }, + want: "draft", + }, + { + name: "enabled returns apply mode", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "apply", + }, + want: "apply", + }, + { + name: "enabled normalizes uppercase apply", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "APPLY", + }, + want: "apply", + }, + { + name: "enabled unknown mode falls back to observe", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "propose", + }, + want: "observe", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.cfg.EffectiveMode()) + }) + } +} + +func TestEvolutionConfig_ModeSemantics(t *testing.T) { + tests := []struct { + name string + cfg EvolutionConfig + wantRunsCold bool + wantAutoApply bool + }{ + { + name: "disabled does not run cold path", + cfg: EvolutionConfig{ + Enabled: false, + Mode: "apply", + }, + wantRunsCold: false, + wantAutoApply: false, + }, + { + name: "observe only records hot path", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, + wantRunsCold: false, + wantAutoApply: false, + }, + { + name: "draft runs cold path without applying", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "draft", + }, + wantRunsCold: true, + wantAutoApply: false, + }, + { + name: "draft scheduled runs cold path without after turn", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "draft", + ColdPathTrigger: "scheduled", + ColdPathTimes: []string{"03:00"}, + }, + wantRunsCold: true, + wantAutoApply: false, + }, + { + name: "apply runs cold path and auto applies", + cfg: EvolutionConfig{ + Enabled: true, + Mode: "apply", + }, + wantRunsCold: true, + wantAutoApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.wantRunsCold, tt.cfg.RunsColdPathAutomatically()) + assert.Equal(t, tt.wantAutoApply, tt.cfg.AutoAppliesDrafts()) + }) + } +} + +func TestEvolutionConfig_ColdPathTriggerMode(t *testing.T) { + assert.Equal(t, "after_turn", (EvolutionConfig{Enabled: true, Mode: "draft"}).ColdPathTriggerMode()) + assert.True(t, (EvolutionConfig{Enabled: true, Mode: "draft"}).RunsColdPathAfterTurn()) + assert.False(t, (EvolutionConfig{Enabled: true, Mode: "draft"}).RunsColdPathScheduled()) + + scheduled := EvolutionConfig{ + Enabled: true, + Mode: "apply", + ColdPathTrigger: "scheduled", + ColdPathTimes: []string{"03:00"}, + } + assert.Equal(t, "scheduled", scheduled.ColdPathTriggerMode()) + assert.False(t, scheduled.RunsColdPathAfterTurn()) + assert.True(t, scheduled.RunsColdPathScheduled()) + + manual := EvolutionConfig{Enabled: true, Mode: "draft", ColdPathTrigger: "manual"} + assert.Equal(t, "manual", manual.ColdPathTriggerMode()) + assert.False(t, manual.RunsColdPathAutomatically()) +} + +func TestEvolutionConfig_NewThresholdNamesPreferLegacyAliases(t *testing.T) { + cfg := EvolutionConfig{MinTaskCount: 4, MinSuccessRatio: 0.9, MinCaseCount: 1, MinSuccessRate: 0.2} + assert.Equal(t, 4, cfg.EffectiveMinTaskCount()) + assert.Equal(t, 0.9, cfg.EffectiveMinSuccessRatio()) + + legacy := EvolutionConfig{MinCaseCount: 5, MinSuccessRate: 0.8} + assert.Equal(t, 5, legacy.EffectiveMinTaskCount()) + assert.Equal(t, 0.8, legacy.EffectiveMinSuccessRatio()) +} + +func TestEvolutionConfig_MarshalUsesNewThresholdNames(t *testing.T) { + data, err := json.Marshal(EvolutionConfig{ + Enabled: true, + Mode: "draft", + MinCaseCount: 5, + MinSuccessRate: 0.8, + }) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + if raw["min_task_count"] != float64(5) { + t.Fatalf("min_task_count = %#v, want 5", raw["min_task_count"]) + } + if raw["min_success_ratio"] != 0.8 { + t.Fatalf("min_success_ratio = %#v, want 0.8", raw["min_success_ratio"]) + } + if _, ok := raw["min_case_count"]; ok { + t.Fatalf("min_case_count should not be marshaled: %#v", raw) + } + if _, ok := raw["min_success_rate"]; ok { + t.Fatalf("min_success_rate should not be marshaled: %#v", raw) + } +} + +func TestLoadConfig_EvolutionEnabledWithoutModeUsesObserveSemantics(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.json") + raw := `{ + "version": 3, + "evolution": { + "enabled": true + } + }` + if err := os.WriteFile(configPath, []byte(raw), 0o644); err != nil { + t.Fatalf("WriteFile(configPath): %v", err) + } + + cfg, err := LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error: %v", err) + } + + assert.True(t, cfg.Evolution.Enabled) + assert.Equal(t, "", cfg.Evolution.Mode) + assert.Equal(t, "observe", cfg.Evolution.EffectiveMode()) + assert.False(t, cfg.Evolution.RunsColdPathAutomatically()) + assert.False(t, cfg.Evolution.AutoAppliesDrafts()) +} + +func TestLoadConfig_EvolutionExplicitApplyModeAutoApplies(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.json") + raw := `{ + "version": 3, + "evolution": { + "enabled": true, + "mode": "apply" + } + }` + if err := os.WriteFile(configPath, []byte(raw), 0o644); err != nil { + t.Fatalf("WriteFile(configPath): %v", err) + } + + cfg, err := LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error: %v", err) + } + + assert.True(t, cfg.Evolution.Enabled) + assert.Equal(t, "apply", cfg.Evolution.Mode) + assert.Equal(t, "apply", cfg.Evolution.EffectiveMode()) + assert.True(t, cfg.Evolution.RunsColdPathAutomatically()) + assert.True(t, cfg.Evolution.AutoAppliesDrafts()) +} + +func TestSaveConfig_DisabledEvolutionOmitsApplyMode(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.json") + cfg := DefaultConfig() + + if err := SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error: %v", err) + } + + data, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("ReadFile(configPath): %v", err) + } + + var raw map[string]any + if unmarshalErr := json.Unmarshal(data, &raw); unmarshalErr != nil { + t.Fatalf("Unmarshal saved config: %v", unmarshalErr) + } + evolutionRaw, ok := raw["evolution"].(map[string]any) + if !ok { + t.Fatalf("saved evolution config = %#v, want object", raw["evolution"]) + } + if _, ok := evolutionRaw["mode"]; ok { + t.Fatalf("disabled evolution should not persist mode: %#v", evolutionRaw) + } + + evolutionRaw["enabled"] = true + edited, err := json.Marshal(raw) + if err != nil { + t.Fatalf("Marshal edited config: %v", err) + } + if writeErr := os.WriteFile(configPath, edited, 0o600); writeErr != nil { + t.Fatalf("WriteFile(configPath): %v", writeErr) + } + + loaded, err := LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error: %v", err) + } + assert.True(t, loaded.Evolution.Enabled) + assert.Equal(t, "observe", loaded.Evolution.EffectiveMode()) + assert.False(t, loaded.Evolution.AutoAppliesDrafts()) +} + func TestLoadConfig_MCPMaxInlineTextChars(t *testing.T) { dir := t.TempDir() configPath := filepath.Join(dir, "config.json") @@ -808,7 +1119,9 @@ func TestLoadConfig_ToolFeedbackDefaultsFalseWhenUnset(t *testing.T) { t.Fatalf("LoadConfig() error: %v", err) } if cfg.Agents.Defaults.ToolFeedback.Enabled { - t.Fatal("agents.defaults.tool_feedback.enabled should remain false when unset in config file") + t.Fatal( + "agents.defaults.tool_feedback.enabled should remain false when unset in config file", + ) } if cfg.Agents.Defaults.ToolFeedback.SeparateMessages { t.Fatal("agents.defaults.tool_feedback.separate_messages should remain false when unset in config file") @@ -1131,7 +1444,10 @@ func TestDefaultConfig_SummarizationThresholds(t *testing.T) { cfg := DefaultConfig() if cfg.Agents.Defaults.SummarizeMessageThreshold != 20 { - t.Errorf("SummarizeMessageThreshold = %d, want 20", cfg.Agents.Defaults.SummarizeMessageThreshold) + t.Errorf( + "SummarizeMessageThreshold = %d, want 20", + cfg.Agents.Defaults.SummarizeMessageThreshold, + ) } if cfg.Agents.Defaults.SummarizeTokenPercent != 75 { t.Errorf("SummarizeTokenPercent = %d, want 75", cfg.Agents.Defaults.SummarizeTokenPercent) @@ -1173,7 +1489,11 @@ func TestDefaultConfig_WorkspacePath_WithPicoclawHome(t *testing.T) { want := filepath.Join("/custom/picoclaw/home", "workspace") if cfg.Agents.Defaults.Workspace != want { - t.Errorf("Workspace path with PICOCLAW_HOME = %q, want %q", cfg.Agents.Defaults.Workspace, want) + t.Errorf( + "Workspace path with PICOCLAW_HOME = %q, want %q", + cfg.Agents.Defaults.Workspace, + want, + ) } } @@ -1283,7 +1603,12 @@ func TestFlexibleStringSlice_UnmarshalText(t *testing.T) { } if len(f) != len(tt.expected) { - t.Errorf("UnmarshalText(%q) length = %d, want %d", tt.input, len(f), len(tt.expected)) + t.Errorf( + "UnmarshalText(%q) length = %d, want %d", + tt.input, + len(f), + len(tt.expected), + ) return } @@ -1592,9 +1917,21 @@ func TestSaveConfig_MixedKeys(t *testing.T) { cfg := &Config{ Version: CurrentVersion, ModelList: []*ModelConfig{ - {ModelName: "plain", Model: "openai/gpt-4", APIKeys: SimpleSecureStrings("sk-new-plaintext")}, - {ModelName: "enc", Model: "openai/gpt-4", APIKeys: SimpleSecureStrings(alreadyEncrypted)}, - {ModelName: "file", Model: "openai/gpt-4", APIKeys: SimpleSecureStrings("file://api.key")}, + { + ModelName: "plain", + Model: "openai/gpt-4", + APIKeys: SimpleSecureStrings("sk-new-plaintext"), + }, + { + ModelName: "enc", + Model: "openai/gpt-4", + APIKeys: SimpleSecureStrings(alreadyEncrypted), + }, + { + ModelName: "file", + Model: "openai/gpt-4", + APIKeys: SimpleSecureStrings("file://api.key"), + }, }, } if err := SaveConfig(cfgPath, cfg); err != nil { @@ -1731,7 +2068,10 @@ func TestSaveConfig_UsesPassphraseProvider(t *testing.T) { raw, _ := os.ReadFile(filepath.Join(dir, SecurityConfigFile)) if !strings.Contains(string(raw), "enc://") { - t.Errorf("SaveConfig should have encrypted plaintext key via PassphraseProvider; got:\n%s", raw) + t.Errorf( + "SaveConfig should have encrypted plaintext key via PassphraseProvider; got:\n%s", + raw, + ) } } @@ -1998,6 +2338,36 @@ func TestModelConfig_CustomHeadersRoundTrip(t *testing.T) { } } +func TestModelConfig_ToolSchemaTransformRoundTrip(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + + cfg := &Config{ + Version: CurrentVersion, + ModelList: []*ModelConfig{ + { + ModelName: "test-model", + Model: "openai/test", + APIKeys: SimpleSecureStrings("sk-test"), + ToolSchemaTransform: "simple", + }, + }, + } + + if err := SaveConfig(cfgPath, cfg); err != nil { + t.Fatalf("SaveConfig error: %v", err) + } + + loaded, err := LoadConfig(cfgPath) + if err != nil { + t.Fatalf("LoadConfig error: %v", err) + } + + if got := loaded.ModelList[0].ToolSchemaTransform; got != "simple" { + t.Fatalf("ToolSchemaTransform = %q, want %q", got, "simple") + } +} + func TestDefaultConfig_MinimaxExtraBody(t *testing.T) { cfg := DefaultConfig() @@ -2110,9 +2480,13 @@ func TestFilterSensitiveData_AllTokenTypes(t *testing.T) { FilterMinLength: 8, // Web tool API keys Web: WebToolsConfig{ - Brave: BraveConfig{APIKeys: SecureStrings{NewSecureString("brave-api-key")}}, - Tavily: TavilyConfig{APIKeys: SecureStrings{NewSecureString("tavily-api-key")}}, - Perplexity: PerplexityConfig{APIKeys: SecureStrings{NewSecureString("perplexity-api-key")}}, + Brave: BraveConfig{APIKeys: SecureStrings{NewSecureString("brave-api-key")}}, + Tavily: TavilyConfig{ + APIKeys: SecureStrings{NewSecureString("tavily-api-key")}, + }, + Perplexity: PerplexityConfig{ + APIKeys: SecureStrings{NewSecureString("perplexity-api-key")}, + }, GLMSearch: GLMSearchConfig{APIKey: *NewSecureString("glm-search-key")}, BaiduSearch: BaiduSearchConfig{APIKey: *NewSecureString("baidu-search-key")}, }, diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 13a6b567f..37498ff1c 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -39,13 +39,22 @@ func DefaultConfig() *Config { MaxArgsLength: 300, SeparateMessages: false, }, - StreamingEnabled: true, - SplitOnMarker: false, + StreamingEnabled: true, + SplitOnMarker: false, + MaxLLMRetries: 2, + LLMRetryBackoffSecs: 2, }, }, Session: SessionConfig{ Dimensions: []string{"chat"}, }, + Evolution: EvolutionConfig{ + Enabled: false, + Mode: "observe", + MinTaskCount: 2, + MinSuccessRatio: 0.7, + ColdPathTrigger: "after_turn", + }, Channels: defaultChannels(), Hooks: HooksConfig{ Enabled: true, @@ -295,6 +304,9 @@ func DefaultConfig() *Config { HotReload: false, LogLevel: DefaultGatewayLogLevel, }, + Events: EventsConfig{ + Logging: defaultEventLoggingConfig(), + }, Tools: ToolsConfig{ FilterSensitiveData: true, FilterMinLength: 8, @@ -330,6 +342,11 @@ func DefaultConfig() *Config { Enabled: false, MaxResults: 5, }, + Gemini: GeminiSearchConfig{ + Enabled: false, + Model: "gemini-2.5-flash", + MaxResults: 5, + }, Perplexity: PerplexityConfig{ Enabled: false, MaxResults: 5, @@ -492,8 +509,9 @@ func defaultChannels() ChannelsConfig { "typing": map[string]any{"enabled": true}, "placeholder": map[string]any{"enabled": true, "text": []string{"Thinking... 💭"}}, "settings": map[string]any{ - "streaming": map[string]any{"enabled": true, "throttle_seconds": 3, "min_growth_chars": 200}, - "use_markdown_v2": false, + "streaming": map[string]any{"enabled": true, "throttle_seconds": 3, "min_growth_chars": 200}, + "use_markdown_v2": false, + "media_group_delay_ms": 500, }, }, "feishu": map[string]any{}, diff --git a/pkg/config/events.go b/pkg/config/events.go new file mode 100644 index 000000000..54a2ce709 --- /dev/null +++ b/pkg/config/events.go @@ -0,0 +1,48 @@ +package config + +// EventsConfig groups runtime event configuration. +type EventsConfig struct { + Logging EventLoggingConfig `json:"logging,omitempty" envPrefix:"PICOCLAW_EVENTS_LOGGING_"` +} + +// EventLoggingConfig controls centralized runtime event logging. +type EventLoggingConfig struct { + // Enabled controls whether runtime events are printed by the built-in logger. + Enabled bool `json:"enabled" env:"ENABLED"` + // Include contains exact event kinds or glob patterns such as "agent.*" or "*". + Include []string `json:"include,omitempty" env:"INCLUDE"` + // Exclude contains exact event kinds or glob patterns to suppress after Include matches. + Exclude []string `json:"exclude,omitempty" env:"EXCLUDE"` + // MinSeverity filters out events below the configured severity: debug, info, warn, or error. + MinSeverity string `json:"min_severity,omitempty" env:"MIN_SEVERITY"` + // IncludePayload adds the raw payload to logs. Leave disabled unless detailed diagnostics are needed. + IncludePayload bool `json:"include_payload,omitempty" env:"INCLUDE_PAYLOAD"` +} + +// DefaultEventLoggingInclude keeps the pre-existing behavior where agent events +// are printed, while non-agent runtime events are published for subscribers only. +var DefaultEventLoggingInclude = []string{"agent.*"} + +// EffectiveEventLoggingConfig returns a logging config with stable defaults. +func EffectiveEventLoggingConfig(cfg *Config) EventLoggingConfig { + if cfg == nil { + return defaultEventLoggingConfig() + } + + out := cfg.Events.Logging + if out.MinSeverity == "" { + out.MinSeverity = "info" + } + if len(out.Include) == 0 { + out.Include = append([]string(nil), DefaultEventLoggingInclude...) + } + return out +} + +func defaultEventLoggingConfig() EventLoggingConfig { + return EventLoggingConfig{ + Enabled: true, + Include: append([]string(nil), DefaultEventLoggingInclude...), + MinSeverity: "info", + } +} diff --git a/pkg/config/events_test.go b/pkg/config/events_test.go new file mode 100644 index 000000000..6cd410492 --- /dev/null +++ b/pkg/config/events_test.go @@ -0,0 +1,103 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestDefaultEventLoggingConfig(t *testing.T) { + cfg := DefaultConfig() + logCfg := EffectiveEventLoggingConfig(cfg) + + if !logCfg.Enabled { + t.Fatal("default event logging should be enabled") + } + if !reflect.DeepEqual(logCfg.Include, []string{"agent.*"}) { + t.Fatalf("default include = %#v, want agent.*", logCfg.Include) + } + if logCfg.MinSeverity != "info" { + t.Fatalf("default min severity = %q, want info", logCfg.MinSeverity) + } + if logCfg.IncludePayload { + t.Fatal("default event logging should not include raw payloads") + } +} + +func TestLoadConfigEventLoggingOverrides(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + data := []byte(`{ + "version": 3, + "events": { + "logging": { + "enabled": false, + "include": ["gateway.*"], + "exclude": ["gateway.ready"], + "min_severity": "warn", + "include_payload": true + } + } + }`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig() error: %v", err) + } + logCfg := EffectiveEventLoggingConfig(cfg) + + if logCfg.Enabled { + t.Fatal("loaded event logging enabled = true, want false") + } + if !reflect.DeepEqual(logCfg.Include, []string{"gateway.*"}) { + t.Fatalf("loaded include = %#v, want gateway.*", logCfg.Include) + } + if !reflect.DeepEqual(logCfg.Exclude, []string{"gateway.ready"}) { + t.Fatalf("loaded exclude = %#v, want gateway.ready", logCfg.Exclude) + } + if logCfg.MinSeverity != "warn" { + t.Fatalf("loaded min severity = %q, want warn", logCfg.MinSeverity) + } + if !logCfg.IncludePayload { + t.Fatal("loaded include_payload = false, want true") + } +} + +func TestLoadConfigEventLoggingEnvOverrides(t *testing.T) { + t.Setenv("PICOCLAW_EVENTS_LOGGING_ENABLED", "false") + t.Setenv("PICOCLAW_EVENTS_LOGGING_INCLUDE", "gateway.*,channel.lifecycle.*") + t.Setenv("PICOCLAW_EVENTS_LOGGING_EXCLUDE", "gateway.ready") + t.Setenv("PICOCLAW_EVENTS_LOGGING_MIN_SEVERITY", "error") + t.Setenv("PICOCLAW_EVENTS_LOGGING_INCLUDE_PAYLOAD", "true") + + path := filepath.Join(t.TempDir(), "config.json") + data := []byte(`{"version": 3}`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig() error: %v", err) + } + logCfg := EffectiveEventLoggingConfig(cfg) + + if logCfg.Enabled { + t.Fatal("env enabled override = true, want false") + } + if !reflect.DeepEqual(logCfg.Include, []string{"gateway.*", "channel.lifecycle.*"}) { + t.Fatalf("env include = %#v, want gateway/channel lifecycle", logCfg.Include) + } + if !reflect.DeepEqual(logCfg.Exclude, []string{"gateway.ready"}) { + t.Fatalf("env exclude = %#v, want gateway.ready", logCfg.Exclude) + } + if logCfg.MinSeverity != "error" { + t.Fatalf("env min severity = %q, want error", logCfg.MinSeverity) + } + if !logCfg.IncludePayload { + t.Fatal("env include_payload = false, want true") + } +} diff --git a/pkg/config/migration.go b/pkg/config/migration.go index 96914819e..40ef1a5a2 100644 --- a/pkg/config/migration.go +++ b/pkg/config/migration.go @@ -83,6 +83,8 @@ func migrateLegacyAgentDefaultsModel(m map[string]any) { // loadConfigV1 loads a version 1 config (current schema) func loadConfig(data []byte) (*Config, error) { cfg := DefaultConfig() + evolutionModeExplicit := configObjectHasField(data, "evolution", "mode") + evolutionExplicitWithoutMode := configObjectHasTopLevelField(data, "evolution") && !evolutionModeExplicit // Pre-scan the JSON to check how many model_list entries the user provided. // Go's JSON decoder reuses existing slice backing-array elements rather than @@ -101,9 +103,38 @@ func loadConfig(data []byte) (*Config, error) { if err := decodeJSONWithDiagnostics(data, cfg, "config.json"); err != nil { return nil, err } + if evolutionExplicitWithoutMode { + cfg.Evolution.Mode = "" + } return cfg, nil } +func configObjectHasTopLevelField(data []byte, field string) bool { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return false + } + _, ok := raw[field] + return ok +} + +func configObjectHasField(data []byte, objectField, nestedField string) bool { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return false + } + objectData, ok := raw[objectField] + if !ok { + return false + } + var object map[string]json.RawMessage + if err := json.Unmarshal(objectData, &object); err != nil { + return false + } + _, ok = object[nestedField] + return ok +} + func mergeAPIKeys(apiKey string, apiKeys []string) []string { seen := make(map[string]struct{}) var all []string diff --git a/pkg/config/model_config_test.go b/pkg/config/model_config_test.go index 8fd501155..d22eb290f 100644 --- a/pkg/config/model_config_test.go +++ b/pkg/config/model_config_test.go @@ -158,6 +158,15 @@ func TestModelConfig_Validate(t *testing.T) { }, wantErr: false, }, + { + name: "valid tool schema transform", + config: ModelConfig{ + ModelName: "test", + Model: "openai/gpt-4o", + ToolSchemaTransform: "simple", + }, + wantErr: false, + }, { name: "missing model_name", config: ModelConfig{ @@ -177,6 +186,15 @@ func TestModelConfig_Validate(t *testing.T) { config: ModelConfig{}, wantErr: true, }, + { + name: "invalid tool schema transform", + config: ModelConfig{ + ModelName: "test", + Model: "openai/gpt-4o", + ToolSchemaTransform: "invalid", + }, + wantErr: true, + }, } for _, tt := range tests { diff --git a/pkg/config/multikey_test.go b/pkg/config/multikey_test.go index cb55db938..073cb7826 100644 --- a/pkg/config/multikey_test.go +++ b/pkg/config/multikey_test.go @@ -187,15 +187,16 @@ func TestExpandMultiKeyModels_Deduplication(t *testing.T) { func TestExpandMultiKeyModels_PreservesOtherFields(t *testing.T) { modelCfg := &ModelConfig{ - ModelName: "gpt-4", - Provider: "openrouter", - Model: "openai/gpt-4o", - APIBase: "https://api.example.com", - Proxy: "http://proxy:8080", - RPM: 60, - MaxTokensField: "max_completion_tokens", - RequestTimeout: 30, - ThinkingLevel: "high", + ModelName: "gpt-4", + Provider: "openrouter", + Model: "openai/gpt-4o", + APIBase: "https://api.example.com", + Proxy: "http://proxy:8080", + RPM: 60, + MaxTokensField: "max_completion_tokens", + RequestTimeout: 30, + ThinkingLevel: "high", + ToolSchemaTransform: "simple", } modelCfg.APIKeys = SimpleSecureStrings("key0", "key1") // Use internal field for multi-key testing models := []*ModelConfig{modelCfg} @@ -225,6 +226,9 @@ func TestExpandMultiKeyModels_PreservesOtherFields(t *testing.T) { if primary.ThinkingLevel != "high" { t.Errorf("expected thinking_level preserved, got %q", primary.ThinkingLevel) } + if primary.ToolSchemaTransform != "simple" { + t.Errorf("expected tool_schema_transform preserved, got %q", primary.ToolSchemaTransform) + } // Check additional entry also preserves fields additional := result[0] @@ -237,6 +241,9 @@ func TestExpandMultiKeyModels_PreservesOtherFields(t *testing.T) { if additional.RPM != 60 { t.Errorf("expected additional rpm preserved, got %d", additional.RPM) } + if additional.ToolSchemaTransform != "simple" { + t.Errorf("expected additional tool_schema_transform preserved, got %q", additional.ToolSchemaTransform) + } } func TestExpandMultiKeyModels_IsVirtualFlag(t *testing.T) { diff --git a/pkg/events/bus.go b/pkg/events/bus.go new file mode 100644 index 000000000..f193ccb74 --- /dev/null +++ b/pkg/events/bus.go @@ -0,0 +1,243 @@ +package events + +import ( + "context" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" +) + +var globalEventSeq atomic.Uint64 + +// Bus publishes runtime events and creates filtered channels. +type Bus interface { + Publish(ctx context.Context, evt Event) PublishResult + PublishNonBlocking(evt Event) PublishResult + Channel() EventChannel + Close() error + Stats() Stats +} + +// PublishResult reports per-publish delivery outcomes. +type PublishResult struct { + Matched int + Delivered int + Dropped int + Blocked int + Closed bool +} + +// EventBus is an in-process runtime event broadcaster. +type EventBus struct { + mu sync.RWMutex + subs map[uint64]*eventSubscription + orderedSubs []*eventSubscription + closed bool + + nextSubID atomic.Uint64 + published atomic.Uint64 + matched atomic.Uint64 + delivered atomic.Uint64 + dropped atomic.Uint64 + blocked atomic.Uint64 +} + +var _ Bus = (*EventBus)(nil) + +// NewBus creates an in-process runtime event bus. +func NewBus() *EventBus { + return &EventBus{ + subs: make(map[uint64]*eventSubscription), + } +} + +// Publish broadcasts evt to subscriptions whose filters match it. +func (b *EventBus) Publish(ctx context.Context, evt Event) PublishResult { + return b.publish(ctx, evt, false) +} + +// PublishNonBlocking broadcasts evt without waiting for subscriber queue capacity. +func (b *EventBus) PublishNonBlocking(evt Event) PublishResult { + return b.publish(context.Background(), evt, true) +} + +func (b *EventBus) publish(ctx context.Context, evt Event, nonBlocking bool) PublishResult { + if b == nil { + return PublishResult{Closed: true} + } + if ctx == nil { + ctx = context.Background() + } + if evt.Time.IsZero() { + evt.Time = time.Now() + } + if evt.ID == "" { + evt.ID = nextEventID() + } + + subs, closed := b.snapshotSubscribers() + if closed { + return PublishResult{Closed: true} + } + + b.published.Add(1) + result := PublishResult{} + + for _, sub := range subs { + if !matchesFilters(sub.filters, evt) { + continue + } + + result.Matched++ + b.matched.Add(1) + + delivery := sub.enqueue(ctx, evt, nonBlocking) + if delivery.closed { + continue + } + result.Delivered += delivery.delivered + result.Dropped += delivery.dropped + result.Blocked += delivery.blocked + b.delivered.Add(uint64(delivery.delivered)) + b.dropped.Add(uint64(delivery.dropped)) + b.blocked.Add(uint64(delivery.blocked)) + } + + return result +} + +// Channel returns the root event channel for this bus. +func (b *EventBus) Channel() EventChannel { + return eventChannel{bus: b} +} + +// Close closes the bus and all active subscriptions. +func (b *EventBus) Close() error { + if b == nil { + return nil + } + + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return nil + } + b.closed = true + subs := b.orderedSubs + b.subs = nil + b.orderedSubs = nil + b.mu.Unlock() + + for _, sub := range subs { + sub.closeInput() + } + return nil +} + +// Stats returns a snapshot of bus and subscription counters. +func (b *EventBus) Stats() Stats { + if b == nil { + return Stats{Closed: true} + } + + b.mu.RLock() + closed := b.closed + subs := b.orderedSubs + b.mu.RUnlock() + + stats := Stats{ + Published: b.published.Load(), + Matched: b.matched.Load(), + Delivered: b.delivered.Load(), + Dropped: b.dropped.Load(), + Blocked: b.blocked.Load(), + Closed: closed, + Subscribers: len(subs), + SubscriberStats: make([]SubscriberStats, 0, len(subs)), + } + for _, sub := range subs { + stats.SubscriberStats = append(stats.SubscriberStats, sub.Stats()) + } + return stats +} + +func (b *EventBus) subscribe( + ctx context.Context, + filters []Filter, + opts SubscribeOptions, + handler Handler, + once bool, +) (Subscription, error) { + if b == nil { + return nil, ErrBusClosed + } + + id := b.nextSubID.Add(1) + sub := newSubscription(b, id, filters, opts, handler, once) + + b.mu.Lock() + if b.closed { + b.mu.Unlock() + sub.closeInput() + return nil, ErrBusClosed + } + b.subs[id] = sub + b.rebuildOrderedSubscribersLocked() + b.mu.Unlock() + + if handler != nil { + go sub.run(ctx) + } + sub.watchContext(ctx) + return sub, nil +} + +func (b *EventBus) unsubscribe(id uint64) { + b.mu.Lock() + sub, ok := b.subs[id] + if ok { + delete(b.subs, id) + b.rebuildOrderedSubscribersLocked() + } + b.mu.Unlock() + + if ok { + sub.closeInput() + } +} + +func (b *EventBus) snapshotSubscribers() ([]*eventSubscription, bool) { + b.mu.RLock() + defer b.mu.RUnlock() + + if b.closed { + return nil, true + } + + return b.orderedSubs, false +} + +func (b *EventBus) rebuildOrderedSubscribersLocked() { + subs := make([]*eventSubscription, 0, len(b.subs)) + for _, sub := range b.subs { + subs = append(subs, sub) + } + sortSubscriptions(subs) + b.orderedSubs = subs +} + +func sortSubscriptions(subs []*eventSubscription) { + sort.Slice(subs, func(i, j int) bool { + if subs[i].opts.Priority == subs[j].opts.Priority { + return subs[i].id < subs[j].id + } + return subs[i].opts.Priority > subs[j].opts.Priority + }) +} + +func nextEventID() string { + id := globalEventSeq.Add(1) + return "evt-" + strconv.FormatUint(id, 10) +} diff --git a/pkg/events/channel.go b/pkg/events/channel.go new file mode 100644 index 000000000..9cf6d8d8c --- /dev/null +++ b/pkg/events/channel.go @@ -0,0 +1,75 @@ +package events + +import "context" + +// EventChannel is a filtered view over an EventBus. +type EventChannel interface { + Filter(filter Filter) EventChannel + OfKind(kinds ...Kind) EventChannel + KindPrefix(prefix string) EventChannel + Source(component string, names ...string) EventChannel + Scope(scope ScopeFilter) EventChannel + + Subscribe(ctx context.Context, opts SubscribeOptions, handler Handler) (Subscription, error) + SubscribeChan(ctx context.Context, opts SubscribeOptions) (Subscription, <-chan Event, error) + SubscribeOnce(ctx context.Context, opts SubscribeOptions, handler Handler) (Subscription, error) +} + +type eventChannel struct { + bus *EventBus + filters []Filter +} + +// Filter returns a new EventChannel with filter appended. +func (c eventChannel) Filter(filter Filter) EventChannel { + filters := append([]Filter(nil), c.filters...) + if filter != nil { + filters = append(filters, filter) + } + return eventChannel{bus: c.bus, filters: filters} +} + +// OfKind returns a new EventChannel matching any of kinds. +func (c eventChannel) OfKind(kinds ...Kind) EventChannel { + return c.Filter(MatchKind(kinds...)) +} + +// KindPrefix returns a new EventChannel matching events with the kind prefix. +func (c eventChannel) KindPrefix(prefix string) EventChannel { + return c.Filter(MatchKindPrefix(prefix)) +} + +// Source returns a new EventChannel matching source component and optional names. +func (c eventChannel) Source(component string, names ...string) EventChannel { + return c.Filter(MatchSource(component, names...)) +} + +// Scope returns a new EventChannel matching non-empty scope fields. +func (c eventChannel) Scope(scope ScopeFilter) EventChannel { + return c.Filter(MatchScope(scope)) +} + +// Subscribe registers handler for events matching this channel. +func (c eventChannel) Subscribe(ctx context.Context, opts SubscribeOptions, handler Handler) (Subscription, error) { + if handler == nil { + return nil, ErrNilHandler + } + return c.bus.subscribe(ctx, c.filters, opts, handler, false) +} + +// SubscribeChan registers a channel subscription for events matching this channel. +func (c eventChannel) SubscribeChan(ctx context.Context, opts SubscribeOptions) (Subscription, <-chan Event, error) { + sub, err := c.bus.subscribe(ctx, c.filters, opts, nil, false) + if err != nil { + return nil, nil, err + } + return sub, sub.(*eventSubscription).ch, nil +} + +// SubscribeOnce registers handler and closes the subscription after the first event. +func (c eventChannel) SubscribeOnce(ctx context.Context, opts SubscribeOptions, handler Handler) (Subscription, error) { + if handler == nil { + return nil, ErrNilHandler + } + return c.bus.subscribe(ctx, c.filters, opts, handler, true) +} diff --git a/pkg/events/doc.go b/pkg/events/doc.go new file mode 100644 index 000000000..dc2f55631 --- /dev/null +++ b/pkg/events/doc.go @@ -0,0 +1,3 @@ +// Package events provides the process-local runtime event bus used to observe +// PicoClaw components without coupling them to agent-specific event envelopes. +package events diff --git a/pkg/events/events_test.go b/pkg/events/events_test.go new file mode 100644 index 000000000..6991e8291 --- /dev/null +++ b/pkg/events/events_test.go @@ -0,0 +1,254 @@ +package events + +import ( + "context" + "testing" + "time" +) + +func TestPublishDeliversToMatchingSubscriber(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + _, ch, err := bus.Channel().OfKind(KindAgentTurnStart).SubscribeChan( + context.Background(), + SubscribeOptions{Name: "turn-starts", Buffer: 1}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + unmatched := bus.Publish(context.Background(), Event{Kind: KindAgentTurnEnd}) + if unmatched.Matched != 0 || unmatched.Delivered != 0 { + t.Fatalf("unmatched Publish = %+v, want no delivery", unmatched) + } + + result := bus.Publish(context.Background(), Event{Kind: KindAgentTurnStart}) + if result.Matched != 1 || result.Delivered != 1 || result.Dropped != 0 { + t.Fatalf("Publish = %+v, want one delivered event", result) + } + + evt := receiveEvent(t, ch) + if evt.Kind != KindAgentTurnStart { + t.Fatalf("event kind = %q, want %q", evt.Kind, KindAgentTurnStart) + } + if evt.ID == "" { + t.Fatal("event ID is empty") + } + if evt.Time.IsZero() { + t.Fatal("event Time is zero") + } +} + +func TestDropNewestIncrementsStats(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + sub, _, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "drop-newest", Buffer: 1, Backpressure: DropNewest}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + first := bus.Publish(context.Background(), Event{Kind: KindAgentTurnStart}) + if first.Delivered != 1 || first.Dropped != 0 { + t.Fatalf("first Publish = %+v, want one delivered event", first) + } + + second := bus.Publish(context.Background(), Event{Kind: KindAgentTurnEnd}) + if second.Delivered != 0 || second.Dropped != 1 { + t.Fatalf("second Publish = %+v, want one dropped event", second) + } + + if got := sub.Stats().Dropped; got != 1 { + t.Fatalf("subscription dropped = %d, want 1", got) + } + if got := bus.Stats().Dropped; got != 1 { + t.Fatalf("bus dropped = %d, want 1", got) + } +} + +func TestDropOldestKeepsNewestEvent(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + sub, ch, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "drop-oldest", Buffer: 1, Backpressure: DropOldest}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + bus.Publish(context.Background(), Event{Kind: Kind("test.old"), Payload: "old"}) + result := bus.Publish(context.Background(), Event{Kind: Kind("test.new"), Payload: "new"}) + if result.Delivered != 1 || result.Dropped != 1 { + t.Fatalf("Publish = %+v, want replacement delivery", result) + } + + evt := receiveEvent(t, ch) + if evt.Payload != "new" { + t.Fatalf("payload = %v, want new", evt.Payload) + } + if got := sub.Stats().Dropped; got != 1 { + t.Fatalf("subscription dropped = %d, want 1", got) + } +} + +func TestBlockRespectsContext(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + _, _, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "block", Buffer: 1, Backpressure: Block}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + first := bus.Publish(context.Background(), Event{Kind: Kind("test.first")}) + if first.Delivered != 1 { + t.Fatalf("first Publish = %+v, want one delivered event", first) + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + second := bus.Publish(ctx, Event{Kind: Kind("test.second")}) + if second.Blocked != 1 || second.Dropped != 1 || second.Delivered != 0 { + t.Fatalf("second Publish = %+v, want one blocked drop", second) + } +} + +func TestPublishNonBlockingDropsForFullBlockSubscriber(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + sub, _, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "block", Buffer: 1, Backpressure: Block}, + ) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + first := bus.PublishNonBlocking(Event{Kind: Kind("test.first")}) + if first.Delivered != 1 { + t.Fatalf("first PublishNonBlocking = %+v, want one delivered event", first) + } + + resultCh := make(chan PublishResult, 1) + go func() { + resultCh <- bus.PublishNonBlocking(Event{Kind: Kind("test.second")}) + }() + + select { + case second := <-resultCh: + if second.Matched != 1 || second.Delivered != 0 || second.Dropped != 1 || second.Blocked != 0 { + t.Fatalf("second PublishNonBlocking = %+v, want non-blocking drop", second) + } + case <-time.After(100 * time.Millisecond): + t.Fatal("PublishNonBlocking blocked on full Block subscriber") + } + + if got := sub.Stats().Dropped; got != 1 { + t.Fatalf("subscription dropped = %d, want 1", got) + } +} + +func TestStatsSubscribersKeepPriorityOrder(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + low, _, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "low", Priority: -1}, + ) + if err != nil { + t.Fatalf("SubscribeChan low failed: %v", err) + } + high, _, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "high", Priority: 10}, + ) + if err != nil { + t.Fatalf("SubscribeChan high failed: %v", err) + } + peer, _, err := bus.Channel().SubscribeChan( + context.Background(), + SubscribeOptions{Name: "peer", Priority: 10}, + ) + if err != nil { + t.Fatalf("SubscribeChan peer failed: %v", err) + } + + stats := bus.Stats() + got := []string{ + stats.SubscriberStats[0].Name, + stats.SubscriberStats[1].Name, + stats.SubscriberStats[2].Name, + } + want := []string{"high", "peer", "low"} + if got[0] != want[0] || got[1] != want[1] || got[2] != want[2] { + t.Fatalf("subscriber order = %v, want %v", got, want) + } + + if err := high.Close(); err != nil { + t.Fatalf("Close high failed: %v", err) + } + + stats = bus.Stats() + got = []string{ + stats.SubscriberStats[0].Name, + stats.SubscriberStats[1].Name, + } + want = []string{"peer", "low"} + if got[0] != want[0] || got[1] != want[1] { + t.Fatalf("subscriber order after unsubscribe = %v, want %v", got, want) + } + + if err := peer.Close(); err != nil { + t.Fatalf("Close peer failed: %v", err) + } + if err := low.Close(); err != nil { + t.Fatalf("Close low failed: %v", err) + } +} + +func receiveEvent(t *testing.T, ch <-chan Event) Event { + t.Helper() + + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("event channel closed before receive") + } + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for event") + return Event{} + } +} + +func closeBus(t *testing.T, bus *EventBus) { + t.Helper() + + if err := bus.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } +} diff --git a/pkg/events/filter.go b/pkg/events/filter.go new file mode 100644 index 000000000..0af2c85c1 --- /dev/null +++ b/pkg/events/filter.go @@ -0,0 +1,131 @@ +package events + +import "strings" + +// Filter decides whether an event should pass through an EventChannel. +type Filter func(Event) bool + +// ScopeFilter matches selected non-empty fields against Event.Scope. +type ScopeFilter struct { + AgentID string + SessionKey string + TurnID string + Channel string + ChatID string + MessageID string +} + +// MatchKind matches events whose kind is in kinds. Empty kinds match all events. +func MatchKind(kinds ...Kind) Filter { + if len(kinds) == 0 { + return matchAll + } + + allowed := make(map[Kind]struct{}, len(kinds)) + for _, kind := range kinds { + allowed[kind] = struct{}{} + } + + return func(evt Event) bool { + _, ok := allowed[evt.Kind] + return ok + } +} + +// MatchKindPrefix matches events whose kind starts with prefix. +func MatchKindPrefix(prefix string) Filter { + if prefix == "" { + return matchAll + } + return func(evt Event) bool { + return strings.HasPrefix(evt.Kind.String(), prefix) + } +} + +// MatchSource matches events emitted by component and, optionally, one of names. +func MatchSource(component string, names ...string) Filter { + if component == "" && len(names) == 0 { + return matchAll + } + + allowedNames := make(map[string]struct{}, len(names)) + for _, name := range names { + allowedNames[name] = struct{}{} + } + + return func(evt Event) bool { + if component != "" && evt.Source.Component != component { + return false + } + if len(allowedNames) == 0 { + return true + } + _, ok := allowedNames[evt.Source.Name] + return ok + } +} + +// MatchScope matches events whose Scope contains all non-empty filter fields. +func MatchScope(scope ScopeFilter) Filter { + if scope == (ScopeFilter{}) { + return matchAll + } + + return func(evt Event) bool { + return matchesString(scope.AgentID, evt.Scope.AgentID) && + matchesString(scope.SessionKey, evt.Scope.SessionKey) && + matchesString(scope.TurnID, evt.Scope.TurnID) && + matchesString(scope.Channel, evt.Scope.Channel) && + matchesString(scope.ChatID, evt.Scope.ChatID) && + matchesString(scope.MessageID, evt.Scope.MessageID) + } +} + +// And combines filters and short-circuits on the first non-match. +func And(filters ...Filter) Filter { + if len(filters) == 0 { + return matchAll + } + + return func(evt Event) bool { + for _, filter := range filters { + if filter != nil && !filter(evt) { + return false + } + } + return true + } +} + +// Or combines filters and short-circuits on the first match. +func Or(filters ...Filter) Filter { + if len(filters) == 0 { + return matchAll + } + + return func(evt Event) bool { + for _, filter := range filters { + if filter == nil || filter(evt) { + return true + } + } + return false + } +} + +func matchAll(Event) bool { + return true +} + +func matchesString(want, got string) bool { + return want == "" || want == got +} + +func matchesFilters(filters []Filter, evt Event) bool { + for _, filter := range filters { + if filter != nil && !filter(evt) { + return false + } + } + return true +} diff --git a/pkg/events/filter_test.go b/pkg/events/filter_test.go new file mode 100644 index 000000000..9b0112754 --- /dev/null +++ b/pkg/events/filter_test.go @@ -0,0 +1,96 @@ +package events + +import "testing" + +func TestFilterKindPrefix(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + prefix string + event Event + want bool + }{ + { + name: "matches agent prefix", + prefix: "agent.", + event: Event{Kind: KindAgentTurnStart}, + want: true, + }, + { + name: "rejects different prefix", + prefix: "channel.", + event: Event{Kind: KindAgentTurnStart}, + want: false, + }, + { + name: "empty prefix matches all", + prefix: "", + event: Event{Kind: KindAgentTurnStart}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := MatchKindPrefix(tt.prefix)(tt.event); got != tt.want { + t.Fatalf("MatchKindPrefix(%q) = %v, want %v", tt.prefix, got, tt.want) + } + }) + } +} + +func TestFilterScope(t *testing.T) { + t.Parallel() + + evt := Event{ + Scope: Scope{ + AgentID: "agent-a", + SessionKey: "session-1", + TurnID: "turn-1", + Channel: "telegram", + ChatID: "chat-1", + MessageID: "msg-1", + }, + } + + tests := []struct { + name string + scope ScopeFilter + want bool + }{ + { + name: "empty filter matches", + scope: ScopeFilter{}, + want: true, + }, + { + name: "matches selected fields", + scope: ScopeFilter{ + AgentID: "agent-a", + ChatID: "chat-1", + }, + want: true, + }, + { + name: "rejects mismatched field", + scope: ScopeFilter{ + AgentID: "agent-a", + MessageID: "msg-2", + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := MatchScope(tt.scope)(evt); got != tt.want { + t.Fatalf("MatchScope(%+v) = %v, want %v", tt.scope, got, tt.want) + } + }) + } +} diff --git a/pkg/events/kind.go b/pkg/events/kind.go new file mode 100644 index 000000000..b9327e155 --- /dev/null +++ b/pkg/events/kind.go @@ -0,0 +1,156 @@ +package events + +const ( + // KindAgentTurnStart is emitted when an agent turn starts. + KindAgentTurnStart Kind = "agent.turn.start" + // KindAgentTurnEnd is emitted when an agent turn ends. + KindAgentTurnEnd Kind = "agent.turn.end" + + // KindAgentLLMRequest is emitted before an LLM request. + KindAgentLLMRequest Kind = "agent.llm.request" + // KindAgentLLMDelta is emitted for streaming LLM deltas. + KindAgentLLMDelta Kind = "agent.llm.delta" + // KindAgentLLMResponse is emitted after an LLM response. + KindAgentLLMResponse Kind = "agent.llm.response" + // KindAgentLLMRetry is emitted before retrying an LLM request. + KindAgentLLMRetry Kind = "agent.llm.retry" + + // KindAgentContextCompress is emitted when agent context is compressed. + KindAgentContextCompress Kind = "agent.context.compress" + // KindAgentSessionSummarize is emitted when session summarization completes. + KindAgentSessionSummarize Kind = "agent.session.summarize" + + // KindAgentToolExecStart is emitted before a tool executes. + KindAgentToolExecStart Kind = "agent.tool.exec_start" + // KindAgentToolExecEnd is emitted after a tool finishes. + KindAgentToolExecEnd Kind = "agent.tool.exec_end" + // KindAgentToolExecSkipped is emitted when a tool call is skipped. + KindAgentToolExecSkipped Kind = "agent.tool.exec_skipped" + + // KindAgentSteeringInjected is emitted when steering is injected into context. + KindAgentSteeringInjected Kind = "agent.steering.injected" + // KindAgentFollowUpQueued is emitted when async follow-up input is queued. + KindAgentFollowUpQueued Kind = "agent.follow_up.queued" + // KindAgentInterruptReceived is emitted when a turn interrupt is accepted. + KindAgentInterruptReceived Kind = "agent.interrupt.received" + + // KindAgentSubTurnSpawn is emitted when a sub-turn is spawned. + KindAgentSubTurnSpawn Kind = "agent.subturn.spawn" + // KindAgentSubTurnEnd is emitted when a sub-turn ends. + KindAgentSubTurnEnd Kind = "agent.subturn.end" + // KindAgentSubTurnResultDelivered is emitted when a sub-turn result is delivered. + KindAgentSubTurnResultDelivered Kind = "agent.subturn.result_delivered" + // KindAgentSubTurnOrphan is emitted when a sub-turn result cannot be delivered. + KindAgentSubTurnOrphan Kind = "agent.subturn.orphan" + // KindAgentError is emitted when agent execution reports an error. + KindAgentError Kind = "agent.error" + + // KindChannelLifecycleStarted is emitted when a channel starts. + KindChannelLifecycleStarted Kind = "channel.lifecycle.started" + // KindChannelLifecycleInitialized is emitted when a channel is initialized. + KindChannelLifecycleInitialized Kind = "channel.lifecycle.initialized" + // KindChannelLifecycleStartFailed is emitted when a channel fails to start. + KindChannelLifecycleStartFailed Kind = "channel.lifecycle.start_failed" + // KindChannelLifecycleStopped is emitted when a channel stops. + KindChannelLifecycleStopped Kind = "channel.lifecycle.stopped" + // KindChannelWebhookRegistered is emitted when a channel webhook is registered. + KindChannelWebhookRegistered Kind = "channel.webhook.registered" + // KindChannelWebhookUnregistered is emitted when a channel webhook is unregistered. + KindChannelWebhookUnregistered Kind = "channel.webhook.unregistered" + // KindChannelMessageOutboundQueued is emitted when an outbound message is queued. + KindChannelMessageOutboundQueued Kind = "channel.message.outbound_queued" + // KindChannelMessageOutboundSent is emitted when an outbound channel message is sent. + KindChannelMessageOutboundSent Kind = "channel.message.outbound_sent" + // KindChannelMessageOutboundFailed is emitted when an outbound channel message fails. + KindChannelMessageOutboundFailed Kind = "channel.message.outbound_failed" + // KindChannelRateLimited is emitted when channel rate limiting blocks delivery. + KindChannelRateLimited Kind = "channel.rate_limited" + + // KindBusPublishFailed is emitted when message bus publish fails. + KindBusPublishFailed Kind = "bus.publish.failed" + // KindBusCloseStarted is emitted when message bus close starts. + KindBusCloseStarted Kind = "bus.close.started" + // KindBusCloseCompleted is emitted when message bus close completes. + KindBusCloseCompleted Kind = "bus.close.completed" + // KindBusCloseDrained is emitted when message bus close drains buffered messages. + KindBusCloseDrained Kind = "bus.close.drained" + + // KindGatewayStart is emitted when gateway startup reaches runtime bootstrap. + KindGatewayStart Kind = "gateway.start" + // KindGatewayReady is emitted when gateway services are started and ready. + KindGatewayReady Kind = "gateway.ready" + // KindGatewayShutdown is emitted when gateway shutdown starts. + KindGatewayShutdown Kind = "gateway.shutdown" + // KindGatewayReloadStarted is emitted when gateway reload starts. + KindGatewayReloadStarted Kind = "gateway.reload.started" + // KindGatewayReloadCompleted is emitted when gateway reload completes. + KindGatewayReloadCompleted Kind = "gateway.reload.completed" + // KindGatewayReloadFailed is emitted when gateway reload fails. + KindGatewayReloadFailed Kind = "gateway.reload.failed" + + // KindMCPServerConnected is emitted when an MCP server connects. + KindMCPServerConnected Kind = "mcp.server.connected" + // KindMCPServerConnecting is emitted before connecting to an MCP server. + KindMCPServerConnecting Kind = "mcp.server.connecting" + // KindMCPServerFailed is emitted when an MCP server fails. + KindMCPServerFailed Kind = "mcp.server.failed" + // KindMCPToolDiscovered is emitted when an MCP tool is discovered. + KindMCPToolDiscovered Kind = "mcp.tool.discovered" + // KindMCPToolCallStart is emitted when an MCP tool call starts. + KindMCPToolCallStart Kind = "mcp.tool.call.start" + // KindMCPToolCallEnd is emitted when an MCP tool call ends. + KindMCPToolCallEnd Kind = "mcp.tool.call.end" +) + +var knownKinds = []Kind{ + KindAgentTurnStart, + KindAgentTurnEnd, + KindAgentLLMRequest, + KindAgentLLMDelta, + KindAgentLLMResponse, + KindAgentLLMRetry, + KindAgentContextCompress, + KindAgentSessionSummarize, + KindAgentToolExecStart, + KindAgentToolExecEnd, + KindAgentToolExecSkipped, + KindAgentSteeringInjected, + KindAgentFollowUpQueued, + KindAgentInterruptReceived, + KindAgentSubTurnSpawn, + KindAgentSubTurnEnd, + KindAgentSubTurnResultDelivered, + KindAgentSubTurnOrphan, + KindAgentError, + KindChannelLifecycleStarted, + KindChannelLifecycleInitialized, + KindChannelLifecycleStartFailed, + KindChannelLifecycleStopped, + KindChannelWebhookRegistered, + KindChannelWebhookUnregistered, + KindChannelMessageOutboundQueued, + KindChannelMessageOutboundSent, + KindChannelMessageOutboundFailed, + KindChannelRateLimited, + KindBusPublishFailed, + KindBusCloseStarted, + KindBusCloseCompleted, + KindBusCloseDrained, + KindGatewayStart, + KindGatewayReady, + KindGatewayShutdown, + KindGatewayReloadStarted, + KindGatewayReloadCompleted, + KindGatewayReloadFailed, + KindMCPServerConnected, + KindMCPServerConnecting, + KindMCPServerFailed, + KindMCPToolDiscovered, + KindMCPToolCallStart, + KindMCPToolCallEnd, +} + +// KnownKinds returns the runtime event kinds declared by this package. +func KnownKinds() []Kind { + return append([]Kind(nil), knownKinds...) +} diff --git a/pkg/events/stats.go b/pkg/events/stats.go new file mode 100644 index 000000000..7931c5ef3 --- /dev/null +++ b/pkg/events/stats.go @@ -0,0 +1,26 @@ +package events + +// Stats reports aggregate EventBus counters. +type Stats struct { + Published uint64 + Matched uint64 + Delivered uint64 + Dropped uint64 + Blocked uint64 + Closed bool + Subscribers int + + SubscriberStats []SubscriberStats +} + +// SubscriberStats reports counters for one subscription. +type SubscriberStats struct { + ID uint64 + Name string + Received uint64 + Handled uint64 + Failed uint64 + Dropped uint64 + Panicked uint64 + TimedOut uint64 +} diff --git a/pkg/events/subscription.go b/pkg/events/subscription.go new file mode 100644 index 000000000..6619707a7 --- /dev/null +++ b/pkg/events/subscription.go @@ -0,0 +1,459 @@ +package events + +import ( + "context" + "errors" + "log" + "sync" + "sync/atomic" + "time" +) + +const defaultSubscriberBuffer = 16 + +var ( + // ErrBusClosed is returned when subscribing to a closed event bus. + ErrBusClosed = errors.New("events: bus is closed") + // ErrNilHandler is returned when subscribing without a handler. + ErrNilHandler = errors.New("events: handler is nil") +) + +// Handler processes a runtime event delivered to a subscription. +type Handler func(context.Context, Event) error + +// SubscribeOptions controls how a subscription receives events. +type SubscribeOptions struct { + Name string + Buffer int + Priority int + Concurrency ConcurrencyKind + Backpressure BackpressurePolicy + // Timeout bounds how long the subscription worker waits for one handler call. + // Handlers should still honor ctx cancellation; timed-out calls keep running + // until their handler returns. + Timeout time.Duration + PanicPolicy PanicPolicy +} + +// ConcurrencyKind controls how handler subscriptions process queued events. +type ConcurrencyKind string + +const ( + // Concurrent processes each event in its own goroutine. + Concurrent ConcurrencyKind = "concurrent" + // Locked processes events sequentially in subscription order. + Locked ConcurrencyKind = "locked" + // Keyed is reserved for keyed sequential processing and currently behaves as Locked. + Keyed ConcurrencyKind = "keyed" +) + +// BackpressurePolicy controls delivery when a subscription queue is full. +type BackpressurePolicy string + +const ( + // DropNewest drops the event being published when the queue is full. + DropNewest BackpressurePolicy = "drop_newest" + // DropOldest drops one queued event and enqueues the event being published. + DropOldest BackpressurePolicy = "drop_oldest" + // Block waits for queue capacity until Publish's context is canceled. + Block BackpressurePolicy = "block" +) + +// PanicPolicy controls handler panic behavior. +type PanicPolicy string + +const ( + // RecoverAndLog recovers handler panics and records them in subscription stats. + RecoverAndLog PanicPolicy = "recover_and_log" + // Crash lets handler panics propagate from the worker goroutine. + Crash PanicPolicy = "crash" +) + +// Subscription represents an active event subscription. +type Subscription interface { + ID() uint64 + Name() string + Close() error + Done() <-chan struct{} + Stats() SubscriberStats +} + +type subscriberCounters struct { + received atomic.Uint64 + handled atomic.Uint64 + failed atomic.Uint64 + dropped atomic.Uint64 + panicked atomic.Uint64 + timedOut atomic.Uint64 +} + +type eventSubscription struct { + bus *EventBus + id uint64 + name string + opts SubscribeOptions + filters []Filter + handler Handler + once bool + + ch chan Event + done chan struct{} + closing chan struct{} + + closeOnce sync.Once + doneOnce sync.Once + mu sync.RWMutex + closed bool + wg sync.WaitGroup + blockWG sync.WaitGroup + + counters subscriberCounters +} + +type handlerResult struct { + err error + panicked bool +} + +func normalizeSubscribeOptions(opts SubscribeOptions) SubscribeOptions { + if opts.Buffer <= 0 { + opts.Buffer = defaultSubscriberBuffer + } + if opts.Concurrency == "" { + opts.Concurrency = Locked + } + if opts.Backpressure == "" { + opts.Backpressure = DropNewest + } + if opts.PanicPolicy == "" { + opts.PanicPolicy = RecoverAndLog + } + return opts +} + +func newSubscription( + bus *EventBus, + id uint64, + filters []Filter, + opts SubscribeOptions, + handler Handler, + once bool, +) *eventSubscription { + opts = normalizeSubscribeOptions(opts) + return &eventSubscription{ + bus: bus, + id: id, + name: opts.Name, + opts: opts, + filters: append([]Filter(nil), filters...), + handler: handler, + once: once, + ch: make(chan Event, opts.Buffer), + done: make(chan struct{}), + closing: make(chan struct{}), + } +} + +// ID returns the subscription identifier. +func (s *eventSubscription) ID() uint64 { + if s == nil { + return 0 + } + return s.id +} + +// Name returns the subscription name. +func (s *eventSubscription) Name() string { + if s == nil { + return "" + } + return s.name +} + +// Close removes the subscription and closes its delivery channel. +func (s *eventSubscription) Close() error { + if s == nil || s.bus == nil { + return nil + } + s.bus.unsubscribe(s.id) + return nil +} + +// Done returns a channel closed after the subscription has stopped processing. +func (s *eventSubscription) Done() <-chan struct{} { + if s == nil { + ch := make(chan struct{}) + close(ch) + return ch + } + return s.done +} + +// Stats returns a snapshot of the subscription counters. +func (s *eventSubscription) Stats() SubscriberStats { + if s == nil { + return SubscriberStats{} + } + return SubscriberStats{ + ID: s.id, + Name: s.name, + Received: s.counters.received.Load(), + Handled: s.counters.handled.Load(), + Failed: s.counters.failed.Load(), + Dropped: s.counters.dropped.Load(), + Panicked: s.counters.panicked.Load(), + TimedOut: s.counters.timedOut.Load(), + } +} + +func (s *eventSubscription) run(ctx context.Context) { + defer func() { + s.wg.Wait() + s.closeDone() + }() + + for evt := range s.ch { + s.dispatch(ctx, evt) + if s.once { + _ = s.Close() + return + } + } +} + +func (s *eventSubscription) dispatch(ctx context.Context, evt Event) { + switch s.opts.Concurrency { + case Concurrent: + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.handle(ctx, evt) + }() + case Keyed: + // TODO: replace this with keyed executors when runtime events need + // per-scope ordering with cross-scope concurrency. + s.handle(ctx, evt) + default: + s.handle(ctx, evt) + } +} + +func (s *eventSubscription) handle(ctx context.Context, evt Event) { + if ctx == nil { + ctx = context.Background() + } + + if s.opts.Timeout <= 0 { + s.recordHandlerResult(ctx, s.invokeHandler(ctx, evt)) + return + } + + ctx, cancel := context.WithTimeout(ctx, s.opts.Timeout) + defer cancel() + + done := make(chan handlerResult, 1) + go func() { + done <- s.invokeHandler(ctx, evt) + }() + + select { + case result := <-done: + s.recordHandlerResult(ctx, result) + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + s.counters.timedOut.Add(1) + } + s.counters.failed.Add(1) + } +} + +func (s *eventSubscription) invokeHandler(ctx context.Context, evt Event) (result handlerResult) { + if s.opts.PanicPolicy != Crash { + defer func() { + if recovered := recover(); recovered != nil { + s.counters.panicked.Add(1) + result.panicked = true + log.Printf("events: subscriber %q recovered panic: %v", s.name, recovered) + } + }() + } + + result.err = s.handler(ctx, evt) + return result +} + +func (s *eventSubscription) recordHandlerResult(ctx context.Context, result handlerResult) { + if result.panicked { + return + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + s.counters.timedOut.Add(1) + } + if result.err != nil { + s.counters.failed.Add(1) + return + } + s.counters.handled.Add(1) +} + +func (s *eventSubscription) watchContext(ctx context.Context) { + if ctx == nil { + return + } + + go func() { + select { + case <-ctx.Done(): + _ = s.Close() + case <-s.done: + } + }() +} + +func (s *eventSubscription) closeInput() { + s.closeOnce.Do(func() { + close(s.closing) + s.mu.Lock() + s.closed = true + s.mu.Unlock() + s.blockWG.Wait() + s.mu.Lock() + close(s.ch) + s.mu.Unlock() + if s.handler == nil { + s.closeDone() + } + }) +} + +func (s *eventSubscription) closeDone() { + s.doneOnce.Do(func() { + close(s.done) + }) +} + +type deliveryResult struct { + delivered int + dropped int + blocked int + closed bool +} + +func (s *eventSubscription) enqueue(ctx context.Context, evt Event, nonBlocking bool) deliveryResult { + if ctx == nil { + ctx = context.Background() + } + + if nonBlocking { + return s.enqueueNonBlocking(evt) + } + + if s.opts.Backpressure == Block { + return s.enqueueBlocking(ctx, evt) + } + + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return deliveryResult{closed: true} + } + + s.counters.received.Add(1) + + switch s.opts.Backpressure { + case DropOldest: + return s.enqueueDropOldest(evt) + default: + return s.enqueueDropNewest(evt) + } +} + +func (s *eventSubscription) enqueueBlocking(ctx context.Context, evt Event) deliveryResult { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return deliveryResult{closed: true} + } + s.blockWG.Add(1) + s.counters.received.Add(1) + s.mu.Unlock() + + defer s.blockWG.Done() + return s.enqueueBlock(ctx, evt) +} + +func (s *eventSubscription) enqueueNonBlocking(evt Event) deliveryResult { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return deliveryResult{closed: true} + } + + s.counters.received.Add(1) + if s.opts.Backpressure == DropOldest { + return s.enqueueDropOldest(evt) + } + return s.enqueueDropNewest(evt) +} + +func (s *eventSubscription) enqueueDropNewest(evt Event) deliveryResult { + select { + case <-s.closing: + return deliveryResult{closed: true} + default: + } + + select { + case s.ch <- evt: + return deliveryResult{delivered: 1} + default: + s.counters.dropped.Add(1) + return deliveryResult{dropped: 1} + } +} + +func (s *eventSubscription) enqueueDropOldest(evt Event) deliveryResult { + select { + case <-s.closing: + return deliveryResult{closed: true} + default: + } + + select { + case s.ch <- evt: + return deliveryResult{delivered: 1} + default: + } + + dropped := 0 + select { + case <-s.ch: + s.counters.dropped.Add(1) + dropped = 1 + default: + } + + select { + case <-s.closing: + return deliveryResult{dropped: dropped, closed: true} + case s.ch <- evt: + return deliveryResult{delivered: 1, dropped: dropped} + default: + s.counters.dropped.Add(1) + return deliveryResult{dropped: dropped + 1} + } +} + +func (s *eventSubscription) enqueueBlock(ctx context.Context, evt Event) deliveryResult { + select { + case <-s.closing: + return deliveryResult{closed: true} + case s.ch <- evt: + return deliveryResult{delivered: 1} + case <-ctx.Done(): + s.counters.dropped.Add(1) + return deliveryResult{dropped: 1, blocked: 1} + } +} diff --git a/pkg/events/subscription_test.go b/pkg/events/subscription_test.go new file mode 100644 index 000000000..8fde731cc --- /dev/null +++ b/pkg/events/subscription_test.go @@ -0,0 +1,254 @@ +package events + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +func TestSubscribeOnceClosesAfterFirstEvent(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + var handled atomic.Uint64 + sub, err := bus.Channel().SubscribeOnce( + context.Background(), + SubscribeOptions{Name: "once", Buffer: 2}, + func(context.Context, Event) error { + handled.Add(1) + return nil + }, + ) + if err != nil { + t.Fatalf("SubscribeOnce failed: %v", err) + } + + bus.Publish(context.Background(), Event{Kind: KindAgentTurnStart}) + waitForSubscriptionDone(t, sub) + bus.Publish(context.Background(), Event{Kind: KindAgentTurnEnd}) + + if got := handled.Load(); got != 1 { + t.Fatalf("handled = %d, want 1", got) + } + if got := sub.Stats().Handled; got != 1 { + t.Fatalf("subscription handled = %d, want 1", got) + } +} + +func TestUnsubscribeClosesChannel(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + sub, ch, err := bus.Channel().SubscribeChan(context.Background(), SubscribeOptions{Name: "chan"}) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + if err := sub.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + select { + case _, ok := <-ch: + if ok { + t.Fatal("channel is open, want closed") + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for channel close") + } + waitForSubscriptionDone(t, sub) +} + +func TestBlockBackpressureCloseUnblocksPublisher(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + sub, _, err := bus.Channel().SubscribeChan(context.Background(), SubscribeOptions{ + Name: "block-close", + Buffer: 1, + Backpressure: Block, + }) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + first := bus.Publish(context.Background(), Event{Kind: Kind("test.first")}) + if first.Delivered != 1 { + t.Fatalf("first Publish = %+v, want one delivered event", first) + } + + publishStarted := make(chan struct{}) + publishReturned := make(chan PublishResult, 1) + go func() { + close(publishStarted) + publishReturned <- bus.Publish(context.Background(), Event{Kind: Kind("test.second")}) + }() + + <-publishStarted + waitForStat(t, func() uint64 { + return sub.Stats().Received + }, 2) + select { + case result := <-publishReturned: + t.Fatalf("blocking Publish returned before close: %+v", result) + default: + } + + closeReturned := make(chan error, 1) + go func() { + closeReturned <- sub.Close() + }() + + select { + case err := <-closeReturned: + if err != nil { + t.Fatalf("Close failed: %v", err) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for Close to unblock") + } + + select { + case <-publishReturned: + case <-time.After(time.Second): + t.Fatal("timed out waiting for blocking Publish to return after close") + } + waitForSubscriptionDone(t, sub) +} + +func TestHandlerPanicRecovered(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + sub, err := bus.Channel().Subscribe( + context.Background(), + SubscribeOptions{Name: "panic", Buffer: 1}, + func(context.Context, Event) error { + panic("boom") + }, + ) + if err != nil { + t.Fatalf("Subscribe failed: %v", err) + } + + bus.Publish(context.Background(), Event{Kind: KindAgentError}) + waitForStat(t, func() uint64 { + return sub.Stats().Panicked + }, 1) +} + +func TestLockedHandlerProcessesSequentially(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + var active atomic.Int64 + var maxActive atomic.Int64 + sub, err := bus.Channel().Subscribe( + context.Background(), + SubscribeOptions{Name: "locked", Buffer: 8, Concurrency: Locked}, + func(context.Context, Event) error { + current := active.Add(1) + for { + currentMax := maxActive.Load() + if current <= currentMax || maxActive.CompareAndSwap(currentMax, current) { + break + } + } + time.Sleep(10 * time.Millisecond) + active.Add(-1) + return nil + }, + ) + if err != nil { + t.Fatalf("Subscribe failed: %v", err) + } + + for i := 0; i < 5; i++ { + bus.Publish(context.Background(), Event{Kind: KindAgentLLMDelta}) + } + waitForStat(t, func() uint64 { + return sub.Stats().Handled + }, 5) + + if got := maxActive.Load(); got != 1 { + t.Fatalf("max active handlers = %d, want 1", got) + } +} + +func TestHandlerTimeoutDoesNotWedgeLockedSubscription(t *testing.T) { + t.Parallel() + + bus := NewBus() + defer closeBus(t, bus) + + releaseFirst := make(chan struct{}) + defer close(releaseFirst) + + var calls atomic.Uint64 + sub, err := bus.Channel().Subscribe( + context.Background(), + SubscribeOptions{Name: "timeout", Buffer: 2, Concurrency: Locked, Timeout: 20 * time.Millisecond}, + func(context.Context, Event) error { + if calls.Add(1) == 1 { + <-releaseFirst + } + return nil + }, + ) + if err != nil { + t.Fatalf("Subscribe failed: %v", err) + } + + bus.Publish(context.Background(), Event{Kind: Kind("test.first")}) + waitForStat(t, func() uint64 { + return sub.Stats().TimedOut + }, 1) + + bus.Publish(context.Background(), Event{Kind: Kind("test.second")}) + waitForStat(t, func() uint64 { + return sub.Stats().Handled + }, 1) + + if got := sub.Stats().Failed; got != 1 { + t.Fatalf("subscription failed = %d, want timeout failure", got) + } +} + +func waitForSubscriptionDone(t *testing.T, sub Subscription) { + t.Helper() + + select { + case <-sub.Done(): + case <-time.After(time.Second): + t.Fatal("timed out waiting for subscription to stop") + } +} + +func waitForStat(t *testing.T, stat func() uint64, want uint64) { + t.Helper() + + deadline := time.After(time.Second) + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + + for { + if got := stat(); got >= want { + return + } + select { + case <-ticker.C: + case <-deadline: + t.Fatalf("timed out waiting for stat >= %d", want) + } + } +} diff --git a/pkg/events/types.go b/pkg/events/types.go new file mode 100644 index 000000000..2cfc0eaac --- /dev/null +++ b/pkg/events/types.go @@ -0,0 +1,77 @@ +package events + +import "time" + +// Kind identifies a runtime event category. +type Kind string + +// String returns the string representation of the event kind. +func (k Kind) String() string { + return string(k) +} + +// Event is the runtime event envelope shared across PicoClaw components. +type Event struct { + ID string `json:"id"` + Kind Kind `json:"kind"` + Time time.Time `json:"time"` + Source Source `json:"source"` + Scope Scope `json:"scope,omitempty"` + Correlation Correlation `json:"correlation,omitempty"` + Severity Severity `json:"severity,omitempty"` + Payload any `json:"payload,omitempty"` + Attrs map[string]any `json:"attrs,omitempty"` +} + +// Source identifies the component that emitted an event. +type Source struct { + Component string `json:"component"` + Name string `json:"name,omitempty"` +} + +// Scope identifies the runtime ownership of an event. +// +// Scope is intentionally limited to agent, session, turn, channel, chat, +// message, and sender identity. Tool, provider, model, and MCP details belong +// in Source, Payload, or Attrs. +type Scope struct { + RuntimeID string `json:"runtime_id,omitempty"` + + AgentID string `json:"agent_id,omitempty"` + SessionKey string `json:"session_key,omitempty"` + TurnID string `json:"turn_id,omitempty"` + + Channel string `json:"channel,omitempty"` + Account string `json:"account,omitempty"` + ChatID string `json:"chat_id,omitempty"` + TopicID string `json:"topic_id,omitempty"` + + SpaceID string `json:"space_id,omitempty"` + SpaceType string `json:"space_type,omitempty"` + ChatType string `json:"chat_type,omitempty"` + + SenderID string `json:"sender_id,omitempty"` + MessageID string `json:"message_id,omitempty"` +} + +// Correlation carries cross-event tracing fields. +type Correlation struct { + TraceID string `json:"trace_id,omitempty"` + ParentTurnID string `json:"parent_turn_id,omitempty"` + RequestID string `json:"request_id,omitempty"` + ReplyToID string `json:"reply_to_id,omitempty"` +} + +// Severity describes the operational severity of an event. +type Severity string + +const ( + // SeverityDebug is used for verbose diagnostic events. + SeverityDebug Severity = "debug" + // SeverityInfo is used for normal lifecycle and activity events. + SeverityInfo Severity = "info" + // SeverityWarn is used for recoverable abnormal events. + SeverityWarn Severity = "warn" + // SeverityError is used for failed operations and unrecoverable events. + SeverityError Severity = "error" +) diff --git a/pkg/evolution/apply.go b/pkg/evolution/apply.go new file mode 100644 index 000000000..7cb1b9b5e --- /dev/null +++ b/pkg/evolution/apply.go @@ -0,0 +1,308 @@ +package evolution + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "gopkg.in/yaml.v3" + + "github.com/sipeed/picoclaw/pkg/fileutil" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type Applier struct { + paths Paths + now func() time.Time +} + +func NewApplier(paths Paths, now func() time.Time) *Applier { + if now == nil { + now = time.Now + } + return &Applier{ + paths: paths, + now: now, + } +} + +func (a *Applier) ApplyDraft(ctx context.Context, workspace string, draft SkillDraft) error { + rollback, err := a.applyDraftWithRollback(ctx, workspace, draft) + if err != nil { + return err + } + _ = rollback + return nil +} + +func (a *Applier) applyDraftWithRollback( + ctx context.Context, + workspace string, + draft SkillDraft, +) (func() error, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + if validateErr := skills.ValidateSkillName(draft.TargetSkillName); validateErr != nil { + return nil, validateErr + } + + existingBody, backupPath, hadOriginal, err := a.backupCurrentSkill(workspace, draft.TargetSkillName) + if err != nil { + return nil, err + } + + renderedBody, err := renderAppliedBody(draft, existingBody, hadOriginal) + if err != nil { + return nil, err + } + + if err := validateAppliedSkillBody( + renderedBody, + draft.TargetSkillName, + allowsExistingFrontmatterFields(draft.ChangeKind, hadOriginal), + ); err != nil { + return nil, err + } + + skillDir := filepath.Join(workspace, "skills", draft.TargetSkillName) + if mkdirErr := os.MkdirAll(skillDir, 0o755); mkdirErr != nil { + return nil, mkdirErr + } + + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := fileutil.WriteFileAtomic(skillPath, []byte(renderedBody), 0o644); err != nil { + return nil, err + } + + return func() error { + return a.rollbackSkill(skillPath, backupPath, hadOriginal) + }, nil +} + +func (a *Applier) backupCurrentSkill( + workspace, skillName string, +) (currentBody, backupPath string, hadOriginal bool, err error) { + if validateErr := skills.ValidateSkillName(skillName); validateErr != nil { + return "", "", false, validateErr + } + + skillPath := filepath.Join(workspace, "skills", skillName, "SKILL.md") + data, err := os.ReadFile(skillPath) + if os.IsNotExist(err) { + return "", "", false, nil + } + if err != nil { + return "", "", false, err + } + + backupDir := filepath.Join( + a.paths.BackupsDir, + workspaceScopeDir(workspace), + skillName, + a.now().Format("20060102-150405.000000000"), + ) + if err := os.MkdirAll(backupDir, 0o755); err != nil { + return "", "", false, err + } + + backupPath = filepath.Join(backupDir, "SKILL.md") + if err := fileutil.WriteFileAtomic(backupPath, data, 0o644); err != nil { + return "", "", false, err + } + return string(data), backupPath, true, nil +} + +func (a *Applier) rollbackSkill(skillPath, backupPath string, hadOriginal bool) error { + if hadOriginal { + data, err := os.ReadFile(backupPath) + if err != nil { + return err + } + return fileutil.WriteFileAtomic(skillPath, data, 0o644) + } + if err := os.Remove(skillPath); err != nil && !os.IsNotExist(err) { + return err + } + skillDir := filepath.Dir(skillPath) + if err := os.Remove(skillDir); err != nil && !os.IsNotExist(err) && !isDirNotEmptyError(err) { + return err + } + return nil +} + +func isDirNotEmptyError(err error) bool { + if err == nil { + return false + } + return strings.Contains(strings.ToLower(err.Error()), "directory not empty") +} + +func validateAppliedSkillBody(body, targetSkillName string, allowExtraFrontmatterFields bool) error { + body = strings.TrimSpace(body) + if !strings.HasPrefix(body, "---\n") { + return fmt.Errorf("skill frontmatter is required") + } + if !strings.Contains(body, "\n# ") { + return fmt.Errorf("skill heading is required") + } + frontmatter, _ := splitSkillFrontmatter(body) + fields, err := parseSkillFrontmatterFields(frontmatter, allowExtraFrontmatterFields) + if err != nil { + return err + } + name := strings.TrimSpace(fields["name"]) + if name == "" { + return fmt.Errorf("skill frontmatter name is required") + } + if name != targetSkillName { + return fmt.Errorf("skill frontmatter name %q does not match target skill %q", name, targetSkillName) + } + if strings.TrimSpace(fields["description"]) == "" { + return fmt.Errorf("skill frontmatter description is required") + } + return nil +} + +func allowsExistingFrontmatterFields(kind ChangeKind, hadOriginal bool) bool { + return hadOriginal && (kind == ChangeKindAppend || kind == ChangeKindMerge) +} + +func renderAppliedBody(draft SkillDraft, existingBody string, hadOriginal bool) (string, error) { + switch draft.ChangeKind { + case ChangeKindCreate: + if hadOriginal { + return "", fmt.Errorf("cannot create skill %q: skill already exists", draft.TargetSkillName) + } + return renderDeployableSkillBody(draft.BodyOrPatch), nil + case ChangeKindReplace: + if !hadOriginal { + return "", fmt.Errorf("cannot replace skill %q: skill does not exist", draft.TargetSkillName) + } + return renderDeployableSkillBody(draft.BodyOrPatch), nil + case ChangeKindAppend: + patch, err := renderDeployablePatchBody(draft.BodyOrPatch, draft.TargetSkillName) + if err != nil { + return "", err + } + if !hadOriginal || strings.TrimSpace(existingBody) == "" { + return renderDeployableSkillBody(draft.BodyOrPatch), nil + } + return strings.TrimRight(existingBody, "\n") + "\n\n" + strings.TrimLeft(patch, "\n"), nil + case ChangeKindMerge: + patch, err := renderDeployablePatchBody(draft.BodyOrPatch, draft.TargetSkillName) + if err != nil { + return "", err + } + if !hadOriginal || strings.TrimSpace(existingBody) == "" { + return renderDeployableSkillBody(draft.BodyOrPatch), nil + } + mergedSection := strings.Join([]string{ + "", + "## Merged Knowledge", + strings.TrimSpace(patch), + "", + }, "\n") + return strings.TrimRight(existingBody, "\n") + mergedSection, nil + default: + return "", fmt.Errorf("unsupported change_kind %q", draft.ChangeKind) + } +} + +func renderDeployablePatchBody(body, targetSkillName string) (string, error) { + body = renderDeployableSkillBody(body) + frontmatter, markdownBody := splitSkillFrontmatter(body) + if frontmatter == "" { + markdownBody = body + } else { + fields, err := parseSkillFrontmatterFields(frontmatter, true) + if err != nil { + return "", err + } + if name := strings.TrimSpace(fields["name"]); name != "" && name != targetSkillName { + return "", fmt.Errorf( + "skill patch frontmatter name %q does not match target skill %q", + name, + targetSkillName, + ) + } + } + return strings.TrimSpace(stripLeadingH1(markdownBody)), nil +} + +func splitSkillFrontmatter(body string) (frontmatter, markdownBody string) { + normalized := strings.ReplaceAll(strings.TrimSpace(body), "\r\n", "\n") + lines := strings.Split(normalized, "\n") + if len(lines) == 0 || strings.TrimSpace(lines[0]) != "---" { + return "", body + } + end := -1 + for i := 1; i < len(lines); i++ { + if strings.TrimSpace(lines[i]) == "---" { + end = i + break + } + } + if end < 0 { + return "", body + } + return strings.Join(lines[1:end], "\n"), strings.TrimLeft(strings.Join(lines[end+1:], "\n"), "\n") +} + +func parseSkillFrontmatterFields(frontmatter string, allowExtraFields bool) (map[string]string, error) { + var raw map[string]any + if err := yaml.Unmarshal([]byte(frontmatter), &raw); err != nil { + return nil, fmt.Errorf("invalid skill frontmatter: %w", err) + } + for key := range raw { + if key != "name" && key != "description" { + if allowExtraFields { + continue + } + return nil, fmt.Errorf("unsupported skill frontmatter field %q", key) + } + } + + var typed struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + } + if err := yaml.Unmarshal([]byte(frontmatter), &typed); err != nil { + return nil, fmt.Errorf("invalid skill frontmatter: %w", err) + } + return map[string]string{ + "name": typed.Name, + "description": typed.Description, + }, nil +} + +func stripLeadingH1(body string) string { + lines := strings.Split(strings.TrimLeft(body, "\n"), "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) == "" { + lines = lines[1:] + } + if len(lines) > 0 && strings.HasPrefix(strings.TrimSpace(lines[0]), "# ") { + lines = lines[1:] + } + return strings.Join(lines, "\n") +} + +func errorsJoin(errs ...error) error { + var first error + for _, err := range errs { + if err == nil { + continue + } + if first == nil { + first = err + continue + } + first = fmt.Errorf("%w; %v", first, err) + } + return first +} diff --git a/pkg/evolution/apply_test.go b/pkg/evolution/apply_test.go new file mode 100644 index 000000000..36e4e21e5 --- /dev/null +++ b/pkg/evolution/apply_test.go @@ -0,0 +1,785 @@ +package evolution_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestApplier_CreateDraftWritesSkillFile(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: workspace, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n", + Status: evolution.DraftStatusAccepted, + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + skillPath := filepath.Join(workspace, "skills", "weather", "SKILL.md") + data, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !strings.Contains(string(data), "# Weather") { + t.Fatalf("unexpected content: %s", string(data)) + } +} + +func TestApplier_CreateDraftRendersDeployableSkillWithoutLearningTrace(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: workspace, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: strings.Join([]string{ + "---", + "name: weather", + "description: Create combined shortcut perform-mathematical-calculations-by-via-theorems for: Perform mathematical calculations by applying specific theorems and their associated rules.", + "---", + "# Weather", + "", + "## Learned Context", + "- Learned task: use native-name weather lookup.", + "", + "## Source Evidence", + "- Evidence: learned from task records: task-1", + "", + "## Procedure", + "Use native-name query first.", + "", + }, "\n"), + Status: evolution.DraftStatusAccepted, + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + data, err := os.ReadFile(filepath.Join(workspace, "skills", "weather", "SKILL.md")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(data) + for _, forbidden := range []string{ + "Create combined shortcut", + "perform-mathematical-calculations-by-via-theorems for:", + "Learned Context", + "Learned task", + "Source Evidence", + "task records", + } { + if strings.Contains(content, forbidden) { + t.Fatalf("deployed skill contains %q:\n%s", forbidden, content) + } + } + if !strings.Contains(content, "Use native-name query first.") { + t.Fatalf("deployed skill lost procedure:\n%s", content) + } + if !strings.Contains( + content, + "description: Perform mathematical calculations by applying specific theorems and their associated rules.", + ) { + t.Fatalf("deployed skill did not clean description:\n%s", content) + } +} + +func TestApplier_CreateDraftDoesNotRewriteEvolutionDomainTextOrFrontmatter(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-evolution-domain", + WorkspaceID: workspace, + SourceRecordID: "rule-evolution-domain", + TargetSkillName: "agent-evolution-helper", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "agent evolution helper", + BodyOrPatch: "---\nname: agent-evolution-helper\ndescription: Explain agent evolution workflows.\n---\n# Agent Evolution Helper\nUse this skill to reason about agent evolution behavior.\n", + Status: evolution.DraftStatusAccepted, + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + data, err := os.ReadFile(filepath.Join(workspace, "skills", "agent-evolution-helper", "SKILL.md")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(data) + if !strings.Contains(content, "name: agent-evolution-helper") { + t.Fatalf("frontmatter name was rewritten:\n%s", content) + } + if strings.Contains(content, "agent-update-helper") { + t.Fatalf("frontmatter name should not be rewritten:\n%s", content) + } + if !strings.Contains(content, "agent evolution behavior") { + t.Fatalf("domain text should preserve evolution wording:\n%s", content) + } +} + +func TestApplier_CreateDraftFailsWhenSkillAlreadyExists(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\nold body\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-create-existing", + WorkspaceID: workspace, + SourceRecordID: "rule-create-existing", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: replacement\n---\n# Weather\nnew body\n", + } + + err := applier.ApplyDraft(context.Background(), workspace, draft) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("error = %v, want already exists", err) + } + + got, readErr := os.ReadFile(skillPath) + if readErr != nil { + t.Fatalf("ReadFile: %v", readErr) + } + if string(got) != original { + t.Fatalf("skill content changed unexpectedly:\n%s", string(got)) + } +} + +func TestApplier_CreateDraftRejectsMismatchedFrontmatterName(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-mismatched-name", + WorkspaceID: workspace, + SourceRecordID: "rule-mismatched-name", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: other-skill\ndescription: other helper\n---\n# Other\nUse something else.\n", + } + + err := applier.ApplyDraft(context.Background(), workspace, draft) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + if !strings.Contains(err.Error(), "frontmatter name") { + t.Fatalf("error = %v, want frontmatter name mismatch", err) + } + if _, statErr := os.Stat(filepath.Join(workspace, "skills", "weather", "SKILL.md")); !os.IsNotExist(statErr) { + t.Fatalf("expected no skill file, got err=%v", statErr) + } +} + +func TestApplier_RollsBackOnInvalidSkillBody(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\nold body\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-2", + WorkspaceID: workspace, + SourceRecordID: "rule-2", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindReplace, + HumanSummary: "broken draft", + BodyOrPatch: "invalid-frontmatter", + Status: evolution.DraftStatusAccepted, + } + + err := applier.ApplyDraft(context.Background(), workspace, draft) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + + got, readErr := os.ReadFile(skillPath) + if readErr != nil { + t.Fatalf("ReadFile: %v", readErr) + } + if string(got) != original { + t.Fatalf("skill content changed after rollback:\n%s", string(got)) + } +} + +func TestApplier_FailedNewSkillDoesNotLeaveEmptyDirectory(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-invalid-new-skill", + WorkspaceID: workspace, + SourceRecordID: "rule-invalid-new-skill", + TargetSkillName: "calculate-100-via-theorems", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "broken new skill", + BodyOrPatch: "invalid-frontmatter", + Status: evolution.DraftStatusAccepted, + } + + err := applier.ApplyDraft(context.Background(), workspace, draft) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + + skillPath := filepath.Join(workspace, "skills", "calculate-100-via-theorems", "SKILL.md") + if _, statErr := os.Stat(skillPath); !os.IsNotExist(statErr) { + t.Fatalf("expected no skill file, got err=%v", statErr) + } + skillDir := filepath.Dir(skillPath) + if _, statErr := os.Stat(skillDir); !os.IsNotExist(statErr) { + t.Fatalf("expected no leftover skill dir, got err=%v", statErr) + } +} + +func TestApplier_ReplaceDraftFailsWhenSkillDoesNotExist(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-replace-missing", + WorkspaceID: workspace, + SourceRecordID: "rule-replace-missing", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindReplace, + HumanSummary: "replace missing skill", + BodyOrPatch: "---\nname: weather\ndescription: replacement\n---\n# Weather\nnew body\n", + } + + err := applier.ApplyDraft(context.Background(), workspace, draft) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + if !strings.Contains(err.Error(), "does not exist") { + t.Fatalf("error = %v, want does not exist", err) + } + + skillPath := filepath.Join(workspace, "skills", "weather", "SKILL.md") + if _, statErr := os.Stat(skillPath); !os.IsNotExist(statErr) { + t.Fatalf("expected no skill file, got err=%v", statErr) + } +} + +func TestApplier_AppendDraftPreservesOriginalBody(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\n## Start Here\nUse city names.\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-append", + WorkspaceID: workspace, + SourceRecordID: "rule-append", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "append draft", + BodyOrPatch: "\n## Learned Pattern\nPrefer native-name query first.\n", + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(got) + if !strings.Contains(content, "Use city names.") { + t.Fatalf("appended content lost original body:\n%s", content) + } + if !strings.Contains(content, "Prefer native-name query first.") { + t.Fatalf("appended content missing new body:\n%s", content) + } +} + +func TestApplier_AppendDraftAllowsExistingExtraFrontmatterFields(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := strings.Join([]string{ + "---", + "name: weather", + "description: valid", + "# Human-authored metadata should not block append updates.", + "homepage: https://example.com/weather", + "aliases:", + "- forecast", + "metadata:", + " owner: human", + "---", + "# Weather", + "## Start Here", + "Use city names.", + "", + }, "\n") + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-append-extra-frontmatter", + WorkspaceID: workspace, + SourceRecordID: "rule-append-extra-frontmatter", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "append draft", + BodyOrPatch: "\n## Learned Pattern\nPrefer native-name query first.\n", + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(got) + for _, want := range []string{ + "homepage: https://example.com/weather", + "aliases:", + "- forecast", + "metadata:", + " owner: human", + "Prefer native-name query first.", + } { + if !strings.Contains(content, want) { + t.Fatalf("appended content missing %q:\n%s", want, content) + } + } +} + +func TestApplier_CreateDraftRejectsExtraFrontmatterFields(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-create-extra-frontmatter", + WorkspaceID: workspace, + SourceRecordID: "rule-create-extra-frontmatter", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\nhomepage: https://example.com/weather\n---\n# Weather\nUse weather.\n", + } + + err := applier.ApplyDraft(context.Background(), workspace, draft) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + if !strings.Contains(err.Error(), "unsupported skill frontmatter field") { + t.Fatalf("error = %v, want unsupported field", err) + } +} + +func TestApplier_AppendDraftDoesNotRewriteExistingLearningTerms(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\n## Evolution Notes\nKeep this manually-authored Learned phrase unchanged.\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-append-clean", + WorkspaceID: workspace, + SourceRecordID: "rule-append-clean", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "append draft", + BodyOrPatch: "\n## Learned Pattern\nPrefer native-name query first.\n", + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(got) + if !strings.Contains(content, "## Evolution Notes") { + t.Fatalf("existing heading was rewritten:\n%s", content) + } + if !strings.Contains(content, "Keep this manually-authored Learned phrase unchanged.") { + t.Fatalf("existing body was rewritten:\n%s", content) + } + if strings.Contains(content, "## Learned Pattern") { + t.Fatalf("new patch should be deploy-sanitized:\n%s", content) + } + if !strings.Contains(content, "## Usage Pattern") { + t.Fatalf("new patch missing sanitized heading:\n%s", content) + } +} + +func TestApplier_AppendDraftStripsPlainMarkdownTopLevelHeading(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\n## Start Here\nUse city names.\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-append-plain-doc", + WorkspaceID: workspace, + SourceRecordID: "rule-append-plain-doc", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "append draft", + BodyOrPatch: "# Weather\n## Procedure\nPrefer native-name query first.\n", + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(got) + if strings.Count(content, "# Weather") != 1 { + t.Fatalf("appended content should not duplicate top-level heading:\n%s", content) + } + if !strings.Contains(content, "## Procedure") || !strings.Contains(content, "Prefer native-name query first.") { + t.Fatalf("appended content lost patch body:\n%s", content) + } +} + +func TestApplier_AppendAndMergeRejectFullDocumentPatchWithMismatchedName(t *testing.T) { + for _, kind := range []evolution.ChangeKind{evolution.ChangeKindAppend, evolution.ChangeKindMerge} { + t.Run(string(kind), func(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\n## Start Here\nUse city names.\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + err := applier.ApplyDraft(context.Background(), workspace, evolution.SkillDraft{ + ID: "draft-mismatched-patch", + WorkspaceID: workspace, + SourceRecordID: "rule-mismatched-patch", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: kind, + HumanSummary: "append draft", + BodyOrPatch: "---\nname: other-skill\ndescription: wrong target\n---\n# Other Skill\n## Procedure\nDo something else.\n", + }) + if err == nil { + t.Fatal("expected ApplyDraft to fail") + } + if !strings.Contains(err.Error(), "patch frontmatter name") { + t.Fatalf("error = %v, want patch frontmatter name mismatch", err) + } + got, readErr := os.ReadFile(skillPath) + if readErr != nil { + t.Fatalf("ReadFile: %v", readErr) + } + if string(got) != original { + t.Fatalf("skill content changed unexpectedly:\n%s", string(got)) + } + }) + } +} + +func TestApplier_AppendDraftStripsFullSkillDocumentPatch(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\n## Start Here\nUse city names.\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-append-full-doc", + WorkspaceID: workspace, + SourceRecordID: "rule-append-full-doc", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "append draft", + BodyOrPatch: "---\nname: weather\ndescription: duplicate document\n---\n# Weather\n## Procedure\nPrefer native-name query first.\n", + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(got) + if strings.Count(content, "---") != 2 { + t.Fatalf("appended content should keep only original frontmatter:\n%s", content) + } + if strings.Count(content, "# Weather") != 1 { + t.Fatalf("appended content should not duplicate top-level heading:\n%s", content) + } + if !strings.Contains(content, "## Procedure") || !strings.Contains(content, "Prefer native-name query first.") { + t.Fatalf("appended content lost patch body:\n%s", content) + } +} + +func TestApplier_BackupsAreScopedByWorkspace(t *testing.T) { + sharedState := t.TempDir() + workspaceA := t.TempDir() + workspaceB := t.TempDir() + now := time.Unix(1700000000, 0).UTC() + + for workspace, body := range map[string]string{ + workspaceA: "---\nname: weather\ndescription: valid\n---\n# Weather\nworkspace A\n", + workspaceB: "---\nname: weather\ndescription: valid\n---\n# Weather\nworkspace B\n", + } { + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll(%s): %v", workspace, err) + } + if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(body), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", workspace, err) + } + } + + for _, workspace := range []string{workspaceA, workspaceB} { + applier := evolution.NewApplier(evolution.NewPaths(workspace, sharedState), func() time.Time { + return now + }) + if err := applier.ApplyDraft(context.Background(), workspace, evolution.SkillDraft{ + ID: "draft-replace", + WorkspaceID: workspace, + SourceRecordID: "rule-replace", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindReplace, + HumanSummary: "replace weather", + BodyOrPatch: "---\nname: weather\ndescription: replacement\n---\n# Weather\nreplacement\n", + }); err != nil { + t.Fatalf("ApplyDraft(%s): %v", workspace, err) + } + } + + var backupBodies []string + if err := filepath.WalkDir( + filepath.Join(sharedState, "backups"), + func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || entry.Name() != "SKILL.md" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + backupBodies = append(backupBodies, string(data)) + return nil + }, + ); err != nil { + t.Fatalf("WalkDir(backups): %v", err) + } + + if len(backupBodies) != 2 { + t.Fatalf("backup count = %d, want 2", len(backupBodies)) + } + joined := strings.Join(backupBodies, "\n") + if !strings.Contains(joined, "workspace A") || !strings.Contains(joined, "workspace B") { + t.Fatalf("backups should preserve both workspace bodies:\n%s", joined) + } +} + +func TestApplier_MergeDraftAddsMergedKnowledgeSection(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + original := "---\nname: weather\ndescription: valid\n---\n# Weather\n## Start Here\nUse city names.\n" + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + draft := evolution.SkillDraft{ + ID: "draft-merge", + WorkspaceID: workspace, + SourceRecordID: "rule-merge", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindMerge, + HumanSummary: "merge draft", + BodyOrPatch: "Prefer native-name query first.", + } + + if err := applier.ApplyDraft(context.Background(), workspace, draft); err != nil { + t.Fatalf("ApplyDraft: %v", err) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(got) + if !strings.Contains(content, "Use city names.") { + t.Fatalf("merged content lost original body:\n%s", content) + } + if !strings.Contains(content, "## Merged Knowledge") { + t.Fatalf("merged content missing merged section:\n%s", content) + } + if !strings.Contains(content, "Prefer native-name query first.") { + t.Fatalf("merged content missing new knowledge:\n%s", content) + } +} + +func TestApplier_RejectsInvalidSkillName(t *testing.T) { + workspace := t.TempDir() + applier := evolution.NewApplier(evolution.NewPaths(workspace, ""), func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + + for _, name := range []string{"../escape", "/tmp/escape"} { + err := applier.ApplyDraft(context.Background(), workspace, evolution.SkillDraft{ + ID: "draft-invalid-name", + WorkspaceID: workspace, + SourceRecordID: "rule-invalid-name", + TargetSkillName: name, + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "bad name", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\nbody\n", + }) + if err == nil { + t.Fatalf("TargetSkillName %q expected error", name) + } + } +} diff --git a/pkg/evolution/case_writer.go b/pkg/evolution/case_writer.go new file mode 100644 index 000000000..6948ff69e --- /dev/null +++ b/pkg/evolution/case_writer.go @@ -0,0 +1,21 @@ +package evolution + +import ( + "context" +) + +type CaseWriter struct { + paths Paths + store *Store +} + +func NewCaseWriter(paths Paths) *CaseWriter { + return &CaseWriter{ + paths: paths, + store: NewStore(paths), + } +} + +func (w *CaseWriter) AppendCase(ctx context.Context, record LearningRecord) error { + return w.store.AppendTaskRecord(ctx, record) +} diff --git a/pkg/evolution/case_writer_test.go b/pkg/evolution/case_writer_test.go new file mode 100644 index 000000000..e6d0742e8 --- /dev/null +++ b/pkg/evolution/case_writer_test.go @@ -0,0 +1,77 @@ +package evolution_test + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestCaseWriter_AppendsOneRecord(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + writer := evolution.NewCaseWriter(paths) + + record1 := testRecord("rec-1", "ws-1", true) + record2 := testRecord("rec-2", "ws-2", false) + + if err := writer.AppendCase(context.Background(), record1); err != nil { + t.Fatalf("AppendCase: %v", err) + } + if err := writer.AppendCase(context.Background(), record2); err != nil { + t.Fatalf("AppendCase second record: %v", err) + } + + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + text := string(data) + if !strings.HasSuffix(text, "\n") { + t.Fatalf("record file should end with newline, got %q", text) + } + + lines := strings.Split(strings.TrimSpace(text), "\n") + if len(lines) != 2 { + t.Fatalf("record file line count = %d, want 2", len(lines)) + } + + records := []evolution.LearningRecord{record1, record2} + for i, line := range lines { + var got evolution.LearningRecord + if err := json.Unmarshal([]byte(line), &got); err != nil { + t.Fatalf("Unmarshal line %d: %v", i, err) + } + + want := records[i] + if got.ID != want.ID { + t.Fatalf("record %d ID = %q, want %q", i, got.ID, want.ID) + } + if got.Kind != evolution.RecordKindCase { + t.Fatalf("record %d kind = %q, want %q", i, got.Kind, evolution.RecordKindCase) + } + if got.Summary != want.Summary { + t.Fatalf("record %d summary = %q, want %q", i, got.Summary, want.Summary) + } + if got.Success == nil || *got.Success != *want.Success { + t.Fatalf("record %d success = %v, want %v", i, got.Success, want.Success) + } + } +} + +func testRecord(id, workspaceID string, success bool) evolution.LearningRecord { + return evolution.LearningRecord{ + ID: id, + Kind: evolution.RecordKindCase, + WorkspaceID: workspaceID, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "cli turn completed", + Status: evolution.RecordStatus("new"), + Success: &success, + } +} diff --git a/pkg/evolution/cold_path_runner.go b/pkg/evolution/cold_path_runner.go new file mode 100644 index 000000000..696c706fb --- /dev/null +++ b/pkg/evolution/cold_path_runner.go @@ -0,0 +1,121 @@ +package evolution + +import ( + "context" + "errors" + "sync" +) + +type coldPathRuntime interface { + RunColdPathOnce(ctx context.Context, workspace string) error +} + +type ColdPathRunner struct { + runtime coldPathRuntime + async func(func()) + onError func(error) + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + wg sync.WaitGroup + closeOnce sync.Once + closed bool + running map[string]workspaceRunState +} + +func NewColdPathRunner(runtime coldPathRuntime) *ColdPathRunner { + return NewColdPathRunnerWithErrorHandler(runtime, nil) +} + +func NewColdPathRunnerWithErrorHandler(runtime coldPathRuntime, onError func(error)) *ColdPathRunner { + if onError == nil { + onError = func(error) {} + } + ctx, cancel := context.WithCancel(context.Background()) + + return &ColdPathRunner{ + runtime: runtime, + async: func(run func()) { + go run() + }, + onError: onError, + ctx: ctx, + cancel: cancel, + running: make(map[string]workspaceRunState), + } +} + +type workspaceRunState struct { + running bool + pending bool +} + +func (r *ColdPathRunner) Trigger(workspace string) bool { + if r == nil || r.runtime == nil || workspace == "" { + return false + } + + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return false + } + state, exists := r.running[workspace] + if exists && state.running { + state.pending = true + r.running[workspace] = state + r.mu.Unlock() + return true + } + r.running[workspace] = workspaceRunState{running: true} + r.wg.Add(1) + r.mu.Unlock() + + r.async(func() { + defer r.wg.Done() + r.runWorkspace(workspace) + }) + + return true +} + +func (r *ColdPathRunner) runWorkspace(workspace string) { + for { + if err := r.runtime.RunColdPathOnce(r.ctx, workspace); err != nil && !errors.Is(err, context.Canceled) { + r.onError(err) + } + + r.mu.Lock() + state, exists := r.running[workspace] + if !exists || r.closed { + delete(r.running, workspace) + r.mu.Unlock() + return + } + if state.pending { + state.pending = false + r.running[workspace] = state + r.mu.Unlock() + continue + } + delete(r.running, workspace) + r.mu.Unlock() + return + } +} + +func (r *ColdPathRunner) Close() error { + if r == nil { + return nil + } + + r.closeOnce.Do(func() { + r.mu.Lock() + r.closed = true + r.mu.Unlock() + r.cancel() + }) + r.wg.Wait() + return nil +} diff --git a/pkg/evolution/cold_path_runner_test.go b/pkg/evolution/cold_path_runner_test.go new file mode 100644 index 000000000..2a0b28309 --- /dev/null +++ b/pkg/evolution/cold_path_runner_test.go @@ -0,0 +1,142 @@ +package evolution + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +type blockingColdPathRuntime struct { + runCount atomic.Int32 + cancelCount atomic.Int32 + started chan string + release chan struct{} +} + +func (r *blockingColdPathRuntime) RunColdPathOnce(ctx context.Context, workspace string) error { + r.runCount.Add(1) + r.started <- workspace + select { + case <-r.release: + return nil + case <-ctx.Done(): + r.cancelCount.Add(1) + return ctx.Err() + } +} + +func TestColdPathRunner_QueuesPendingRunForWorkspace(t *testing.T) { + runtime := &blockingColdPathRuntime{ + started: make(chan string, 4), + release: make(chan struct{}, 4), + } + runner := NewColdPathRunner(runtime) + defer runner.Close() + + if scheduled := runner.Trigger("workspace-a"); !scheduled { + t.Fatal("expected first trigger to be scheduled") + } + + select { + case workspace := <-runtime.started: + if workspace != "workspace-a" { + t.Fatalf("workspace = %q, want workspace-a", workspace) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for first cold path run") + } + + if scheduled := runner.Trigger("workspace-a"); !scheduled { + t.Fatal("expected second trigger to queue a pending run") + } + + select { + case workspace := <-runtime.started: + t.Fatalf("unexpected early pending cold path run for %q", workspace) + case <-time.After(150 * time.Millisecond): + } + + runtime.release <- struct{}{} + + select { + case workspace := <-runtime.started: + if workspace != "workspace-a" { + t.Fatalf("workspace = %q, want workspace-a", workspace) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for pending cold path run") + } + + runtime.release <- struct{}{} + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if runtime.runCount.Load() == 2 { + return + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("runCount = %d, want 2", runtime.runCount.Load()) +} + +func TestColdPathRunner_CloseCancelsActiveRunAndDropsPendingWork(t *testing.T) { + runtime := &blockingColdPathRuntime{ + started: make(chan string, 4), + release: make(chan struct{}, 4), + } + runner := NewColdPathRunner(runtime) + + if scheduled := runner.Trigger("workspace-a"); !scheduled { + t.Fatal("expected first trigger to be scheduled") + } + + select { + case <-runtime.started: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for first cold path run") + } + + if scheduled := runner.Trigger("workspace-a"); !scheduled { + t.Fatal("expected second trigger to mark pending work") + } + + closeDone := make(chan struct{}) + go func() { + defer close(closeDone) + if err := runner.Close(); err != nil { + t.Errorf("Close() error = %v", err) + } + }() + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if !runner.Trigger("workspace-a") { + break + } + time.Sleep(10 * time.Millisecond) + } + if runner.Trigger("workspace-a") { + t.Fatal("expected Trigger to reject new work after Close") + } + + select { + case <-closeDone: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for Close to finish") + } + + select { + case workspace := <-runtime.started: + t.Fatalf("unexpected pending cold path run after Close for %q", workspace) + case <-time.After(150 * time.Millisecond): + } + + if got := runtime.runCount.Load(); got != 1 { + t.Fatalf("runCount = %d, want 1", got) + } + if got := runtime.cancelCount.Load(); got != 1 { + t.Fatalf("cancelCount = %d, want 1", got) + } +} diff --git a/pkg/evolution/draft_review.go b/pkg/evolution/draft_review.go new file mode 100644 index 000000000..da44d6365 --- /dev/null +++ b/pkg/evolution/draft_review.go @@ -0,0 +1,38 @@ +package evolution + +import "strings" + +type DraftReviewResult struct { + Status DraftStatus + Findings []string + ReviewNotes []string +} + +func ReviewDraft(draft SkillDraft) DraftReviewResult { + findings := append([]string(nil), ValidateDraft(draft)...) + findings = append(findings, scanDraftContent(draft)...) + + result := DraftReviewResult{ + Status: DraftStatusCandidate, + Findings: findings, + ReviewNotes: []string{"local structural validation completed"}, + } + if len(findings) > 0 { + result.Status = DraftStatusQuarantined + } + return result +} + +func scanDraftContent(draft SkillDraft) []string { + body := strings.ToLower(draft.BodyOrPatch) + findings := make([]string, 0, 2) + + if strings.Contains(body, "sk-live-") || strings.Contains(body, "sk_test_") || strings.Contains(body, "api_key=") { + findings = append(findings, "secret-like token detected in body_or_patch") + } + if strings.Contains(body, "-----begin private key-----") { + findings = append(findings, "private key material detected in body_or_patch") + } + + return findings +} diff --git a/pkg/evolution/draft_review_test.go b/pkg/evolution/draft_review_test.go new file mode 100644 index 000000000..70bcc6a6f --- /dev/null +++ b/pkg/evolution/draft_review_test.go @@ -0,0 +1,67 @@ +package evolution_test + +import ( + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestReviewDraft_QuarantinesInvalidDraft(t *testing.T) { + result := evolution.ReviewDraft(evolution.SkillDraft{ + ID: "draft-1", + TargetSkillName: "", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "broken", + BodyOrPatch: "", + }) + + if result.Status != evolution.DraftStatusQuarantined { + t.Fatalf("Status = %q, want %q", result.Status, evolution.DraftStatusQuarantined) + } + if len(result.Findings) == 0 { + t.Fatal("expected findings for invalid draft") + } +} + +func TestReviewDraft_QuarantinesSecretLikeContent(t *testing.T) { + result := evolution.ReviewDraft(evolution.SkillDraft{ + ID: "draft-2", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "contains credentials", + BodyOrPatch: "Use token sk-live-secret for direct calls.", + }) + + if result.Status != evolution.DraftStatusQuarantined { + t.Fatalf("Status = %q, want %q", result.Status, evolution.DraftStatusQuarantined) + } + if len(result.Findings) == 0 { + t.Fatal("expected findings for secret-like content") + } + if !strings.Contains(strings.Join(result.Findings, "\n"), "secret-like") { + t.Fatalf("findings = %v, want secret-like finding", result.Findings) + } +} + +func TestReviewDraft_QuarantinesInvalidTargetSkillName(t *testing.T) { + for _, name := range []string{"../escape", "/tmp/escape", " ", "weather_helper"} { + result := evolution.ReviewDraft(evolution.SkillDraft{ + ID: "draft-invalid-name", + TargetSkillName: name, + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "bad name", + BodyOrPatch: "body", + }) + + if result.Status != evolution.DraftStatusQuarantined { + t.Fatalf("TargetSkillName %q status = %q, want %q", name, result.Status, evolution.DraftStatusQuarantined) + } + if len(result.Findings) == 0 { + t.Fatalf("TargetSkillName %q expected findings", name) + } + } +} diff --git a/pkg/evolution/drafts.go b/pkg/evolution/drafts.go new file mode 100644 index 000000000..0d48d6605 --- /dev/null +++ b/pkg/evolution/drafts.go @@ -0,0 +1,511 @@ +package evolution + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type DraftGenerator interface { + GenerateDraft(ctx context.Context, rule LearningRecord, matches []skills.SkillInfo) (SkillDraft, error) +} + +type EvidenceAwareDraftGenerator interface { + GenerateDraftWithEvidence( + ctx context.Context, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, + ) (SkillDraft, error) +} + +type DraftEvidence struct { + TaskRecords []LearningRecord +} + +func ValidateDraft(draft SkillDraft) []string { + findings := make([]string, 0, 5) + + if strings.TrimSpace(draft.TargetSkillName) == "" { + findings = append(findings, "target_skill_name is required") + } else if err := skills.ValidateSkillName(draft.TargetSkillName); err != nil { + findings = append(findings, "target_skill_name is invalid: "+err.Error()) + } else if isNumericToken(strings.TrimSpace(draft.TargetSkillName)) { + findings = append(findings, "target_skill_name must be descriptive, not numeric-only") + } + if strings.TrimSpace(draft.HumanSummary) == "" { + findings = append(findings, "human_summary is required") + } + if strings.TrimSpace(draft.BodyOrPatch) == "" { + findings = append(findings, "body_or_patch is required") + } + + switch draft.DraftType { + case DraftTypeWorkflow, DraftTypeShortcut: + default: + findings = append(findings, "draft_type is invalid") + } + + switch draft.ChangeKind { + case ChangeKindCreate, ChangeKindAppend, ChangeKindReplace, ChangeKindMerge: + default: + findings = append(findings, "change_kind is invalid") + } + + return findings +} + +type DefaultDraftGenerator struct { + loader *skills.SkillsLoader +} + +func NewDefaultDraftGenerator(workspace string) *DefaultDraftGenerator { + builtinSkillsDir := strings.TrimSpace(os.Getenv(config.EnvBuiltinSkills)) + if builtinSkillsDir == "" { + wd, _ := os.Getwd() + builtinSkillsDir = filepath.Join(wd, "skills") + } + + globalSkillsDir := filepath.Join(config.GetHome(), "skills") + return &DefaultDraftGenerator{ + loader: skills.NewSkillsLoader(workspace, globalSkillsDir, builtinSkillsDir), + } +} + +func (g *DefaultDraftGenerator) GenerateDraft( + _ context.Context, + rule LearningRecord, + matches []skills.SkillInfo, +) (SkillDraft, error) { + return g.GenerateDraftWithEvidence(context.Background(), rule, matches, DraftEvidence{}) +} + +func (g *DefaultDraftGenerator) GenerateDraftWithEvidence( + _ context.Context, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) (SkillDraft, error) { + rule = enrichRuleWithDraftEvidence(rule, evidence) + target := inferTargetSkillName(rule, matches) + if target == "" { + target = "learned-skill" + } + + _, hasExisting, err := g.loadBaseSkillContent(target, matches) + if err != nil { + return SkillDraft{}, err + } + + draftType := DraftTypeWorkflow + if len(rule.WinningPath) <= 1 { + draftType = DraftTypeShortcut + } + + changeKind := ChangeKindCreate + body := g.buildNewSkillBody(target, rule, evidence, matches) + if hasExisting { + changeKind = ChangeKindAppend + body = g.buildAppendBody(rule, evidence, matches) + } + + return SkillDraft{ + TargetSkillName: target, + DraftType: draftType, + ChangeKind: changeKind, + HumanSummary: g.buildHumanSummary(target, rule, hasExisting), + IntendedUseCases: inferIntendedUseCases(rule), + PreferredEntryPath: inferPreferredEntryPath(rule), + AvoidPatterns: inferAvoidPatterns(rule), + BodyOrPatch: body, + }, nil +} + +func inferTargetSkillName(rule LearningRecord, matches []skills.SkillInfo) string { + if target := inferCombinedSkillName(rule); target != "" { + return target + } + if label := validSkillNameOrEmpty(rule.Label); label != "" { + return label + } + if len(matches) > 0 && strings.TrimSpace(matches[0].Name) != "" { + return strings.TrimSpace(matches[0].Name) + } + if len(rule.LateAddedSkills) > 0 && strings.TrimSpace(rule.LateAddedSkills[0]) != "" { + return strings.TrimSpace(rule.LateAddedSkills[0]) + } + if len(rule.WinningPath) > 0 && strings.TrimSpace(rule.WinningPath[0]) != "" { + return strings.TrimSpace(rule.WinningPath[0]) + } + if len(rule.MatchedSkillNames) > 0 && strings.TrimSpace(rule.MatchedSkillNames[0]) != "" { + return strings.TrimSpace(rule.MatchedSkillNames[0]) + } + + tokens := tokenizeForEvolution(rule.Summary) + if len(tokens) > 0 { + if len(tokens) == 1 && isNumericToken(tokens[0]) { + return "learned-" + tokens[0] + } + return tokens[0] + } + return "" +} + +func enrichRuleWithDraftEvidence(rule LearningRecord, evidence DraftEvidence) LearningRecord { + if len(evidence.TaskRecords) == 0 { + return rule + } + usedSkillNames := make([]string, 0) + pathCounts := make(map[string]int) + pathByKey := make(map[string][]string) + for _, task := range evidence.TaskRecords { + path := uniqueTrimmedNames(task.UsedSkillNames) + if len(path) == 0 { + continue + } + usedSkillNames = append(usedSkillNames, path...) + key := strings.Join(path, "\x00") + pathCounts[key]++ + pathByKey[key] = path + } + rule.MatchedSkillNames = appendUniqueStrings(rule.MatchedSkillNames, uniqueTrimmedNames(usedSkillNames)...) + if len(rule.WinningPath) == 0 { + bestKey := "" + bestCount := 0 + for key, count := range pathCounts { + if count > bestCount || (count == bestCount && key < bestKey) { + bestKey = key + bestCount = count + } + } + if bestKey != "" { + rule.WinningPath = append([]string(nil), pathByKey[bestKey]...) + } + } + return rule +} + +func inferCombinedSkillName(rule LearningRecord) string { + path := normalizePath(rule.WinningPath) + if len(path) < 2 { + return "" + } + + tokens := tokenizeForEvolution(rule.Summary) + suffix := commonWinningPathSuffix(path) + if len(tokens) == 1 && isNumericToken(tokens[0]) && suffix != "" { + if candidate := validSkillNameOrEmpty( + "calculate-" + tokens[0] + "-via-" + pluralizeSuffix(suffix), + ); candidate != "" { + return candidate + } + } + if len(tokens) >= 2 { + prefix := strings.Join(tokens[:minInt(len(tokens), 4)], "-") + if suffix != "" { + if candidate := validSkillNameOrEmpty(prefix + "-via-" + pluralizeSuffix(suffix)); candidate != "" { + return candidate + } + } + if candidate := validSkillNameOrEmpty(prefix + "-shortcut"); candidate != "" { + return candidate + } + } + + compressedPath := compressedWinningPathName(path) + if candidate := validSkillNameOrEmpty("combined-" + compressedPath); candidate != "" { + return candidate + } + if candidate := validSkillNameOrEmpty(path[0] + "-to-" + path[len(path)-1] + "-shortcut"); candidate != "" { + return candidate + } + return "" +} + +func commonWinningPathSuffix(path []string) string { + if len(path) < 2 { + return "" + } + + var suffix string + for i, name := range path { + parts := strings.Split(strings.TrimSpace(name), "-") + if len(parts) == 0 { + return "" + } + last := strings.TrimSpace(parts[len(parts)-1]) + if last == "" { + return "" + } + if i == 0 { + suffix = last + continue + } + if suffix != last { + return "" + } + } + return suffix +} + +func compressedWinningPathName(path []string) string { + suffix := commonWinningPathSuffix(path) + fragments := make([]string, 0, len(path)+1) + for _, name := range path { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + continue + } + if suffix != "" { + trimmed = strings.TrimSuffix(trimmed, "-"+suffix) + trimmed = strings.TrimSuffix(trimmed, suffix) + trimmed = strings.Trim(trimmed, "-") + } + if trimmed != "" { + fragments = append(fragments, trimmed) + } + } + if suffix != "" { + fragments = append(fragments, pluralizeSuffix(suffix)) + } + if len(fragments) == 0 { + return strings.Join(path, "-") + } + return strings.Join(fragments, "-") +} + +func pluralizeSuffix(suffix string) string { + suffix = strings.TrimSpace(strings.ToLower(suffix)) + if suffix == "" { + return "" + } + if strings.HasSuffix(suffix, "s") { + return suffix + } + return suffix + "s" +} + +func isNumericToken(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func validSkillNameOrEmpty(candidate string) string { + candidate = strings.Trim(candidate, "-") + candidate = strings.Join(strings.FieldsFunc(candidate, func(r rune) bool { + return !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') + }), "-") + candidate = strings.ToLower(strings.Trim(candidate, "-")) + if candidate == "" { + return "" + } + if len(candidate) > skills.MaxNameLength { + return "" + } + if err := skills.ValidateSkillName(candidate); err != nil { + return "" + } + return candidate +} + +func (g *DefaultDraftGenerator) loadBaseSkillContent(target string, matches []skills.SkillInfo) (string, bool, error) { + for _, match := range matches { + if match.Name != target || strings.TrimSpace(match.Path) == "" { + continue + } + data, err := os.ReadFile(match.Path) + if err != nil { + return "", false, err + } + return string(data), true, nil + } + + if g.loader == nil { + return "", false, nil + } + content, ok := g.loader.LoadSkill(target) + if !ok { + return "", false, nil + } + description := fmt.Sprintf("Use this skill to %s when the task requires this workflow.", sentenceFragment(target)) + return buildSkillDocument(target, description, content), true, nil +} + +func (g *DefaultDraftGenerator) buildHumanSummary(target string, rule LearningRecord, hasExisting bool) string { + if hasExisting { + return fmt.Sprintf("Refresh %s with learned pattern: %s", target, rule.Summary) + } + return fmt.Sprintf("Create %s from learned pattern: %s", target, rule.Summary) +} + +func (g *DefaultDraftGenerator) buildNewSkillBody( + target string, + rule LearningRecord, + evidence DraftEvidence, + matches []skills.SkillInfo, +) string { + description := fmt.Sprintf( + "Use this skill to %s when the task matches this workflow.", + sentenceFragment(fallbackString(rule.Summary, target)), + ) + body := strings.Join([]string{ + "# " + titleCaseSkillName(target), + "", + "## Start Here", + g.startHereLine(rule), + "", + "## When To Use", + fmt.Sprintf("Use this skill when the task matches `%s`.", strings.TrimSpace(rule.Summary)), + "", + "## Learned Pattern", + g.learnedPatternLine(rule), + "", + "## Procedure", + g.procedureLine(rule, evidence), + "", + "## Expected Result", + g.expectedResultLine(evidence), + "", + "## Source Skills", + synthesizedComponentBreakdown(matches), + "", + "## Source Evidence", + g.evidenceLine(rule, evidence), + }, "\n") + return buildSkillDocument(target, description, body) +} + +func (g *DefaultDraftGenerator) buildAppendBody( + rule LearningRecord, + evidence DraftEvidence, + matches []skills.SkillInfo, +) string { + return strings.Join([]string{ + "## Learned Evolution", + fmt.Sprintf("- Summary: %s", strings.TrimSpace(rule.Summary)), + fmt.Sprintf("- Learned pattern: %s", g.learnedPatternLine(rule)), + fmt.Sprintf("- Procedure: %s", g.procedureLine(rule, evidence)), + fmt.Sprintf("- Expected result: %s", g.expectedResultLine(evidence)), + fmt.Sprintf("- Evidence: %s", g.evidenceLine(rule, evidence)), + "", + "### Source Skills", + synthesizedComponentBreakdown(matches), + "", + }, "\n") +} + +func buildSkillDocument(name, description, body string) string { + return strings.Join([]string{ + "---", + "name: " + strings.TrimSpace(name), + "description: " + strings.TrimSpace(description), + "---", + "", + strings.TrimSpace(body), + "", + }, "\n") +} + +func titleCaseSkillName(name string) string { + parts := strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == ' ' }) + for i, part := range parts { + if part == "" { + continue + } + parts[i] = strings.ToUpper(part[:1]) + part[1:] + } + if len(parts) == 0 { + return "Learned Skill" + } + return strings.Join(parts, " ") +} + +func (g *DefaultDraftGenerator) startHereLine(rule LearningRecord) string { + if len(rule.WinningPath) > 0 { + return fmt.Sprintf("Start with `%s` before trying other paths.", strings.Join(rule.WinningPath, " -> ")) + } + return fmt.Sprintf("Start from the learned path for `%s`.", strings.TrimSpace(rule.Summary)) +} + +func (g *DefaultDraftGenerator) learnedPatternLine(rule LearningRecord) string { + if len(rule.LateAddedSkills) > 0 { + return fmt.Sprintf( + "Late-added skill `%s` was repeatedly introduced immediately before success%s.", + strings.Join(rule.LateAddedSkills, " -> "), + triggerSuffix(rule.FinalSnapshotTrigger), + ) + } + if len(rule.WinningPath) > 0 { + return fmt.Sprintf( + "Prefer `%s` because it was the most reliable recent path.", + strings.Join(rule.WinningPath, " -> "), + ) + } + return fmt.Sprintf("Prefer the pattern summarized as `%s`.", strings.TrimSpace(rule.Summary)) +} + +func (g *DefaultDraftGenerator) procedureLine(rule LearningRecord, evidence DraftEvidence) string { + if len(rule.WinningPath) > 0 { + return fmt.Sprintf( + "Follow `%s`, applying the concrete operation from each source skill, then return the final result directly.", + strings.Join(rule.WinningPath, " -> "), + ) + } + if excerpt := firstFinalOutputExcerpt(evidence, 260); excerpt != "" { + return "Use the same operation demonstrated by the source task result: " + excerpt + } + return fmt.Sprintf( + "Solve tasks matching `%s` using the learned successful workflow, then return the final result directly.", + strings.TrimSpace(rule.Summary), + ) +} + +func (g *DefaultDraftGenerator) expectedResultLine(evidence DraftEvidence) string { + if excerpt := firstFinalOutputExcerpt(evidence, 320); excerpt != "" { + return excerpt + } + return "Return the completed result for the matched task without restating unrelated discovery steps." +} + +func (g *DefaultDraftGenerator) evidenceLine(rule LearningRecord, evidence DraftEvidence) string { + if len(evidence.TaskRecords) > 0 { + ids := make([]string, 0, len(evidence.TaskRecords)) + for _, task := range evidence.TaskRecords { + ids = append(ids, task.ID) + } + return fmt.Sprintf("Learned from task records: %s", strings.Join(ids, ", ")) + } + if len(rule.TaskRecordIDs) > 0 { + return fmt.Sprintf("Learned from task records: %s", strings.Join(rule.TaskRecordIDs, ", ")) + } + return "Learned from the pattern record." +} + +func firstFinalOutputExcerpt(evidence DraftEvidence, maxLen int) string { + for _, task := range evidence.TaskRecords { + if excerpt := summarizeText(task.FinalOutput, maxLen); excerpt != "" { + return excerpt + } + } + return "" +} + +func triggerSuffix(trigger string) string { + trigger = strings.TrimSpace(trigger) + if trigger == "" { + return "" + } + return fmt.Sprintf(" during `%s`", trigger) +} diff --git a/pkg/evolution/drafts_test.go b/pkg/evolution/drafts_test.go new file mode 100644 index 000000000..c45c3b69f --- /dev/null +++ b/pkg/evolution/drafts_test.go @@ -0,0 +1,232 @@ +package evolution_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/evolution" + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/skills" +) + +func TestDefaultDraftGenerator_PrefersLateAddedSkillAsTargetWhenNoMatches(t *testing.T) { + generator := evolution.NewDefaultDraftGenerator(t.TempDir()) + + draft, err := generator.GenerateDraft(context.Background(), evolution.LearningRecord{ + Summary: "weather lookup", + WinningPath: []string{"weather"}, + LateAddedSkills: []string{"weather"}, + FinalSnapshotTrigger: "context_retry_rebuild", + EventCount: 4, + SuccessRate: 1, + }, nil) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + if draft.TargetSkillName != "weather" { + t.Fatalf("TargetSkillName = %q, want weather", draft.TargetSkillName) + } + if !strings.Contains(draft.BodyOrPatch, "Late-added skill") { + t.Fatalf("BodyOrPatch = %q, want late-added skill guidance", draft.BodyOrPatch) + } +} + +func TestDefaultDraftGenerator_PrefersCombinedSkillForStableMultiSkillPath(t *testing.T) { + workspace := t.TempDir() + generator := evolution.NewDefaultDraftGenerator(workspace) + + draft, err := generator.GenerateDraft(context.Background(), evolution.LearningRecord{ + Summary: "调用三一定理计算100", + WinningPath: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + LateAddedSkills: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + EventCount: 3, + SuccessRate: 1, + }, []skills.SkillInfo{ + { + Name: "three-one-theorem", + Path: filepath.Join(workspace, "skills", "three-one-theorem", "SKILL.md"), + Source: "workspace", + }, + { + Name: "four-two-theorem", + Path: filepath.Join(workspace, "skills", "four-two-theorem", "SKILL.md"), + Source: "workspace", + }, + { + Name: "five-three-theorem", + Path: filepath.Join(workspace, "skills", "five-three-theorem", "SKILL.md"), + Source: "workspace", + }, + }) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + if draft.TargetSkillName != "calculate-100-via-theorems" { + t.Fatalf("TargetSkillName = %q, want calculate-100-via-theorems", draft.TargetSkillName) + } + if draft.ChangeKind != evolution.ChangeKindCreate { + t.Fatalf("ChangeKind = %q, want create", draft.ChangeKind) + } + if !strings.Contains(draft.BodyOrPatch, "---\nname: calculate-100-via-theorems") { + t.Fatalf("BodyOrPatch should contain full skill document:\n%s", draft.BodyOrPatch) + } +} + +func TestDefaultDraftGenerator_CombinedSkillIncludesEvidenceAndSourceOperations(t *testing.T) { + workspace := t.TempDir() + generator := evolution.NewDefaultDraftGenerator(workspace) + sourceSkills := []struct { + name string + body string + }{ + {name: "three-one-theorem", body: "Add 31 to the input value."}, + {name: "four-two-theorem", body: "Add 42 to the current value."}, + {name: "five-three-theorem", body: "Subtract 53 from the current value."}, + } + + matches := make([]skills.SkillInfo, 0, len(sourceSkills)) + for _, source := range sourceSkills { + skillPath := filepath.Join(workspace, "skills", source.name, "SKILL.md") + if err := os.MkdirAll(filepath.Dir(skillPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + content := "---\nname: " + source.name + "\ndescription: theorem helper\n---\n# " + source.name + "\n" + source.body + "\n" + if err := os.WriteFile(skillPath, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + matches = append( + matches, + skills.SkillInfo{Name: source.name, Path: skillPath, Source: "workspace", Description: "theorem helper"}, + ) + } + + draft, err := generator.GenerateDraftWithEvidence(context.Background(), evolution.LearningRecord{ + ID: "pattern-1", + Summary: "调用三一定理计算100", + TaskRecordIDs: []string{"task-1"}, + }, matches, evolution.DraftEvidence{ + TaskRecords: []evolution.LearningRecord{ + { + ID: "task-1", + Summary: "调用三一定理计算100", + FinalOutput: "100 + 31 = 131; 131 + 42 = 173; 173 - 53 = 120", + UsedSkillNames: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + }, + }, + }) + if err != nil { + t.Fatalf("GenerateDraftWithEvidence: %v", err) + } + for _, want := range []string{ + "calculate-100-via-theorems", + "Add 31 to the input value", + "Add 42 to the current value", + "Subtract 53 from the current value", + "100 + 31 = 131", + "task-1", + } { + if !strings.Contains(draft.BodyOrPatch, want) && draft.TargetSkillName != want { + t.Fatalf("draft missing %q:\nname=%s\n%s", want, draft.TargetSkillName, draft.BodyOrPatch) + } + } +} + +func TestDefaultDraftGenerator_DoesNotInferNumericOnlyTargetFromSummary(t *testing.T) { + generator := evolution.NewDefaultDraftGenerator(t.TempDir()) + + draft, err := generator.GenerateDraft(context.Background(), evolution.LearningRecord{ + Summary: "100", + EventCount: 1, + SuccessRate: 1, + }, nil) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + if draft.TargetSkillName != "learned-100" { + t.Fatalf("TargetSkillName = %q, want learned-100", draft.TargetSkillName) + } +} + +func TestDefaultDraftGenerator_UsesAppendWhenExtendingExistingSkill(t *testing.T) { + workspace := t.TempDir() + generator := evolution.NewDefaultDraftGenerator(workspace) + + existingPath := filepath.Join(workspace, "skills", "weather", "SKILL.md") + if err := os.MkdirAll(filepath.Dir(existingPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + existing := "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse city names.\n" + if err := os.WriteFile(existingPath, []byte(existing), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + draft, err := generator.GenerateDraft(context.Background(), evolution.LearningRecord{ + Summary: "weather native-name path", + WinningPath: []string{"weather"}, + EventCount: 4, + SuccessRate: 1, + }, []skills.SkillInfo{ + {Name: "weather", Path: existingPath, Source: "workspace", Description: "Weather helper"}, + }) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + if draft.ChangeKind != evolution.ChangeKindAppend { + t.Fatalf("ChangeKind = %q, want append", draft.ChangeKind) + } + if strings.Contains(draft.BodyOrPatch, "---\nname: weather") { + t.Fatalf("BodyOrPatch should contain only appended section, got full document:\n%s", draft.BodyOrPatch) + } + if !strings.Contains(draft.BodyOrPatch, "## Learned Evolution") { + t.Fatalf("BodyOrPatch = %q, want learned evolution section", draft.BodyOrPatch) + } + if len(draft.IntendedUseCases) != 1 || draft.IntendedUseCases[0] != "weather native-name path" { + t.Fatalf("IntendedUseCases = %v, want [weather native-name path]", draft.IntendedUseCases) + } + if len(draft.PreferredEntryPath) != 1 || draft.PreferredEntryPath[0] != "weather" { + t.Fatalf("PreferredEntryPath = %v, want [weather]", draft.PreferredEntryPath) + } +} + +func TestLLMDraftGenerator_BuildPromptIncludesLateAddedSkillHint(t *testing.T) { + provider := &llmDraftTestProvider{ + defaultModel: "test-model", + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name lookup first","body_or_patch":"## Start Here\nUse native-name first."}`, + }, + } + generator := evolution.NewLLMDraftGenerator(provider, "", &recordingDraftGenerator{}) + + _, err := generator.GenerateDraft(context.Background(), evolution.LearningRecord{ + ID: "rule-1", + Summary: "weather native-name path", + EventCount: 7, + SuccessRate: 0.86, + WinningPath: []string{"geocode", "weather"}, + MatchedSkillNames: []string{"weather"}, + LateAddedSkills: []string{"weather"}, + FinalSnapshotTrigger: "context_retry_rebuild", + }, []skills.SkillInfo{ + {Name: "weather", Path: "/tmp/weather/SKILL.md", Source: "workspace", Description: "Find weather details."}, + }) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + + prompt := provider.lastMessages[1].Content + if !strings.Contains(prompt, "Late-added successful skills: weather") { + t.Fatalf("prompt missing late-added skill hint:\n%s", prompt) + } + if !strings.Contains(prompt, "Final snapshot trigger: context_retry_rebuild") { + t.Fatalf("prompt missing final snapshot trigger:\n%s", prompt) + } + if !strings.Contains(prompt, "Prefer creating a new combined shortcut skill") { + t.Fatalf("prompt missing combined skill guidance:\n%s", prompt) + } + if !strings.Contains(prompt, "Suggested target skill name:") { + t.Fatalf("prompt missing suggested target skill name:\n%s", prompt) + } +} diff --git a/pkg/evolution/generator_factory.go b/pkg/evolution/generator_factory.go new file mode 100644 index 000000000..1baafae79 --- /dev/null +++ b/pkg/evolution/generator_factory.go @@ -0,0 +1,11 @@ +package evolution + +import "github.com/sipeed/picoclaw/pkg/providers" + +func NewDraftGeneratorForWorkspace(workspace string, provider providers.LLMProvider, modelID string) DraftGenerator { + fallback := NewDefaultDraftGenerator(workspace) + if provider == nil { + return fallback + } + return NewLLMDraftGenerator(provider, modelID, fallback) +} diff --git a/pkg/evolution/lifecycle.go b/pkg/evolution/lifecycle.go new file mode 100644 index 000000000..f9cad26bd --- /dev/null +++ b/pkg/evolution/lifecycle.go @@ -0,0 +1,133 @@ +package evolution + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/sipeed/picoclaw/pkg/skills" +) + +type LifecycleRunSummary struct { + EvaluatedProfiles int + TransitionedProfiles int + DeletedSkills int +} + +func NextLifecycleState(profile SkillProfile, now time.Time) SkillStatus { + if profile.Origin == "manual" || profile.LastUsedAt.IsZero() { + return profile.Status + } + + idle := now.Sub(profile.LastUsedAt) + switch profile.Status { + case SkillStatusActive: + if idle > 90*24*time.Hour && profile.RetentionScore < 0.3 { + return SkillStatusCold + } + case SkillStatusCold: + if idle > 180*24*time.Hour && profile.RetentionScore < 0.2 { + return SkillStatusArchived + } + case SkillStatusArchived: + if idle > 365*24*time.Hour && profile.RetentionScore < 0.1 { + return SkillStatusDeleted + } + } + + return profile.Status +} + +func ApplyLifecycleState(paths Paths, profile SkillProfile, next SkillStatus) error { + if next != SkillStatusDeleted { + return nil + } + + workspace := profile.WorkspaceID + if workspace == "" { + workspace = inferWorkspaceFromPaths(paths) + } + if workspace == "" { + return fmt.Errorf("resolve lifecycle delete workspace for skill %q: workspace is required", profile.SkillName) + } + if err := skills.ValidateSkillName(profile.SkillName); err != nil { + return fmt.Errorf("resolve lifecycle delete skill name: %w", err) + } + + skillPath := filepath.Join(workspace, "skills", profile.SkillName, "SKILL.md") + err := os.Remove(skillPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +func RunLifecycleOnce(store *Store, paths Paths, workspace string, now time.Time) (LifecycleRunSummary, error) { + if store == nil { + return LifecycleRunSummary{}, nil + } + + profiles, err := store.LoadProfiles() + if err != nil { + return LifecycleRunSummary{}, err + } + + summary := LifecycleRunSummary{} + for _, profile := range profiles { + if !profileBelongsToWorkspace(paths, workspace, profile) { + continue + } + + summary.EvaluatedProfiles++ + next := NextLifecycleState(profile, now) + if next == profile.Status { + continue + } + + if err := ApplyLifecycleState(paths, profile, next); err != nil { + return summary, err + } + profile.VersionHistory = append(profile.VersionHistory, SkillVersionEntry{ + Version: profile.CurrentVersion, + Action: "lifecycle:" + string(next), + Timestamp: now, + Summary: fmt.Sprintf("lifecycle transition: %s -> %s", profile.Status, next), + }) + profile.Status = next + if err := store.SaveProfile(profile); err != nil { + return summary, err + } + + summary.TransitionedProfiles++ + if next == SkillStatusDeleted { + summary.DeletedSkills++ + } + } + + return summary, nil +} + +func inferWorkspaceFromPaths(paths Paths) string { + root := filepath.Clean(paths.RootDir) + if filepath.Base(root) != "evolution" { + return "" + } + stateDir := filepath.Dir(root) + if filepath.Base(stateDir) != "state" { + return "" + } + return filepath.Dir(stateDir) +} + +func profileBelongsToWorkspace(paths Paths, workspace string, profile SkillProfile) bool { + if profile.WorkspaceID == workspace { + return true + } + return profile.WorkspaceID == "" && usesDefaultWorkspaceState(paths, workspace) +} + +func usesDefaultWorkspaceState(paths Paths, workspace string) bool { + return paths.RootDir == NewPaths(workspace, "").RootDir +} diff --git a/pkg/evolution/lifecycle_actions_test.go b/pkg/evolution/lifecycle_actions_test.go new file mode 100644 index 000000000..8bd65d87d --- /dev/null +++ b/pkg/evolution/lifecycle_actions_test.go @@ -0,0 +1,72 @@ +package evolution_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestApplyLifecycleStateDeletedRemovesSkillFile(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile(skillPath, []byte("# weather\n"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + err := evolution.ApplyLifecycleState( + evolution.NewPaths(workspace, ""), + evolution.SkillProfile{SkillName: "weather"}, + evolution.SkillStatusDeleted, + ) + if err != nil { + t.Fatalf("ApplyLifecycleState: %v", err) + } + + if _, err := os.Stat(skillPath); !os.IsNotExist(err) { + t.Fatalf("skill file should be removed, stat err = %v", err) + } +} + +func TestApplyLifecycleStateDeletedRequiresResolvedWorkspace(t *testing.T) { + err := evolution.ApplyLifecycleState( + evolution.Paths{RootDir: filepath.Join(t.TempDir(), "shared-evolution")}, + evolution.SkillProfile{SkillName: "weather"}, + evolution.SkillStatusDeleted, + ) + if err == nil { + t.Fatal("expected error when workspace cannot be resolved") + } +} + +func TestApplyLifecycleStateDeletedRequiresSkillName(t *testing.T) { + workspace := t.TempDir() + + err := evolution.ApplyLifecycleState( + evolution.NewPaths(workspace, ""), + evolution.SkillProfile{WorkspaceID: workspace}, + evolution.SkillStatusDeleted, + ) + if err == nil { + t.Fatal("expected error when skill name is empty") + } +} + +func TestApplyLifecycleStateDeletedRejectsTraversalSkillName(t *testing.T) { + workspace := t.TempDir() + + err := evolution.ApplyLifecycleState( + evolution.NewPaths(workspace, ""), + evolution.SkillProfile{WorkspaceID: workspace, SkillName: "../escape"}, + evolution.SkillStatusDeleted, + ) + if err == nil { + t.Fatal("expected error for traversal skill name") + } +} diff --git a/pkg/evolution/lifecycle_test.go b/pkg/evolution/lifecycle_test.go new file mode 100644 index 000000000..25bec518d --- /dev/null +++ b/pkg/evolution/lifecycle_test.go @@ -0,0 +1,247 @@ +package evolution_test + +import ( + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestStore_SaveAndLoadProfile(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + profile := evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: root, + CurrentVersion: "v2", + Status: evolution.SkillStatusActive, + Origin: "evolved", + HumanSummary: "weather lookup helper", + LastUsedAt: time.Unix(1700000000, 0).UTC(), + UseCount: 3, + RetentionScore: 0.8, + VersionHistory: []evolution.SkillVersionEntry{ + { + Version: "v1", + Action: "create", + Timestamp: time.Unix(1699990000, 0).UTC(), + Summary: "initial learned version", + }, + }, + } + + if err := store.SaveProfile(profile); err != nil { + t.Fatalf("SaveProfile: %v", err) + } + + loaded, err := store.LoadProfile("weather") + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if loaded.SkillName != "weather" { + t.Fatalf("SkillName = %q, want weather", loaded.SkillName) + } + if loaded.Status != evolution.SkillStatusActive { + t.Fatalf("Status = %q, want %q", loaded.Status, evolution.SkillStatusActive) + } + if len(loaded.VersionHistory) != 1 { + t.Fatalf("len(VersionHistory) = %d, want 1", len(loaded.VersionHistory)) + } +} + +func TestNextLifecycleState_ActiveToCold(t *testing.T) { + now := time.Now().UTC() + profile := evolution.SkillProfile{ + SkillName: "release-flow", + Status: evolution.SkillStatusActive, + Origin: "evolved", + LastUsedAt: now.AddDate(0, -6, 0), + RetentionScore: 0.1, + } + + got := evolution.NextLifecycleState(profile, now) + if got != evolution.SkillStatusCold { + t.Fatalf("NextLifecycleState = %q, want %q", got, evolution.SkillStatusCold) + } +} + +func TestNextLifecycleState_ManualSkillStaysActive(t *testing.T) { + now := time.Now().UTC() + profile := evolution.SkillProfile{ + SkillName: "manual-weather", + Status: evolution.SkillStatusActive, + Origin: "manual", + LastUsedAt: now.AddDate(-1, 0, 0), + RetentionScore: 0, + } + + got := evolution.NextLifecycleState(profile, now) + if got != evolution.SkillStatusActive { + t.Fatalf("NextLifecycleState = %q, want %q", got, evolution.SkillStatusActive) + } +} + +func TestStore_SaveProfileRejectsInvalidSkillName(t *testing.T) { + store := evolution.NewStore(evolution.NewPaths(t.TempDir(), "")) + + err := store.SaveProfile(evolution.SkillProfile{SkillName: "../escape"}) + if err == nil { + t.Fatal("expected SaveProfile to reject invalid skill name") + } +} + +func TestStore_LoadProfileRejectsInvalidSkillName(t *testing.T) { + store := evolution.NewStore(evolution.NewPaths(t.TempDir(), "")) + + _, err := store.LoadProfile("/tmp/escape") + if err == nil { + t.Fatal("expected LoadProfile to reject invalid skill name") + } +} + +func TestStore_SharedStateProfilesRemainIsolatedPerWorkspace(t *testing.T) { + sharedState := t.TempDir() + workspaceA := t.TempDir() + workspaceB := t.TempDir() + + storeA := evolution.NewStore(evolution.NewPaths(workspaceA, sharedState)) + storeB := evolution.NewStore(evolution.NewPaths(workspaceB, sharedState)) + + profileA := evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: workspaceA, + CurrentVersion: "v-a", + Status: evolution.SkillStatusActive, + Origin: "evolved", + HumanSummary: "workspace A weather helper", + LastUsedAt: time.Unix(1700000000, 0).UTC(), + UseCount: 2, + RetentionScore: 0.6, + } + profileB := evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: workspaceB, + CurrentVersion: "v-b", + Status: evolution.SkillStatusCold, + Origin: "manual", + HumanSummary: "workspace B weather helper", + LastUsedAt: time.Unix(1700000500, 0).UTC(), + UseCount: 9, + RetentionScore: 0.2, + } + + if err := storeA.SaveProfile(profileA); err != nil { + t.Fatalf("storeA.SaveProfile: %v", err) + } + if err := storeB.SaveProfile(profileB); err != nil { + t.Fatalf("storeB.SaveProfile: %v", err) + } + + loadedA, err := storeA.LoadProfile("weather") + if err != nil { + t.Fatalf("storeA.LoadProfile: %v", err) + } + if loadedA.WorkspaceID != workspaceA { + t.Fatalf("storeA workspace = %q, want %q", loadedA.WorkspaceID, workspaceA) + } + if loadedA.CurrentVersion != "v-a" { + t.Fatalf("storeA CurrentVersion = %q, want v-a", loadedA.CurrentVersion) + } + + loadedB, err := storeB.LoadProfile("weather") + if err != nil { + t.Fatalf("storeB.LoadProfile: %v", err) + } + if loadedB.WorkspaceID != workspaceB { + t.Fatalf("storeB workspace = %q, want %q", loadedB.WorkspaceID, workspaceB) + } + if loadedB.CurrentVersion != "v-b" { + t.Fatalf("storeB CurrentVersion = %q, want v-b", loadedB.CurrentVersion) + } + + allProfiles, err := storeA.LoadProfiles() + if err != nil { + t.Fatalf("LoadProfiles: %v", err) + } + if len(allProfiles) != 2 { + t.Fatalf("len(LoadProfiles()) = %d, want 2", len(allProfiles)) + } +} + +func TestStore_LoadProfileDoesNotBorrowAnotherWorkspaceProfile(t *testing.T) { + sharedState := t.TempDir() + workspaceA := t.TempDir() + workspaceB := t.TempDir() + + storeA := evolution.NewStore(evolution.NewPaths(workspaceA, sharedState)) + storeB := evolution.NewStore(evolution.NewPaths(workspaceB, sharedState)) + + if err := storeA.SaveProfile(evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: workspaceA, + CurrentVersion: "v-a", + Status: evolution.SkillStatusActive, + Origin: "evolved", + HumanSummary: "workspace A weather helper", + LastUsedAt: time.Unix(1700000000, 0).UTC(), + UseCount: 4, + RetentionScore: 0.8, + }); err != nil { + t.Fatalf("storeA.SaveProfile: %v", err) + } + + _, err := storeB.LoadProfile("weather") + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("storeB.LoadProfile should not borrow workspace A profile, got err=%v", err) + } +} + +func TestStore_UpdateProfileIsAtomicPerWorkspaceSkill(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + const workers = 64 + var wg sync.WaitGroup + errs := make(chan error, workers) + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- store.UpdateProfile(root, "weather", func(profile *evolution.SkillProfile, exists bool) error { + if !exists { + *profile = evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: root, + Status: evolution.SkillStatusActive, + Origin: "manual", + HumanSummary: "weather", + RetentionScore: 0.2, + } + } + profile.UseCount++ + return nil + }) + }() + } + + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("UpdateProfile: %v", err) + } + } + + profile, err := store.LoadProfile("weather") + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if profile.UseCount != workers { + t.Fatalf("UseCount = %d, want %d", profile.UseCount, workers) + } +} diff --git a/pkg/evolution/llm_draft_generator.go b/pkg/evolution/llm_draft_generator.go new file mode 100644 index 000000000..2db27004c --- /dev/null +++ b/pkg/evolution/llm_draft_generator.go @@ -0,0 +1,235 @@ +package evolution + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type LLMDraftGenerator struct { + provider providers.LLMProvider + model string + fallback DraftGenerator +} + +type llmDraftResponse struct { + TargetSkillName string `json:"target_skill_name"` + DraftType string `json:"draft_type"` + ChangeKind string `json:"change_kind"` + HumanSummary string `json:"human_summary"` + IntendedUseCases []string `json:"intended_use_cases"` + PreferredEntryPath []string `json:"preferred_entry_path"` + AvoidPatterns []string `json:"avoid_patterns"` + BodyOrPatch string `json:"body_or_patch"` +} + +func NewLLMDraftGenerator(provider providers.LLMProvider, model string, fallback DraftGenerator) *LLMDraftGenerator { + return &LLMDraftGenerator{ + provider: provider, + model: strings.TrimSpace(model), + fallback: fallback, + } +} + +func (g *LLMDraftGenerator) GenerateDraft( + ctx context.Context, + rule LearningRecord, + matches []skills.SkillInfo, +) (SkillDraft, error) { + return g.GenerateDraftWithEvidence(ctx, rule, matches, DraftEvidence{}) +} + +func (g *LLMDraftGenerator) GenerateDraftWithEvidence( + ctx context.Context, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) (SkillDraft, error) { + rule = enrichRuleWithDraftEvidence(rule, evidence) + if g == nil || g.provider == nil { + return g.generateFallback(ctx, rule, matches, evidence) + } + + model := g.model + if model == "" { + model = strings.TrimSpace(g.provider.GetDefaultModel()) + } + if model == "" { + return g.generateFallback(ctx, rule, matches, evidence) + } + + callCtx, cancel := withLLMCallTimeout(ctx, llmDraftGenerationTimeout) + defer cancel() + resp, err := g.provider.Chat(callCtx, []providers.Message{ + { + Role: "system", + Content: "Return exactly one JSON object for a skill draft. Do not use markdown fences.", + }, + { + Role: "user", + Content: g.buildPrompt(rule, matches, evidence), + }, + }, nil, model, map[string]any{"temperature": 0.2}) + if err != nil || resp == nil { + return g.generateFallback(ctx, rule, matches, evidence) + } + + content := strings.TrimSpace(resp.Content) + if content == "" { + return g.generateFallback(ctx, rule, matches, evidence) + } + + draft, ok := parseLLMDraft(content) + if !ok || len(ValidateDraft(draft)) > 0 { + return g.generateFallback(ctx, rule, matches, evidence) + } + + return draft, nil +} + +func (g *LLMDraftGenerator) generateFallback( + ctx context.Context, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) (SkillDraft, error) { + if g == nil || g.fallback == nil { + return SkillDraft{}, nil + } + if generator, ok := g.fallback.(EvidenceAwareDraftGenerator); ok { + return generator.GenerateDraftWithEvidence(ctx, rule, matches, evidence) + } + return g.fallback.GenerateDraft(ctx, rule, matches) +} + +func (g *LLMDraftGenerator) buildPrompt( + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) string { + return strings.Join([]string{ + "Generate a skill draft JSON object with these required string fields:", + `target_skill_name, draft_type, change_kind, human_summary, body_or_patch.`, + "Optional array fields: intended_use_cases, preferred_entry_path, avoid_patterns.", + "", + "Allowed values:", + "- draft_type: workflow | shortcut", + "- change_kind: create | append | replace | merge", + "- target_skill_name: lowercase hyphenated skill name that describes the functional purpose; it must not be numeric-only", + "", + "Rule summary: " + strings.TrimSpace(rule.Summary), + "Winning path: " + joinOrFallback(rule.WinningPath, "none"), + "Late-added successful skills: " + joinOrFallback(rule.LateAddedSkills, "none"), + "Final snapshot trigger: " + fallbackString(rule.FinalSnapshotTrigger, "none"), + fmt.Sprintf("Event count: %d", rule.EventCount), + fmt.Sprintf("Success rate: %.2f", rule.SuccessRate), + "Matched skill refs: " + summarizeSkillMatches(matches), + "Matched skill names: " + joinOrFallback(rule.MatchedSkillNames, "none"), + "Source task evidence:", + summarizeDraftTaskEvidence(evidence), + "Matched skill content excerpts:", + summarizeMatchedSkillExcerpts(matches), + "", + combinedSkillGuidance(rule), + skillDraftPromptText(), + }, "\n") +} + +func summarizeDraftTaskEvidence(evidence DraftEvidence) string { + if len(evidence.TaskRecords) == 0 { + return "none" + } + lines := make([]string, 0, minInt(len(evidence.TaskRecords), 5)) + for i, task := range evidence.TaskRecords { + if i >= 5 { + break + } + parts := []string{ + "- id: " + fallbackString(task.ID, "unknown"), + " summary: " + fallbackString(task.Summary, "none"), + " final_output_excerpt: " + fallbackString(summarizeText(task.FinalOutput, 700), "none"), + " used_skill_names: " + joinOrFallback(task.UsedSkillNames, "none"), + } + lines = append(lines, strings.Join(parts, "\n")) + } + return strings.Join(lines, "\n") +} + +func combinedSkillGuidance(rule LearningRecord) string { + if target := inferCombinedSkillName(rule); target != "" { + return strings.Join([]string{ + "This rule represents a stable multi-step successful path.", + "Prefer creating a new combined shortcut skill instead of modifying one component skill.", + "Suggested target skill name: " + target, + }, "\n") + } + return "Prefer updating an existing skill only when the learned pattern clearly belongs inside that single skill." +} + +func parseLLMDraft(content string) (SkillDraft, bool) { + normalized := strings.TrimSpace(content) + normalized = strings.TrimPrefix(normalized, "```json") + normalized = strings.TrimPrefix(normalized, "```") + normalized = strings.TrimSuffix(normalized, "```") + normalized = strings.TrimSpace(normalized) + + var payload llmDraftResponse + if err := json.Unmarshal([]byte(normalized), &payload); err != nil { + return SkillDraft{}, false + } + + draft := SkillDraft{ + TargetSkillName: strings.TrimSpace(payload.TargetSkillName), + DraftType: DraftType(strings.TrimSpace(payload.DraftType)), + ChangeKind: ChangeKind(strings.TrimSpace(payload.ChangeKind)), + HumanSummary: strings.TrimSpace(payload.HumanSummary), + IntendedUseCases: append([]string(nil), payload.IntendedUseCases...), + PreferredEntryPath: append([]string(nil), payload.PreferredEntryPath...), + AvoidPatterns: append([]string(nil), payload.AvoidPatterns...), + BodyOrPatch: strings.TrimSpace(payload.BodyOrPatch), + } + return draft, true +} + +func summarizeSkillMatches(matches []skills.SkillInfo) string { + if len(matches) == 0 { + return "none" + } + + parts := make([]string, 0, len(matches)) + for _, match := range matches { + part := strings.TrimSpace(match.Name) + if desc := strings.TrimSpace(match.Description); desc != "" { + part += ": " + desc + } + if path := strings.TrimSpace(match.Path); path != "" { + part += " (" + path + ")" + } + if part != "" { + parts = append(parts, part) + } + } + if len(parts) == 0 { + return "none" + } + return strings.Join(parts, "; ") +} + +func joinOrFallback(parts []string, fallback string) string { + if len(parts) == 0 { + return fallback + } + return strings.Join(parts, " -> ") +} + +func fallbackString(value, fallback string) string { + value = strings.TrimSpace(value) + if value == "" { + return fallback + } + return value +} diff --git a/pkg/evolution/llm_draft_generator_test.go b/pkg/evolution/llm_draft_generator_test.go new file mode 100644 index 000000000..ffba62b84 --- /dev/null +++ b/pkg/evolution/llm_draft_generator_test.go @@ -0,0 +1,367 @@ +package evolution_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/evolution" + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type recordingDraftGenerator struct { + draft evolution.SkillDraft + err error + calls int +} + +func (g *recordingDraftGenerator) GenerateDraft( + _ context.Context, + _ evolution.LearningRecord, + _ []skills.SkillInfo, +) (evolution.SkillDraft, error) { + g.calls++ + return g.draft, g.err +} + +type llmDraftTestProvider struct { + response *providers.LLMResponse + err error + defaultModel string + lastModel string + lastMessages []providers.Message + chatCallCount int +} + +func (p *llmDraftTestProvider) Chat( + _ context.Context, + messages []providers.Message, + _ []providers.ToolDefinition, + model string, + _ map[string]any, +) (*providers.LLMResponse, error) { + p.chatCallCount++ + p.lastModel = model + p.lastMessages = append([]providers.Message(nil), messages...) + return p.response, p.err +} + +func (p *llmDraftTestProvider) GetDefaultModel() string { + return p.defaultModel +} + +func testLearningRule() evolution.LearningRecord { + return evolution.LearningRecord{ + ID: "rule-1", + Summary: "weather native-name path", + EventCount: 7, + SuccessRate: 0.86, + WinningPath: []string{"weather", "native-name"}, + MatchedSkillNames: []string{"weather"}, + } +} + +func testSkillMatches() []skills.SkillInfo { + return []skills.SkillInfo{ + { + Name: "weather", + Path: "/tmp/weather/SKILL.md", + Source: "workspace", + Description: "Find weather details.", + }, + } +} + +func TestLLMDraftGenerator_GenerateDraft_ParsesJSONResponse(t *testing.T) { + provider := &llmDraftTestProvider{ + defaultModel: "test-model", + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name lookup first","body_or_patch":"## Start Here\nUse native-name first."}`, + }, + } + fallback := &recordingDraftGenerator{ + draft: evolution.SkillDraft{TargetSkillName: "fallback"}, + } + generator := evolution.NewLLMDraftGenerator(provider, "", fallback) + + draft, err := generator.GenerateDraft(context.Background(), testLearningRule(), testSkillMatches()) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + + if provider.chatCallCount != 1 { + t.Fatalf("chatCallCount = %d, want 1", provider.chatCallCount) + } + if provider.lastModel != "test-model" { + t.Fatalf("lastModel = %q, want test-model", provider.lastModel) + } + if len(provider.lastMessages) == 0 { + t.Fatal("expected prompt messages") + } + if fallback.calls != 0 { + t.Fatalf("fallback.calls = %d, want 0", fallback.calls) + } + if draft.TargetSkillName != "weather" { + t.Fatalf("TargetSkillName = %q, want weather", draft.TargetSkillName) + } + if draft.DraftType != evolution.DraftTypeShortcut { + t.Fatalf("DraftType = %q, want %q", draft.DraftType, evolution.DraftTypeShortcut) + } + if draft.ChangeKind != evolution.ChangeKindAppend { + t.Fatalf("ChangeKind = %q, want %q", draft.ChangeKind, evolution.ChangeKindAppend) + } + if draft.HumanSummary == "" || draft.BodyOrPatch == "" { + t.Fatal("expected non-empty draft content") + } +} + +func TestLLMDraftGenerator_BuildPromptIncludesMatchedSkillContent(t *testing.T) { + dir := t.TempDir() + skillPath := filepath.Join(dir, "skills", "three-one-theorem", "SKILL.md") + if err := os.MkdirAll(filepath.Dir(skillPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile( + skillPath, + []byte( + "---\nname: three-one-theorem\ndescription: Add 31 then delegate\n---\n# Three One\nAdd 31 to the input, then continue with the next theorem.\n", + ), + 0o644, + ); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + provider := &llmDraftTestProvider{ + defaultModel: "test-model", + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"calculate-100-via-theorems","draft_type":"shortcut","change_kind":"create","human_summary":"Combine theorem chain","body_or_patch":"## Start Here\nAdd 31, then continue."}`, + }, + } + generator := evolution.NewLLMDraftGenerator(provider, "", &recordingDraftGenerator{}) + + _, err := generator.GenerateDraft(context.Background(), evolution.LearningRecord{ + ID: "rule-1", + Summary: "calculate 100", + WinningPath: []string{"three-one-theorem", "four-two-theorem"}, + EventCount: 2, + SuccessRate: 1, + }, []skills.SkillInfo{{ + Name: "three-one-theorem", + Path: skillPath, + Source: "workspace", + Description: "Add 31 then delegate", + }}) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + if len(provider.lastMessages) < 2 { + t.Fatal("expected user prompt") + } + prompt := provider.lastMessages[1].Content + if !strings.Contains(prompt, "Matched skill content excerpts") { + t.Fatalf("prompt missing content section:\n%s", prompt) + } + if !strings.Contains(prompt, "Add 31 to the input") { + t.Fatalf("prompt missing matched skill body:\n%s", prompt) + } + if !strings.Contains(prompt, "summarize the functional purpose and result") { + t.Fatalf("prompt missing synthesis instruction:\n%s", prompt) + } + if !strings.Contains(prompt, "complete SKILL.md file with exactly two parts") { + t.Fatalf("prompt missing complete skill instruction:\n%s", prompt) + } + if !strings.Contains(prompt, "The YAML frontmatter must contain only name and description fields") { + t.Fatalf("prompt missing frontmatter instruction:\n%s", prompt) + } + if !strings.Contains( + prompt, + "The description field must and only describe what this skill can do and when to use it", + ) { + t.Fatalf("prompt missing description field instruction:\n%s", prompt) + } + if !strings.Contains( + prompt, + "The deployable Markdown body should only contain what the skill is useful for and how to use it", + ) { + t.Fatalf("prompt missing deployable body scope instruction:\n%s", prompt) + } + if !strings.Contains( + prompt, + "provide detailed step-by-step instructions for the exact operation or execution process", + ) { + t.Fatalf("prompt missing step-by-step instruction:\n%s", prompt) + } + if !strings.Contains(prompt, "body_or_patch is an internal draft and review artifact") { + t.Fatalf("prompt missing internal draft instruction:\n%s", prompt) + } + if !strings.Contains(prompt, "the final deployed SKILL.md will be rendered without learning traces") { + t.Fatalf("prompt missing deploy-clean instruction:\n%s", prompt) + } + if !strings.Contains(prompt, "do not copy or directly include other skills' instructions") { + t.Fatalf("prompt missing no-copy instruction:\n%s", prompt) + } +} + +func TestLLMDraftGenerator_BuildPromptIncludesTaskEvidence(t *testing.T) { + provider := &llmDraftTestProvider{ + defaultModel: "test-model", + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"calculate-with-three-one-theorem","draft_type":"shortcut","change_kind":"create","human_summary":"Calculate using theorem chain","body_or_patch":"---\nname: calculate-with-three-one-theorem\ndescription: Calculate with theorem chain.\n---\n# Calculate With Three One Theorem\n\n## Procedure\nAdd 31, add 42, then subtract 53."}`, + }, + } + generator := evolution.NewLLMDraftGenerator(provider, "", &recordingDraftGenerator{}) + + _, err := generator.GenerateDraftWithEvidence(context.Background(), evolution.LearningRecord{ + ID: "rule-1", + Label: "calculate-with-three-one-theorem", + Summary: "调用三一定理计算", + }, nil, evolution.DraftEvidence{ + TaskRecords: []evolution.LearningRecord{ + { + ID: "main-turn-6", + Summary: "调用三一定理计算100", + FinalOutput: "100 + 31 = 131; 131 + 42 = 173; 173 - 53 = 120", + UsedSkillNames: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + }, + }, + }) + if err != nil { + t.Fatalf("GenerateDraftWithEvidence: %v", err) + } + if len(provider.lastMessages) < 2 { + t.Fatal("expected user prompt") + } + prompt := provider.lastMessages[1].Content + for _, want := range []string{ + "Source task evidence", + "main-turn-6", + "调用三一定理计算100", + "100 + 31 = 131", + "three-one-theorem -> four-two-theorem -> five-three-theorem", + "directly usable by a future agent", + } { + if !strings.Contains(prompt, want) { + t.Fatalf("prompt missing %q:\n%s", want, prompt) + } + } +} + +func TestLLMDraftGenerator_GenerateDraft_PrefersExplicitModelIDOverProviderDefault(t *testing.T) { + provider := &llmDraftTestProvider{ + defaultModel: "provider-default-model", + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name lookup first","body_or_patch":"## Start Here\nUse native-name first."}`, + }, + } + generator := evolution.NewLLMDraftGenerator(provider, "explicit-model-id", &recordingDraftGenerator{}) + + _, err := generator.GenerateDraft(context.Background(), testLearningRule(), testSkillMatches()) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + if provider.lastModel != "explicit-model-id" { + t.Fatalf("lastModel = %q, want explicit-model-id", provider.lastModel) + } +} + +func TestLLMDraftGenerator_GenerateDraft_FallsBackOnProviderError(t *testing.T) { + fallback := &recordingDraftGenerator{ + draft: evolution.SkillDraft{ + TargetSkillName: "weather-fallback", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "fallback summary", + BodyOrPatch: "fallback body", + }, + } + generator := evolution.NewLLMDraftGenerator(&llmDraftTestProvider{ + defaultModel: "test-model", + err: errors.New("provider unavailable"), + }, "", fallback) + + draft, err := generator.GenerateDraft(context.Background(), testLearningRule(), testSkillMatches()) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + + if fallback.calls != 1 { + t.Fatalf("fallback.calls = %d, want 1", fallback.calls) + } + if draft.TargetSkillName != "weather-fallback" { + t.Fatalf("TargetSkillName = %q, want weather-fallback", draft.TargetSkillName) + } +} + +func TestLLMDraftGenerator_GenerateDraft_FallsBackOnInvalidOrEmptyContent(t *testing.T) { + testCases := []struct { + name string + content string + }{ + {name: "invalid json", content: `not-json`}, + {name: "empty content", content: ``}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + fallback := &recordingDraftGenerator{ + draft: evolution.SkillDraft{ + TargetSkillName: "weather-fallback", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "fallback summary", + BodyOrPatch: "fallback body", + }, + } + generator := evolution.NewLLMDraftGenerator(&llmDraftTestProvider{ + defaultModel: "test-model", + response: &providers.LLMResponse{Content: tt.content}, + }, "", fallback) + + draft, err := generator.GenerateDraft(context.Background(), testLearningRule(), testSkillMatches()) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + + if fallback.calls != 1 { + t.Fatalf("fallback.calls = %d, want 1", fallback.calls) + } + if draft.TargetSkillName != "weather-fallback" { + t.Fatalf("TargetSkillName = %q, want weather-fallback", draft.TargetSkillName) + } + }) + } +} + +func TestLLMDraftGenerator_GenerateDraft_FallsBackOnNumericOnlyTargetSkillName(t *testing.T) { + fallback := &recordingDraftGenerator{ + draft: evolution.SkillDraft{ + TargetSkillName: "learned-100", + DraftType: evolution.DraftTypeWorkflow, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "fallback summary", + BodyOrPatch: "fallback body", + }, + } + generator := evolution.NewLLMDraftGenerator(&llmDraftTestProvider{ + defaultModel: "test-model", + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"100","draft_type":"shortcut","change_kind":"create","human_summary":"Calculate 100","body_or_patch":"## Start Here\nCalculate 100."}`, + }, + }, "", fallback) + + draft, err := generator.GenerateDraft(context.Background(), testLearningRule(), testSkillMatches()) + if err != nil { + t.Fatalf("GenerateDraft: %v", err) + } + + if fallback.calls != 1 { + t.Fatalf("fallback.calls = %d, want 1", fallback.calls) + } + if draft.TargetSkillName != "learned-100" { + t.Fatalf("TargetSkillName = %q, want learned-100", draft.TargetSkillName) + } +} diff --git a/pkg/evolution/llm_timeout.go b/pkg/evolution/llm_timeout.go new file mode 100644 index 000000000..0c5700a60 --- /dev/null +++ b/pkg/evolution/llm_timeout.go @@ -0,0 +1,25 @@ +package evolution + +import ( + "context" + "time" +) + +const ( + llmTaskSuccessJudgeTimeout = 15 * time.Second + llmPatternClusterTimeout = 45 * time.Second + llmDraftGenerationTimeout = 60 * time.Second +) + +func withLLMCallTimeout(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + if parent == nil { + parent = context.Background() + } + if timeout <= 0 { + return context.WithCancel(parent) + } + if deadline, ok := parent.Deadline(); ok && time.Until(deadline) <= timeout { + return context.WithCancel(parent) + } + return context.WithTimeout(parent, timeout) +} diff --git a/pkg/evolution/organizer.go b/pkg/evolution/organizer.go new file mode 100644 index 000000000..d8b8a2e67 --- /dev/null +++ b/pkg/evolution/organizer.go @@ -0,0 +1,397 @@ +package evolution + +import ( + "crypto/sha1" + "encoding/hex" + "sort" + "strings" + "time" +) + +type OrganizerOptions struct { + MinCaseCount int + MinSuccessRate float64 + Now func() time.Time +} + +type Organizer struct { + minCaseCount int + minSuccessRate float64 + now func() time.Time +} + +func NewOrganizer(opts OrganizerOptions) *Organizer { + now := opts.Now + if now == nil { + now = time.Now + } + + minCaseCount := opts.MinCaseCount + if minCaseCount <= 0 { + minCaseCount = 3 + } + + minSuccessRate := opts.MinSuccessRate + if minSuccessRate <= 0 { + minSuccessRate = 0.7 + } + + return &Organizer{ + minCaseCount: minCaseCount, + minSuccessRate: minSuccessRate, + now: now, + } +} + +func (o *Organizer) BuildRules(records []LearningRecord) ([]LearningRecord, error) { + clusters := make(map[string][]LearningRecord) + keys := make([]string, 0) + + for _, record := range records { + if !isTaskRecordKind(record.Kind) { + continue + } + + key := normalizeRuleKey(record) + if key == "" { + continue + } + + clusterKey := record.WorkspaceID + "\x00" + key + if _, ok := clusters[clusterKey]; !ok { + keys = append(keys, clusterKey) + } + clusters[clusterKey] = append(clusters[clusterKey], record) + } + + sort.Strings(keys) + + rules := make([]LearningRecord, 0, len(keys)) + for _, clusterKey := range keys { + cluster := append([]LearningRecord(nil), clusters[clusterKey]...) + sortCaseCluster(cluster) + + if len(cluster) < o.minCaseCount { + continue + } + + successRate := clusterSuccessRate(cluster) + if successRate < o.minSuccessRate { + continue + } + + ruleKey := clusterKey[strings.Index(clusterKey, "\x00")+1:] + winningPath := clusterWinningPath(cluster) + lateAddedSkills, finalSnapshotTrigger := clusterLateAddedSkills(cluster, winningPath) + matchedSkillNames := append([]string(nil), winningPath...) + + rules = append(rules, LearningRecord{ + ID: stableRuleID(cluster[0].WorkspaceID, ruleKey), + Kind: RecordKindPattern, + WorkspaceID: cluster[0].WorkspaceID, + CreatedAt: o.now(), + Summary: buildRuleSummary(cluster, ruleKey, winningPath), + Source: map[string]any{"cluster_key": ruleKey}, + Status: RecordStatus("ready"), + SourceRecordIDs: collectRecordIDs(cluster), + EventCount: len(cluster), + SuccessRate: successRate, + MaturityScore: computeMaturityScore(len(cluster), successRate), + WinningPath: winningPath, + LateAddedSkills: lateAddedSkills, + FinalSnapshotTrigger: finalSnapshotTrigger, + MatchedSkillNames: matchedSkillNames, + }) + } + + return rules, nil +} + +func normalizeRuleKey(record LearningRecord) string { + if path := preferredRulePath(record); len(path) > 0 { + return strings.Join(path, " ") + } + if path := normalizePath(record.ToolKinds); len(path) > 0 { + return strings.Join(path, " ") + } + + tokens := tokenizeForEvolution(record.Summary) + if len(tokens) == 0 { + return "" + } + if len(tokens) > 6 { + tokens = tokens[:6] + } + return strings.Join(tokens, " ") +} + +func preferredRulePath(record LearningRecord) []string { + if path := normalizeFinalSuccessfulPath(record); len(path) > 0 { + return path + } + if path := normalizePath(record.UsedSkillNames); len(path) > 0 { + return path + } + if path := normalizePath(record.AddedSkillNames); len(path) > 0 { + return path + } + if path := normalizeAttemptedSkills(record); len(path) > 0 { + return path + } + if path := normalizePath(record.ActiveSkillNames); len(path) > 0 { + return path + } + if path := normalizePath(record.MatchedSkillNames); len(path) > 0 { + return path + } + return nil +} + +func normalizePath(values []string) []string { + if len(values) == 0 { + return nil + } + + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + continue + } + out = append(out, value) + } + if len(out) == 0 { + return nil + } + return out +} + +func normalizeFinalSuccessfulPath(record LearningRecord) []string { + if record.AttemptTrail == nil { + return nil + } + return normalizePath(record.AttemptTrail.FinalSuccessfulPath) +} + +func normalizeAttemptedSkills(record LearningRecord) []string { + if record.AttemptTrail == nil { + return nil + } + return normalizePath(record.AttemptTrail.AttemptedSkills) +} + +func sortCaseCluster(cluster []LearningRecord) { + sort.Slice(cluster, func(i, j int) bool { + if !cluster[i].CreatedAt.Equal(cluster[j].CreatedAt) { + return cluster[i].CreatedAt.Before(cluster[j].CreatedAt) + } + return cluster[i].ID < cluster[j].ID + }) +} + +func clusterSuccessRate(cluster []LearningRecord) float64 { + if len(cluster) == 0 { + return 0 + } + + successes := 0 + for _, record := range cluster { + if record.Success != nil && *record.Success { + successes++ + } + } + return float64(successes) / float64(len(cluster)) +} + +func clusterWinningPath(cluster []LearningRecord) []string { + type pathScore struct { + path []string + count int + } + + bestKey := "" + best := pathScore{} + paths := make(map[string]pathScore) + order := make([]string, 0) + + for _, record := range cluster { + path := preferredRulePath(record) + if len(path) == 0 { + path = normalizePath(record.ToolKinds) + } + if len(path) == 0 { + continue + } + + key := strings.Join(path, "\x00") + score := paths[key] + if score.path == nil { + score.path = append([]string(nil), path...) + order = append(order, key) + } + score.count++ + paths[key] = score + } + + for _, key := range order { + score := paths[key] + if score.count > best.count { + best = score + bestKey = key + } + } + + if bestKey == "" { + return nil + } + return best.path +} + +func clusterLateAddedSkills(cluster []LearningRecord, winningPath []string) ([]string, string) { + type lateAddedScore struct { + skills []string + trigger string + count int + } + + bestKey := "" + best := lateAddedScore{} + scores := make(map[string]lateAddedScore) + order := make([]string, 0) + + for _, record := range cluster { + skills, trigger := lateAddedSkillsFromRecord(record) + if len(skills) == 0 { + continue + } + if len(winningPath) > 0 && !pathsEqual(skills, tailAddedWithinWinningPath(winningPath, skills)) { + continue + } + + key := trigger + "\x00" + strings.Join(skills, "\x00") + score := scores[key] + if score.skills == nil { + score.skills = append([]string(nil), skills...) + score.trigger = trigger + order = append(order, key) + } + score.count++ + scores[key] = score + } + + for _, key := range order { + score := scores[key] + if score.count > best.count { + bestKey = key + best = score + } + } + + if bestKey == "" { + return nil, "" + } + return best.skills, best.trigger +} + +func lateAddedSkillsFromRecord(record LearningRecord) ([]string, string) { + if skills := normalizePath(record.AddedSkillNames); len(skills) > 0 { + return skills, "loaded_during_task" + } + if record.AttemptTrail == nil || len(record.AttemptTrail.SkillContextSnapshots) == 0 { + return nil, "" + } + + snapshots := record.AttemptTrail.SkillContextSnapshots + last := snapshots[len(snapshots)-1] + if len(last.SkillNames) == 0 { + return nil, "" + } + if len(snapshots) == 1 { + return nil, strings.TrimSpace(last.Trigger) + } + + prev := snapshots[len(snapshots)-2] + prevSet := make(map[string]struct{}, len(prev.SkillNames)) + for _, skill := range normalizePath(prev.SkillNames) { + prevSet[skill] = struct{}{} + } + + added := make([]string, 0, len(last.SkillNames)) + for _, skill := range normalizePath(last.SkillNames) { + if _, ok := prevSet[skill]; ok { + continue + } + added = append(added, skill) + } + if len(added) == 0 { + return nil, strings.TrimSpace(last.Trigger) + } + return added, strings.TrimSpace(last.Trigger) +} + +func tailAddedWithinWinningPath(winningPath, lateAdded []string) []string { + if len(winningPath) == 0 || len(lateAdded) == 0 || len(lateAdded) > len(winningPath) { + return nil + } + tail := winningPath[len(winningPath)-len(lateAdded):] + if !pathsEqual(tail, lateAdded) { + return nil + } + return tail +} + +func pathsEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func collectRecordIDs(cluster []LearningRecord) []string { + ids := make([]string, 0, len(cluster)) + for _, record := range cluster { + ids = append(ids, record.ID) + } + return ids +} + +func computeMaturityScore(caseCount int, successRate float64) float64 { + return float64(caseCount) * successRate +} + +func stableRuleID(workspaceID, key string) string { + sum := sha1.Sum([]byte(workspaceID + "\x00" + key)) + return "rule-" + hex.EncodeToString(sum[:6]) +} + +func buildRuleSummary(cluster []LearningRecord, key string, winningPath []string) string { + if goal := representativeGoal(cluster); goal != "" && len(winningPath) > 0 { + return goal + " via " + strings.Join(winningPath, " -> ") + } + if goal := representativeGoal(cluster); goal != "" { + return goal + } + if len(winningPath) > 0 { + return strings.Join(winningPath, " -> ") + } + return key +} + +func representativeGoal(cluster []LearningRecord) string { + for _, record := range cluster { + if goal := strings.TrimSpace(record.UserGoal); goal != "" { + return goal + } + } + for _, record := range cluster { + if summary := strings.TrimSpace(record.Summary); summary != "" { + return summary + } + } + return "" +} diff --git a/pkg/evolution/organizer_test.go b/pkg/evolution/organizer_test.go new file mode 100644 index 000000000..397bfa5be --- /dev/null +++ b/pkg/evolution/organizer_test.go @@ -0,0 +1,310 @@ +package evolution_test + +import ( + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestOrganizer_BuildRulesCreatesRuleRecord(t *testing.T) { + ok := true + cases := []evolution.LearningRecord{ + { + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather shanghai", + Status: evolution.RecordStatus("new"), + Success: &ok, + ActiveSkillNames: []string{"weather"}, + }, + { + ID: "case-2", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather beijing", + Status: evolution.RecordStatus("new"), + Success: &ok, + ActiveSkillNames: []string{"weather"}, + }, + { + ID: "case-3", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather hangzhou", + Status: evolution.RecordStatus("new"), + Success: &ok, + ActiveSkillNames: []string{"weather"}, + }, + } + + org := evolution.NewOrganizer(evolution.OrganizerOptions{ + MinCaseCount: 3, + MinSuccessRate: 0.7, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + }) + + rules, err := org.BuildRules(cases) + if err != nil { + t.Fatalf("BuildRules: %v", err) + } + if len(rules) != 1 { + t.Fatalf("len(rules) = %d, want 1", len(rules)) + } + + rule := rules[0] + if rule.Kind != evolution.RecordKindRule { + t.Fatalf("Kind = %q, want %q", rule.Kind, evolution.RecordKindRule) + } + if rule.EventCount != 3 { + t.Fatalf("EventCount = %d, want 3", rule.EventCount) + } + if len(rule.SourceRecordIDs) != 3 { + t.Fatalf("SourceRecordIDs = %v", rule.SourceRecordIDs) + } + if rule.MaturityScore <= 0 { + t.Fatalf("MaturityScore = %v, want > 0", rule.MaturityScore) + } + if len(rule.WinningPath) != 1 || rule.WinningPath[0] != "weather" { + t.Fatalf("WinningPath = %v, want [weather]", rule.WinningPath) + } +} + +func TestOrganizer_BuildRulesSkipsImmatureCluster(t *testing.T) { + ok := true + cases := []evolution.LearningRecord{ + { + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "release build linux", + Status: evolution.RecordStatus("new"), + Success: &ok, + }, + } + + org := evolution.NewOrganizer(evolution.OrganizerOptions{ + MinCaseCount: 3, + MinSuccessRate: 0.7, + }) + + rules, err := org.BuildRules(cases) + if err != nil { + t.Fatalf("BuildRules: %v", err) + } + if len(rules) != 0 { + t.Fatalf("len(rules) = %d, want 0", len(rules)) + } +} + +func TestOrganizer_BuildRulesPrefersFinalSuccessfulPathFromAttemptTrail(t *testing.T) { + ok := true + cases := []evolution.LearningRecord{ + { + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather shanghai", + Status: evolution.RecordStatus("new"), + Success: &ok, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"geocode", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + }, + ActiveSkillNames: []string{"geocode", "weather"}, + }, + { + ID: "case-2", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather beijing", + Status: evolution.RecordStatus("new"), + Success: &ok, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"browser", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + }, + ActiveSkillNames: []string{"browser", "weather"}, + }, + { + ID: "case-3", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather hangzhou", + Status: evolution.RecordStatus("new"), + Success: &ok, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"maps", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + }, + ActiveSkillNames: []string{"maps", "weather"}, + }, + } + + org := evolution.NewOrganizer(evolution.OrganizerOptions{ + MinCaseCount: 3, + MinSuccessRate: 0.7, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + }) + + rules, err := org.BuildRules(cases) + if err != nil { + t.Fatalf("BuildRules: %v", err) + } + if len(rules) != 1 { + t.Fatalf("len(rules) = %d, want 1", len(rules)) + } + if got := rules[0].WinningPath; len(got) != 2 || got[0] != "geocode" || got[1] != "weather" { + t.Fatalf("WinningPath = %v, want [geocode weather]", got) + } +} + +func TestOrganizer_BuildRulesCapturesLateAddedSkillHintFromSnapshots(t *testing.T) { + ok := true + cases := []evolution.LearningRecord{ + { + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather shanghai", + Status: evolution.RecordStatus("new"), + Success: &ok, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"geocode", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + SkillContextSnapshots: []evolution.SkillContextSnapshot{ + {Sequence: 1, Trigger: "initial_build", SkillNames: []string{"geocode"}}, + {Sequence: 2, Trigger: "context_retry_rebuild", SkillNames: []string{"geocode", "weather"}}, + }, + }, + }, + { + ID: "case-2", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather beijing", + Status: evolution.RecordStatus("new"), + Success: &ok, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"browser", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + SkillContextSnapshots: []evolution.SkillContextSnapshot{ + {Sequence: 1, Trigger: "initial_build", SkillNames: []string{"geocode"}}, + {Sequence: 2, Trigger: "context_retry_rebuild", SkillNames: []string{"geocode", "weather"}}, + }, + }, + }, + { + ID: "case-3", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather hangzhou", + Status: evolution.RecordStatus("new"), + Success: &ok, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"maps", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + SkillContextSnapshots: []evolution.SkillContextSnapshot{ + {Sequence: 1, Trigger: "initial_build", SkillNames: []string{"geocode"}}, + {Sequence: 2, Trigger: "context_retry_rebuild", SkillNames: []string{"geocode", "weather"}}, + }, + }, + }, + } + + org := evolution.NewOrganizer(evolution.OrganizerOptions{ + MinCaseCount: 3, + MinSuccessRate: 0.7, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + }) + + rules, err := org.BuildRules(cases) + if err != nil { + t.Fatalf("BuildRules: %v", err) + } + if len(rules) != 1 { + t.Fatalf("len(rules) = %d, want 1", len(rules)) + } + if got := rules[0].LateAddedSkills; len(got) != 1 || got[0] != "weather" { + t.Fatalf("LateAddedSkills = %v, want [weather]", got) + } + if got := rules[0].FinalSnapshotTrigger; got != "context_retry_rebuild" { + t.Fatalf("FinalSnapshotTrigger = %q, want context_retry_rebuild", got) + } +} + +func TestOrganizer_BuildRulesUsesAddedSkillNamesWithoutSnapshots(t *testing.T) { + ok := true + cases := []evolution.LearningRecord{ + { + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather shanghai", + UserGoal: "check weather in shanghai", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"geocode", "weather"}, + AddedSkillNames: []string{"weather"}, + }, + { + ID: "case-2", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather beijing", + UserGoal: "check weather in beijing", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"geocode", "weather"}, + AddedSkillNames: []string{"weather"}, + }, + { + ID: "case-3", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather hangzhou", + UserGoal: "check weather in hangzhou", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"geocode", "weather"}, + AddedSkillNames: []string{"weather"}, + }, + } + + org := evolution.NewOrganizer(evolution.OrganizerOptions{ + MinCaseCount: 3, + MinSuccessRate: 0.7, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + }) + + rules, err := org.BuildRules(cases) + if err != nil { + t.Fatalf("BuildRules: %v", err) + } + if len(rules) != 1 { + t.Fatalf("len(rules) = %d, want 1", len(rules)) + } + if got := rules[0].WinningPath; len(got) != 2 || got[0] != "geocode" || got[1] != "weather" { + t.Fatalf("WinningPath = %v, want [geocode weather]", got) + } + if got := rules[0].LateAddedSkills; len(got) != 1 || got[0] != "weather" { + t.Fatalf("LateAddedSkills = %v, want [weather]", got) + } + if got := rules[0].FinalSnapshotTrigger; got != "loaded_during_task" { + t.Fatalf("FinalSnapshotTrigger = %q, want loaded_during_task", got) + } +} diff --git a/pkg/evolution/paths.go b/pkg/evolution/paths.go new file mode 100644 index 000000000..631d5cd93 --- /dev/null +++ b/pkg/evolution/paths.go @@ -0,0 +1,35 @@ +package evolution + +import ( + "path/filepath" + "strings" +) + +type Paths struct { + Workspace string + RootDir string + LearningRecords string + TaskRecords string + PatternRecords string + SkillDrafts string + ProfilesDir string + BackupsDir string +} + +func NewPaths(workspace, override string) Paths { + root := strings.TrimSpace(override) + if root == "" { + root = filepath.Join(workspace, "state", "evolution") + } + + return Paths{ + Workspace: workspace, + RootDir: root, + LearningRecords: filepath.Join(root, "learning-records.jsonl"), + TaskRecords: filepath.Join(root, "task-records.jsonl"), + PatternRecords: filepath.Join(root, "pattern-records.jsonl"), + SkillDrafts: filepath.Join(root, "skill-drafts.json"), + ProfilesDir: filepath.Join(root, "profiles"), + BackupsDir: filepath.Join(root, "backups"), + } +} diff --git a/pkg/evolution/paths_test.go b/pkg/evolution/paths_test.go new file mode 100644 index 000000000..309ff012d --- /dev/null +++ b/pkg/evolution/paths_test.go @@ -0,0 +1,86 @@ +package evolution + +import ( + "path/filepath" + "testing" +) + +func TestNewPaths_DefaultRoot(t *testing.T) { + workspace := "/tmp/workspace" + + paths := NewPaths(workspace, "") + + wantRoot := filepath.Join(workspace, "state", "evolution") + if paths.RootDir != wantRoot { + t.Fatalf("RootDir = %q, want %q", paths.RootDir, wantRoot) + } + if paths.LearningRecords != filepath.Join(wantRoot, "learning-records.jsonl") { + t.Fatalf("LearningRecords = %q", paths.LearningRecords) + } + if paths.TaskRecords != filepath.Join(wantRoot, "task-records.jsonl") { + t.Fatalf("TaskRecords = %q", paths.TaskRecords) + } + if paths.PatternRecords != filepath.Join(wantRoot, "pattern-records.jsonl") { + t.Fatalf("PatternRecords = %q", paths.PatternRecords) + } + if paths.SkillDrafts != filepath.Join(wantRoot, "skill-drafts.json") { + t.Fatalf("SkillDrafts = %q", paths.SkillDrafts) + } + if paths.ProfilesDir != filepath.Join(wantRoot, "profiles") { + t.Fatalf("ProfilesDir = %q", paths.ProfilesDir) + } + if paths.BackupsDir != filepath.Join(wantRoot, "backups") { + t.Fatalf("BackupsDir = %q", paths.BackupsDir) + } +} + +func TestNewPaths_UsesOverride(t *testing.T) { + workspace := "/tmp/workspace" + override := "/tmp/custom-evolution" + + paths := NewPaths(workspace, override) + + if paths.RootDir != override { + t.Fatalf("RootDir = %q, want %q", paths.RootDir, override) + } + if paths.LearningRecords != filepath.Join(override, "learning-records.jsonl") { + t.Fatalf("LearningRecords = %q", paths.LearningRecords) + } + if paths.TaskRecords != filepath.Join(override, "task-records.jsonl") { + t.Fatalf("TaskRecords = %q", paths.TaskRecords) + } + if paths.PatternRecords != filepath.Join(override, "pattern-records.jsonl") { + t.Fatalf("PatternRecords = %q", paths.PatternRecords) + } + if paths.SkillDrafts != filepath.Join(override, "skill-drafts.json") { + t.Fatalf("SkillDrafts = %q", paths.SkillDrafts) + } + if paths.ProfilesDir != filepath.Join(override, "profiles") { + t.Fatalf("ProfilesDir = %q", paths.ProfilesDir) + } + if paths.BackupsDir != filepath.Join(override, "backups") { + t.Fatalf("BackupsDir = %q", paths.BackupsDir) + } +} + +func TestNewPaths_BlankOverrideFallsBackToDefaultRoot(t *testing.T) { + workspace := "/tmp/workspace" + + paths := NewPaths(workspace, " \t\n ") + + wantRoot := filepath.Join(workspace, "state", "evolution") + if paths.RootDir != wantRoot { + t.Fatalf("RootDir = %q, want %q", paths.RootDir, wantRoot) + } +} + +func TestNewPaths_TrimmedOverrideIsUsed(t *testing.T) { + workspace := "/tmp/workspace" + override := " /tmp/custom-evolution " + + paths := NewPaths(workspace, override) + + if paths.RootDir != "/tmp/custom-evolution" { + t.Fatalf("RootDir = %q, want %q", paths.RootDir, "/tmp/custom-evolution") + } +} diff --git a/pkg/evolution/pattern_clusterer.go b/pkg/evolution/pattern_clusterer.go new file mode 100644 index 000000000..b167c7432 --- /dev/null +++ b/pkg/evolution/pattern_clusterer.go @@ -0,0 +1,732 @@ +package evolution + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + "unicode" + + "github.com/sipeed/picoclaw/pkg/providers" +) + +type PatternClusterer interface { + BuildPatterns( + ctx context.Context, + workspace string, + tasks []LearningRecord, + existing []LearningRecord, + ) ([]LearningRecord, []string, error) +} + +type evidencePatternClusterer interface { + BuildPatternsWithEvidence( + ctx context.Context, + workspace string, + successfulTasks []LearningRecord, + evidenceTasks []LearningRecord, + existing []LearningRecord, + minSuccessRatio float64, + ) ([]LearningRecord, []string, error) +} + +type HeuristicPatternClusterer struct { + minCaseCount int + now func() time.Time +} + +func NewHeuristicPatternClusterer(minCaseCount int, now func() time.Time) *HeuristicPatternClusterer { + if minCaseCount <= 0 { + minCaseCount = 3 + } + if now == nil { + now = time.Now + } + return &HeuristicPatternClusterer{minCaseCount: minCaseCount, now: now} +} + +func (c *HeuristicPatternClusterer) BuildPatterns( + _ context.Context, + workspace string, + tasks []LearningRecord, + existing []LearningRecord, +) ([]LearningRecord, []string, error) { + groups := make(map[string][]LearningRecord) + keys := make([]string, 0) + for _, task := range tasks { + if task.WorkspaceID != workspace { + continue + } + key := heuristicClusterKey(task) + if key == "" { + continue + } + if _, ok := groups[key]; !ok { + keys = append(keys, key) + } + groups[key] = append(groups[key], task) + } + sort.Strings(keys) + + existingByLabel := patternsByLabel(existing, workspace) + patterns := make([]LearningRecord, 0, len(keys)) + clusteredIDs := make([]string, 0) + for _, key := range keys { + cluster := groups[key] + label := heuristicClusterLabelForGroup(key, cluster) + if label == "" { + continue + } + existingPattern, hasExisting := existingByLabel[label] + if !hasExisting && len(cluster) < c.minCaseCount { + continue + } + pattern := buildPatternFromCluster( + workspace, + label, + heuristicClusterSummary(label, cluster), + "heuristic cluster by normalized task summary", + cluster, + existingPattern, + c.now(), + ) + patterns = append(patterns, pattern) + clusteredIDs = append(clusteredIDs, collectRecordIDs(cluster)...) + } + return patterns, clusteredIDs, nil +} + +type LLMPatternClusterer struct { + provider providers.LLMProvider + model string + fallback PatternClusterer + minCount int + now func() time.Time +} + +type llmClusterResponse struct { + Clusters []llmCluster `json:"clusters"` +} + +type llmCluster struct { + Label string `json:"label"` + Summary string `json:"summary"` + TaskRecordIDs []string `json:"task_record_ids"` + Reason string `json:"cluster_reason"` +} + +func NewLLMPatternClusterer( + provider providers.LLMProvider, + model string, + fallback PatternClusterer, + minCount int, + now func() time.Time, +) *LLMPatternClusterer { + if fallback == nil { + fallback = NewHeuristicPatternClusterer(minCount, now) + } + if minCount <= 0 { + minCount = 3 + } + if now == nil { + now = time.Now + } + return &LLMPatternClusterer{ + provider: provider, + model: strings.TrimSpace(model), + fallback: fallback, + minCount: minCount, + now: now, + } +} + +func (c *LLMPatternClusterer) BuildPatterns( + ctx context.Context, + workspace string, + tasks []LearningRecord, + existing []LearningRecord, +) ([]LearningRecord, []string, error) { + if c == nil { + return NewHeuristicPatternClusterer(0, nil).BuildPatterns(ctx, workspace, tasks, existing) + } + fallback := c.fallback + if fallback == nil { + fallback = NewHeuristicPatternClusterer(c.minCount, c.now) + } + if c.provider == nil { + return fallback.BuildPatterns(ctx, workspace, tasks, existing) + } + model := strings.TrimSpace(c.model) + if model == "" { + model = strings.TrimSpace(c.provider.GetDefaultModel()) + } + if model == "" { + return fallback.BuildPatterns(ctx, workspace, tasks, existing) + } + + callCtx, cancel := withLLMCallTimeout(ctx, llmPatternClusterTimeout) + defer cancel() + resp, err := c.provider.Chat(callCtx, []providers.Message{ + { + Role: "system", + Content: "Cluster agent task records by task meaning. Return exactly one JSON object with clusters:[{label,summary,task_record_ids,cluster_reason}]. No markdown fences.", + }, + { + Role: "user", + Content: buildPatternClusterPrompt(workspace, tasks, existing), + }, + }, nil, model, map[string]any{"temperature": 0}) + if err != nil || resp == nil || strings.TrimSpace(resp.Content) == "" { + return fallback.BuildPatterns(ctx, workspace, tasks, existing) + } + + payload, ok := parseLLMClusterResponse(resp.Content) + if !ok { + return fallback.BuildPatterns(ctx, workspace, tasks, existing) + } + patterns, clusteredIDs := c.validateAndBuildPatterns(workspace, payload.Clusters, tasks, existing) + if len(patterns) == 0 { + return fallback.BuildPatterns(ctx, workspace, tasks, existing) + } + return patterns, clusteredIDs, nil +} + +func (c *LLMPatternClusterer) BuildPatternsWithEvidence( + ctx context.Context, + workspace string, + successfulTasks []LearningRecord, + evidenceTasks []LearningRecord, + existing []LearningRecord, + minSuccessRatio float64, +) ([]LearningRecord, []string, error) { + if c == nil { + return NewHeuristicPatternClusterer(0, nil).BuildPatterns(ctx, workspace, successfulTasks, existing) + } + fallback := c.fallback + if fallback == nil { + fallback = NewHeuristicPatternClusterer(c.minCount, c.now) + } + if c.provider == nil { + return buildFallbackPatternsWithEvidence( + ctx, + fallback, + workspace, + successfulTasks, + evidenceTasks, + existing, + minSuccessRatio, + ) + } + model := strings.TrimSpace(c.model) + if model == "" { + model = strings.TrimSpace(c.provider.GetDefaultModel()) + } + if model == "" { + return buildFallbackPatternsWithEvidence( + ctx, + fallback, + workspace, + successfulTasks, + evidenceTasks, + existing, + minSuccessRatio, + ) + } + if len(evidenceTasks) == 0 { + evidenceTasks = successfulTasks + } + + callCtx, cancel := withLLMCallTimeout(ctx, llmPatternClusterTimeout) + defer cancel() + resp, err := c.provider.Chat(callCtx, []providers.Message{ + { + Role: "system", + Content: "Cluster agent task records by task meaning. Include successful and failed task IDs in the same cluster when they share the same reusable meaning. Return exactly one JSON object with clusters:[{label,summary,task_record_ids,cluster_reason}]. No markdown fences.", + }, + { + Role: "user", + Content: buildPatternClusterPrompt(workspace, evidenceTasks, existing), + }, + }, nil, model, map[string]any{"temperature": 0}) + if err != nil || resp == nil || strings.TrimSpace(resp.Content) == "" { + return buildFallbackPatternsWithEvidence( + ctx, + fallback, + workspace, + successfulTasks, + evidenceTasks, + existing, + minSuccessRatio, + ) + } + + payload, ok := parseLLMClusterResponse(resp.Content) + if !ok { + return buildFallbackPatternsWithEvidence( + ctx, + fallback, + workspace, + successfulTasks, + evidenceTasks, + existing, + minSuccessRatio, + ) + } + if len(payload.Clusters) == 0 { + return buildFallbackPatternsWithEvidence( + ctx, + fallback, + workspace, + successfulTasks, + evidenceTasks, + existing, + minSuccessRatio, + ) + } + patterns, clusteredIDs := c.validateAndBuildPatternsWithEvidence( + workspace, + payload.Clusters, + successfulTasks, + evidenceTasks, + existing, + minSuccessRatio, + ) + return patterns, clusteredIDs, nil +} + +func buildFallbackPatternsWithEvidence( + ctx context.Context, + fallback PatternClusterer, + workspace string, + successfulTasks []LearningRecord, + evidenceTasks []LearningRecord, + existing []LearningRecord, + minSuccessRatio float64, +) ([]LearningRecord, []string, error) { + if fallback == nil { + fallback = NewHeuristicPatternClusterer(0, nil) + } + patterns, _, err := fallback.BuildPatterns(ctx, workspace, successfulTasks, existing) + if err != nil || len(patterns) == 0 { + return patterns, nil, err + } + if len(evidenceTasks) == 0 { + evidenceTasks = successfulTasks + } + + successByID := make(map[string]LearningRecord, len(successfulTasks)) + for _, task := range successfulTasks { + successByID[task.ID] = task + } + evidenceByKey := make(map[string][]LearningRecord) + for _, task := range evidenceTasks { + if task.WorkspaceID != workspace { + continue + } + key := heuristicClusterKey(task) + if key == "" { + continue + } + evidenceByKey[key] = append(evidenceByKey[key], task) + } + + filteredPatterns := make([]LearningRecord, 0, len(patterns)) + clusteredIDs := make([]string, 0) + for _, pattern := range patterns { + keys := make(map[string]struct{}) + for _, id := range pattern.TaskRecordIDs { + task, ok := successByID[id] + if !ok { + continue + } + key := heuristicClusterKey(task) + if key == "" { + continue + } + keys[key] = struct{}{} + } + + clusterEvidenceByID := make(map[string]LearningRecord) + for key := range keys { + for _, task := range evidenceByKey[key] { + clusterEvidenceByID[task.ID] = task + } + } + if len(clusterEvidenceByID) == 0 { + for _, id := range pattern.TaskRecordIDs { + if task, ok := successByID[id]; ok { + clusterEvidenceByID[task.ID] = task + } + } + } + if len(clusterEvidenceByID) == 0 { + continue + } + + successes := 0 + clusterEvidence := make([]LearningRecord, 0, len(clusterEvidenceByID)) + for _, task := range clusterEvidenceByID { + clusterEvidence = append(clusterEvidence, task) + if task.Success != nil && *task.Success { + successes++ + } + } + sort.Slice(clusterEvidence, func(i, j int) bool { + leftSuccess := clusterEvidence[i].Success != nil && *clusterEvidence[i].Success + rightSuccess := clusterEvidence[j].Success != nil && *clusterEvidence[j].Success + if leftSuccess != rightSuccess { + return leftSuccess + } + return clusterEvidence[i].ID < clusterEvidence[j].ID + }) + if successes == 0 { + continue + } + if minSuccessRatio > 0 { + ratio := float64(successes) / float64(len(clusterEvidence)) + if ratio < minSuccessRatio { + continue + } + } + + filteredPatterns = append(filteredPatterns, pattern) + clusteredIDs = append(clusteredIDs, collectRecordIDs(clusterEvidence)...) + } + return filteredPatterns, appendUniqueStrings(nil, clusteredIDs...), nil +} + +func (c *LLMPatternClusterer) validateAndBuildPatterns( + workspace string, + clusters []llmCluster, + tasks []LearningRecord, + existing []LearningRecord, +) ([]LearningRecord, []string) { + taskByID := make(map[string]LearningRecord, len(tasks)) + for _, task := range tasks { + taskByID[task.ID] = task + } + existingByLabel := patternsByLabel(existing, workspace) + assigned := make(map[string]struct{}, len(tasks)) + patterns := make([]LearningRecord, 0, len(clusters)) + clusteredIDs := make([]string, 0) + + for _, cluster := range clusters { + label := validSkillNameOrEmpty(cluster.Label) + if label == "" { + continue + } + clusterTasks := make([]LearningRecord, 0, len(cluster.TaskRecordIDs)) + for _, id := range cluster.TaskRecordIDs { + id = strings.TrimSpace(id) + if id == "" { + continue + } + if _, exists := assigned[id]; exists { + continue + } + task, ok := taskByID[id] + if !ok { + continue + } + clusterTasks = append(clusterTasks, task) + assigned[id] = struct{}{} + } + existingPattern, hasExisting := existingByLabel[label] + if !hasExisting && len(clusterTasks) < c.minCount { + continue + } + if len(clusterTasks) == 0 { + continue + } + pattern := buildPatternFromCluster( + workspace, + label, + cluster.Summary, + cluster.Reason, + clusterTasks, + existingPattern, + c.now(), + ) + patterns = append(patterns, pattern) + clusteredIDs = append(clusteredIDs, collectRecordIDs(clusterTasks)...) + } + return patterns, clusteredIDs +} + +func (c *LLMPatternClusterer) validateAndBuildPatternsWithEvidence( + workspace string, + clusters []llmCluster, + successfulTasks []LearningRecord, + evidenceTasks []LearningRecord, + existing []LearningRecord, + minSuccessRatio float64, +) ([]LearningRecord, []string) { + evidenceByID := make(map[string]LearningRecord, len(evidenceTasks)) + for _, task := range evidenceTasks { + evidenceByID[task.ID] = task + } + successfulByID := make(map[string]LearningRecord, len(successfulTasks)) + for _, task := range successfulTasks { + successfulByID[task.ID] = task + } + existingByLabel := patternsByLabel(existing, workspace) + assigned := make(map[string]struct{}, len(evidenceTasks)) + patterns := make([]LearningRecord, 0, len(clusters)) + clusteredIDs := make([]string, 0) + + for _, cluster := range clusters { + label := validSkillNameOrEmpty(cluster.Label) + if label == "" { + continue + } + clusterEvidence := make([]LearningRecord, 0, len(cluster.TaskRecordIDs)) + clusterSuccesses := make([]LearningRecord, 0, len(cluster.TaskRecordIDs)) + for _, id := range cluster.TaskRecordIDs { + id = strings.TrimSpace(id) + if id == "" { + continue + } + if _, exists := assigned[id]; exists { + continue + } + task, ok := evidenceByID[id] + if !ok { + continue + } + clusterEvidence = append(clusterEvidence, task) + if successTask, ok := successfulByID[id]; ok { + clusterSuccesses = append(clusterSuccesses, successTask) + } + assigned[id] = struct{}{} + } + if len(clusterEvidence) == 0 || len(clusterSuccesses) == 0 { + continue + } + if minSuccessRatio > 0 { + ratio := float64(len(clusterSuccesses)) / float64(len(clusterEvidence)) + if ratio < minSuccessRatio { + continue + } + } + existingPattern, hasExisting := existingByLabel[label] + if !hasExisting && len(clusterSuccesses) < c.minCount { + continue + } + pattern := buildPatternFromCluster( + workspace, + label, + cluster.Summary, + cluster.Reason, + clusterSuccesses, + existingPattern, + c.now(), + ) + patterns = append(patterns, pattern) + clusteredIDs = append(clusteredIDs, collectRecordIDs(clusterEvidence)...) + } + if len(assigned) != len(evidenceByID) { + return nil, nil + } + return patterns, clusteredIDs +} + +func parseLLMClusterResponse(content string) (llmClusterResponse, bool) { + normalized := strings.TrimSpace(content) + normalized = strings.TrimPrefix(normalized, "```json") + normalized = strings.TrimPrefix(normalized, "```") + normalized = strings.TrimSuffix(normalized, "```") + normalized = strings.TrimSpace(normalized) + var payload llmClusterResponse + if err := json.Unmarshal([]byte(normalized), &payload); err != nil { + return llmClusterResponse{}, false + } + return payload, true +} + +func buildPatternClusterPrompt(workspace string, tasks []LearningRecord, existing []LearningRecord) string { + type taskPayload struct { + ID string `json:"id"` + Summary string `json:"summary"` + FinalOutputExcerpt string `json:"final_output_excerpt"` + Success *bool `json:"success,omitempty"` + } + type patternPayload struct { + Label string `json:"label"` + Summary string `json:"summary"` + } + payload := struct { + Instruction string `json:"instruction"` + ExistingPatterns []patternPayload `json:"existing_patterns,omitempty"` + Tasks []taskPayload `json:"tasks"` + }{ + Instruction: "Group tasks that have the same reusable task meaning. Use existing pattern labels when they fit. Labels must be lowercase hyphenated and must not include concrete values.", + } + for _, pattern := range existing { + if pattern.WorkspaceID != workspace { + continue + } + if strings.TrimSpace(pattern.Label) == "" { + continue + } + payload.ExistingPatterns = append(payload.ExistingPatterns, patternPayload{ + Label: strings.TrimSpace(pattern.Label), + Summary: strings.TrimSpace(pattern.Summary), + }) + } + for _, task := range tasks { + payload.Tasks = append(payload.Tasks, taskPayload{ + ID: task.ID, + Summary: task.Summary, + FinalOutputExcerpt: summarizeText(task.FinalOutput, 800), + Success: task.Success, + }) + } + data, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return fmt.Sprintf("tasks: %d", len(tasks)) + } + return string(data) +} + +func buildPatternFromCluster( + workspace, label, summary, reason string, + tasks []LearningRecord, + existing LearningRecord, + now time.Time, +) LearningRecord { + taskIDs := append([]string(nil), existing.TaskRecordIDs...) + taskIDs = appendUniqueStrings(taskIDs, collectRecordIDs(tasks)...) + if summary = strings.TrimSpace(summary); summary == "" { + summary = labelSummary(label) + } + pattern := existing + if strings.TrimSpace(pattern.ID) == "" { + pattern = LearningRecord{ + ID: stableRuleID(workspace, label), + Kind: RecordKindPattern, + WorkspaceID: workspace, + CreatedAt: now, + Status: RecordStatus("ready"), + } + } else { + updatedAt := now + pattern.UpdatedAt = &updatedAt + } + pattern.Label = label + pattern.Summary = summary + pattern.TaskRecordIDs = taskIDs + pattern.ClusterReason = strings.TrimSpace(reason) + pattern.Status = RecordStatus("ready") + pattern.Source = nil + pattern.SourceRecordIDs = nil + pattern.EventCount = 0 + pattern.SuccessRate = 0 + pattern.MaturityScore = 0 + pattern.WinningPath = nil + pattern.LateAddedSkills = nil + pattern.FinalSnapshotTrigger = "" + pattern.MatchedSkillNames = nil + return pattern +} + +func patternsByLabel(patterns []LearningRecord, workspace string) map[string]LearningRecord { + out := make(map[string]LearningRecord, len(patterns)) + for _, pattern := range patterns { + if pattern.WorkspaceID != workspace { + continue + } + label := strings.TrimSpace(pattern.Label) + if label == "" { + label = validSkillNameOrEmpty(pattern.Summary) + } + if label == "" { + continue + } + out[label] = pattern + } + return out +} + +func heuristicClusterLabel(record LearningRecord) string { + if label := heuristicASCIIClusterLabel(record.Summary); label != "" { + return label + } + if normalized := normalizeUnicodeTaskSummary(record.Summary); normalized != "" { + return hashedTaskLabel(normalized) + } + return "" +} + +func heuristicClusterKey(record LearningRecord) string { + if label := heuristicASCIIClusterLabel(record.Summary); label != "" { + return "ascii:" + label + } + if normalized := normalizeUnicodeTaskSummary(record.Summary); normalized != "" { + return "unicode:" + hashedTaskLabel(normalized) + } + return "" +} + +func heuristicClusterLabelForGroup(key string, cluster []LearningRecord) string { + if strings.HasPrefix(key, "ascii:") || strings.HasPrefix(key, "unicode:") { + return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(key, "ascii:"), "unicode:")) + } + for _, record := range cluster { + if label := heuristicClusterLabel(record); label != "" { + return label + } + } + return "" +} + +func heuristicClusterSummary(label string, cluster []LearningRecord) string { + for _, record := range cluster { + if summary := strings.TrimSpace(record.Summary); summary != "" { + return summary + } + } + return labelSummary(label) +} + +func heuristicASCIIClusterLabel(summary string) string { + tokens := tokenizeForEvolution(summary) + out := make([]string, 0, len(tokens)) + for _, token := range tokens { + if isNumericToken(token) { + continue + } + out = append(out, token) + if len(out) >= 5 { + break + } + } + return validSkillNameOrEmpty(strings.Join(out, "-")) +} + +func normalizeUnicodeTaskSummary(summary string) string { + var b strings.Builder + for _, r := range strings.ToLower(strings.TrimSpace(summary)) { + if unicode.IsDigit(r) || unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r) { + continue + } + b.WriteRune(r) + } + return b.String() +} + +func hashedTaskLabel(value string) string { + sum := sha1.Sum([]byte(value)) + return "task-" + hex.EncodeToString(sum[:4]) +} + +func labelSummary(label string) string { + label = strings.ReplaceAll(strings.TrimSpace(label), "-", " ") + if label == "" { + return "Learned task pattern." + } + return strings.ToUpper(label[:1]) + label[1:] + "." +} diff --git a/pkg/evolution/pattern_clusterer_test.go b/pkg/evolution/pattern_clusterer_test.go new file mode 100644 index 000000000..0e0c91128 --- /dev/null +++ b/pkg/evolution/pattern_clusterer_test.go @@ -0,0 +1,402 @@ +package evolution_test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/evolution" + "github.com/sipeed/picoclaw/pkg/providers" +) + +type llmClusterTestProvider struct { + content string + defaultModel string + messages []providers.Message +} + +func (p *llmClusterTestProvider) Chat( + _ context.Context, + messages []providers.Message, + _ []providers.ToolDefinition, + _ string, + _ map[string]any, +) (*providers.LLMResponse, error) { + p.messages = append([]providers.Message(nil), messages...) + return &providers.LLMResponse{Content: p.content}, nil +} + +func (p *llmClusterTestProvider) GetDefaultModel() string { + return p.defaultModel +} + +func TestHeuristicPatternClusterer_GroupsChineseSummariesWithoutLLM(t *testing.T) { + clusterer := evolution.NewHeuristicPatternClusterer(3, func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + success := true + tasks := []evolution.LearningRecord{ + { + ID: "task-1", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace", + Summary: "调用三一定理计算100", + FinalOutput: "100 + 31 = 131; 131 + 42 = 173; 173 - 53 = 120", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + { + ID: "task-2", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace", + Summary: "调用三一定理计算200", + FinalOutput: "200 + 31 = 231; 231 + 42 = 273; 273 - 53 = 220", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + { + ID: "task-3", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace", + Summary: "调用三一定理计算300", + FinalOutput: "300 + 31 = 331; 331 + 42 = 373; 373 - 53 = 320", + Status: evolution.RecordStatus("new"), + Success: &success, + UsedSkillNames: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + }, + } + + patterns, clusteredIDs, err := clusterer.BuildPatterns(context.Background(), "workspace", tasks, nil) + if err != nil { + t.Fatalf("BuildPatterns: %v", err) + } + if len(patterns) != 1 { + t.Fatalf("len(patterns) = %d, want 1: %#v", len(patterns), patterns) + } + if !strings.HasPrefix(patterns[0].Label, "task-") { + t.Fatalf("Label = %q, want task-* fallback label", patterns[0].Label) + } + if patterns[0].Summary != "调用三一定理计算100" { + t.Fatalf("Summary = %q, want representative Chinese summary", patterns[0].Summary) + } + if len(patterns[0].TaskRecordIDs) != 3 { + t.Fatalf("TaskRecordIDs = %v, want 3 ids", patterns[0].TaskRecordIDs) + } + if len(clusteredIDs) != 3 { + t.Fatalf("clusteredIDs = %v, want 3 ids", clusteredIDs) + } +} + +func TestLLMPatternClusterer_FallsBackWhenLLMReturnsNoUsableClusters(t *testing.T) { + fallback := evolution.NewHeuristicPatternClusterer(2, func() time.Time { + return time.Unix(1700000000, 0).UTC() + }) + clusterer := evolution.NewLLMPatternClusterer( + &llmClusterTestProvider{content: `{"clusters":[]}`, defaultModel: "test-model"}, + "test-model", + fallback, + 2, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + success := true + tasks := []evolution.LearningRecord{ + { + ID: "task-1", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace", + Summary: "调用三一定理计算100", + FinalOutput: "100 + 31 = 131", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + { + ID: "task-2", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace", + Summary: "调用三一定理计算200", + FinalOutput: "200 + 31 = 231", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + } + + patterns, clusteredIDs, err := clusterer.BuildPatterns(context.Background(), "workspace", tasks, nil) + if err != nil { + t.Fatalf("BuildPatterns: %v", err) + } + if len(patterns) != 1 { + t.Fatalf("len(patterns) = %d, want fallback pattern: %#v", len(patterns), patterns) + } + if len(clusteredIDs) != 2 { + t.Fatalf("clusteredIDs = %v, want 2 task IDs", clusteredIDs) + } +} + +func TestLLMPatternClusterer_PromptFiltersExistingPatternsByWorkspace(t *testing.T) { + provider := &llmClusterTestProvider{ + content: `{"clusters":[{"label":"current-weather-path","summary":"current summary","task_record_ids":["task-1"],"cluster_reason":"same goal"}]}`, + defaultModel: "test-model", + } + clusterer := evolution.NewLLMPatternClusterer( + provider, + "test-model", + evolution.NewHeuristicPatternClusterer(1, nil), + 1, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + success := true + tasks := []evolution.LearningRecord{ + { + ID: "task-1", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: "weather lookup", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + } + existing := []evolution.LearningRecord{ + { + ID: "rule-a", + Kind: evolution.RecordKindPattern, + WorkspaceID: "workspace-a", + Label: "current-weather-path", + Summary: "current workspace pattern", + }, + { + ID: "rule-b", + Kind: evolution.RecordKindPattern, + WorkspaceID: "workspace-b", + Label: "other-workspace-secret-path", + Summary: "other workspace pattern", + }, + } + + if _, _, err := clusterer.BuildPatterns(context.Background(), "workspace-a", tasks, existing); err != nil { + t.Fatalf("BuildPatterns: %v", err) + } + if len(provider.messages) != 2 { + t.Fatalf("len(messages) = %d, want 2", len(provider.messages)) + } + prompt := provider.messages[1].Content + if !strings.Contains(prompt, "current-weather-path") { + t.Fatalf("prompt = %q, want current workspace pattern", prompt) + } + if strings.Contains(prompt, "other-workspace-secret-path") || strings.Contains(prompt, "other workspace pattern") { + t.Fatalf("prompt leaked other workspace pattern: %s", prompt) + } +} + +func TestLLMPatternClusterer_RejectsClusterBelowEvidenceSuccessRatio(t *testing.T) { + provider := &llmClusterTestProvider{ + content: `{"clusters":[{"label":"weather-lookup","summary":"lookup weather","task_record_ids":["task-success","task-failed"],"cluster_reason":"same weather lookup goal"}]}`, + defaultModel: "test-model", + } + clusterer := evolution.NewLLMPatternClusterer( + provider, + "test-model", + evolution.NewHeuristicPatternClusterer(1, nil), + 1, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + success := true + failed := false + successfulTasks := []evolution.LearningRecord{ + { + ID: "task-success", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: "weather lookup shanghai", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + } + evidenceTasks := []evolution.LearningRecord{ + successfulTasks[0], + { + ID: "task-failed", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: "forecast for shanghai", + FinalOutput: "could not complete", + Status: evolution.RecordStatus("new"), + Success: &failed, + }, + } + + patterns, clusteredIDs, err := clusterer.BuildPatternsWithEvidence( + context.Background(), + "workspace-a", + successfulTasks, + evidenceTasks, + nil, + 0.8, + ) + if err != nil { + t.Fatalf("BuildPatternsWithEvidence: %v", err) + } + if len(patterns) != 0 { + t.Fatalf("len(patterns) = %d, want 0: %#v", len(patterns), patterns) + } + if len(clusteredIDs) != 0 { + t.Fatalf("clusteredIDs = %v, want none", clusteredIDs) + } + prompt := provider.messages[1].Content + if !strings.Contains(prompt, `"success": true`) || !strings.Contains(prompt, `"success": false`) { + t.Fatalf("prompt should include success and failure evidence:\n%s", prompt) + } +} + +func TestLLMPatternClusterer_RejectsIncompleteEvidenceAssignment(t *testing.T) { + provider := &llmClusterTestProvider{ + content: `{"clusters":[{"label":"weather-lookup","summary":"lookup weather","task_record_ids":["task-success"],"cluster_reason":"same weather lookup goal"}]}`, + defaultModel: "test-model", + } + clusterer := evolution.NewLLMPatternClusterer( + provider, + "test-model", + evolution.NewHeuristicPatternClusterer(1, nil), + 1, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + success := true + failed := false + successfulTasks := []evolution.LearningRecord{ + { + ID: "task-success", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: "weather lookup shanghai", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + } + evidenceTasks := []evolution.LearningRecord{ + successfulTasks[0], + { + ID: "task-failed", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: "forecast for shanghai", + FinalOutput: "could not complete", + Status: evolution.RecordStatus("new"), + Success: &failed, + }, + } + + patterns, clusteredIDs, err := clusterer.BuildPatternsWithEvidence( + context.Background(), + "workspace-a", + successfulTasks, + evidenceTasks, + nil, + 0.8, + ) + if err != nil { + t.Fatalf("BuildPatternsWithEvidence: %v", err) + } + if len(patterns) != 0 { + t.Fatalf("len(patterns) = %d, want 0: %#v", len(patterns), patterns) + } + if len(clusteredIDs) != 0 { + t.Fatalf("clusteredIDs = %v, want none", clusteredIDs) + } +} + +func TestLLMPatternClusterer_MarksAllAcceptedEvidenceClusteredButStoresSuccessfulTaskIDs(t *testing.T) { + provider := &llmClusterTestProvider{ + content: `{"clusters":[{"label":"weather-lookup","summary":"lookup weather","task_record_ids":["task-success","task-failed"],"cluster_reason":"same weather lookup goal"}]}`, + defaultModel: "test-model", + } + assertClustererMarksAllAcceptedEvidenceClustered( + t, + provider, + "weather lookup shanghai", + "forecast for shanghai", + "could not complete", + "1", + ) +} + +func TestLLMPatternClusterer_FallbackMarksAllAcceptedEvidenceClustered(t *testing.T) { + provider := &llmClusterTestProvider{ + content: `not-json`, + defaultModel: "test-model", + } + assertClustererMarksAllAcceptedEvidenceClustered( + t, + provider, + "weather lookup 100", + "weather lookup 200", + "partial result", + "fallback pattern", + ) +} + +func assertClustererMarksAllAcceptedEvidenceClustered( + t *testing.T, + provider *llmClusterTestProvider, + successSummary string, + failedSummary string, + failedOutput string, + wantPatternDescription string, +) { + t.Helper() + clusterer := evolution.NewLLMPatternClusterer( + provider, + "test-model", + evolution.NewHeuristicPatternClusterer(1, nil), + 1, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + success := true + failed := false + successfulTasks := []evolution.LearningRecord{ + { + ID: "task-success", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: successSummary, + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &success, + }, + } + evidenceTasks := []evolution.LearningRecord{ + successfulTasks[0], + { + ID: "task-failed", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + Summary: failedSummary, + FinalOutput: failedOutput, + Status: evolution.RecordStatus("new"), + Success: &failed, + }, + } + + patterns, clusteredIDs, err := clusterer.BuildPatternsWithEvidence( + context.Background(), + "workspace-a", + successfulTasks, + evidenceTasks, + nil, + 0.5, + ) + if err != nil { + t.Fatalf("BuildPatternsWithEvidence: %v", err) + } + if len(patterns) != 1 { + t.Fatalf("len(patterns) = %d, want %s: %#v", len(patterns), wantPatternDescription, patterns) + } + if got := strings.Join(patterns[0].TaskRecordIDs, ","); got != "task-success" { + t.Fatalf("pattern TaskRecordIDs = %v, want only successful task", patterns[0].TaskRecordIDs) + } + if got := strings.Join(clusteredIDs, ","); got != "task-success,task-failed" { + t.Fatalf("clusteredIDs = %v, want all accepted evidence IDs", clusteredIDs) + } +} diff --git a/pkg/evolution/preview.go b/pkg/evolution/preview.go new file mode 100644 index 000000000..ee2774136 --- /dev/null +++ b/pkg/evolution/preview.go @@ -0,0 +1,154 @@ +package evolution + +import ( + "os" + "path/filepath" + "strconv" + "strings" +) + +type DraftPreview struct { + CurrentBody string + RenderedBody string + DiffPreview string +} + +func BuildDraftPreview(workspace string, draft SkillDraft) (DraftPreview, error) { + currentBody, hadOriginal, err := loadCurrentSkillBody(workspace, draft.TargetSkillName) + if err != nil { + return DraftPreview{}, err + } + + renderedBody, err := renderAppliedBody(draft, currentBody, hadOriginal) + if err != nil { + return DraftPreview{}, err + } + + return DraftPreview{ + CurrentBody: currentBody, + RenderedBody: renderedBody, + DiffPreview: buildLineDiffPreview(currentBody, renderedBody), + }, nil +} + +func loadCurrentSkillBody(workspace, skillName string) (string, bool, error) { + skillPath := filepath.Join(workspace, "skills", skillName, "SKILL.md") + data, err := os.ReadFile(skillPath) + if os.IsNotExist(err) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return string(data), true, nil +} + +func buildLineDiffPreview(currentBody, renderedBody string) string { + before := strings.Split(strings.TrimRight(currentBody, "\n"), "\n") + after := strings.Split(strings.TrimRight(renderedBody, "\n"), "\n") + + if len(before) == 1 && before[0] == "" { + before = nil + } + if len(after) == 1 && after[0] == "" { + after = nil + } + + prefixLen := sharedPrefixLen(before, after) + suffixLen := sharedSuffixLen(before[prefixLen:], after[prefixLen:]) + const contextRadius = 2 + + beforeChangeStart := prefixLen + beforeChangeEnd := len(before) - suffixLen + afterChangeStart := prefixLen + afterChangeEnd := len(after) - suffixLen + + hunkBeforeStart := previewMaxInt(0, beforeChangeStart-contextRadius) + hunkAfterStart := previewMaxInt(0, afterChangeStart-contextRadius) + hunkBeforeEnd := previewMinInt(len(before), beforeChangeEnd+contextRadius) + hunkAfterEnd := previewMinInt(len(after), afterChangeEnd+contextRadius) + + removed := before[prefixLen : len(before)-suffixLen] + added := after[prefixLen : len(after)-suffixLen] + if len(removed) == 0 && len(added) == 0 { + return "(no content change)" + } + + lines := make([]string, 0, (hunkBeforeEnd-hunkBeforeStart)+(hunkAfterEnd-hunkAfterStart)) + header := make([]string, 0, 3+len(lines)) + header = append(header, + "--- current", + "+++ rendered", + formatUnifiedHunkHeader( + hunkBeforeStart, + hunkBeforeEnd-hunkBeforeStart, + hunkAfterStart, + hunkAfterEnd-hunkAfterStart, + ), + ) + for _, line := range before[hunkBeforeStart:beforeChangeStart] { + lines = append(lines, " "+line) + } + for _, line := range removed { + lines = append(lines, "-"+line) + } + for _, line := range added { + lines = append(lines, "+"+line) + } + for _, line := range after[afterChangeEnd:hunkAfterEnd] { + lines = append(lines, " "+line) + } + return strings.Join(append(header, lines...), "\n") +} + +func formatUnifiedHunkHeader(beforeStart, beforeCount, afterStart, afterCount int) string { + return "@@ -" + formatUnifiedRange( + beforeStart+1, + beforeCount, + ) + " +" + formatUnifiedRange( + afterStart+1, + afterCount, + ) + " @@" +} + +func formatUnifiedRange(start, count int) string { + return strconv.Itoa(start) + "," + strconv.Itoa(count) +} + +func sharedPrefixLen(left, right []string) int { + limit := len(left) + if len(right) < limit { + limit = len(right) + } + n := 0 + for n < limit && left[n] == right[n] { + n++ + } + return n +} + +func sharedSuffixLen(left, right []string) int { + limit := len(left) + if len(right) < limit { + limit = len(right) + } + n := 0 + for n < limit && left[len(left)-1-n] == right[len(right)-1-n] { + n++ + } + return n +} + +func previewMinInt(a, b int) int { + if a < b { + return a + } + return b +} + +func previewMaxInt(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/pkg/evolution/preview_test.go b/pkg/evolution/preview_test.go new file mode 100644 index 000000000..e6f0a9ce5 --- /dev/null +++ b/pkg/evolution/preview_test.go @@ -0,0 +1,111 @@ +package evolution + +import ( + "strings" + "testing" +) + +func TestBuildLineDiffPreview_UsesUnifiedDiffStyle(t *testing.T) { + current := strings.Join([]string{ + "---", + "name: weather", + "description: weather helper", + "---", + "# Weather", + "## Start Here", + "Use city names first.", + "", + }, "\n") + rendered := strings.Join([]string{ + "---", + "name: weather", + "description: weather helper", + "---", + "# Weather", + "## Start Here", + "Use city names first.", + "", + "## Start Here", + "Use native-name query first.", + "", + }, "\n") + + diff := buildLineDiffPreview(current, rendered) + + for _, want := range []string{ + "--- current", + "+++ rendered", + "@@", + "+## Start Here", + "+Use native-name query first.", + } { + if !strings.Contains(diff, want) { + t.Fatalf("diff missing %q:\n%s", want, diff) + } + } +} + +func TestBuildLineDiffPreview_NoContentChange(t *testing.T) { + body := "---\nname: weather\n---\n# Weather\n" + diff := buildLineDiffPreview(body, body) + if diff != "(no content change)" { + t.Fatalf("diff = %q, want no-content marker", diff) + } +} + +func TestBuildLineDiffPreview_LimitsContextAroundChanges(t *testing.T) { + current := strings.Join([]string{ + "line-01", + "line-02", + "line-03", + "line-04", + "line-05", + "line-06", + "line-07", + "line-08", + "line-09", + "line-10", + "", + }, "\n") + rendered := strings.Join([]string{ + "line-01", + "line-02", + "line-03", + "line-04", + "line-05", + "line-06", + "inserted-a", + "inserted-b", + "line-07", + "line-08", + "line-09", + "line-10", + "", + }, "\n") + + diff := buildLineDiffPreview(current, rendered) + + for _, want := range []string{ + "@@", + " line-05", + " line-06", + "+inserted-a", + "+inserted-b", + " line-07", + " line-08", + } { + if !strings.Contains(diff, want) { + t.Fatalf("diff missing %q:\n%s", want, diff) + } + } + for _, unwanted := range []string{ + "line-01", + "line-02", + "line-09", + "line-10", + } { + if strings.Contains(diff, unwanted) { + t.Fatalf("diff should omit distant context %q:\n%s", unwanted, diff) + } + } +} diff --git a/pkg/evolution/profile_sync.go b/pkg/evolution/profile_sync.go new file mode 100644 index 000000000..1499a6d44 --- /dev/null +++ b/pkg/evolution/profile_sync.go @@ -0,0 +1,75 @@ +package evolution + +import ( + "strings" + "time" +) + +func SaveAppliedProfile(store *Store, workspace string, draft SkillDraft, now time.Time) error { + return store.UpdateProfile(workspace, draft.TargetSkillName, func(profile *SkillProfile, exists bool) error { + if !exists { + *profile = SkillProfile{ + SkillName: draft.TargetSkillName, + WorkspaceID: workspace, + Origin: "evolved", + } + } + + profile.SkillName = draft.TargetSkillName + profile.WorkspaceID = workspace + profile.CurrentVersion = draft.ID + profile.Status = SkillStatusActive + profile.Origin = profileOrigin(profile.Origin) + profile.HumanSummary = draft.HumanSummary + profile.ChangeReason = draft.HumanSummary + profile.IntendedUseCases = append([]string(nil), draft.IntendedUseCases...) + profile.PreferredEntryPath = append([]string(nil), draft.PreferredEntryPath...) + profile.AvoidPatterns = append([]string(nil), draft.AvoidPatterns...) + profile.LastUsedAt = now + if profile.RetentionScore <= 0 { + profile.RetentionScore = 1 + } + profile.VersionHistory = append(profile.VersionHistory, SkillVersionEntry{ + Version: draft.ID, + Action: string(draft.ChangeKind), + Timestamp: now, + DraftID: draft.ID, + Summary: draft.HumanSummary, + }) + return nil + }) +} + +func inferIntendedUseCases(rule LearningRecord) []string { + summary := strings.TrimSpace(rule.Summary) + if summary == "" { + return nil + } + return []string{summary} +} + +func inferPreferredEntryPath(rule LearningRecord) []string { + if len(rule.WinningPath) == 0 { + return nil + } + return append([]string(nil), rule.WinningPath...) +} + +func inferAvoidPatterns(rule LearningRecord) []string { + if len(rule.LateAddedSkills) == 0 || len(rule.WinningPath) <= len(rule.LateAddedSkills) { + return nil + } + prefix := rule.WinningPath[:len(rule.WinningPath)-len(rule.LateAddedSkills)] + if len(prefix) == 0 { + return nil + } + return []string{ + "avoid starting with " + strings.Join( + prefix, + " -> ", + ) + " before using " + strings.Join( + rule.LateAddedSkills, + " -> ", + ), + } +} diff --git a/pkg/evolution/record_kinds.go b/pkg/evolution/record_kinds.go new file mode 100644 index 000000000..db9af93a3 --- /dev/null +++ b/pkg/evolution/record_kinds.go @@ -0,0 +1,9 @@ +package evolution + +func isTaskRecordKind(kind RecordKind) bool { + return kind == RecordKindTask || kind == legacyRecordKindCase +} + +func isPatternRecordKind(kind RecordKind) bool { + return kind == RecordKindPattern || kind == legacyRecordKindRule +} diff --git a/pkg/evolution/runtime.go b/pkg/evolution/runtime.go new file mode 100644 index 000000000..cc88433f4 --- /dev/null +++ b/pkg/evolution/runtime.go @@ -0,0 +1,1579 @@ +package evolution + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/skills" +) + +var ErrApplyDraftFailed = errors.New("apply draft failed") + +type RuntimeOptions struct { + Config config.EvolutionConfig + Now func() time.Time + Store *Store + Organizer *Organizer + PatternClusterer PatternClusterer + SuccessJudge SuccessJudge + SkillsRecaller *SkillsRecaller + DraftGenerator DraftGenerator + GeneratorFactory func(workspace string) DraftGenerator + SuccessJudgeFactory func(workspace string) SuccessJudge + Applier *Applier + ApplierFactory func(workspace string) *Applier +} + +type Runtime struct { + cfg config.EvolutionConfig + mu sync.Mutex + now func() time.Time + writer *CaseWriter + store *Store + organizer *Organizer + patternClusterer PatternClusterer + successJudge SuccessJudge + skillsRecaller *SkillsRecaller + draftGenerator DraftGenerator + generatorFactory func(workspace string) DraftGenerator + successJudgeFactory func(workspace string) SuccessJudge + applier *Applier + applierFactory func(workspace string) *Applier +} + +type TurnCaseInput struct { + Workspace string + WorkspaceID string + TurnID string + SessionKey string + AgentID string + Status string + UserMessage string + FinalContent string + ToolKinds []string + ToolExecutions []ToolExecutionRecord + ActiveSkillNames []string + AttemptedSkillNames []string + FinalSuccessfulPath []string + SkillContextSnapshots []SkillContextSnapshot +} + +func NewRuntime(opts RuntimeOptions) (*Runtime, error) { + now := opts.Now + if now == nil { + now = time.Now + } + + organizer := opts.Organizer + if organizer == nil { + organizer = NewOrganizer(OrganizerOptions{ + MinCaseCount: opts.Config.EffectiveMinTaskCount(), + MinSuccessRate: opts.Config.EffectiveMinSuccessRatio(), + Now: now, + }) + } + + patternClusterer := opts.PatternClusterer + if patternClusterer == nil { + patternClusterer = NewHeuristicPatternClusterer(opts.Config.EffectiveMinTaskCount(), now) + } + + return &Runtime{ + cfg: opts.Config, + now: now, + store: opts.Store, + organizer: organizer, + patternClusterer: patternClusterer, + successJudge: opts.SuccessJudge, + skillsRecaller: opts.SkillsRecaller, + draftGenerator: opts.DraftGenerator, + generatorFactory: opts.GeneratorFactory, + successJudgeFactory: opts.SuccessJudgeFactory, + applier: opts.Applier, + applierFactory: opts.ApplierFactory, + }, nil +} + +func (rt *Runtime) FinalizeTurn(ctx context.Context, input TurnCaseInput) error { + if rt == nil || !rt.cfg.Enabled || input.Workspace == "" || shouldSkipLearningRecord(input) { + return nil + } + + success := input.Status == "completed" + usedSkillNames := buildUsedSkillNames(input) + workspaceID := input.Workspace + createdAt := rt.now() + + record := LearningRecord{ + ID: buildTaskRecordID(input, createdAt), + Kind: RecordKindTask, + WorkspaceID: workspaceID, + CreatedAt: createdAt, + SessionKey: input.SessionKey, + Summary: buildRecordSummary(input), + FinalOutput: summarizeText(input.FinalContent, 1200), + Status: RecordStatus("new"), + Success: &success, + UsedSkillNames: append([]string(nil), usedSkillNames...), + } + + paths := NewPaths(input.Workspace, rt.cfg.StateDir) + + rt.mu.Lock() + if rt.writer == nil || rt.writer.paths.RootDir != paths.RootDir { + rt.writer = NewCaseWriter(paths) + } + writer := rt.writer + rt.mu.Unlock() + + if err := writer.AppendCase(ctx, record); err != nil { + return err + } + + if err := rt.recordSkillUsage(input, success); err != nil { + return err + } + + logger.DebugCF("evolution", "Recorded hot path learning record", map[string]any{ + "workspace": input.Workspace, + "turn_id": input.TurnID, + "success": success, + "used_skills": len(record.UsedSkillNames), + }) + return nil +} + +func buildTaskRecordID(input TurnCaseInput, createdAt time.Time) string { + base := strings.TrimSpace(input.TurnID) + if base == "" { + base = "turn" + } + base = validSkillNameOrEmpty(base) + if base == "" { + base = "turn" + } + seed := strings.Join([]string{ + input.Workspace, + input.SessionKey, + input.AgentID, + input.TurnID, + createdAt.UTC().Format(time.RFC3339Nano), + }, "\x00") + sum := sha1.Sum([]byte(seed)) + return base + "-" + hex.EncodeToString(sum[:6]) +} + +func buildRecordSummary(input TurnCaseInput) string { + if goal := summarizeText(input.UserMessage, 160); goal != "" { + return goal + } + return fmt.Sprintf("turn %s finished with status=%s", input.TurnID, input.Status) +} + +func summarizeText(text string, maxLen int) string { + text = strings.TrimSpace(text) + if text == "" || maxLen <= 0 { + return text + } + if utf8.RuneCountInString(text) <= maxLen { + return text + } + if maxLen <= 3 { + runes := []rune(text) + return string(runes[:maxLen]) + } + runes := []rune(text) + return string(runes[:maxLen-3]) + "..." +} + +func buildUsedSkillNames(input TurnCaseInput) []string { + if final := uniqueTrimmedNames(input.FinalSuccessfulPath); len(final) > 0 { + return final + } + out := make([]string, 0) + for _, exec := range input.ToolExecutions { + if !exec.Success { + continue + } + out = append(out, exec.SkillNames...) + } + return uniqueTrimmedNames(out) +} + +func shouldSkipLearningRecord(input TurnCaseInput) bool { + if strings.EqualFold(strings.TrimSpace(input.SessionKey), "heartbeat") { + return true + } + return false +} + +func uniqueTrimmedNames(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + key := strings.ToLower(value) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, value) + } + return out +} + +func (rt *Runtime) RunColdPathOnce(ctx context.Context, workspace string) error { + if rt == nil || !rt.cfg.Enabled || workspace == "" { + return nil + } + + mode := rt.cfg.EffectiveMode() + runID := fmt.Sprintf("%d", rt.now().UnixNano()) + if mode == "" || mode == "observe" { + logger.DebugCF("evolution", "Skipped cold path run", map[string]any{ + "workspace": workspace, + "mode": mode, + "run_id": runID, + }) + return nil + } + + logger.InfoCF("evolution", "Started cold path run", map[string]any{ + "workspace": workspace, + "mode": mode, + "run_id": runID, + }) + + store := rt.storeForWorkspace(workspace) + taskRecords, err := store.LoadTaskRecords() + if err != nil { + return err + } + patternRecords, err := store.LoadPatternRecords() + if err != nil { + return err + } + logger.DebugCF("evolution", "Loaded evolution records", map[string]any{ + "workspace": workspace, + "task_count": len(taskRecords), + "pattern_count": len(patternRecords), + "run_id": runID, + }) + + admittedCount := 0 + newRuleCount := 0 + if rt.patternClusterer != nil { + recordsForOrganizer, evidenceRecordsForOrganizer, inputErr := rt.recordsForColdPathInputs( + ctx, + workspace, + taskRecords, + ) + if inputErr != nil { + return inputErr + } + recordsForOrganizer = rt.filterRecordsByMinSuccessRatio( + workspace, + evidenceRecordsForOrganizer, + recordsForOrganizer, + ) + admittedCount = countTaskLearningRecords(recordsForOrganizer) + logger.DebugCF("evolution", "Admitted task records for cold path", map[string]any{ + "workspace": workspace, + "admitted_tasks": admittedCount, + "organizer_input": len(recordsForOrganizer), + "task_ids": joinRecordIDs(recordsForOrganizer), + "run_id": runID, + }) + var rules []LearningRecord + var clusteredTaskIDs []string + if clusterer, ok := rt.patternClusterer.(evidencePatternClusterer); ok { + rules, clusteredTaskIDs, err = clusterer.BuildPatternsWithEvidence( + ctx, + workspace, + recordsForOrganizer, + evidenceRecordsForOrganizer, + patternRecords, + rt.cfg.EffectiveMinSuccessRatio(), + ) + } else { + rules, clusteredTaskIDs, err = rt.patternClusterer.BuildPatterns( + ctx, + workspace, + recordsForOrganizer, + patternRecords, + ) + } + if err != nil { + return err + } + newRuleCount = countNewPatterns(patternRecords, rules, workspace) + logger.DebugCF("evolution", "Built learning patterns", map[string]any{ + "workspace": workspace, + "pattern_count": len(rules), + "new_patterns": newRuleCount, + "admitted_tasks": admittedCount, + "patterns": summarizePatternRecords(rules), + "run_id": runID, + }) + if len(rules) > 0 { + merged := mergePatternRecords(patternRecords, rules, workspace) + if mergeErr := store.MergePatternRecords(rules); mergeErr != nil { + return mergeErr + } + patternRecords = merged + } + if len(clusteredTaskIDs) > 0 { + if markErr := markTaskRecordsClustered(store, clusteredTaskIDs); markErr != nil { + return markErr + } + } + } + + generator := rt.draftGeneratorForWorkspace(workspace) + if generator == nil { + logger.DebugCF("evolution", "Skipped drafting because no draft generator is available", map[string]any{ + "workspace": workspace, + "run_id": runID, + }) + return rt.runLifecycleMaintenance(workspace, store, runID) + } + + recaller := rt.skillsRecallerForWorkspace(workspace) + applier := rt.applierForWorkspace(workspace) + readyRules := filterReadyRules(patternRecords, workspace) + readyRules = enrichReadyRulesForDrafts(readyRules, taskRecords) + if len(readyRules) == 0 { + logger.DebugCF("evolution", "Finished cold path run without ready patterns", map[string]any{ + "workspace": workspace, + "record_count": len(taskRecords), + "new_patterns": newRuleCount, + "admitted_tasks": admittedCount, + "run_id": runID, + }) + return rt.runLifecycleMaintenance(workspace, store, runID) + } + + existingDrafts, err := store.LoadDrafts() + if err != nil { + return err + } + readyRuleByID := make(map[string]LearningRecord, len(readyRules)) + for _, rule := range readyRules { + readyRuleByID[rule.ID] = rule + } + appliedExistingDrafts := 0 + changedExistingDrafts := false + for _, draft := range existingDrafts { + if draft.WorkspaceID != workspace || draft.Status != DraftStatusCandidate { + continue + } + rule, ok := readyRuleByID[draft.SourceRecordID] + if !ok { + logger.DebugCF( + "evolution", + "Skipped existing candidate draft because its source pattern is not ready", + map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "source_record_id": draft.SourceRecordID, + "run_id": runID, + }, + ) + continue + } + matches, recallErr := recaller.RecallSimilarSkills(rule) + if recallErr != nil { + return recallErr + } + draft.MatchedSkillRefs = collectSkillRefs(matches) + var normalizationNotes []string + evidence := draftEvidenceForRule(rule, taskRecords) + draft, normalizationNotes = rt.normalizeDraftForWorkspace(workspace, rule, matches, evidence, draft) + review := ReviewDraft(draft) + draft.Status = review.Status + draft.ReviewNotes = appendUniqueStrings(draft.ReviewNotes, append(review.ReviewNotes, normalizationNotes...)...) + draft.ScanFindings = appendUniqueStrings(draft.ScanFindings, review.Findings...) + changedExistingDrafts = true + if draft.Status != DraftStatusCandidate || mode != "apply" || applier == nil { + if saveErr := store.SaveDrafts([]SkillDraft{draft}); saveErr != nil { + return saveErr + } + continue + } + updatedDraft, applyErr := rt.applyCandidateDraft(ctx, workspace, store, applier, draft, runID) + if applyErr != nil { + return applyErr + } + if updatedDraft.Status == DraftStatusAccepted { + appliedExistingDrafts++ + changedExistingDrafts = true + } + } + if changedExistingDrafts { + existingDrafts, err = store.LoadDrafts() + if err != nil { + return err + } + } + existingBySource := existingDraftSourceSet(existingDrafts, workspace) + logger.DebugCF("evolution", "Selected ready patterns for drafting", map[string]any{ + "workspace": workspace, + "ready_patterns": len(readyRules), + "existing_draft_count": len(existingBySource), + "applied_existing": appliedExistingDrafts, + "ready_pattern_ids": joinRecordIDs(readyRules), + "ready_patterns_info": summarizePatternRecords(readyRules), + "run_id": runID, + }) + + processedRules := 0 + for _, rule := range readyRules { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if _, exists := existingBySource[rule.ID]; exists { + logger.DebugCF( + "evolution", + "Skipped pattern because a non-quarantined draft already exists", + map[string]any{ + "workspace": workspace, + "pattern_id": rule.ID, + "pattern_info": summarizePatternRecord(rule), + "run_id": runID, + }, + ) + continue + } + + evidence := draftEvidenceForRule(rule, taskRecords) + rule = enrichRuleWithDraftEvidence(rule, evidence) + matches, err := recaller.RecallSimilarSkills(rule) + if err != nil { + return err + } + logger.DebugCF("evolution", "Generating skill draft", map[string]any{ + "workspace": workspace, + "pattern_id": rule.ID, + "matched_skill_count": len(matches), + "pattern_info": summarizePatternRecord(rule), + "run_id": runID, + }) + + draft, err := generateDraftWithEvidence(ctx, generator, rule, matches, evidence) + if err != nil { + return err + } + + draft = rt.finalizeDraft(workspace, rule, matches, evidence, draft) + draftSaved := false + logger.DebugCF("evolution", "Finalized skill draft", map[string]any{ + "workspace": workspace, + "pattern_id": rule.ID, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "change_kind": string(draft.ChangeKind), + "status": string(draft.Status), + "run_id": runID, + }) + if mode == "apply" && applier != nil && draft.Status == DraftStatusCandidate { + var err error + draft, err = rt.applyCandidateDraft(ctx, workspace, store, applier, draft, runID) + if err != nil { + return err + } + draftSaved = true + } + + if !draftSaved { + if err := store.SaveDrafts([]SkillDraft{draft}); err != nil { + return err + } + } + logger.DebugCF("evolution", "Saved skill draft", map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "status": string(draft.Status), + "run_id": runID, + }) + existingBySource[rule.ID] = struct{}{} + processedRules++ + } + + logger.InfoCF("evolution", "Finished cold path run", map[string]any{ + "workspace": workspace, + "ready_patterns": len(readyRules), + "processed_patterns": processedRules, + "new_patterns": newRuleCount, + "run_id": runID, + }) + return rt.runLifecycleMaintenance(workspace, store, runID) +} + +func (rt *Runtime) recordsForColdPathInputs( + ctx context.Context, + workspace string, + records []LearningRecord, +) ([]LearningRecord, []LearningRecord, error) { + admitted := make([]LearningRecord, 0, len(records)) + evidence := make([]LearningRecord, 0, len(records)) + judge := rt.successJudgeForWorkspace(workspace) + + for _, record := range records { + if !isTaskRecordKind(record.Kind) || record.WorkspaceID != workspace { + continue + } + if reason := coldPathEvidenceRejectReason(record); reason != "" { + logger.DebugCF("evolution", "Rejected task record for cold path", map[string]any{ + "workspace": workspace, + "record_id": record.ID, + "reason": reason, + }) + continue + } + + evidenceRecord := record + if record.Success != nil && *record.Success && judge != nil { + decision, err := judge.JudgeTaskRecord(ctx, record) + if err != nil { + return nil, nil, err + } + judgedSuccess := decision.Success + evidenceRecord.Success = &judgedSuccess + if !decision.Success { + logger.DebugCF("evolution", "Rejected task record by success judge", map[string]any{ + "workspace": workspace, + "record_id": record.ID, + "reason": strings.TrimSpace(decision.Reason), + }) + } + } + evidence = append(evidence, evidenceRecord) + if evidenceRecord.Success == nil || !*evidenceRecord.Success { + continue + } + admitted = append(admitted, evidenceRecord) + } + return admitted, evidence, nil +} + +func (rt *Runtime) filterRecordsByMinSuccessRatio( + workspace string, + allRecords []LearningRecord, + admittedRecords []LearningRecord, +) []LearningRecord { + minRatio := rt.cfg.EffectiveMinSuccessRatio() + if minRatio <= 0 { + return admittedRecords + } + + type successStats struct { + success int + total int + } + statsByKey := make(map[string]successStats) + for _, record := range allRecords { + key, ok := coldPathSuccessRatioKey(workspace, record) + if !ok { + continue + } + stats := statsByKey[key] + stats.total++ + if record.Success != nil && *record.Success { + stats.success++ + } + statsByKey[key] = stats + } + + out := make([]LearningRecord, 0, len(admittedRecords)) + for _, record := range admittedRecords { + if !isTaskRecordKind(record.Kind) { + out = append(out, record) + continue + } + key, ok := coldPathSuccessRatioKey(workspace, record) + if !ok { + continue + } + stats := statsByKey[key] + if stats.total == 0 { + continue + } + ratio := float64(stats.success) / float64(stats.total) + if ratio < minRatio { + logger.DebugCF("evolution", "Rejected task record below cold path success ratio", map[string]any{ + "workspace": workspace, + "record_id": record.ID, + "success_ratio": ratio, + "min_success_ratio": minRatio, + "success_count": stats.success, + "total_count": stats.total, + }) + continue + } + out = append(out, record) + } + return out +} + +func coldPathSuccessRatioKey(workspace string, record LearningRecord) (string, bool) { + if !isTaskRecordKind(record.Kind) || record.WorkspaceID != workspace { + return "", false + } + if record.Status != "" && record.Status != RecordStatus("new") { + return "", false + } + if strings.EqualFold(strings.TrimSpace(record.SessionKey), "heartbeat") { + return "", false + } + if strings.EqualFold(strings.TrimSpace(record.FinalOutput), "HEARTBEAT_OK") { + return "", false + } + if strings.TrimSpace(record.Summary) == "" { + return "", false + } + key := heuristicClusterKey(record) + if key == "" { + return "", false + } + return key, true +} + +func coldPathEvidenceRejectReason(record LearningRecord) string { + if !isTaskRecordKind(record.Kind) { + return "not a task record" + } + if record.Success == nil { + return "task success unknown" + } + if record.Status != "" && record.Status != RecordStatus("new") { + return "task already processed" + } + if strings.EqualFold(strings.TrimSpace(record.SessionKey), "heartbeat") { + return "heartbeat session" + } + if strings.EqualFold(strings.TrimSpace(record.FinalOutput), "HEARTBEAT_OK") { + return "heartbeat output" + } + if strings.TrimSpace(record.Summary) == "" { + return "missing summary" + } + if strings.TrimSpace(record.FinalOutput) == "" { + return "missing final output" + } + return "" +} + +func (rt *Runtime) storeForWorkspace(workspace string) *Store { + paths := NewPaths(workspace, rt.cfg.StateDir) + if rt.store != nil && rt.store.paths.RootDir == paths.RootDir && rt.store.paths.Workspace == paths.Workspace { + return rt.store + } + return NewStore(paths) +} + +func (rt *Runtime) skillsRecallerForWorkspace(workspace string) *SkillsRecaller { + rt.mu.Lock() + defer rt.mu.Unlock() + + if rt.skillsRecaller == nil || rt.skillsRecaller.workspace != workspace { + rt.skillsRecaller = NewSkillsRecaller(workspace) + } + return rt.skillsRecaller +} + +func (rt *Runtime) draftGeneratorForWorkspace(workspace string) DraftGenerator { + if rt.generatorFactory != nil { + if generator := rt.generatorFactory(workspace); generator != nil { + return generator + } + } + if rt.draftGenerator != nil { + return rt.draftGenerator + } + return NewDefaultDraftGenerator(workspace) +} + +func (rt *Runtime) successJudgeForWorkspace(workspace string) SuccessJudge { + if rt.successJudgeFactory != nil { + if judge := rt.successJudgeFactory(workspace); judge != nil { + return judge + } + } + if rt.successJudge != nil { + return rt.successJudge + } + return &HeuristicSuccessJudge{} +} + +func (rt *Runtime) applierForWorkspace(workspace string) *Applier { + if rt.applierFactory != nil { + if applier := rt.applierFactory(workspace); applier != nil { + return applier + } + } + return rt.applier +} + +func (rt *Runtime) finalizeDraft( + workspace string, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, + draft SkillDraft, +) SkillDraft { + if draft.ID == "" { + draft.ID = "draft-" + rule.ID + } + if draft.CreatedAt.IsZero() { + draft.CreatedAt = rt.now() + } + draft.WorkspaceID = workspace + draft.SourceRecordID = rule.ID + draft.MatchedSkillRefs = collectSkillRefs(matches) + + draft, normalizationNotes := rt.normalizeDraftForWorkspace(workspace, rule, matches, evidence, draft) + review := ReviewDraft(draft) + draft.Status = review.Status + draft.ReviewNotes = append([]string(nil), review.ReviewNotes...) + draft.ReviewNotes = append(draft.ReviewNotes, normalizationNotes...) + if len(review.Findings) == 0 { + draft.ScanFindings = nil + return draft + } + draft.ScanFindings = append([]string(nil), review.Findings...) + return draft +} + +func (rt *Runtime) normalizeDraftForWorkspace( + workspace string, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, + draft SkillDraft, +) (SkillDraft, []string) { + target := strings.TrimSpace(draft.TargetSkillName) + if workspace == "" || target == "" { + return draft, nil + } + + notes := make([]string, 0, 4) + if combinedTarget := inferCombinedSkillName(rule); combinedTarget != "" && combinedTarget != target { + originalTarget := target + draft.TargetSkillName = combinedTarget + target = combinedTarget + notes = append(notes, fmt.Sprintf( + "retargeted draft from %q to combined shortcut skill %q because the winning path was a stable multi-skill chain", + originalTarget, + combinedTarget, + )) + } + + skillPath := filepath.Join(workspace, "skills", target, "SKILL.md") + _, err := os.Stat(skillPath) + hasExisting := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return draft, notes + } + + if combinedTarget := inferCombinedSkillName(rule); combinedTarget != "" && combinedTarget == target { + draft.HumanSummary = buildCombinedSkillHumanSummary(target, rule, hasExisting) + draft.PreferredEntryPath = []string{target} + draft.AvoidPatterns = appendUniqueStrings( + draft.AvoidPatterns, + buildCombinedSkillAvoidPattern(target, rule), + ) + if hasExisting { + draft.ChangeKind = ChangeKindAppend + draft.BodyOrPatch = synthesizeCombinedSkillAppendBody(target, draft, rule, matches, evidence) + notes = append(notes, "normalized combined shortcut draft to append onto the existing combined skill") + } else { + draft.ChangeKind = ChangeKindCreate + draft.BodyOrPatch = synthesizeCombinedSkillDocument(target, draft, rule, matches, evidence) + notes = append(notes, "normalized combined shortcut draft to create a new standalone shortcut skill") + } + return draft, notes + } + + if !hasExisting { + switch draft.ChangeKind { + case ChangeKindAppend, ChangeKindMerge, ChangeKindReplace: + draft.ChangeKind = ChangeKindCreate + notes = append(notes, "normalized change_kind to create because target skill did not exist") + if !looksLikeSkillDocument(draft.BodyOrPatch) { + draft.BodyOrPatch = synthesizeSkillDocumentFromPartialDraft(target, draft, rule, evidence) + notes = append(notes, "synthesized full skill document because draft body was partial") + } + } + return draft, notes + } + + if draft.ChangeKind == ChangeKindCreate && !looksLikeSkillDocument(draft.BodyOrPatch) { + draft.ChangeKind = ChangeKindAppend + notes = append(notes, "normalized change_kind to append because target skill already existed") + } + return draft, notes +} + +func looksLikeSkillDocument(body string) bool { + body = strings.TrimSpace(body) + return strings.HasPrefix(body, "---\n") && strings.Contains(body, "\n# ") +} + +func synthesizeSkillDocumentFromPartialDraft( + target string, + draft SkillDraft, + rule LearningRecord, + evidence DraftEvidence, +) string { + description := strings.TrimSpace(draft.HumanSummary) + if description == "" { + description = fmt.Sprintf("Learned workflow for %s.", target) + } + + bodyContent := strings.TrimSpace(draft.BodyOrPatch) + if bodyContent == "" { + bodyContent = "No learned content was generated." + } + if strings.HasPrefix(bodyContent, "# ") { + return buildSkillDocument(target, description, bodyContent) + } + + body := strings.Join([]string{ + "# " + titleCaseSkillName(target), + "", + "## Start Here", + synthesizedStartHereLine(rule, target), + "", + "## Learned Evolution", + bodyContent, + "", + "## Expected Result", + synthesizedExpectedResultLine(evidence), + "", + "## Source Evidence", + synthesizedEvidenceLine(rule, evidence), + "", + }, "\n") + return buildSkillDocument(target, description, body) +} + +func synthesizeCombinedSkillDocument( + target string, + draft SkillDraft, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) string { + description := strings.TrimSpace(draft.HumanSummary) + if description == "" { + description = buildCombinedSkillHumanSummary(target, rule, false) + } + + body := strings.Join([]string{ + "# " + titleCaseSkillName(target), + "", + "## When To Use", + synthesizedCombinedWhenToUseLine(rule, target), + "", + "## Procedure", + synthesizedCombinedStartHereLine(rule, target), + synthesizedCombinedProcedure(matches, rule), + "", + "## Source Skills", + synthesizedComponentBreakdown(matches), + "", + "## Learned Context", + synthesizedCombinedLearnedContent(draft.BodyOrPatch, rule), + "", + "## Expected Result", + synthesizedExpectedResultLine(evidence), + "", + "## Source Evidence", + synthesizedEvidenceLine(rule, evidence), + "", + }, "\n") + return buildSkillDocument(target, description, body) +} + +func synthesizeCombinedSkillAppendBody( + target string, + draft SkillDraft, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) string { + lines := []string{ + "## Learned Shortcut Update", + fmt.Sprintf("- Shortcut skill: `%s`", target), + fmt.Sprintf("- Task summary: %s", fallbackEvolutionSummary(rule)), + fmt.Sprintf("- Wrapped path: %s", synthesizedWrappedPathLine(rule)), + "- Guidance: prefer this shortcut directly instead of replaying the whole path when the task matches.", + fmt.Sprintf("- Expected result: %s", synthesizedExpectedResultLine(evidence)), + fmt.Sprintf("- Evidence: %s", synthesizedEvidenceLine(rule, evidence)), + "", + "### Source Skills", + synthesizedComponentBreakdown(matches), + "", + synthesizedCombinedLearnedContent(draft.BodyOrPatch, rule), + "", + } + return strings.Join(lines, "\n") +} + +func synthesizedStartHereLine(rule LearningRecord, target string) string { + if len(rule.WinningPath) > 0 { + return fmt.Sprintf( + "Start with `%s` for tasks like `%s`.", + strings.Join(rule.WinningPath, " -> "), + strings.TrimSpace(rule.Summary), + ) + } + if summary := strings.TrimSpace(rule.Summary); summary != "" { + return fmt.Sprintf("Use `%s` when the task matches `%s`.", target, summary) + } + return fmt.Sprintf("Use `%s` for the learned task pattern.", target) +} + +func synthesizedCombinedStartHereLine(rule LearningRecord, target string) string { + return fmt.Sprintf("Use `%s` directly when the task matches `%s`.", target, fallbackEvolutionSummary(rule)) +} + +func synthesizedCombinedWhenToUseLine(rule LearningRecord, target string) string { + if len(rule.WinningPath) == 0 { + return fmt.Sprintf("Use `%s` when the learned task pattern appears again.", target) + } + return fmt.Sprintf( + "Use `%s` as a direct shortcut instead of replaying `%s` step by step.", + target, + strings.Join(rule.WinningPath, " -> "), + ) +} + +func synthesizedCombinedProcedure(matches []skills.SkillInfo, rule LearningRecord) string { + components := synthesizedComponentBreakdown(matches) + if !strings.HasPrefix(strings.TrimSpace(components), "- `") { + if len(rule.WinningPath) == 0 { + return "Use the learned shortcut directly and keep the response focused on the requested result." + } + return fmt.Sprintf( + "Apply the recorded path `%s`, then return the final result with only the necessary explanation.", + strings.Join(rule.WinningPath, " -> "), + ) + } + return "Follow the source skill guidance below as one compact procedure, then return the final result without replaying unnecessary discovery steps." +} + +func synthesizedExpectedResultLine(evidence DraftEvidence) string { + if excerpt := firstFinalOutputExcerpt(evidence, 360); excerpt != "" { + return excerpt + } + return "Return the completed result for the matched task without restating unrelated discovery steps." +} + +func synthesizedEvidenceLine(rule LearningRecord, evidence DraftEvidence) string { + if len(evidence.TaskRecords) > 0 { + ids := make([]string, 0, len(evidence.TaskRecords)) + for _, task := range evidence.TaskRecords { + if id := strings.TrimSpace(task.ID); id != "" { + ids = append(ids, id) + } + } + if len(ids) > 0 { + return "learned from task records: " + strings.Join(ids, ", ") + } + } + if len(rule.TaskRecordIDs) > 0 { + return "learned from task records: " + strings.Join(rule.TaskRecordIDs, ", ") + } + return "learned from the pattern record." +} + +func synthesizedWrappedPathLine(rule LearningRecord) string { + if len(rule.WinningPath) == 0 { + return "No explicit wrapped path was recorded." + } + return strings.Join(rule.WinningPath, " -> ") +} + +func synthesizedCombinedLearnedContent(body string, rule LearningRecord) string { + content := strings.TrimSpace(stripSkillFrontmatter(body)) + if content == "" { + return fmt.Sprintf( + "Learned from `%s`; use this shortcut directly when the same task pattern appears again.", + fallbackEvolutionSummary(rule), + ) + } + content = removeVerboseCombinedSections(content) + content = strings.Join(strings.Fields(content), " ") + if content == "" { + return fmt.Sprintf( + "Learned from `%s`; use this shortcut directly when the same task pattern appears again.", + fallbackEvolutionSummary(rule), + ) + } + content = trimAtReadableBoundary(content, 1200) + return "- Learned task: " + fallbackEvolutionSummary(rule) + "\n- Reusable guidance: " + content +} + +func stripSkillFrontmatter(body string) string { + trimmed := strings.TrimSpace(body) + if !strings.HasPrefix(trimmed, "---\n") { + return trimmed + } + rest := strings.TrimPrefix(trimmed, "---\n") + end := strings.Index(rest, "\n---\n") + if end < 0 { + return trimmed + } + return strings.TrimSpace(rest[end+5:]) +} + +func removeVerboseCombinedSections(content string) string { + lines := strings.Split(content, "\n") + out := make([]string, 0, len(lines)) + skip := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") { + title := strings.TrimSpace(strings.TrimLeft(trimmed, "#")) + normalized := strings.ToLower(title) + switch normalized { + case "component skill breakdown", "source skills", "wrapped path", "start here", "when to use", "procedure": + skip = true + continue + default: + skip = false + } + } + if skip { + continue + } + out = append(out, line) + } + return strings.TrimSpace(strings.Join(out, "\n")) +} + +func fallbackEvolutionSummary(rule LearningRecord) string { + if summary := strings.TrimSpace(rule.Summary); summary != "" { + return summary + } + if len(rule.WinningPath) > 0 { + return strings.Join(rule.WinningPath, " -> ") + } + return "the learned task pattern" +} + +func buildCombinedSkillHumanSummary(target string, rule LearningRecord, hasExisting bool) string { + _ = hasExisting + summary := fallbackEvolutionSummary(rule) + if strings.TrimSpace(summary) == "" || summary == "the learned task pattern" { + summary = titleCaseSkillName(target) + } + return fmt.Sprintf("Use this skill to %s when the task requires this workflow.", sentenceFragment(summary)) +} + +func buildCombinedSkillAvoidPattern(target string, rule LearningRecord) string { + if len(rule.WinningPath) == 0 { + return fmt.Sprintf("avoid bypassing `%s` when the same learned task pattern appears again", target) + } + return fmt.Sprintf("avoid replaying %s before trying `%s` directly", strings.Join(rule.WinningPath, " -> "), target) +} + +func collectSkillRefs(matches []skills.SkillInfo) []string { + if len(matches) == 0 { + return nil + } + + refs := make([]string, 0, len(matches)) + for _, match := range matches { + if strings := match.Path; strings != "" { + refs = append(refs, strings) + continue + } + refs = append(refs, match.Source+":"+match.Name) + } + return refs +} + +func countTaskLearningRecords(records []LearningRecord) int { + count := 0 + for _, record := range records { + if isTaskRecordKind(record.Kind) { + count++ + } + } + return count +} + +func (rt *Runtime) runLifecycleMaintenance(workspace string, store *Store, runID string) error { + if rt == nil || store == nil || workspace == "" { + return nil + } + + paths := NewPaths(workspace, rt.cfg.StateDir) + logger.DebugCF("evolution", "Started lifecycle maintenance", map[string]any{ + "workspace": workspace, + "run_id": runID, + }) + + summary, err := RunLifecycleOnce(store, paths, workspace, rt.now()) + if err != nil { + logger.WarnCF("evolution", "Lifecycle maintenance failed", map[string]any{ + "workspace": workspace, + "run_id": runID, + "error": err.Error(), + }) + return err + } + + logger.DebugCF("evolution", "Finished lifecycle maintenance", map[string]any{ + "workspace": workspace, + "run_id": runID, + "evaluated_profiles": summary.EvaluatedProfiles, + "transitioned_profiles": summary.TransitionedProfiles, + "deleted_skills": summary.DeletedSkills, + }) + return nil +} + +func joinRecordIDs(records []LearningRecord) string { + if len(records) == 0 { + return "" + } + ids := make([]string, 0, len(records)) + for _, record := range records { + if strings.TrimSpace(record.ID) == "" { + continue + } + ids = append(ids, record.ID) + } + return strings.Join(ids, ",") +} + +func summarizePatternRecords(records []LearningRecord) string { + if len(records) == 0 { + return "" + } + parts := make([]string, 0, len(records)) + for _, record := range records { + parts = append(parts, summarizePatternRecord(record)) + } + return strings.Join(parts, " | ") +} + +func summarizePatternRecord(record LearningRecord) string { + label := strings.TrimSpace(record.ID) + if label == "" { + label = "unknown-pattern" + } + + path := strings.Join(record.WinningPath, " -> ") + if path == "" { + path = strings.TrimSpace(record.Summary) + } + if path == "" { + path = "no-summary" + } + + return fmt.Sprintf("%s[%s]", label, path) +} + +func enrichReadyRulesForDrafts(rules, taskRecords []LearningRecord) []LearningRecord { + if len(rules) == 0 || len(taskRecords) == 0 { + return rules + } + out := make([]LearningRecord, 0, len(rules)) + for _, rule := range rules { + evidence := draftEvidenceForRule(rule, taskRecords) + out = append(out, enrichRuleWithDraftEvidence(rule, evidence)) + } + return out +} + +func draftEvidenceForRule(rule LearningRecord, taskRecords []LearningRecord) DraftEvidence { + if len(rule.TaskRecordIDs) == 0 || len(taskRecords) == 0 { + return DraftEvidence{} + } + idSet := make(map[string]struct{}, len(rule.TaskRecordIDs)) + for _, id := range rule.TaskRecordIDs { + id = strings.TrimSpace(id) + if id == "" { + continue + } + idSet[id] = struct{}{} + } + if len(idSet) == 0 { + return DraftEvidence{} + } + tasks := make([]LearningRecord, 0, len(idSet)) + for _, task := range taskRecords { + if rule.WorkspaceID != "" && task.WorkspaceID != rule.WorkspaceID { + continue + } + if _, ok := idSet[task.ID]; !ok { + continue + } + tasks = append(tasks, task) + } + return DraftEvidence{TaskRecords: tasks} +} + +func generateDraftWithEvidence( + ctx context.Context, + generator DraftGenerator, + rule LearningRecord, + matches []skills.SkillInfo, + evidence DraftEvidence, +) (SkillDraft, error) { + if generator == nil { + return SkillDraft{}, nil + } + if evidenceAware, ok := generator.(EvidenceAwareDraftGenerator); ok { + return evidenceAware.GenerateDraftWithEvidence(ctx, rule, matches, evidence) + } + return generator.GenerateDraft(ctx, rule, matches) +} + +func countNewPatterns(existing, patterns []LearningRecord, workspace string) int { + existingIDs := make(map[string]struct{}, len(existing)) + for _, pattern := range existing { + if !isPatternRecordKind(pattern.Kind) || pattern.WorkspaceID != workspace { + continue + } + existingIDs[pattern.ID] = struct{}{} + } + count := 0 + for _, pattern := range patterns { + if pattern.WorkspaceID != workspace { + continue + } + if _, ok := existingIDs[pattern.ID]; ok { + continue + } + count++ + } + return count +} + +func mergePatternRecords(existing, updates []LearningRecord, workspace string) []LearningRecord { + out := append([]LearningRecord(nil), existing...) + indexByID := make(map[string]int, len(out)) + for i, pattern := range out { + indexByID[pattern.ID] = i + } + for _, update := range updates { + if update.WorkspaceID != workspace { + continue + } + if idx, ok := indexByID[update.ID]; ok { + out[idx] = update + continue + } + indexByID[update.ID] = len(out) + out = append(out, update) + } + return out +} + +func markTaskRecordsClustered(store *Store, ids []string) error { + if store == nil || len(ids) == 0 { + return nil + } + return store.MarkTaskRecordsClustered(ids) +} + +func filterReadyRules(records []LearningRecord, workspace string) []LearningRecord { + seen := make(map[string]LearningRecord) + for _, record := range records { + if !isPatternRecordKind(record.Kind) || record.WorkspaceID != workspace || + record.Status != RecordStatus("ready") { + continue + } + seen[record.ID] = record + } + + out := make([]LearningRecord, 0, len(seen)) + for _, record := range seen { + out = append(out, record) + } + sort.Slice(out, func(i, j int) bool { + if !out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].CreatedAt.Before(out[j].CreatedAt) + } + return out[i].ID < out[j].ID + }) + return out +} + +func existingDraftSourceSet(drafts []SkillDraft, workspace string) map[string]struct{} { + out := make(map[string]struct{}, len(drafts)) + for _, draft := range drafts { + if draft.WorkspaceID != workspace || draft.SourceRecordID == "" { + continue + } + if draft.Status == DraftStatusQuarantined { + continue + } + out[draft.SourceRecordID] = struct{}{} + } + return out +} + +func (rt *Runtime) saveAppliedProfile(store *Store, workspace string, draft SkillDraft) error { + now := rt.now() + + return SaveAppliedProfile(store, workspace, draft, now) +} + +func (rt *Runtime) applyCandidateDraft( + ctx context.Context, + workspace string, + store *Store, + applier *Applier, + draft SkillDraft, + runID string, +) (SkillDraft, error) { + logger.InfoCF("evolution", "Applying skill draft", map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "change_kind": string(draft.ChangeKind), + "run_id": runID, + }) + rollbackApply, err := applier.applyDraftWithRollback(ctx, workspace, draft) + if err != nil { + logger.WarnCF("evolution", "Skill draft apply failed", map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "error": err.Error(), + "run_id": runID, + }) + draft.Status = DraftStatusQuarantined + draft.ScanFindings = appendUniqueStrings(draft.ScanFindings, fmt.Sprintf("apply failed: %v", err)) + if auditErr := rt.recordRollbackAudit(store, draft, err); auditErr != nil { + draft.ScanFindings = appendUniqueStrings( + draft.ScanFindings, + fmt.Sprintf("rollback audit failed: %v", auditErr), + ) + if saveErr := store.SaveDrafts([]SkillDraft{draft}); saveErr != nil { + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, err), auditErr, saveErr) + } + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, err), auditErr) + } + if saveErr := store.SaveDrafts([]SkillDraft{draft}); saveErr != nil { + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, err), saveErr) + } + return draft, fmt.Errorf("%w: %v", ErrApplyDraftFailed, err) + } + + draft.Status = DraftStatusAccepted + if saveErr := store.SaveDrafts([]SkillDraft{draft}); saveErr != nil { + logger.WarnCF("evolution", "Skill draft save failed after apply", map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "error": saveErr.Error(), + "run_id": runID, + }) + if rollbackErr := rollbackApply(); rollbackErr != nil { + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, saveErr), rollbackErr) + } + return draft, fmt.Errorf("%w: %v", ErrApplyDraftFailed, saveErr) + } + + if err := rt.saveAppliedProfile(store, workspace, draft); err != nil { + logger.WarnCF("evolution", "Skill profile save failed after apply", map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "error": err.Error(), + "run_id": runID, + }) + draft.Status = DraftStatusQuarantined + draft.ScanFindings = appendUniqueStrings(draft.ScanFindings, fmt.Sprintf("profile save failed: %v", err)) + if rollbackErr := rollbackApply(); rollbackErr != nil { + draft.ScanFindings = appendUniqueStrings( + draft.ScanFindings, + fmt.Sprintf("apply rollback failed: %v", rollbackErr), + ) + if saveErr := store.SaveDrafts([]SkillDraft{draft}); saveErr != nil { + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, err), rollbackErr, saveErr) + } + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, err), rollbackErr) + } + if saveErr := store.SaveDrafts([]SkillDraft{draft}); saveErr != nil { + return draft, errorsJoin(fmt.Errorf("%w: %v", ErrApplyDraftFailed, err), saveErr) + } + return draft, fmt.Errorf("%w: %v", ErrApplyDraftFailed, err) + } + logger.InfoCF("evolution", "Applied skill draft successfully", map[string]any{ + "workspace": workspace, + "draft_id": draft.ID, + "target_skill": draft.TargetSkillName, + "run_id": runID, + }) + return draft, nil +} + +func (rt *Runtime) recordRollbackAudit(store *Store, draft SkillDraft, applyErr error) error { + now := rt.now() + return store.UpdateProfile( + draft.WorkspaceID, + draft.TargetSkillName, + func(profile *SkillProfile, exists bool) error { + if !exists { + return nil + } + profile.VersionHistory = append(profile.VersionHistory, SkillVersionEntry{ + Version: profile.CurrentVersion, + Action: "rollback", + Timestamp: now, + DraftID: draft.ID, + Summary: fmt.Sprintf("Rolled back failed draft apply: %s", draft.HumanSummary), + Rollback: true, + RollbackReason: applyErr.Error(), + }) + return nil + }, + ) +} + +func profileOrigin(origin string) string { + if origin == "manual" { + return origin + } + return "evolved" +} + +func appendUniqueStrings(existing []string, values ...string) []string { + seen := make(map[string]struct{}, len(existing)) + for _, value := range existing { + seen[value] = struct{}{} + } + for _, value := range values { + if strings.TrimSpace(value) == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + existing = append(existing, value) + seen[value] = struct{}{} + } + return existing +} + +type skillUsageSummary struct { + All []string +} + +func buildSkillUsage(input TurnCaseInput) skillUsageSummary { + capacity := len(input.ActiveSkillNames) + len(input.AttemptedSkillNames) + len(input.FinalSuccessfulPath) + for _, snapshot := range input.SkillContextSnapshots { + capacity += len(snapshot.SkillNames) + } + for _, exec := range input.ToolExecutions { + capacity += len(exec.SkillNames) + } + + all := make([]string, 0, capacity) + all = append(all, input.ActiveSkillNames...) + all = append(all, input.AttemptedSkillNames...) + all = append(all, input.FinalSuccessfulPath...) + for _, snapshot := range input.SkillContextSnapshots { + all = append(all, snapshot.SkillNames...) + } + for _, exec := range input.ToolExecutions { + all = append(all, exec.SkillNames...) + } + return skillUsageSummary{All: uniqueTrimmedNames(all)} +} + +func (rt *Runtime) recordSkillUsage(input TurnCaseInput, success bool) error { + usage := buildSkillUsage(input) + if len(usage.All) == 0 { + return nil + } + + store := rt.storeForWorkspace(input.Workspace) + seen := make(map[string]struct{}, len(usage.All)) + for _, skillName := range usage.All { + skillName = strings.TrimSpace(skillName) + if skillName == "" { + continue + } + if _, ok := seen[skillName]; ok { + continue + } + seen[skillName] = struct{}{} + + if err := rt.touchSkillProfile(store, input, skillName, success); err != nil { + return err + } + } + return nil +} + +func (rt *Runtime) touchSkillProfile(store *Store, input TurnCaseInput, skillName string, success bool) error { + now := rt.now() + return store.UpdateProfile(input.Workspace, skillName, func(profile *SkillProfile, exists bool) error { + if !exists { + *profile = SkillProfile{ + SkillName: skillName, + WorkspaceID: input.Workspace, + Status: SkillStatusActive, + Origin: "manual", + HumanSummary: skillName, + RetentionScore: 0.2, + } + } + + profile.SkillName = skillName + profile.WorkspaceID = input.Workspace + if profile.Status == SkillStatusCold || profile.Status == SkillStatusArchived || profile.Status == "" { + profile.Status = SkillStatusActive + } + if profile.Origin == "" { + profile.Origin = "manual" + } + if strings.TrimSpace(profile.HumanSummary) == "" { + profile.HumanSummary = skillName + } + profile.LastUsedAt = now + profile.UseCount++ + profile.RetentionScore = nextRetentionScore(profile.RetentionScore, success) + return nil + }) +} + +func nextRetentionScore(current float64, success bool) float64 { + increment := 0.05 + if success { + increment = 0.1 + } + current += increment + if current > 1 { + return 1 + } + return current +} diff --git a/pkg/evolution/runtime_apply_test.go b/pkg/evolution/runtime_apply_test.go new file mode 100644 index 000000000..5f5a53185 --- /dev/null +++ b/pkg/evolution/runtime_apply_test.go @@ -0,0 +1,1170 @@ +package evolution_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestRuntime_RunColdPathOnce_ApplyModeWritesSkillAndProfile(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + IntendedUseCases: []string{ + "weather native-name path", + }, + PreferredEntryPath: []string{"weather"}, + AvoidPatterns: []string{"avoid translating city names before querying weather"}, + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + skillPath := filepath.Join(root, "skills", "weather", "SKILL.md") + if _, statErr := os.Stat(skillPath); statErr != nil { + t.Fatalf("expected skill file: %v", statErr) + } + + profile, err := store.LoadProfile("weather") + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if profile.Status != evolution.SkillStatusActive { + t.Fatalf("Status = %q, want %q", profile.Status, evolution.SkillStatusActive) + } + if profile.CurrentVersion == "" { + t.Fatal("CurrentVersion should not be empty") + } + if profile.ChangeReason != "weather helper" { + t.Fatalf("ChangeReason = %q, want weather helper", profile.ChangeReason) + } + if len(profile.IntendedUseCases) != 1 || profile.IntendedUseCases[0] != "weather native-name path" { + t.Fatalf("IntendedUseCases = %v, want [weather native-name path]", profile.IntendedUseCases) + } + if len(profile.PreferredEntryPath) != 1 || profile.PreferredEntryPath[0] != "weather" { + t.Fatalf("PreferredEntryPath = %v, want [weather]", profile.PreferredEntryPath) + } + if len(profile.AvoidPatterns) != 1 || + profile.AvoidPatterns[0] != "avoid translating city names before querying weather" { + t.Fatalf("AvoidPatterns = %v, want populated metadata", profile.AvoidPatterns) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusAccepted { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusAccepted) + } +} + +func TestRuntime_RunColdPathOnce_DraftModeKeepsCandidateDraft(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + if _, statErr := os.Stat(filepath.Join(root, "skills", "weather", "SKILL.md")); !os.IsNotExist(statErr) { + t.Fatalf("expected no applied skill file, got err=%v", statErr) + } + if _, loadErr := store.LoadProfile("weather"); !os.IsNotExist(loadErr) { + t.Fatalf("expected no profile, got err=%v", loadErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusCandidate { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusCandidate) + } +} + +func TestRuntime_RunColdPathOnce_DraftModeRefreshesExistingCandidateWithEvidence(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + for _, source := range []struct { + name string + body string + }{ + {name: "three-one-theorem", body: "Add 31 to the input value."}, + {name: "four-two-theorem", body: "Add 42 to the current value."}, + {name: "five-three-theorem", body: "Subtract 53 from the current value."}, + } { + skillPath := filepath.Join(root, "skills", source.name, "SKILL.md") + if err := os.MkdirAll(filepath.Dir(skillPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + content := "---\nname: " + source.name + "\ndescription: theorem helper\n---\n# " + source.name + "\n" + source.body + "\n" + if err := os.WriteFile(skillPath, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + } + + success := true + if err := store.SaveTaskRecords([]evolution.LearningRecord{{ + ID: "task-1", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "调用三一定理计算100", + FinalOutput: "100 + 31 = 131; 131 + 42 = 173; 173 - 53 = 120", + Status: evolution.RecordStatus("clustered"), + Success: &success, + UsedSkillNames: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + }}); err != nil { + t.Fatalf("SaveTaskRecords: %v", err) + } + if err := store.SavePatternRecords([]evolution.LearningRecord{{ + ID: "pattern-1", + Kind: evolution.RecordKindPattern, + WorkspaceID: root, + CreatedAt: time.Unix(1700000001, 0).UTC(), + Summary: "调用三一定理计算100", + Status: evolution.RecordStatus("ready"), + TaskRecordIDs: []string{"task-1"}, + }}); err != nil { + t.Fatalf("SavePatternRecords: %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{ + { + ID: "draft-pattern-1", + WorkspaceID: root, + SourceRecordID: "pattern-1", + TargetSkillName: "learned-skill", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "old generic draft", + BodyOrPatch: "---\nname: learned-skill\ndescription: old\n---\n# Learned Skill\n\nNo explicit winning path was recorded.\n", + Status: evolution.DraftStatusCandidate, + }, + }); err != nil { + t.Fatalf("SaveDrafts: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusCandidate { + t.Fatalf("draft status = %q, want candidate", drafts[0].Status) + } + for _, want := range []string{ + "calculate-100-via-theorems", + "Add 31 to the input value", + "Subtract 53 from the current value", + "100 + 31 = 131", + "task-1", + } { + if !strings.Contains(drafts[0].BodyOrPatch, want) && drafts[0].TargetSkillName != want { + t.Fatalf("refreshed draft missing %q:\nname=%s\n%s", want, drafts[0].TargetSkillName, drafts[0].BodyOrPatch) + } + } +} + +func TestRuntime_RunColdPathOnce_ApplyModeAppliesExistingCandidateDraft(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{ + { + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n", + Status: evolution.DraftStatusCandidate, + }, + }); err != nil { + t.Fatalf("SaveDrafts: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-unused", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "unused-weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "unused", + BodyOrPatch: "---\nname: unused-weather\ndescription: unused\n---\n# Unused\n", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + if _, statErr := os.Stat(filepath.Join(root, "skills", "weather", "SKILL.md")); statErr != nil { + t.Fatalf("expected existing candidate to be applied: %v", statErr) + } + if _, statErr := os.Stat(filepath.Join(root, "skills", "unused-weather", "SKILL.md")); !os.IsNotExist(statErr) { + t.Fatalf("expected source rule to stay skipped after applying existing draft, got err=%v", statErr) + } + profile, err := store.LoadProfile("weather") + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if profile.CurrentVersion != "draft-1" { + t.Fatalf("CurrentVersion = %q, want draft-1", profile.CurrentVersion) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusAccepted { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusAccepted) + } +} + +func TestRuntime_RunColdPathOnce_ApplyModeSkipsOrphanCandidateDraft(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{ + { + ID: "draft-orphan", + WorkspaceID: root, + SourceRecordID: "missing-rule", + TargetSkillName: "orphan-weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "orphan weather helper", + BodyOrPatch: "---\nname: orphan-weather\ndescription: orphan weather helper\n---\n# Orphan Weather\n## Start Here\nUse stale guidance.\n", + Status: evolution.DraftStatusCandidate, + }, + }); err != nil { + t.Fatalf("SaveDrafts: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-valid", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "valid-weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "valid weather helper", + BodyOrPatch: "---\nname: valid-weather\ndescription: valid weather helper\n---\n# Valid Weather\n", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + if _, statErr := os.Stat(filepath.Join(root, "skills", "orphan-weather", "SKILL.md")); !os.IsNotExist(statErr) { + t.Fatalf("orphan candidate draft should not be applied, got err=%v", statErr) + } + if _, statErr := os.Stat(filepath.Join(root, "skills", "valid-weather", "SKILL.md")); statErr != nil { + t.Fatalf("expected current ready rule draft to be applied: %v", statErr) + } + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + statusByID := map[string]evolution.DraftStatus{} + for _, draft := range drafts { + statusByID[draft.ID] = draft.Status + } + if statusByID["draft-orphan"] != evolution.DraftStatusCandidate { + t.Fatalf("orphan draft status = %q, want candidate", statusByID["draft-orphan"]) + } +} + +func TestRuntime_RunColdPathOnce_ApplyModeNormalizesExistingCombinedCandidateDraft(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + writeSkillForCombinedShortcutTest(t, root, "three-one-theorem", "Add 31 to the input before continuing.") + writeSkillForCombinedShortcutTest(t, root, "four-two-theorem", "Add 42 to the intermediate result.") + writeSkillForCombinedShortcutTest(t, root, "five-three-theorem", "Subtract 53 to produce the final result.") + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "calculate 100", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{ + { + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "five-three-theorem", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "messy generated combined skill", + BodyOrPatch: strings.Join([]string{ + "Prefer the full theorem chain directly.", + "", + "## Component Skill Breakdown", + "messy raw component dump should be removed before apply.", + "", + "## Learned Shortcut", + "Net effect: input + 20.", + }, "\n"), + Status: evolution.DraftStatusCandidate, + }, + }); err != nil { + t.Fatalf("SaveDrafts: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier( + evolution.NewPaths(root, ""), + func() time.Time { return time.Unix(1700001000, 0).UTC() }, + ), + DraftGenerator: stubDraftGenerator{}, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + data, err := os.ReadFile(filepath.Join(root, "skills", "calculate-100-via-theorems", "SKILL.md")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(data) + if !strings.Contains(content, "## Procedure Details") || !strings.Contains(content, "## Procedure") { + t.Fatalf("expected clean combined skill sections:\n%s", content) + } + if strings.Contains(content, "Learned") || strings.Contains(content, "Source Evidence") { + t.Fatalf("deployed skill should not expose learning traces:\n%s", content) + } + if strings.Contains(content, "messy raw component dump") || + strings.Contains(content, "## Component Skill Breakdown") { + t.Fatalf("expected old verbose draft content to be cleaned:\n%s", content) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusAccepted { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusAccepted) + } + if drafts[0].TargetSkillName != "calculate-100-via-theorems" { + t.Fatalf("TargetSkillName = %q, want calculate-100-via-theorems", drafts[0].TargetSkillName) + } +} + +func TestRuntime_RunColdPathOnce_ApplyModeRetargetsStableMultiSkillPathIntoCombinedShortcut(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + writeSkillForCombinedShortcutTest(t, root, "three-one-theorem", "Add 31 to the input before continuing.") + writeSkillForCombinedShortcutTest(t, root, "four-two-theorem", "Add 42 to the intermediate result.") + writeSkillForCombinedShortcutTest(t, root, "five-three-theorem", "Subtract 53 to produce the final result.") + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "calculate 100", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "five-three-theorem", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "combine the theorem chain into one shortcut skill", + BodyOrPatch: strings.Join([]string{ + "Prefer the full theorem chain directly.", + "", + "## Component Skill Breakdown", + "messy raw component dump should be removed.", + "", + "## Learned Shortcut", + "Net effect: input + 20.", + }, "\n"), + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + skillPath := filepath.Join(root, "skills", "calculate-100-via-theorems", "SKILL.md") + data, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(data) + if !strings.Contains(content, "name: calculate-100-via-theorems") { + t.Fatalf("unexpected content:\n%s", content) + } + if !strings.Contains(content, "# Calculate 100 Via Theorems") { + t.Fatalf("missing synthesized heading:\n%s", content) + } + if !strings.Contains(content, "Prefer the full theorem chain directly.") { + t.Fatalf("missing learned content:\n%s", content) + } + if !strings.Contains(content, "## Procedure") { + t.Fatalf("missing compact procedure:\n%s", content) + } + if !strings.Contains(content, "## Procedure Details") { + t.Fatalf("missing source skill summary:\n%s", content) + } + if strings.Contains(content, "Learned") || strings.Contains(content, "Source Evidence") { + t.Fatalf("deployed skill should not expose learning traces:\n%s", content) + } + if !strings.Contains(content, "Add 31 to the input") || + !strings.Contains(content, "Subtract 53 to produce the final result") { + t.Fatalf("missing extracted component skill content:\n%s", content) + } + if strings.Contains(content, "Extracted guidance") { + t.Fatalf("component content should be concise, not raw extracted guidance:\n%s", content) + } + if strings.Contains(content, "messy raw component dump") { + t.Fatalf("learned context should remove verbose component dumps:\n%s", content) + } + if !strings.Contains(content, "Use `calculate-100-via-theorems` directly") { + t.Fatalf("missing direct shortcut guidance:\n%s", content) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusAccepted { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusAccepted) + } + if drafts[0].ChangeKind != evolution.ChangeKindCreate { + t.Fatalf("ChangeKind = %q, want %q", drafts[0].ChangeKind, evolution.ChangeKindCreate) + } + if drafts[0].TargetSkillName != "calculate-100-via-theorems" { + t.Fatalf("TargetSkillName = %q, want calculate-100-via-theorems", drafts[0].TargetSkillName) + } + if len(drafts[0].PreferredEntryPath) != 1 || drafts[0].PreferredEntryPath[0] != "calculate-100-via-theorems" { + t.Fatalf("PreferredEntryPath = %v, want [calculate-100-via-theorems]", drafts[0].PreferredEntryPath) + } + if len(drafts[0].ReviewNotes) == 0 { + t.Fatal("expected normalization review notes") + } +} + +func TestRuntime_RunColdPathOnce_CombinedShortcutKeepsReadableLongGuidance(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + longOperation := strings.Join([]string{ + "Step 1: normalize the incoming number and keep the original input available for reporting.", + "Step 2: add 31 to the normalized value and record the intermediate value.", + "Step 3: add 42 to the intermediate value and verify that arithmetic was performed exactly once.", + "Step 4: subtract 53 from the second intermediate value and return only the final value.", + "Step 5: if the user asks for explanation, include the compact arithmetic chain without unrelated context.", + }, " ") + writeSkillForCombinedShortcutTest(t, root, "three-one-theorem", longOperation) + writeSkillForCombinedShortcutTest(t, root, "four-two-theorem", "Add 42 to the intermediate result.") + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "calculate with theorem chain", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"three-one-theorem", "four-two-theorem"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier( + evolution.NewPaths(root, ""), + func() time.Time { return time.Unix(1700001000, 0).UTC() }, + ), + DraftGenerator: stubDraftGenerator{draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "three-one-theorem", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "combine theorem chain", + BodyOrPatch: strings.Join([]string{ + "Prefer the theorem chain directly.", + "Include Step A, Step B, Step C, Step D, Step E, Step F, Step G, Step H, Step I, Step J, Step K, Step L, Step M, Step N, Step O, Step P, Step Q, Step R, Step S, Step T, Step U, Step V, Step W, Step X, Step Y, Step Z, and then return the answer.", + "Finish with a short arithmetic explanation.", + }, " "), + }}, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + data, err := os.ReadFile(filepath.Join(root, "skills", "calculate-with-theorem-chain-via-theorems", "SKILL.md")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + content := string(data) + if !strings.Contains(content, "Step 5: if the user asks for explanation") { + t.Fatalf("procedure details were cut too aggressively:\n%s", content) + } + if !strings.Contains(content, "Step Z, and then return the answer.") { + t.Fatalf("procedure notes were cut too aggressively:\n%s", content) + } +} + +func writeSkillForCombinedShortcutTest(t *testing.T, root, name, body string) { + t.Helper() + + skillPath := filepath.Join(root, "skills", name, "SKILL.md") + if err := os.MkdirAll(filepath.Dir(skillPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + content := strings.Join([]string{ + "---", + "name: " + name, + "description: test component skill", + "---", + "# " + name, + body, + "", + }, "\n") + if err := os.WriteFile(skillPath, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } +} + +func TestRuntime_RunColdPathOnce_ApplyFailureQuarantinesDraftAndWritesRollbackAudit(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + profile := evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: root, + CurrentVersion: "v1", + Status: evolution.SkillStatusActive, + Origin: "evolved", + HumanSummary: "weather helper", + LastUsedAt: time.Unix(1700000000, 0).UTC(), + RetentionScore: 1, + VersionHistory: []evolution.SkillVersionEntry{ + { + Version: "v1", + Action: "create", + Timestamp: time.Unix(1700000000, 0).UTC(), + DraftID: "draft-old", + Summary: "initial", + }, + }, + } + if err := store.SaveProfile(profile); err != nil { + t.Fatalf("SaveProfile: %v", err) + } + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + skillDir := filepath.Join(root, "skills", "weather") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + skillPath := filepath.Join(skillDir, "SKILL.md") + original := "---\nname: weather\ndescription: valid\n---\n# Weather\nold body\n" + if err := os.WriteFile(skillPath, []byte(original), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-rollback", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindReplace, + HumanSummary: "broken weather helper", + BodyOrPatch: "invalid-frontmatter", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + err = rt.RunColdPathOnce(context.Background(), root) + if err == nil { + t.Fatal("expected RunColdPathOnce to fail") + } + if !errors.Is(err, evolution.ErrApplyDraftFailed) { + t.Fatalf("error = %v, want ErrApplyDraftFailed", err) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusQuarantined { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusQuarantined) + } + if len(drafts[0].ScanFindings) == 0 { + t.Fatal("expected apply error in ScanFindings") + } + + loadedProfile, err := store.LoadProfile("weather") + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if len(loadedProfile.VersionHistory) != 2 { + t.Fatalf("len(VersionHistory) = %d, want 2", len(loadedProfile.VersionHistory)) + } + last := loadedProfile.VersionHistory[len(loadedProfile.VersionHistory)-1] + if !last.Rollback { + t.Fatal("expected rollback audit entry") + } + if last.DraftID != "draft-rollback" { + t.Fatalf("DraftID = %q, want draft-rollback", last.DraftID) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(got) != original { + t.Fatalf("skill content changed after runtime rollback:\n%s", string(got)) + } +} + +func TestRuntime_RunColdPathOnce_FirstApplyFailureDoesNotCreateGhostProfile(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(evolution.NewPaths(root, ""), func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-ghost-profile", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "broken weather helper", + BodyOrPatch: "invalid-frontmatter", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + err = rt.RunColdPathOnce(context.Background(), root) + if err == nil { + t.Fatal("expected RunColdPathOnce to fail") + } + if !errors.Is(err, evolution.ErrApplyDraftFailed) { + t.Fatalf("error = %v, want ErrApplyDraftFailed", err) + } + + if _, loadErr := store.LoadProfile("weather"); !os.IsNotExist(loadErr) { + t.Fatalf("expected no profile after first apply failure, got err=%v", loadErr) + } +} + +func TestRuntime_RunColdPathOnce_DraftSaveFailureRollsBackAppliedSkill(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission behavior differs on Windows") + } + + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + if err := os.Chmod(paths.RootDir, 0o555); err != nil { + t.Fatalf("Chmod(root read-only): %v", err) + } + t.Cleanup(func() { + _ = os.Chmod(paths.RootDir, 0o755) + }) + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(paths, func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-save-fail", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + err = rt.RunColdPathOnce(context.Background(), root) + if err == nil { + t.Fatal("expected RunColdPathOnce to fail") + } + if !errors.Is(err, evolution.ErrApplyDraftFailed) { + t.Fatalf("error = %v, want ErrApplyDraftFailed", err) + } + + skillPath := filepath.Join(root, "skills", "weather", "SKILL.md") + if _, statErr := os.Stat(skillPath); !os.IsNotExist(statErr) { + t.Fatalf("expected applied skill to be rolled back, got err=%v", statErr) + } + if _, loadErr := store.LoadProfile("weather"); !os.IsNotExist(loadErr) { + t.Fatalf("expected no profile after draft save failure, got err=%v", loadErr) + } +} + +func TestRuntime_RunColdPathOnce_AutoRunsLifecycleMaintenance(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + now := time.Unix(1700001000, 0).UTC() + + if err := store.SaveProfile(evolution.SkillProfile{ + SkillName: "stale-active-skill", + WorkspaceID: root, + Status: evolution.SkillStatusActive, + Origin: "evolved", + HumanSummary: "stale active skill", + LastUsedAt: now.Add(-91 * 24 * time.Hour), + RetentionScore: 0.1, + }); err != nil { + t.Fatalf("SaveProfile(active): %v", err) + } + + skillDir := filepath.Join(root, "skills", "stale-archived-skill") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + skillPath := filepath.Join(skillDir, "SKILL.md") + if err := os.WriteFile( + skillPath, + []byte("---\nname: stale-archived-skill\ndescription: stale\n---\n# Stale Archived Skill\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile: %v", err) + } + if err := store.SaveProfile(evolution.SkillProfile{ + SkillName: "stale-archived-skill", + WorkspaceID: root, + Status: evolution.SkillStatusArchived, + Origin: "evolved", + HumanSummary: "stale archived skill", + LastUsedAt: now.Add(-366 * 24 * time.Hour), + RetentionScore: 0.05, + }); err != nil { + t.Fatalf("SaveProfile(archived): %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return now }, + Store: store, + Applier: evolution.NewApplier(paths, func() time.Time { + return now + }), + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + activeProfile, err := store.LoadProfile("stale-active-skill") + if err != nil { + t.Fatalf("LoadProfile(active): %v", err) + } + if activeProfile.Status != evolution.SkillStatusCold { + t.Fatalf("active profile Status = %q, want %q", activeProfile.Status, evolution.SkillStatusCold) + } + if len(activeProfile.VersionHistory) != 1 || activeProfile.VersionHistory[0].Action != "lifecycle:cold" { + t.Fatalf("active profile VersionHistory = %+v, want lifecycle:cold entry", activeProfile.VersionHistory) + } + + archivedProfile, err := store.LoadProfile("stale-archived-skill") + if err != nil { + t.Fatalf("LoadProfile(archived): %v", err) + } + if archivedProfile.Status != evolution.SkillStatusDeleted { + t.Fatalf("archived profile Status = %q, want %q", archivedProfile.Status, evolution.SkillStatusDeleted) + } + if len(archivedProfile.VersionHistory) != 1 || archivedProfile.VersionHistory[0].Action != "lifecycle:deleted" { + t.Fatalf("archived profile VersionHistory = %+v, want lifecycle:deleted entry", archivedProfile.VersionHistory) + } + + if _, statErr := os.Stat(skillPath); !os.IsNotExist(statErr) { + t.Fatalf("expected lifecycle delete to remove skill file, stat err = %v", statErr) + } +} + +func TestRuntime_RunColdPathOnce_ProfileSaveFailureRollsBackSkillAndQuarantinesDraft(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + if err := os.MkdirAll(filepath.Dir(paths.ProfilesDir), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.ProfilesDir, []byte("not-a-directory"), 0o644); err != nil { + t.Fatalf("WriteFile(profiles): %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + Store: store, + Applier: evolution.NewApplier(paths, func() time.Time { + return time.Unix(1700001000, 0).UTC() + }), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-profile-fail", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\n## Start Here\nUse native-name query first.\n", + }, + }, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 3, MinSuccessRate: 0.7}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + err = rt.RunColdPathOnce(context.Background(), root) + if err == nil { + t.Fatal("expected RunColdPathOnce to fail") + } + if !errors.Is(err, evolution.ErrApplyDraftFailed) { + t.Fatalf("error = %v, want ErrApplyDraftFailed", err) + } + + skillPath := filepath.Join(root, "skills", "weather", "SKILL.md") + if _, statErr := os.Stat(skillPath); !os.IsNotExist(statErr) { + t.Fatalf("expected rolled back skill file, got err=%v", statErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusQuarantined { + t.Fatalf("draft status = %q, want %q", drafts[0].Status, evolution.DraftStatusQuarantined) + } + if len(drafts[0].ScanFindings) == 0 { + t.Fatal("expected scan findings for profile save failure") + } +} diff --git a/pkg/evolution/runtime_cold_path_test.go b/pkg/evolution/runtime_cold_path_test.go new file mode 100644 index 000000000..19c23ebf0 --- /dev/null +++ b/pkg/evolution/runtime_cold_path_test.go @@ -0,0 +1,1285 @@ +package evolution_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/evolution" + "github.com/sipeed/picoclaw/pkg/providers" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type stubDraftGenerator struct { + draft evolution.SkillDraft + err error +} + +func (g stubDraftGenerator) GenerateDraft( + _ context.Context, + _ evolution.LearningRecord, + _ []skills.SkillInfo, +) (evolution.SkillDraft, error) { + return g.draft, g.err +} + +type sequenceDraftGenerator struct { + results []draftGenerationResult + index int +} + +type draftGenerationResult struct { + draft evolution.SkillDraft + err error +} + +type evidenceCaptureDraftGenerator struct { + evidence evolution.DraftEvidence +} + +func (g *evidenceCaptureDraftGenerator) GenerateDraft( + _ context.Context, + _ evolution.LearningRecord, + _ []skills.SkillInfo, +) (evolution.SkillDraft, error) { + return evolution.SkillDraft{}, nil +} + +func (g *evidenceCaptureDraftGenerator) GenerateDraftWithEvidence( + _ context.Context, + _ evolution.LearningRecord, + _ []skills.SkillInfo, + evidence evolution.DraftEvidence, +) (evolution.SkillDraft, error) { + g.evidence = evidence + return evolution.SkillDraft{ + ID: "draft-evidence", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindCreate, + HumanSummary: "weather helper", + BodyOrPatch: "---\nname: weather\ndescription: weather helper\n---\n# Weather\nUse current workspace evidence.\n", + }, nil +} + +type stubSuccessJudge struct { + decisions map[string]evolution.TaskSuccessDecision + calls []string +} + +func (j *stubSuccessJudge) JudgeTaskRecord( + _ context.Context, + record evolution.LearningRecord, +) (evolution.TaskSuccessDecision, error) { + j.calls = append(j.calls, record.ID) + if decision, ok := j.decisions[record.ID]; ok { + return decision, nil + } + return evolution.TaskSuccessDecision{Success: true, Reason: "default success"}, nil +} + +func (g *sequenceDraftGenerator) GenerateDraft( + _ context.Context, + _ evolution.LearningRecord, + _ []skills.SkillInfo, +) (evolution.SkillDraft, error) { + if g.index >= len(g.results) { + return evolution.SkillDraft{}, nil + } + result := g.results[g.index] + g.index++ + return result.draft, result.err +} + +func TestRuntime_RunColdPathOnce_GeneratesCandidateDraft(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Now: func() time.Time { return time.Unix(1700001000, 0).UTC() }, + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer native-name path first", + BodyOrPatch: "## Start Here\nUse native-name query first.", + }, + }, + Store: store, + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusCandidate { + t.Fatalf("Status = %q, want %q", drafts[0].Status, evolution.DraftStatusCandidate) + } +} + +func TestRuntime_RunColdPathOnce_AdmitsOnlyRecordsApprovedBySuccessJudge(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + ok := true + failed := false + + records := []evolution.LearningRecord{ + { + ID: "task-failed", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "failed weather attempt", + UserGoal: "check weather in shanghai", + FinalOutput: "tool failed", + Status: evolution.RecordStatus("new"), + Success: &failed, + UsedSkillNames: []string{"weather", "native-name"}, + ToolKinds: []string{"read_file"}, + }, + { + ID: "task-rejected", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "partial weather answer", + UserGoal: "check weather in shanghai", + FinalOutput: "I will check it next", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather", "native-name"}, + ToolKinds: []string{"read_file"}, + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "read_file", Success: true}, + {Name: "read_file", Success: true}, + }, + }, + { + ID: "task-admitted", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather answer delivered", + UserGoal: "check weather in shanghai", + FinalOutput: "sunny, 26C", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather", "native-name"}, + AddedSkillNames: []string{"native-name"}, + ToolKinds: []string{"read_file"}, + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "read_file", Success: true}, + {Name: "read_file", Success: true}, + }, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"weather"}, + FinalSuccessfulPath: []string{"weather"}, + }, + }, + } + if err := store.AppendLearningRecords(records); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + judge := &stubSuccessJudge{ + decisions: map[string]evolution.TaskSuccessDecision{ + "task-rejected": {Success: false, Reason: "only partial reasoning"}, + "task-admitted": {Success: true, Reason: "goal achieved"}, + }, + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft", MinTaskCount: 1}, + Store: store, + SuccessJudge: judge, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 1, MinSuccessRate: 1}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-weather", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer the proven weather path", + BodyOrPatch: "## Start Here\nUse the weather path directly.", + }, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + if len(judge.calls) != 2 || judge.calls[0] != "task-rejected" || judge.calls[1] != "task-admitted" { + t.Fatalf("judge calls = %v, want [task-rejected task-admitted]", judge.calls) + } + + allRecords, err := store.LoadLearningRecords() + if err != nil { + t.Fatalf("LoadLearningRecords: %v", err) + } + + var pattern evolution.LearningRecord + foundPattern := false + for _, record := range allRecords { + if record.Kind != evolution.RecordKindPattern { + continue + } + pattern = record + foundPattern = true + break + } + if !foundPattern { + t.Fatal("expected generated pattern record") + } + if len(pattern.TaskRecordIDs) != 1 || pattern.TaskRecordIDs[0] != "task-admitted" { + t.Fatalf("TaskRecordIDs = %v, want [task-admitted]", pattern.TaskRecordIDs) + } + if pattern.Label == "" { + t.Fatal("pattern Label should not be empty") + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].SourceRecordID != pattern.ID { + t.Fatalf("draft SourceRecordID = %q, want %q", drafts[0].SourceRecordID, pattern.ID) + } +} + +func TestRuntime_RunColdPathOnce_RejectsClusterBelowMinSuccessRatio(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + ok := true + failed := false + + records := []evolution.LearningRecord{ + { + ID: "task-success", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather lookup 100", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather"}, + }, + { + ID: "task-failed-1", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather lookup 200", + FinalOutput: "failed", + Status: evolution.RecordStatus("new"), + Success: &failed, + UsedSkillNames: []string{"weather"}, + }, + { + ID: "task-failed-2", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather lookup 300", + FinalOutput: "failed", + Status: evolution.RecordStatus("new"), + Success: &failed, + UsedSkillNames: []string{"weather"}, + }, + } + if err := store.AppendLearningRecords(records); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft", MinTaskCount: 1, MinSuccessRatio: 0.8}, + Store: store, + SuccessJudge: &stubSuccessJudge{}, + SkillsRecaller: evolution.NewSkillsRecaller(root), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-weather", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer the proven weather path", + BodyOrPatch: "## Start Here\nUse the weather path directly.", + }, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + patterns, err := store.LoadPatternRecords() + if err != nil { + t.Fatalf("LoadPatternRecords: %v", err) + } + if len(patterns) != 0 { + t.Fatalf("len(patterns) = %d, want 0", len(patterns)) + } + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 0 { + t.Fatalf("len(drafts) = %d, want 0", len(drafts)) + } +} + +func TestRuntime_RunColdPathOnce_FallbackUsesJudgeAdjustedSuccessRatio(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + ok := true + + records := []evolution.LearningRecord{ + { + ID: "task-success", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather lookup 100", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather"}, + }, + { + ID: "task-judge-rejected", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather lookup 200", + FinalOutput: "partial answer", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather"}, + }, + } + if err := store.AppendLearningRecords(records); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + judge := &stubSuccessJudge{ + decisions: map[string]evolution.TaskSuccessDecision{ + "task-success": {Success: true, Reason: "goal achieved"}, + "task-judge-rejected": {Success: false, Reason: "partial result"}, + }, + } + clusterer := evolution.NewLLMPatternClusterer( + &llmClusterTestProvider{content: `not-json`, defaultModel: "test-model"}, + "test-model", + evolution.NewHeuristicPatternClusterer(1, nil), + 1, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft", MinTaskCount: 1, MinSuccessRatio: 0.8}, + Store: store, + PatternClusterer: clusterer, + SuccessJudge: judge, + SkillsRecaller: evolution.NewSkillsRecaller(root), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-weather", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer the proven weather path", + BodyOrPatch: "## Start Here\nUse the weather path directly.", + }, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + patterns, err := store.LoadPatternRecords() + if err != nil { + t.Fatalf("LoadPatternRecords: %v", err) + } + if len(patterns) != 0 { + t.Fatalf("len(patterns) = %d, want 0", len(patterns)) + } + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 0 { + t.Fatalf("len(drafts) = %d, want 0", len(drafts)) + } +} + +func TestRuntime_RunColdPathOnce_FallbackMarksAcceptedFailureEvidenceClustered(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + ok := true + + records := []evolution.LearningRecord{ + { + ID: "task-success", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000200, 0).UTC(), + Summary: "weather lookup 100", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather"}, + }, + { + ID: "task-judge-rejected", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "weather lookup 200", + FinalOutput: "partial answer", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather"}, + }, + } + if err := store.AppendLearningRecords(records); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + judge := &stubSuccessJudge{ + decisions: map[string]evolution.TaskSuccessDecision{ + "task-success": {Success: true, Reason: "goal achieved"}, + "task-judge-rejected": {Success: false, Reason: "partial result"}, + }, + } + clusterer := evolution.NewLLMPatternClusterer( + &llmClusterTestProvider{content: `not-json`, defaultModel: "test-model"}, + "test-model", + evolution.NewHeuristicPatternClusterer(1, nil), + 1, + func() time.Time { return time.Unix(1700000000, 0).UTC() }, + ) + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft", MinTaskCount: 1, MinSuccessRatio: 0.5}, + Store: store, + PatternClusterer: clusterer, + SuccessJudge: judge, + SkillsRecaller: evolution.NewSkillsRecaller(root), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-weather", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer the proven weather path", + BodyOrPatch: "## Start Here\nUse the weather path directly.", + }, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + patterns, err := store.LoadPatternRecords() + if err != nil { + t.Fatalf("LoadPatternRecords: %v", err) + } + if len(patterns) != 1 { + t.Fatalf("len(patterns) = %d, want 1", len(patterns)) + } + if got := strings.Join(patterns[0].TaskRecordIDs, ","); got != "task-success" { + t.Fatalf("pattern TaskRecordIDs = %v, want only successful task", patterns[0].TaskRecordIDs) + } + taskRecords, err := store.LoadTaskRecords() + if err != nil { + t.Fatalf("LoadTaskRecords: %v", err) + } + statusByID := make(map[string]evolution.RecordStatus) + for _, record := range taskRecords { + statusByID[record.ID] = record.Status + } + for _, id := range []string{"task-success", "task-judge-rejected"} { + if statusByID[id] != evolution.RecordStatus("clustered") { + t.Fatalf("statusByID[%s] = %q, want clustered", id, statusByID[id]) + } + } +} + +func TestRuntime_RunColdPathOnce_DraftEvidenceDoesNotCrossWorkspaceWithDuplicateTaskID(t *testing.T) { + sharedState := t.TempDir() + workspaceA := t.TempDir() + workspaceB := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(workspaceA, sharedState)) + ok := true + + if err := store.AppendTaskRecords(context.Background(), []evolution.LearningRecord{ + { + ID: "main-turn-1", + Kind: evolution.RecordKindTask, + WorkspaceID: workspaceB, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "other workspace weather", + FinalOutput: "foreign workspace output", + Status: evolution.RecordStatus("clustered"), + Success: &ok, + UsedSkillNames: []string{"foreign-skill"}, + }, + { + ID: "main-turn-1", + Kind: evolution.RecordKindTask, + WorkspaceID: workspaceA, + CreatedAt: time.Unix(1700000001, 0).UTC(), + Summary: "current workspace weather", + FinalOutput: "current workspace output", + Status: evolution.RecordStatus("clustered"), + Success: &ok, + UsedSkillNames: []string{"current-skill"}, + }, + }); err != nil { + t.Fatalf("AppendTaskRecords: %v", err) + } + if err := store.AppendPatternRecords([]evolution.LearningRecord{{ + ID: "pattern-workspace-a", + Kind: evolution.RecordKindPattern, + WorkspaceID: workspaceA, + CreatedAt: time.Unix(1700000002, 0).UTC(), + Summary: "current workspace weather", + Status: evolution.RecordStatus("ready"), + TaskRecordIDs: []string{"main-turn-1"}, + }}); err != nil { + t.Fatalf("AppendPatternRecords: %v", err) + } + + generator := &evidenceCaptureDraftGenerator{} + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft", StateDir: sharedState}, + Store: store, + SkillsRecaller: evolution.NewSkillsRecaller(workspaceA), + DraftGenerator: generator, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), workspaceA); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + if len(generator.evidence.TaskRecords) != 1 { + t.Fatalf( + "evidence task count = %d, want 1: %#v", + len(generator.evidence.TaskRecords), + generator.evidence.TaskRecords, + ) + } + task := generator.evidence.TaskRecords[0] + if task.WorkspaceID != workspaceA { + t.Fatalf("evidence workspace = %q, want %q", task.WorkspaceID, workspaceA) + } + if task.FinalOutput != "current workspace output" { + t.Fatalf("evidence FinalOutput = %q, want current workspace output", task.FinalOutput) + } + if len(task.UsedSkillNames) != 1 || task.UsedSkillNames[0] != "current-skill" { + t.Fatalf("evidence UsedSkillNames = %v, want [current-skill]", task.UsedSkillNames) + } +} + +func TestRuntime_RunColdPathOnce_AdmitsSingleSkillTaskButWaitsForMinTaskCount(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + ok := true + + record := evolution.LearningRecord{ + ID: "task-simple", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000250, 0).UTC(), + Summary: "simple weather lookup", + UserGoal: "check weather", + FinalOutput: "sunny", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"weather"}, + AddedSkillNames: []string{"weather"}, + ToolKinds: []string{"read_file"}, + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "read_file", Success: true, SkillNames: []string{"weather"}}, + }, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"weather"}, + FinalSuccessfulPath: []string{"weather"}, + }, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{record}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + judge := &stubSuccessJudge{} + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + SuccessJudge: judge, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 1, MinSuccessRate: 1}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-simple", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "simple draft", + BodyOrPatch: "## Start Here\nUse weather.", + }, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + if len(judge.calls) != 1 || judge.calls[0] != "task-simple" { + t.Fatalf("judge calls = %v, want [task-simple]", judge.calls) + } + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 0 { + t.Fatalf("len(drafts) = %d, want 0", len(drafts)) + } +} + +func TestRuntime_RunColdPathOnce_RejectsTaskWhenSuccessJudgeRejects(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + ok := true + + record := evolution.LearningRecord{ + ID: "task-detailed-path", + Kind: evolution.RecordKindTask, + WorkspaceID: root, + CreatedAt: time.Unix(1700000300, 0).UTC(), + Summary: "computed theorem chain", + UserGoal: "调用三一定理计算100", + FinalOutput: "最终结果：100 通过三一定理计算得到 120", + Status: evolution.RecordStatus("new"), + Success: &ok, + UsedSkillNames: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + AddedSkillNames: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + ToolKinds: []string{"read_file"}, + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "read_file", Success: true, SkillNames: []string{"three-one-theorem"}}, + {Name: "read_file", Success: true, SkillNames: []string{"four-two-theorem"}}, + {Name: "read_file", Success: true, SkillNames: []string{"five-three-theorem"}}, + }, + AttemptTrail: &evolution.AttemptTrail{ + AttemptedSkills: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + FinalSuccessfulPath: []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"}, + }, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{record}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + judge := &stubSuccessJudge{ + decisions: map[string]evolution.TaskSuccessDecision{ + "task-detailed-path": {Success: false, Reason: "llm false negative"}, + }, + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + SuccessJudge: judge, + Organizer: evolution.NewOrganizer(evolution.OrganizerOptions{MinCaseCount: 1, MinSuccessRate: 1}), + SkillsRecaller: evolution.NewSkillsRecaller(root), + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-detailed-path", + TargetSkillName: "three-one-theorem", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer the full theorem chain", + BodyOrPatch: "## Start Here\nUse the full three-one, four-two, five-three theorem chain.", + }, + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + allRecords, err := store.LoadLearningRecords() + if err != nil { + t.Fatalf("LoadLearningRecords: %v", err) + } + + foundPattern := false + for _, record := range allRecords { + if record.Kind != evolution.RecordKindPattern { + continue + } + foundPattern = true + break + } + if foundPattern { + t.Fatal("unexpected pattern record for rejected task") + } +} + +func TestRuntime_RunColdPathOnce_QuarantinesInvalidDraft(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "release path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "broken", + BodyOrPatch: "", + }, + }, + Store: store, + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].Status != evolution.DraftStatusQuarantined { + t.Fatalf("Status = %q, want %q", drafts[0].Status, evolution.DraftStatusQuarantined) + } + if len(drafts[0].ScanFindings) == 0 { + t.Fatal("expected scan findings for invalid draft") + } +} + +func TestRuntime_RunColdPathOnce_DoesNotWriteSkillFile(t *testing.T) { + root := t.TempDir() + skillPath := filepath.Join(root, "skills", "weather", "SKILL.md") + if err := os.MkdirAll(filepath.Dir(skillPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile( + skillPath, + []byte("---\nname: weather\ndescription: test\n---\n# Weather"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + store := evolution.NewStore(evolution.NewPaths(root, "")) + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + original, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile(original): %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: root, + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer native-name path first", + BodyOrPatch: "## Start Here\nUse native-name query first.", + }, + }, + Store: store, + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("ReadFile(after): %v", err) + } + if string(got) != string(original) { + t.Fatalf("skill file changed unexpectedly:\n%s", string(got)) + } +} + +func TestRuntime_RunColdPathOnce_UsesDefaultDraftGenerator(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"weather"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].TargetSkillName != "weather" { + t.Fatalf("TargetSkillName = %q, want weather", drafts[0].TargetSkillName) + } + if drafts[0].Status != evolution.DraftStatusCandidate { + t.Fatalf("Status = %q, want %q", drafts[0].Status, evolution.DraftStatusCandidate) + } + if drafts[0].BodyOrPatch == "" { + t.Fatal("expected generated draft body") + } +} + +func TestRuntime_RunColdPathOnce_UsesLLMDraftGeneratorWhenProviderAvailable(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"weather"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + provider := &llmDraftRuntimeProvider{ + response: &providers.LLMResponse{ + Content: `{"target_skill_name":"weather","draft_type":"shortcut","change_kind":"append","human_summary":"Prefer native-name path first","body_or_patch":"## Start Here\nUse native-name query first."}`, + }, + } + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + DraftGenerator: evolution.NewDraftGeneratorForWorkspace(root, provider, "runtime-explicit-model"), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if provider.calls != 1 { + t.Fatalf("provider.calls = %d, want 1", provider.calls) + } + if drafts[0].HumanSummary != "Prefer native-name path first" { + t.Fatalf("HumanSummary = %q, want %q", drafts[0].HumanSummary, "Prefer native-name path first") + } +} + +func TestRuntime_RunColdPathOnce_UsesDefaultDraftGeneratorWhenFactoryHasNoProvider(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"weather"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + DraftGenerator: evolution.NewDraftGeneratorForWorkspace(root, nil, ""), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].TargetSkillName != "weather" { + t.Fatalf("TargetSkillName = %q, want weather", drafts[0].TargetSkillName) + } + if drafts[0].BodyOrPatch == "" { + t.Fatal("expected generated draft body") + } +} + +func TestRuntime_RunColdPathOnce_UsesGeneratorFactoryWorkspaceForFallback(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + if err := os.MkdirAll(filepath.Join(root, "skills", "weather"), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + skillBody := "---\nname: weather\ndescription: workspace weather helper\n---\n# Weather\n## Start Here\nUse the workspace-specific path.\n" + if err := os.WriteFile(filepath.Join(root, "skills", "weather", "SKILL.md"), []byte(skillBody), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + SuccessRate: 1, + WinningPath: []string{"weather"}, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + provider := &llmDraftRuntimeProvider{ + response: &providers.LLMResponse{Content: `not-json`}, + defaultModel: "runtime-test-model", + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + GeneratorFactory: func(workspace string) evolution.DraftGenerator { + return evolution.NewDraftGeneratorForWorkspace(workspace, provider, "runtime-explicit-model") + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].ChangeKind != evolution.ChangeKindAppend { + t.Fatalf("ChangeKind = %q, want %q", drafts[0].ChangeKind, evolution.ChangeKindAppend) + } + if !strings.Contains(drafts[0].BodyOrPatch, "## Learned Evolution") { + t.Fatalf("BodyOrPatch = %q, want appended learned evolution section", drafts[0].BodyOrPatch) + } +} + +func TestRuntime_RunColdPathOnce_PersistsEarlierDraftWhenLaterRuleFails(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rules := []evolution.LearningRecord{ + { + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + }, + { + ID: "rule-2", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "release path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + }, + } + if err := store.AppendLearningRecords(rules); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + generator := &sequenceDraftGenerator{ + results: []draftGenerationResult{ + { + draft: evolution.SkillDraft{ + ID: "draft-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer native-name path first", + BodyOrPatch: "## Start Here\nUse native-name query first.", + }, + }, + { + err: context.DeadlineExceeded, + }, + }, + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + DraftGenerator: generator, + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + err = rt.RunColdPathOnce(context.Background(), root) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("RunColdPathOnce error = %v, want %v", err, context.DeadlineExceeded) + } + + drafts, loadErr := store.LoadDrafts() + if loadErr != nil { + t.Fatalf("LoadDrafts: %v", loadErr) + } + if len(drafts) != 1 { + t.Fatalf("len(drafts) = %d, want 1", len(drafts)) + } + if drafts[0].SourceRecordID != "rule-1" { + t.Fatalf("SourceRecordID = %q, want rule-1", drafts[0].SourceRecordID) + } +} + +func TestRuntime_RunColdPathOnce_RegeneratesAfterQuarantinedDraft(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + rule := evolution.LearningRecord{ + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: root, + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather native-name path", + Status: evolution.RecordStatus("ready"), + EventCount: 4, + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{rule}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{{ + ID: "draft-old", + WorkspaceID: root, + CreatedAt: time.Unix(1700000100, 0).UTC(), + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "broken attempt", + BodyOrPatch: "## Start Here\nBroken content.", + Status: evolution.DraftStatusQuarantined, + ScanFindings: []string{"apply failed"}, + }}); err != nil { + t.Fatalf("SaveDrafts: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "draft"}, + Store: store, + DraftGenerator: stubDraftGenerator{ + draft: evolution.SkillDraft{ + ID: "draft-new", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "fixed attempt", + BodyOrPatch: "## Start Here\nUse native-name query first.", + }, + }, + SkillsRecaller: evolution.NewSkillsRecaller(root), + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if runErr := rt.RunColdPathOnce(context.Background(), root); runErr != nil { + t.Fatalf("RunColdPathOnce: %v", runErr) + } + + drafts, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(drafts) != 2 { + t.Fatalf("len(drafts) = %d, want 2", len(drafts)) + } + if drafts[1].ID != "draft-new" { + t.Fatalf("drafts[1].ID = %q, want draft-new", drafts[1].ID) + } +} + +type llmDraftRuntimeProvider struct { + response *providers.LLMResponse + err error + calls int + defaultModel string +} + +func (p *llmDraftRuntimeProvider) Chat( + _ context.Context, + _ []providers.Message, + _ []providers.ToolDefinition, + _ string, + _ map[string]any, +) (*providers.LLMResponse, error) { + p.calls++ + return p.response, p.err +} + +func (p *llmDraftRuntimeProvider) GetDefaultModel() string { + if p.defaultModel != "" { + return p.defaultModel + } + return "runtime-test-model" +} diff --git a/pkg/evolution/runtime_test.go b/pkg/evolution/runtime_test.go new file mode 100644 index 000000000..533294ae5 --- /dev/null +++ b/pkg/evolution/runtime_test.go @@ -0,0 +1,672 @@ +package evolution_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestRuntime_FinalizeTurnDisabledDoesNothing(t *testing.T) { + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: false, Mode: "observe"}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + workspace := t.TempDir() + err = rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-1", + Status: "completed", + }) + if err != nil { + t.Fatalf("FinalizeTurn: %v", err) + } + + paths := evolution.NewPaths(workspace, "") + if _, statErr := os.Stat(paths.TaskRecords); !os.IsNotExist(statErr) { + t.Fatalf("task records file should not exist, stat err = %v", statErr) + } +} + +func TestRuntime_FinalizeTurnWithEmptyWorkspaceDoesNothing(t *testing.T) { + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "observe"}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + TurnID: "turn-1", + Status: "completed", + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } +} + +func TestRuntime_FinalizeTurnSkipsHeartbeat(t *testing.T) { + workspace := t.TempDir() + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "heartbeat-turn", + SessionKey: "heartbeat", + Status: "completed", + UserMessage: "# Heartbeat Check", + FinalContent: "HEARTBEAT_OK", + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + paths := evolution.NewPaths(workspace, "") + if _, statErr := os.Stat(paths.TaskRecords); !os.IsNotExist(statErr) { + t.Fatalf("heartbeat should not create task records, stat err = %v", statErr) + } +} + +func TestRuntime_FinalizeTurnWritesRecordWithOverride(t *testing.T) { + workspace := t.TempDir() + override := filepath.Join(t.TempDir(), "custom-state") + now := time.Unix(1700000000, 0).UTC() + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + StateDir: override, + }, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-1", + SessionKey: "session-1", + AgentID: "agent-1", + Status: "completed", + UserMessage: "summarize the release notes", + FinalContent: "Here is the summary.", + ToolKinds: []string{"web", "read_file"}, + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "web", Success: true}, + {Name: "read_file", Success: true}, + }, + ActiveSkillNames: []string{"skill-a"}, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn first call: %v", finalizeErr) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + WorkspaceID: "ws-explicit", + TurnID: "turn-2", + SessionKey: "session-2", + AgentID: "agent-2", + Status: "error", + UserMessage: "run the bash command", + FinalContent: "bash failed", + ToolKinds: []string{"bash"}, + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "bash", Success: false, ErrorSummary: "exit status 1"}, + }, + ActiveSkillNames: []string{"skill-b"}, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn second call: %v", finalizeErr) + } + + paths := evolution.NewPaths(workspace, override) + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != 2 { + t.Fatalf("record file line count = %d, want 2", len(lines)) + } + + var first evolution.LearningRecord + if err := json.Unmarshal([]byte(lines[0]), &first); err != nil { + t.Fatalf("Unmarshal first record: %v", err) + } + if first.WorkspaceID != workspace { + t.Fatalf("first WorkspaceID = %q, want %q", first.WorkspaceID, workspace) + } + if first.CreatedAt != now { + t.Fatalf("first CreatedAt = %v, want %v", first.CreatedAt, now) + } + if first.SessionKey != "session-1" { + t.Fatalf("first SessionKey = %q, want %q", first.SessionKey, "session-1") + } + if first.Summary != "summarize the release notes" { + t.Fatalf("first Summary = %q", first.Summary) + } + if first.FinalOutput != "Here is the summary." { + t.Fatalf("first FinalOutput = %q", first.FinalOutput) + } + if first.Success == nil || !*first.Success { + t.Fatalf("first Success = %v, want true", first.Success) + } + if len(first.AddedSkillNames) != 0 { + t.Fatalf("first AddedSkillNames = %v, want empty", first.AddedSkillNames) + } + if len(first.UsedSkillNames) != 0 { + t.Fatalf("first UsedSkillNames = %v, want empty", first.UsedSkillNames) + } + if len(first.ToolKinds) != 0 || len(first.ToolExecutions) != 0 || first.Source != nil || first.AttemptTrail != nil { + t.Fatalf("first record should be slimmed: %+v", first) + } + if first.TaskHash != "" || len(first.Signals) != 0 { + t.Fatalf("first record should not persist task_hash/signals: %+v", first) + } + + var second evolution.LearningRecord + if err := json.Unmarshal([]byte(lines[1]), &second); err != nil { + t.Fatalf("Unmarshal second record: %v", err) + } + if second.WorkspaceID != workspace { + t.Fatalf("second WorkspaceID = %q, want %q", second.WorkspaceID, workspace) + } + if second.SessionKey != "session-2" { + t.Fatalf("second SessionKey = %q, want %q", second.SessionKey, "session-2") + } + if second.Summary != "run the bash command" { + t.Fatalf("second Summary = %q", second.Summary) + } + if second.Success == nil || *second.Success { + t.Fatalf("second Success = %v, want false", second.Success) + } + if len(second.ToolExecutions) != 0 || second.Source != nil || second.AttemptTrail != nil { + t.Fatalf("second record should be slimmed: %+v", second) + } + if second.TaskHash != "" || len(second.Signals) != 0 { + t.Fatalf("second record should not persist task_hash/signals: %+v", second) + } +} + +func TestRuntime_FinalizeTurnGeneratesUniqueTaskRecordIDsAcrossRestartedTurnSequence(t *testing.T) { + workspace := t.TempDir() + createdAt := time.Unix(1700000000, 0).UTC() + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "observe"}, + Now: func() time.Time { + createdAt = createdAt.Add(time.Second) + return createdAt + }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + input := evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "main-turn-1", + SessionKey: "session-a", + AgentID: "main", + Status: "completed", + UserMessage: "summarize release notes", + FinalContent: "done", + } + if finalizeErr := rt.FinalizeTurn(context.Background(), input); finalizeErr != nil { + t.Fatalf("FinalizeTurn first: %v", finalizeErr) + } + input.SessionKey = "session-b" + if finalizeErr := rt.FinalizeTurn(context.Background(), input); finalizeErr != nil { + t.Fatalf("FinalizeTurn second: %v", finalizeErr) + } + + store := evolution.NewStore(evolution.NewPaths(workspace, "")) + records, err := store.LoadTaskRecords() + if err != nil { + t.Fatalf("LoadTaskRecords: %v", err) + } + if len(records) != 2 { + t.Fatalf("len(records) = %d, want 2: %#v", len(records), records) + } + if records[0].ID == records[1].ID { + t.Fatalf("record IDs should be unique across repeated turn IDs: %#v", records) + } + for _, record := range records { + if !strings.HasPrefix(record.ID, "main-turn-1-") { + t.Fatalf("record ID = %q, want main-turn-1-*", record.ID) + } + } +} + +func TestRuntime_FinalizeTurnSharedStateKeepsSkillProfilesScoped(t *testing.T) { + sharedState := t.TempDir() + workspaceA := t.TempDir() + workspaceB := t.TempDir() + now := time.Unix(1700000000, 0).UTC() + + storeA := evolution.NewStore(evolution.NewPaths(workspaceA, sharedState)) + if err := storeA.SaveProfile(evolution.SkillProfile{ + SkillName: "weather", + WorkspaceID: workspaceA, + CurrentVersion: "draft-a", + Status: evolution.SkillStatusActive, + Origin: "evolved", + HumanSummary: "workspace A weather helper", + LastUsedAt: now, + UseCount: 7, + RetentionScore: 0.9, + }); err != nil { + t.Fatalf("storeA.SaveProfile: %v", err) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + StateDir: sharedState, + }, + Now: func() time.Time { return now.Add(time.Minute) }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspaceA, + TurnID: "turn-a", + SessionKey: "session-a", + Status: "completed", + ActiveSkillNames: []string{"weather"}, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn(workspaceA): %v", finalizeErr) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspaceB, + TurnID: "turn-b", + SessionKey: "session-b", + Status: "completed", + ActiveSkillNames: []string{"weather"}, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn(workspaceB): %v", finalizeErr) + } + + loadedA, err := storeA.LoadProfile("weather") + if err != nil { + t.Fatalf("storeA.LoadProfile: %v", err) + } + if loadedA.WorkspaceID != workspaceA { + t.Fatalf("workspace A profile WorkspaceID = %q, want %q", loadedA.WorkspaceID, workspaceA) + } + if loadedA.UseCount != 8 { + t.Fatalf("workspace A profile UseCount = %d, want 8", loadedA.UseCount) + } + + storeB := evolution.NewStore(evolution.NewPaths(workspaceB, sharedState)) + loadedB, err := storeB.LoadProfile("weather") + if err != nil { + t.Fatalf("storeB.LoadProfile: %v", err) + } + if loadedB.WorkspaceID != workspaceB { + t.Fatalf("workspace B profile WorkspaceID = %q, want %q", loadedB.WorkspaceID, workspaceB) + } + if loadedB.UseCount != 1 { + t.Fatalf("workspace B profile UseCount = %d, want 1", loadedB.UseCount) + } + if loadedB.Origin != "manual" { + t.Fatalf("workspace B profile Origin = %q, want manual", loadedB.Origin) + } + if loadedB.CurrentVersion != "" { + t.Fatalf("workspace B profile CurrentVersion = %q, want empty", loadedB.CurrentVersion) + } +} + +func TestRuntime_FinalizeTurnWritesPotentiallyLearnableSignal(t *testing.T) { + workspace := t.TempDir() + now := time.Unix(1700003000, 0).UTC() + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "observe"}, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-learnable", + SessionKey: "session-learnable", + AgentID: "agent-1", + Status: "completed", + ToolKinds: []string{"web", "bash"}, + ActiveSkillNames: []string{"geocode", "weather"}, + FinalContent: "weather workflow completed", + FinalSuccessfulPath: []string{ + "weather", + }, + SkillContextSnapshots: []evolution.SkillContextSnapshot{ + {Sequence: 1, Trigger: "initial_build", SkillNames: []string{"geocode"}}, + {Sequence: 2, Trigger: "context_retry_rebuild", SkillNames: []string{"geocode", "weather"}}, + }, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + paths := evolution.NewPaths(workspace, "") + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != 1 { + t.Fatalf("record file line count = %d, want 1", len(lines)) + } + + var record evolution.LearningRecord + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatalf("Unmarshal record: %v", err) + } + if len(record.Signals) != 0 { + t.Fatalf("Signals = %v, want empty", record.Signals) + } + if got := record.InitialSkillNames; len(got) != 0 { + t.Fatalf("InitialSkillNames = %v, want empty", got) + } + if got := record.AddedSkillNames; len(got) != 0 { + t.Fatalf("AddedSkillNames = %v, want empty", got) + } + if got := record.UsedSkillNames; len(got) != 1 || got[0] != "weather" { + t.Fatalf("UsedSkillNames = %v, want [weather]", got) + } + if got := record.AllLoadedSkillNames; len(got) != 0 { + t.Fatalf("AllLoadedSkillNames = %v, want empty", got) + } + if record.AttemptTrail != nil { + t.Fatalf("AttemptTrail = %+v, want nil", record.AttemptTrail) + } +} + +func TestRuntime_FinalizeTurnUsesSkillNamesFromToolExecutions(t *testing.T) { + workspace := t.TempDir() + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-skill-chain", + SessionKey: "session-skill-chain", + AgentID: "main", + Status: "completed", + UserMessage: "调用三一定理计算100", + FinalContent: "done", + ToolExecutions: []evolution.ToolExecutionRecord{ + {Name: "read_file", Success: true, SkillNames: []string{"three-one"}}, + {Name: "read_file", Success: true, SkillNames: []string{"four-two"}}, + {Name: "read_file", Success: true, SkillNames: []string{"five-three"}}, + }, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + paths := evolution.NewPaths(workspace, "") + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != 1 { + t.Fatalf("record file line count = %d, want 1", len(lines)) + } + + var record evolution.LearningRecord + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatalf("Unmarshal record: %v", err) + } + if got := record.AddedSkillNames; len(got) != 0 { + t.Fatalf("AddedSkillNames = %v, want empty", got) + } + if got := record.UsedSkillNames; len(got) != 3 || got[0] != "three-one" || got[1] != "four-two" || + got[2] != "five-three" { + t.Fatalf("UsedSkillNames = %v, want [three-one four-two five-three]", got) + } + if got := record.AllLoadedSkillNames; len(got) != 0 { + t.Fatalf("AllLoadedSkillNames = %v, want empty", got) + } +} + +func TestRuntime_FinalizeTurnPreservesUTF8WhenTruncatingChineseOutput(t *testing.T) { + workspace := t.TempDir() + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "apply"}, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + longChinese := strings.Repeat("中文输出", 500) + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-utf8", + SessionKey: "session-utf8", + AgentID: "main", + Status: "completed", + UserMessage: "请处理这段中文输出", + FinalContent: longChinese, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + paths := evolution.NewPaths(workspace, "") + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != 1 { + t.Fatalf("record file line count = %d, want 1", len(lines)) + } + + var record evolution.LearningRecord + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatalf("Unmarshal record: %v", err) + } + if !utf8.ValidString(record.FinalOutput) { + t.Fatalf("FinalOutput is not valid UTF-8: %q", record.FinalOutput) + } + if strings.ContainsRune(record.FinalOutput, '\uFFFD') { + t.Fatalf("FinalOutput contains replacement rune: %q", record.FinalOutput) + } + if !strings.HasSuffix(record.FinalOutput, "...") { + t.Fatalf("FinalOutput = %q, want truncated suffix ...", record.FinalOutput) + } +} + +func TestRuntime_FinalizeTurnPrefersExplicitAttemptTrail(t *testing.T) { + workspace := t.TempDir() + now := time.Unix(1700003500, 0).UTC() + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "observe"}, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-explicit-trail", + SessionKey: "session-explicit-trail", + AgentID: "agent-1", + Status: "completed", + ToolKinds: []string{"web"}, + ActiveSkillNames: []string{"weather"}, + AttemptedSkillNames: []string{"geocode", "weather"}, + FinalSuccessfulPath: []string{"geocode", "weather"}, + SkillContextSnapshots: []evolution.SkillContextSnapshot{ + {Sequence: 1, Trigger: "initial_build", SkillNames: []string{"weather"}}, + {Sequence: 2, Trigger: "context_retry_rebuild", SkillNames: []string{"geocode", "weather"}}, + }, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + paths := evolution.NewPaths(workspace, "") + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != 1 { + t.Fatalf("record file line count = %d, want 1", len(lines)) + } + + var record evolution.LearningRecord + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatalf("Unmarshal record: %v", err) + } + if record.AttemptTrail != nil { + t.Fatalf("AttemptTrail = %+v, want nil", record.AttemptTrail) + } + if got := record.UsedSkillNames; len(got) != 2 || got[0] != "geocode" || got[1] != "weather" { + t.Fatalf("UsedSkillNames = %v, want [geocode weather]", got) + } + if got := record.InitialSkillNames; len(got) != 0 { + t.Fatalf("InitialSkillNames = %v, want empty", got) + } + if got := record.AddedSkillNames; len(got) != 0 { + t.Fatalf("AddedSkillNames = %v, want empty", got) + } + if len(record.Signals) != 0 { + t.Fatalf("Signals = %v, want empty", record.Signals) + } +} + +func TestRuntime_FinalizeTurnUpdatesSkillProfileUsage(t *testing.T) { + workspace := t.TempDir() + now := time.Unix(1700000000, 0).UTC() + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{ + Enabled: true, + Mode: "observe", + }, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-1", + SessionKey: "session-1", + AgentID: "agent-1", + Status: "completed", + ActiveSkillNames: []string{"skill-a", "skill-a"}, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + store := evolution.NewStore(evolution.NewPaths(workspace, "")) + profile, err := store.LoadProfile("skill-a") + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if profile.Origin != "manual" { + t.Fatalf("Origin = %q, want manual", profile.Origin) + } + if profile.UseCount != 1 { + t.Fatalf("UseCount = %d, want 1", profile.UseCount) + } + if profile.LastUsedAt != now { + t.Fatalf("LastUsedAt = %v, want %v", profile.LastUsedAt, now) + } + if profile.RetentionScore <= 0.2 { + t.Fatalf("RetentionScore = %v, want > 0.2", profile.RetentionScore) + } +} + +func TestRuntime_FinalizeTurnReactivatesColdSkill(t *testing.T) { + assertFinalizeTurnReactivatesSkill(t, "skill-cold", evolution.SkillStatusCold, 2, 0.2, 24*time.Hour) +} + +func TestRuntime_FinalizeTurnReactivatesArchivedSkill(t *testing.T) { + assertFinalizeTurnReactivatesSkill(t, "skill-archived", evolution.SkillStatusArchived, 5, 0.1, 48*time.Hour) +} + +func assertFinalizeTurnReactivatesSkill( + t *testing.T, + skillName string, + initialStatus evolution.SkillStatus, + useCount int, + retentionScore float64, + lastUsedAge time.Duration, +) { + t.Helper() + workspace := t.TempDir() + now := time.Unix(1700002000, 0).UTC() + store := evolution.NewStore(evolution.NewPaths(workspace, "")) + + if saveErr := store.SaveProfile(evolution.SkillProfile{ + SkillName: skillName, + WorkspaceID: workspace, + Status: initialStatus, + Origin: "evolved", + HumanSummary: string(initialStatus) + " skill", + LastUsedAt: now.Add(-lastUsedAge), + UseCount: useCount, + RetentionScore: retentionScore, + }); saveErr != nil { + t.Fatalf("SaveProfile: %v", saveErr) + } + + rt, err := evolution.NewRuntime(evolution.RuntimeOptions{ + Config: config.EvolutionConfig{Enabled: true, Mode: "observe"}, + Now: func() time.Time { return now }, + Store: store, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + + if finalizeErr := rt.FinalizeTurn(context.Background(), evolution.TurnCaseInput{ + Workspace: workspace, + TurnID: "turn-" + skillName, + Status: "completed", + ActiveSkillNames: []string{skillName}, + }); finalizeErr != nil { + t.Fatalf("FinalizeTurn: %v", finalizeErr) + } + + profile, err := store.LoadProfile(skillName) + if err != nil { + t.Fatalf("LoadProfile: %v", err) + } + if profile.Status != evolution.SkillStatusActive { + t.Fatalf("Status = %q, want %q", profile.Status, evolution.SkillStatusActive) + } +} diff --git a/pkg/evolution/skill_content.go b/pkg/evolution/skill_content.go new file mode 100644 index 000000000..3aad1fe30 --- /dev/null +++ b/pkg/evolution/skill_content.go @@ -0,0 +1,126 @@ +package evolution + +import ( + "fmt" + "os" + "strings" + + "github.com/sipeed/picoclaw/pkg/skills" +) + +const ( + maxMatchedSkillExcerptCount = 5 + maxMatchedSkillExcerptChars = 1400 + maxComponentGuidanceChars = 520 +) + +type matchedSkillExcerpt struct { + Name string + Description string + Body string +} + +func loadMatchedSkillExcerpts(matches []skills.SkillInfo) []matchedSkillExcerpt { + excerpts := make([]matchedSkillExcerpt, 0, minInt(len(matches), maxMatchedSkillExcerptCount)) + for _, match := range matches { + if len(excerpts) >= maxMatchedSkillExcerptCount { + break + } + body := readSkillBodyExcerpt(match.Path) + if body == "" { + continue + } + excerpts = append(excerpts, matchedSkillExcerpt{ + Name: strings.TrimSpace(match.Name), + Description: strings.TrimSpace(match.Description), + Body: body, + }) + } + return excerpts +} + +func readSkillBodyExcerpt(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + data, err := os.ReadFile(path) + if err != nil { + return "" + } + body := strings.TrimSpace(stripSkillFrontmatter(string(data))) + if body == "" { + return "" + } + body = strings.Join(strings.Fields(body), " ") + if len(body) <= maxMatchedSkillExcerptChars { + return body + } + return strings.TrimSpace(body[:maxMatchedSkillExcerptChars]) + "..." +} + +func summarizeMatchedSkillExcerpts(matches []skills.SkillInfo) string { + excerpts := loadMatchedSkillExcerpts(matches) + if len(excerpts) == 0 { + return "none" + } + + parts := make([]string, 0, len(excerpts)) + for _, excerpt := range excerpts { + header := excerpt.Name + if excerpt.Description != "" { + header += ": " + excerpt.Description + } + parts = append(parts, fmt.Sprintf("### %s\n%s", header, excerpt.Body)) + } + return strings.Join(parts, "\n\n") +} + +func synthesizedComponentBreakdown(matches []skills.SkillInfo) string { + excerpts := loadMatchedSkillExcerpts(matches) + if len(excerpts) == 0 { + return "- No component skill content was available when this shortcut was generated." + } + + lines := make([]string, 0, len(excerpts)) + for _, excerpt := range excerpts { + guidance := conciseComponentGuidance(excerpt) + if guidance == "" { + continue + } + lines = append(lines, fmt.Sprintf("- `%s`: %s", excerpt.Name, guidance)) + } + if len(lines) == 0 { + return "- Component skill content was available, but no concise guidance could be extracted." + } + return strings.Join(lines, "\n") +} + +func conciseComponentGuidance(excerpt matchedSkillExcerpt) string { + description := strings.TrimSpace(excerpt.Description) + body := trimComponentGuidance(excerpt.Body) + switch { + case description != "" && body != "": + return trimComponentGuidance(description + " " + body) + case description != "": + return trimComponentGuidance(description) + default: + return body + } +} + +func trimComponentGuidance(content string) string { + content = strings.TrimSpace(content) + if content == "" { + return "" + } + content = strings.NewReplacer( + "#### ", "", + "### ", "", + "## ", "", + "# ", "", + "**", "", + ).Replace(content) + content = strings.TrimSpace(content) + return trimAtReadableBoundary(content, maxComponentGuidanceChars) +} diff --git a/pkg/evolution/skill_draft_policy.go b/pkg/evolution/skill_draft_policy.go new file mode 100644 index 000000000..b91493cec --- /dev/null +++ b/pkg/evolution/skill_draft_policy.go @@ -0,0 +1,174 @@ +package evolution + +import "strings" + +func skillDraftPromptInstructions() []string { + return []string{ + "body_or_patch must contain the complete draft body or patch content as plain text.", + "body_or_patch is an internal draft and review artifact, so it may include concise learning provenance, source task evidence, and source skill summaries when useful for human review.", + "If change_kind is create, body_or_patch must be a complete SKILL.md file with exactly two parts: YAML frontmatter and a Markdown body.", + "The YAML frontmatter must contain only name and description fields.", + "The description field must and only describe what this skill can do and when to use it.", + "The deployable Markdown body should only contain what the skill is useful for and how to use it.", + "The Markdown body is loaded only after the skill triggers, so focus on concise usage guidance and the execution steps needed to complete the task.", + "When describing an operation process in the body, do not use vague summaries; provide detailed step-by-step instructions for the exact operation or execution process.", + "When creating a combined shortcut skill, summarize the functional purpose and result of the provided SKILL.md inputs; do not copy or directly include other skills' instructions.", + "Extract only the necessary operations from source skills and evidence, such as formulas, ordered transformations, commands, inputs, outputs, and boundary conditions.", + "The operational part of the generated skill must be directly usable by a future agent without reading the original task records or source skills.", + "Keep operational instructions separable from audit/provenance notes because the final deployed SKILL.md will be rendered without learning traces.", + } +} + +func skillDraftPromptText() string { + return strings.Join(skillDraftPromptInstructions(), "\n") +} + +func learningTraceReplacer() *strings.Replacer { + return strings.NewReplacer( + "## Learned Shortcut Update", "## Shortcut Update", + "## Learned Evolution", "## Usage Notes", + "## Learned Pattern", "## Usage Pattern", + "## Learned Context", "## Procedure Notes", + "## Source Evidence", "## Validation", + "## Source Skills", "## Procedure Details", + "### Source Skills", "### Procedure Details", + "## Learned Shortcut", "## Shortcut", + "### Learned Shortcut", "### Shortcut", + "Learned workflow for ", "Workflow for ", + "learned workflow for ", "workflow for ", + "from learned pattern: ", "for: ", + "Learned task:", "Task:", + "learned task:", "task:", + "Learned pattern:", "Pattern:", + "learned pattern:", "pattern:", + "Learned from", "Based on", + "learned from", "based on", + "Source evidence", "Validation", + "source evidence", "validation", + "task records", "validated examples", + "Task records", "Validated examples", + ) +} + +func renderDeployableSkillBody(body string) string { + body = strings.TrimSpace(body) + if body == "" { + return body + } + frontmatter, markdownBody := splitSkillFrontmatter(body) + if frontmatter != "" { + body = "---\n" + frontmatter + "\n---\n" + learningTraceReplacer().Replace(strings.TrimLeft(markdownBody, "\n")) + } else { + body = learningTraceReplacer().Replace(body) + } + body = normalizeDeployableDescription(body) + return removeDeployOnlyProvenanceLines(body) +} + +func normalizeDeployableDescription(body string) string { + lines := strings.Split(body, "\n") + inFrontmatter := false + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if i == 0 && trimmed == "---" { + inFrontmatter = true + continue + } + if inFrontmatter && trimmed == "---" { + break + } + if !inFrontmatter || !strings.HasPrefix(trimmed, "description:") { + continue + } + value := strings.TrimSpace(strings.TrimPrefix(trimmed, "description:")) + value = cleanDeployableDescription(value) + lines[i] = "description: " + value + break + } + return strings.Join(lines, "\n") +} + +func cleanDeployableDescription(description string) string { + description = strings.TrimSpace(strings.Trim(description, `"'`)) + for _, marker := range []string{ + " for: ", + " from learned pattern: ", + " for learned pattern: ", + } { + if idx := strings.Index(strings.ToLower(description), marker); idx >= 0 { + description = strings.TrimSpace(description[idx+len(marker):]) + break + } + } + description = strings.TrimPrefix(description, "Create combined shortcut ") + description = strings.TrimPrefix(description, "Refresh combined shortcut ") + description = strings.TrimPrefix(description, "Create shortcut ") + description = strings.TrimPrefix(description, "Refresh shortcut ") + description = strings.TrimSpace(description) + if description == "" { + return "Use this skill when the task matches its documented workflow." + } + return description +} + +func sentenceFragment(text string) string { + text = strings.TrimSpace(text) + if text == "" { + return "complete the documented workflow" + } + runes := []rune(text) + if len(runes) > 0 && runes[0] >= 'A' && runes[0] <= 'Z' { + runes[0] = runes[0] + ('a' - 'A') + } + return string(runes) +} + +func trimAtReadableBoundary(content string, maxLen int) string { + content = strings.TrimSpace(content) + runes := []rune(content) + if content == "" || maxLen <= 0 || len(runes) <= maxLen { + return content + } + + cut := maxLen + searchStart := maxLen - minInt(maxLen/2, 240) + if searchStart < 0 { + searchStart = 0 + } + for i := maxLen; i >= searchStart; i-- { + switch runes[i-1] { + case '\n', '.', '!', '?', ';', ':', '。', '！', '？', '；', '：': + cut = i + goto done + } + } + for i := maxLen; i >= searchStart; i-- { + if runes[i-1] == ' ' || runes[i-1] == '\t' { + cut = i + goto done + } + } + +done: + return strings.TrimRight(strings.TrimSpace(string(runes[:cut])), ".,;:，。；：") + "..." +} + +func removeDeployOnlyProvenanceLines(body string) string { + lines := strings.Split(body, "\n") + out := make([]string, 0, len(lines)) + for _, line := range lines { + trimmed := strings.TrimSpace(line) + lower := strings.ToLower(trimmed) + if strings.HasPrefix(lower, "- evidence:") { + continue + } + if strings.HasPrefix(lower, "- validated examples:") { + continue + } + if strings.Contains(lower, "source_record_id") || strings.Contains(lower, "source record") { + continue + } + out = append(out, line) + } + return strings.TrimSpace(strings.Join(out, "\n")) +} diff --git a/pkg/evolution/skills_recall.go b/pkg/evolution/skills_recall.go new file mode 100644 index 000000000..fb7d2dfcc --- /dev/null +++ b/pkg/evolution/skills_recall.go @@ -0,0 +1,217 @@ +package evolution + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type SkillsRecaller struct { + workspace string + loader *skills.SkillsLoader +} + +func NewSkillsRecaller(workspace string) *SkillsRecaller { + builtinSkillsDir := strings.TrimSpace(os.Getenv(config.EnvBuiltinSkills)) + if builtinSkillsDir == "" { + wd, _ := os.Getwd() + builtinSkillsDir = filepath.Join(wd, "skills") + } + + globalSkillsDir := filepath.Join(config.GetHome(), "skills") + return &SkillsRecaller{ + workspace: workspace, + loader: skills.NewSkillsLoader(workspace, globalSkillsDir, builtinSkillsDir), + } +} + +func (r *SkillsRecaller) RecallSimilarSkills(rule LearningRecord) ([]skills.SkillInfo, error) { + if r == nil || r.loader == nil { + return nil, nil + } + + all := r.loader.ListSkills() + if names := explicitRecallSkillNames(rule); len(names) > 0 { + return filterSkillsByExplicitNames(all, names), nil + } + + type scored struct { + info skills.SkillInfo + score int + sourceRank int + } + + scoredList := make([]scored, 0, len(all)) + for _, skill := range all { + score := scoreSkillMatch(rule, skill) + if score <= 0 { + continue + } + + if body, ok := r.loader.LoadSkill(skill.Name); ok { + score += scoreSkillBody(rule, body) + } + + scoredList = append(scoredList, scored{ + info: skill, + score: score, + sourceRank: skillSourceRank(skill.Source), + }) + } + + sort.Slice(scoredList, func(i, j int) bool { + if scoredList[i].score != scoredList[j].score { + return scoredList[i].score > scoredList[j].score + } + if scoredList[i].sourceRank != scoredList[j].sourceRank { + return scoredList[i].sourceRank < scoredList[j].sourceRank + } + return scoredList[i].info.Name < scoredList[j].info.Name + }) + + out := make([]skills.SkillInfo, 0, len(scoredList)) + for _, item := range scoredList { + out = append(out, item.info) + } + return out, nil +} + +func explicitRecallSkillNames(rule LearningRecord) []string { + names := make([]string, 0, len(rule.WinningPath)+len(rule.MatchedSkillNames)+len(rule.LateAddedSkills)) + names = append(names, normalizePath(rule.WinningPath)...) + names = append(names, normalizePath(rule.MatchedSkillNames)...) + names = append(names, normalizePath(rule.LateAddedSkills)...) + return uniqueTrimmedNames(names) +} + +func filterSkillsByExplicitNames(all []skills.SkillInfo, names []string) []skills.SkillInfo { + if len(all) == 0 || len(names) == 0 { + return nil + } + + byName := make(map[string]skills.SkillInfo, len(all)) + for _, skill := range all { + name := strings.ToLower(strings.TrimSpace(skill.Name)) + if name == "" { + continue + } + if _, exists := byName[name]; exists { + continue + } + byName[name] = skill + } + + out := make([]skills.SkillInfo, 0, len(names)) + for _, name := range names { + if skill, ok := byName[strings.ToLower(strings.TrimSpace(name))]; ok { + out = append(out, skill) + } + } + return out +} + +func scoreSkillMatch(rule LearningRecord, skill skills.SkillInfo) int { + score := 0 + skillName := strings.ToLower(strings.TrimSpace(skill.Name)) + ruleSummary := strings.ToLower(rule.Summary) + + if skillName != "" { + if containsNormalized(rule.WinningPath, skillName) { + score += 8 + } + if containsNormalized(rule.MatchedSkillNames, skillName) { + score += 6 + } + if strings.Contains(ruleSummary, skillName) { + score += 4 + } + } + + score += 2 * tokenOverlap(ruleTokens(rule), tokenizeForEvolution(skill.Name+" "+skill.Description)) + return score +} + +func scoreSkillBody(rule LearningRecord, body string) int { + return minInt(tokenOverlap(ruleTokens(rule), tokenizeForEvolution(body)), 3) +} + +func skillSourceRank(source string) int { + switch source { + case "workspace": + return 0 + case "global": + return 1 + case "builtin": + return 2 + default: + return 3 + } +} + +func ruleTokens(rule LearningRecord) []string { + parts := make([]string, 0, len(rule.WinningPath)+len(rule.MatchedSkillNames)+4) + parts = append(parts, normalizePath(rule.WinningPath)...) + parts = append(parts, normalizePath(rule.MatchedSkillNames)...) + parts = append(parts, tokenizeForEvolution(rule.Summary)...) + return parts +} + +func containsNormalized(values []string, target string) bool { + target = strings.ToLower(strings.TrimSpace(target)) + for _, value := range values { + if strings.ToLower(strings.TrimSpace(value)) == target { + return true + } + } + return false +} + +func tokenOverlap(left, right []string) int { + if len(left) == 0 || len(right) == 0 { + return 0 + } + + leftSet := make(map[string]struct{}, len(left)) + for _, token := range left { + leftSet[token] = struct{}{} + } + + seen := make(map[string]struct{}, len(right)) + count := 0 + for _, token := range right { + if _, ok := seen[token]; ok { + continue + } + seen[token] = struct{}{} + if _, ok := leftSet[token]; ok { + count++ + } + } + return count +} + +func tokenizeForEvolution(text string) []string { + fields := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool { + return !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') + }) + + out := make([]string, 0, len(fields)) + for _, field := range fields { + if field == "" { + continue + } + out = append(out, field) + } + return out +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/pkg/evolution/skills_recall_test.go b/pkg/evolution/skills_recall_test.go new file mode 100644 index 000000000..13e27abbc --- /dev/null +++ b/pkg/evolution/skills_recall_test.go @@ -0,0 +1,118 @@ +package evolution_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestRecallSimilarSkills_ReturnsWorkspaceSkillFirst(t *testing.T) { + workspace := t.TempDir() + globalHome := t.TempDir() + builtinRoot := t.TempDir() + + t.Setenv("HOME", globalHome) + t.Setenv("PICOCLAW_BUILTIN_SKILLS", builtinRoot) + + mustWriteSkill := func(root, name, content string) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("MkdirAll(%s): %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", name, err) + } + } + + mustWriteSkill( + filepath.Join(workspace, "skills"), + "weather", + "---\nname: weather\ndescription: weather lookup\n---\n# Weather\nUse weather queries.\n", + ) + mustWriteSkill( + filepath.Join(globalHome, ".picoclaw", "skills"), + "release", + "---\nname: release\ndescription: release flow\n---\n# Release\nRelease build.\n", + ) + mustWriteSkill( + builtinRoot, + "weather-fallback", + "---\nname: weather-fallback\ndescription: weather backup\n---\n# Weather Fallback\nBackup weather path.\n", + ) + + recaller := evolution.NewSkillsRecaller(workspace) + matches, err := recaller.RecallSimilarSkills(evolution.LearningRecord{ + Kind: evolution.RecordKindRule, + Summary: "weather native-name path", + EventCount: 4, + }) + if err != nil { + t.Fatalf("RecallSimilarSkills: %v", err) + } + if len(matches) == 0 { + t.Fatal("expected at least one match") + } + if matches[0].Name != "weather" { + t.Fatalf("first match = %q, want weather", matches[0].Name) + } +} + +func TestRecallSimilarSkills_UsesExplicitWinningPathOnly(t *testing.T) { + workspace := t.TempDir() + globalHome := t.TempDir() + builtinRoot := t.TempDir() + + t.Setenv("HOME", globalHome) + t.Setenv("PICOCLAW_BUILTIN_SKILLS", builtinRoot) + + mustWriteSkill := func(root, name, description string) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("MkdirAll(%s): %v", dir, err) + } + content := "---\nname: " + name + "\ndescription: " + description + "\n---\n# " + name + "\nUse this skill.\n" + if err := os.WriteFile(filepath.Join(dir, "SKILL.md"), []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", name, err) + } + } + + workspaceSkills := filepath.Join(workspace, "skills") + mustWriteSkill(workspaceSkills, "three-one-theorem", "Add 31 and continue theorem calculation.") + mustWriteSkill(workspaceSkills, "four-two-theorem", "Add 42 and continue theorem calculation.") + mustWriteSkill(workspaceSkills, "five-three-theorem", "Subtract 53 and finish theorem calculation.") + mustWriteSkill(workspaceSkills, "github", "Interact with GitHub using the gh CLI.") + mustWriteSkill(workspaceSkills, "tmux", "Remote-control tmux sessions by sending keystrokes.") + + recaller := evolution.NewSkillsRecaller(workspace) + matches, err := recaller.RecallSimilarSkills(evolution.LearningRecord{ + Kind: evolution.RecordKindPattern, + Summary: "Calculate a value by applying the Three-One Theorem rules", + WinningPath: []string{ + "three-one-theorem", + "four-two-theorem", + "five-three-theorem", + }, + MatchedSkillNames: []string{ + "three-one-theorem", + "four-two-theorem", + "five-three-theorem", + }, + }) + if err != nil { + t.Fatalf("RecallSimilarSkills: %v", err) + } + + got := make([]string, 0, len(matches)) + for _, match := range matches { + got = append(got, match.Name) + } + want := []string{"three-one-theorem", "four-two-theorem", "five-three-theorem"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("matches = %v, want %v", got, want) + } +} diff --git a/pkg/evolution/store.go b/pkg/evolution/store.go new file mode 100644 index 000000000..2e7890799 --- /dev/null +++ b/pkg/evolution/store.go @@ -0,0 +1,672 @@ +package evolution + +import ( + "bufio" + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/sipeed/picoclaw/pkg/fileutil" + "github.com/sipeed/picoclaw/pkg/skills" +) + +type Store struct { + paths Paths +} + +func NewStore(paths Paths) *Store { + return &Store{paths: paths} +} + +var storeFileLocks sync.Map + +func (s *Store) AppendLearningRecord(ctx context.Context, record LearningRecord) error { + switch record.Kind { + case RecordKindPattern, legacyRecordKindRule: + return s.AppendPatternRecords([]LearningRecord{record}) + default: + return s.AppendTaskRecord(ctx, record) + } +} + +func (s *Store) AppendLearningRecords(records []LearningRecord) error { + taskRecords := make([]LearningRecord, 0, len(records)) + patternRecords := make([]LearningRecord, 0, len(records)) + for _, record := range records { + switch record.Kind { + case RecordKindPattern, legacyRecordKindRule: + patternRecords = append(patternRecords, record) + default: + taskRecords = append(taskRecords, record) + } + } + if err := s.AppendTaskRecords(context.Background(), taskRecords); err != nil { + return err + } + return s.AppendPatternRecords(patternRecords) +} + +func (s *Store) AppendTaskRecord(ctx context.Context, record LearningRecord) error { + return s.AppendTaskRecords(ctx, []LearningRecord{record}) +} + +func (s *Store) AppendTaskRecords(ctx context.Context, records []LearningRecord) error { + return s.appendJSONLRecords(ctx, s.paths.TaskRecords, records) +} + +func (s *Store) AppendPatternRecords(records []LearningRecord) error { + return s.appendJSONLRecords(context.Background(), s.paths.PatternRecords, records) +} + +func (s *Store) appendJSONLRecords(ctx context.Context, path string, records []LearningRecord) error { + if len(records) == 0 { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + unlock := lockStoreFile(path) + defer unlock() + + if mkdirErr := os.MkdirAll(filepath.Dir(path), 0o755); mkdirErr != nil { + return mkdirErr + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + + enc := json.NewEncoder(f) + for _, record := range records { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err := enc.Encode(record); err != nil { + return err + } + } + return nil +} + +func (s *Store) LoadLearningRecords() ([]LearningRecord, error) { + taskRecords, err := s.LoadTaskRecords() + if err != nil { + return nil, err + } + patternRecords, err := s.LoadPatternRecords() + if err != nil { + return nil, err + } + return append(taskRecords, patternRecords...), nil +} + +func (s *Store) LoadTaskRecords() ([]LearningRecord, error) { + records, err := s.loadRecordsFromPath(s.paths.TaskRecords) + if err != nil { + return nil, err + } + legacy, err := s.loadLegacyTaskRecords() + if err != nil { + return nil, err + } + return mergeLearningRecordsByID(legacy, records), nil +} + +func (s *Store) LoadPatternRecords() ([]LearningRecord, error) { + records, err := s.loadRecordsFromPath(s.paths.PatternRecords) + if err != nil { + return nil, err + } + legacy, err := s.loadLegacyPatternRecords() + if err != nil { + return nil, err + } + return mergeLearningRecordsByID(legacy, records), nil +} + +func (s *Store) loadRecordsFromPath(path string) ([]LearningRecord, error) { + var records []LearningRecord + if err := decodeJSONLLines(path, func(line []byte) error { + var record LearningRecord + if err := json.Unmarshal(line, &record); err != nil { + return err + } + records = append(records, record) + return nil + }); err != nil { + return nil, err + } + return records, nil +} + +func (s *Store) loadLegacyTaskRecords() ([]LearningRecord, error) { + records, err := s.loadRecordsFromPath(s.paths.LearningRecords) + if err != nil { + return nil, err + } + out := make([]LearningRecord, 0, len(records)) + for _, record := range records { + if isTaskRecordKind(record.Kind) { + out = append(out, record) + } + } + return out, nil +} + +func (s *Store) loadLegacyPatternRecords() ([]LearningRecord, error) { + records, err := s.loadRecordsFromPath(s.paths.LearningRecords) + if err != nil { + return nil, err + } + out := make([]LearningRecord, 0, len(records)) + for _, record := range records { + if isPatternRecordKind(record.Kind) { + out = append(out, record) + } + } + return out, nil +} + +func (s *Store) SaveTaskRecords(records []LearningRecord) error { + return s.saveJSONLRecords(s.paths.TaskRecords, records) +} + +func (s *Store) MarkTaskRecordsClustered(ids []string) error { + if len(ids) == 0 { + return nil + } + target := make(map[string]struct{}, len(ids)) + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + target[id] = struct{}{} + } + if len(target) == 0 { + return nil + } + + unlock := lockStoreFile(s.paths.TaskRecords) + defer unlock() + + current, err := s.loadRecordsFromPath(s.paths.TaskRecords) + if err != nil { + return err + } + legacy, err := s.loadLegacyTaskRecords() + if err != nil { + return err + } + records := mergeLearningRecordsByID(legacy, current) + + hasTargetRecordInWorkspace := make(map[string]bool, len(target)) + if strings.TrimSpace(s.paths.Workspace) != "" { + for _, record := range records { + if _, ok := target[record.ID]; !ok { + continue + } + if record.WorkspaceID == s.paths.Workspace { + hasTargetRecordInWorkspace[record.ID] = true + } + } + } + + changed := false + for i := range records { + if _, ok := target[records[i].ID]; !ok { + continue + } + if hasTargetRecordInWorkspace[records[i].ID] && records[i].WorkspaceID != s.paths.Workspace { + continue + } + records[i].Status = RecordStatus("clustered") + changed = true + } + if !changed { + return nil + } + return s.saveJSONLRecordsLocked(s.paths.TaskRecords, records) +} + +func (s *Store) SavePatternRecords(records []LearningRecord) error { + return s.saveJSONLRecords(s.paths.PatternRecords, records) +} + +func (s *Store) MergePatternRecords(records []LearningRecord) error { + if len(records) == 0 { + return nil + } + + unlock := lockStoreFile(s.paths.PatternRecords) + defer unlock() + + current, err := s.loadRecordsFromPath(s.paths.PatternRecords) + if err != nil { + return err + } + legacy, err := s.loadLegacyPatternRecords() + if err != nil { + return err + } + merged := mergeLearningRecordsByID(mergeLearningRecordsByID(legacy, current), records) + return s.saveJSONLRecordsLocked(s.paths.PatternRecords, merged) +} + +func (s *Store) saveJSONLRecords(path string, records []LearningRecord) error { + unlock := lockStoreFile(path) + defer unlock() + + return s.saveJSONLRecordsLocked(path, records) +} + +func (s *Store) saveJSONLRecordsLocked(path string, records []LearningRecord) error { + if mkdirErr := os.MkdirAll(filepath.Dir(path), 0o755); mkdirErr != nil { + return mkdirErr + } + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for _, record := range records { + if err := enc.Encode(record); err != nil { + return err + } + } + return fileutil.WriteFileAtomic(path, buf.Bytes(), 0o644) +} + +func mergeLearningRecordsByID(base, updates []LearningRecord) []LearningRecord { + out := append([]LearningRecord(nil), base...) + indexByID := make(map[string]int, len(out)+len(updates)) + for i, record := range out { + key := learningRecordMergeKey(record) + if key == "" { + continue + } + indexByID[key] = i + } + for _, record := range updates { + key := learningRecordMergeKey(record) + if key == "" { + out = append(out, record) + continue + } + if idx, ok := indexByID[key]; ok { + out[idx] = record + continue + } + indexByID[key] = len(out) + out = append(out, record) + } + return out +} + +func learningRecordMergeKey(record LearningRecord) string { + id := strings.TrimSpace(record.ID) + if id == "" { + return "" + } + return strings.TrimSpace(record.WorkspaceID) + "\x00" + id +} + +func (s *Store) SaveDrafts(drafts []SkillDraft) error { + unlock := lockStoreFile(s.paths.SkillDrafts) + defer unlock() + + existing, err := s.LoadDrafts() + if err != nil { + return err + } + + indexByKey := make(map[string]int, len(existing)) + for i, draft := range existing { + indexByKey[draftKey(draft.WorkspaceID, draft.ID)] = i + } + + for _, draft := range drafts { + key := draftKey(draft.WorkspaceID, draft.ID) + if idx, ok := indexByKey[key]; ok { + existing[idx] = draft + continue + } + indexByKey[key] = len(existing) + existing = append(existing, draft) + } + + data, err := json.MarshalIndent(existing, "", " ") + if err != nil { + return err + } + return fileutil.WriteFileAtomic(s.paths.SkillDrafts, data, 0o644) +} + +func (s *Store) LoadDrafts() ([]SkillDraft, error) { + data, err := os.ReadFile(s.paths.SkillDrafts) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + if len(bytes.TrimSpace(data)) == 0 { + return nil, nil + } + + var drafts []SkillDraft + if err := json.Unmarshal(data, &drafts); err != nil { + return nil, err + } + return drafts, nil +} + +func (s *Store) SaveProfile(profile SkillProfile) error { + path, err := s.profilePath(profile.WorkspaceID, profile.SkillName) + if err != nil { + return err + } + unlock := lockStoreFile(path) + defer unlock() + + if mkdirErr := os.MkdirAll(filepath.Dir(path), 0o755); mkdirErr != nil { + return mkdirErr + } + + data, err := json.MarshalIndent(profile, "", " ") + if err != nil { + return err + } + return fileutil.WriteFileAtomic(path, data, 0o644) +} + +func (s *Store) LoadProfile(skillName string) (SkillProfile, error) { + return s.loadProfileForWorkspace(strings.TrimSpace(s.paths.Workspace), skillName) +} + +func (s *Store) UpdateProfile( + workspaceID, skillName string, + update func(profile *SkillProfile, exists bool) error, +) error { + targetPath, err := s.profilePath(workspaceID, skillName) + if err != nil { + return err + } + + unlock := lockStoreFile(targetPath) + defer unlock() + + profile, err := s.loadProfileForWorkspace(workspaceID, skillName) + exists := err == nil + if errors.Is(err, os.ErrNotExist) { + profile = SkillProfile{} + } else if err != nil { + return err + } + + if updateErr := update(&profile, exists); updateErr != nil { + return updateErr + } + if !exists && isZeroSkillProfile(profile) { + return nil + } + if mkdirErr := os.MkdirAll(filepath.Dir(targetPath), 0o755); mkdirErr != nil { + return mkdirErr + } + + data, err := json.MarshalIndent(profile, "", " ") + if err != nil { + return err + } + return fileutil.WriteFileAtomic(targetPath, data, 0o644) +} + +func (s *Store) loadProfileForWorkspace(workspaceID, skillName string) (SkillProfile, error) { + paths, err := s.profileLookupPaths(workspaceID, skillName) + if err != nil { + return SkillProfile{}, err + } + for _, path := range paths { + profile, loadErr := s.loadProfileFromPath(path) + if errors.Is(loadErr, os.ErrNotExist) { + continue + } + if loadErr != nil { + return SkillProfile{}, loadErr + } + return profile, nil + } + return SkillProfile{}, os.ErrNotExist +} + +func isZeroSkillProfile(profile SkillProfile) bool { + return profile.SkillName == "" && + profile.WorkspaceID == "" && + profile.CurrentVersion == "" && + profile.Status == "" && + profile.Origin == "" && + profile.HumanSummary == "" && + profile.ChangeReason == "" && + len(profile.IntendedUseCases) == 0 && + len(profile.PreferredEntryPath) == 0 && + len(profile.AvoidPatterns) == 0 && + profile.LastUsedAt.IsZero() && + profile.UseCount == 0 && + profile.RetentionScore == 0 && + len(profile.VersionHistory) == 0 +} + +func (s *Store) LoadProfiles() ([]SkillProfile, error) { + entries, err := os.ReadDir(s.paths.ProfilesDir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + + profiles := make([]SkillProfile, 0, len(entries)) + for _, entry := range entries { + entryPath := filepath.Join(s.paths.ProfilesDir, entry.Name()) + if entry.IsDir() { + nestedProfiles, loadErr := s.loadProfilesFromDir(entryPath) + if loadErr != nil { + return nil, loadErr + } + profiles = append(profiles, nestedProfiles...) + continue + } + if filepath.Ext(entry.Name()) != ".json" { + continue + } + profile, err := s.loadProfileFromPath(entryPath) + if err != nil { + return nil, err + } + profiles = append(profiles, profile) + } + + sort.Slice(profiles, func(i, j int) bool { + if profiles[i].SkillName != profiles[j].SkillName { + return profiles[i].SkillName < profiles[j].SkillName + } + return profiles[i].WorkspaceID < profiles[j].WorkspaceID + }) + return profiles, nil +} + +func decodeJSONLLines(path string, decode func(line []byte) error) error { + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + var lines [][]byte + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + lines = append(lines, append([]byte(nil), line...)) + } + if err := scanner.Err(); err != nil { + return err + } + + for i, line := range lines { + if err := decode(line); err != nil { + if i == len(lines)-1 && isInvalidJSON(err) { + return nil + } + return err + } + } + return nil +} + +func draftKey(workspaceID, id string) string { + return workspaceID + "\x00" + id +} + +func isInvalidJSON(err error) bool { + var syntaxErr *json.SyntaxError + return errors.As(err, &syntaxErr) +} + +func lockStoreFile(path string) func() { + actual, _ := storeFileLocks.LoadOrStore(path, &sync.Mutex{}) + mu := actual.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +func (s *Store) profilePath(workspaceID, skillName string) (string, error) { + if err := skills.ValidateSkillName(skillName); err != nil { + return "", err + } + workspaceID = strings.TrimSpace(workspaceID) + if workspaceID == "" { + return filepath.Join(s.paths.ProfilesDir, skillName+".json"), nil + } + return filepath.Join(s.paths.ProfilesDir, workspaceScopeDir(workspaceID), skillName+".json"), nil +} + +func (s *Store) loadProfilesFromDir(dir string) ([]SkillProfile, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + profiles := make([]SkillProfile, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + profile, err := s.loadProfileFromPath(filepath.Join(dir, entry.Name())) + if err != nil { + return nil, err + } + profiles = append(profiles, profile) + } + return profiles, nil +} + +func (s *Store) loadProfileFromPath(path string) (SkillProfile, error) { + data, err := os.ReadFile(path) + if err != nil { + return SkillProfile{}, err + } + + var profile SkillProfile + if err := json.Unmarshal(data, &profile); err != nil { + return SkillProfile{}, err + } + return profile, nil +} + +func (s *Store) profileLookupPaths(workspaceID, skillName string) ([]string, error) { + if err := skills.ValidateSkillName(skillName); err != nil { + return nil, err + } + + paths := make([]string, 0, 4) + seen := make(map[string]struct{}, 4) + appendPath := func(path string) { + if path == "" { + return + } + if _, ok := seen[path]; ok { + return + } + paths = append(paths, path) + seen[path] = struct{}{} + } + + workspaceID = strings.TrimSpace(workspaceID) + if workspaceID != "" { + path, err := s.profilePath(workspaceID, skillName) + if err != nil { + return nil, err + } + appendPath(path) + if !usesDefaultWorkspaceState(s.paths, workspaceID) { + return paths, nil + } + } + + legacyPath, err := s.profilePath("", skillName) + if err != nil { + return nil, err + } + appendPath(legacyPath) + return paths, nil +} + +func workspaceScopeDir(workspaceID string) string { + sum := sha1.Sum([]byte(workspaceID)) + base := filepath.Base(filepath.Clean(workspaceID)) + base = sanitizeWorkspaceComponent(base) + if base == "" || base == "." { + base = "workspace" + } + return base + "-" + hex.EncodeToString(sum[:6]) +} + +func sanitizeWorkspaceComponent(value string) string { + var b strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-' || r == '_' || r == '.': + b.WriteRune(r) + default: + b.WriteByte('-') + } + } + return strings.Trim(b.String(), "-") +} diff --git a/pkg/evolution/store_test.go b/pkg/evolution/store_test.go new file mode 100644 index 000000000..7b9a78cb4 --- /dev/null +++ b/pkg/evolution/store_test.go @@ -0,0 +1,438 @@ +package evolution_test + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/sipeed/picoclaw/pkg/evolution" +) + +func TestStore_AppendLearningRecordsPersistsCaseAndRule(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + records := []evolution.LearningRecord{ + { + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather task completed", + Status: evolution.RecordStatus("new"), + }, + { + ID: "rule-1", + Kind: evolution.RecordKindRule, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "prefer native-name weather path", + Status: evolution.RecordStatus("ready"), + }, + } + + if err := store.AppendLearningRecords(records); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + loaded, err := store.LoadLearningRecords() + if err != nil { + t.Fatalf("LoadLearningRecords: %v", err) + } + if len(loaded) != 2 { + t.Fatalf("len(loaded) = %d, want 2", len(loaded)) + } + if loaded[1].Kind != evolution.RecordKindRule { + t.Fatalf("loaded[1].Kind = %q, want %q", loaded[1].Kind, evolution.RecordKindRule) + } + if _, statErr := os.Stat(paths.LearningRecords); !os.IsNotExist(statErr) { + t.Fatalf("legacy learning records file should not be written, stat err = %v", statErr) + } + if _, statErr := os.Stat(paths.TaskRecords); statErr != nil { + t.Fatalf("task records file should exist: %v", statErr) + } + if _, statErr := os.Stat(paths.PatternRecords); statErr != nil { + t.Fatalf("pattern records file should exist: %v", statErr) + } +} + +func TestStore_LoadTaskRecordsMergesLegacyWhenSplitFileExists(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + legacy := evolution.LearningRecord{ + ID: "legacy-task", + Kind: evolution.RecordKindTask, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "legacy task", + Status: evolution.RecordStatus("new"), + } + data, err := json.Marshal(legacy) + if err != nil { + t.Fatalf("Marshal legacy: %v", err) + } + if mkdirErr := os.MkdirAll(paths.RootDir, 0o755); mkdirErr != nil { + t.Fatalf("MkdirAll: %v", mkdirErr) + } + if writeErr := os.WriteFile(paths.LearningRecords, append(data, '\n'), 0o644); writeErr != nil { + t.Fatalf("WriteFile legacy: %v", writeErr) + } + + current := evolution.LearningRecord{ + ID: "current-task", + Kind: evolution.RecordKindTask, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "current task", + Status: evolution.RecordStatus("new"), + } + if appendErr := store.AppendTaskRecord(context.Background(), current); appendErr != nil { + t.Fatalf("AppendTaskRecord: %v", appendErr) + } + + records, err := store.LoadTaskRecords() + if err != nil { + t.Fatalf("LoadTaskRecords: %v", err) + } + if len(records) != 2 { + t.Fatalf("len(records) = %d, want 2: %+v", len(records), records) + } + ids := records[0].ID + "," + records[1].ID + if !strings.Contains(ids, "legacy-task") || !strings.Contains(ids, "current-task") { + t.Fatalf("records should include legacy and current task IDs, got %q", ids) + } +} + +func TestStore_LoadPatternRecordsMergesLegacyWhenSplitFileExists(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + legacy := evolution.LearningRecord{ + ID: "legacy-pattern", + Kind: evolution.RecordKindPattern, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "legacy pattern", + Status: evolution.RecordStatus("ready"), + } + data, err := json.Marshal(legacy) + if err != nil { + t.Fatalf("Marshal legacy: %v", err) + } + if mkdirErr := os.MkdirAll(paths.RootDir, 0o755); mkdirErr != nil { + t.Fatalf("MkdirAll: %v", mkdirErr) + } + if writeErr := os.WriteFile(paths.LearningRecords, append(data, '\n'), 0o644); writeErr != nil { + t.Fatalf("WriteFile legacy: %v", writeErr) + } + + current := evolution.LearningRecord{ + ID: "current-pattern", + Kind: evolution.RecordKindPattern, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "current pattern", + Status: evolution.RecordStatus("ready"), + } + if appendErr := store.AppendPatternRecords([]evolution.LearningRecord{current}); appendErr != nil { + t.Fatalf("AppendPatternRecords: %v", appendErr) + } + + records, err := store.LoadPatternRecords() + if err != nil { + t.Fatalf("LoadPatternRecords: %v", err) + } + if len(records) != 2 { + t.Fatalf("len(records) = %d, want 2: %+v", len(records), records) + } + ids := records[0].ID + "," + records[1].ID + if !strings.Contains(ids, "legacy-pattern") || !strings.Contains(ids, "current-pattern") { + t.Fatalf("records should include legacy and current pattern IDs, got %q", ids) + } +} + +func TestStore_MarkTaskRecordsClusteredPreservesNewerAppendedRecords(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + first := evolution.LearningRecord{ + ID: "task-1", + Kind: evolution.RecordKindTask, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "first task", + Status: evolution.RecordStatus("new"), + } + if err := store.AppendTaskRecord(context.Background(), first); err != nil { + t.Fatalf("AppendTaskRecord(first): %v", err) + } + if _, err := store.LoadTaskRecords(); err != nil { + t.Fatalf("LoadTaskRecords snapshot: %v", err) + } + + second := evolution.LearningRecord{ + ID: "task-2", + Kind: evolution.RecordKindTask, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "second task", + Status: evolution.RecordStatus("new"), + } + if err := store.AppendTaskRecord(context.Background(), second); err != nil { + t.Fatalf("AppendTaskRecord(second): %v", err) + } + + if err := store.MarkTaskRecordsClustered([]string{"task-1"}); err != nil { + t.Fatalf("MarkTaskRecordsClustered: %v", err) + } + + records, err := store.LoadTaskRecords() + if err != nil { + t.Fatalf("LoadTaskRecords: %v", err) + } + if len(records) != 2 { + t.Fatalf("len(records) = %d, want 2: %+v", len(records), records) + } + statusByID := map[string]evolution.RecordStatus{} + for _, record := range records { + statusByID[record.ID] = record.Status + } + if statusByID["task-1"] != evolution.RecordStatus("clustered") { + t.Fatalf("task-1 status = %q, want clustered", statusByID["task-1"]) + } + if statusByID["task-2"] != evolution.RecordStatus("new") { + t.Fatalf("task-2 status = %q, want new", statusByID["task-2"]) + } +} + +func TestStore_MergeKeepsSameRecordIDAcrossWorkspaces(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths("workspace-a", root)) + + records := []evolution.LearningRecord{ + { + ID: "main-turn-1", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-a", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "workspace a task", + Status: evolution.RecordStatus("new"), + }, + { + ID: "main-turn-1", + Kind: evolution.RecordKindTask, + WorkspaceID: "workspace-b", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "workspace b task", + Status: evolution.RecordStatus("new"), + }, + } + if err := store.AppendTaskRecords(context.Background(), records); err != nil { + t.Fatalf("AppendTaskRecords: %v", err) + } + + loaded, err := store.LoadTaskRecords() + if err != nil { + t.Fatalf("LoadTaskRecords: %v", err) + } + if len(loaded) != 2 { + t.Fatalf("len(loaded) = %d, want 2: %+v", len(loaded), loaded) + } + + if markErr := store.MarkTaskRecordsClustered([]string{"main-turn-1"}); markErr != nil { + t.Fatalf("MarkTaskRecordsClustered: %v", markErr) + } + loaded, err = store.LoadTaskRecords() + if err != nil { + t.Fatalf("LoadTaskRecords after clustered: %v", err) + } + statusByWorkspace := map[string]evolution.RecordStatus{} + for _, record := range loaded { + statusByWorkspace[record.WorkspaceID] = record.Status + } + if statusByWorkspace["workspace-a"] != evolution.RecordStatus("clustered") { + t.Fatalf("workspace-a status = %q, want clustered", statusByWorkspace["workspace-a"]) + } + if statusByWorkspace["workspace-b"] != evolution.RecordStatus("new") { + t.Fatalf("workspace-b status = %q, want new", statusByWorkspace["workspace-b"]) + } +} + +func TestStore_MergePatternRecordsPreservesNewerWorkspaceRecords(t *testing.T) { + root := t.TempDir() + store := evolution.NewStore(evolution.NewPaths(root, "")) + + first := evolution.LearningRecord{ + ID: "pattern-a", + Kind: evolution.RecordKindPattern, + WorkspaceID: "workspace-a", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "workspace a pattern", + Status: evolution.RecordStatus("ready"), + } + if err := store.SavePatternRecords([]evolution.LearningRecord{first}); err != nil { + t.Fatalf("SavePatternRecords(first): %v", err) + } + if _, err := store.LoadPatternRecords(); err != nil { + t.Fatalf("LoadPatternRecords snapshot: %v", err) + } + + second := evolution.LearningRecord{ + ID: "pattern-b", + Kind: evolution.RecordKindPattern, + WorkspaceID: "workspace-b", + CreatedAt: time.Unix(1700000100, 0).UTC(), + Summary: "workspace b pattern", + Status: evolution.RecordStatus("ready"), + } + if err := store.MergePatternRecords([]evolution.LearningRecord{second}); err != nil { + t.Fatalf("MergePatternRecords: %v", err) + } + + records, err := store.LoadPatternRecords() + if err != nil { + t.Fatalf("LoadPatternRecords: %v", err) + } + if len(records) != 2 { + t.Fatalf("len(records) = %d, want 2: %+v", len(records), records) + } + ids := records[0].ID + "," + records[1].ID + if !strings.Contains(ids, "pattern-a") || !strings.Contains(ids, "pattern-b") { + t.Fatalf("records should include both workspace patterns, got %q", ids) + } +} + +func TestStore_SaveDraftsOverwritesByID(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + first := evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "prefer native-name path first", + BodyOrPatch: "## Start Here", + Status: evolution.DraftStatusCandidate, + } + second := first + second.HumanSummary = "updated summary" + + if err := store.SaveDrafts([]evolution.SkillDraft{first}); err != nil { + t.Fatalf("SaveDrafts(first): %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{second}); err != nil { + t.Fatalf("SaveDrafts(second): %v", err) + } + + loaded, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(loaded) != 1 { + t.Fatalf("len(loaded) = %d, want 1", len(loaded)) + } + if loaded[0].HumanSummary != "updated summary" { + t.Fatalf("HumanSummary = %q, want %q", loaded[0].HumanSummary, "updated summary") + } +} + +func TestStore_SaveDraftsKeepsSameIDDifferentWorkspace(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + first := evolution.SkillDraft{ + ID: "draft-1", + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + SourceRecordID: "rule-1", + TargetSkillName: "weather", + DraftType: evolution.DraftTypeShortcut, + ChangeKind: evolution.ChangeKindAppend, + HumanSummary: "workspace one", + BodyOrPatch: "## Start Here", + Status: evolution.DraftStatusCandidate, + } + second := first + second.WorkspaceID = "ws-2" + second.HumanSummary = "workspace two" + + if err := store.SaveDrafts([]evolution.SkillDraft{first}); err != nil { + t.Fatalf("SaveDrafts(first): %v", err) + } + if err := store.SaveDrafts([]evolution.SkillDraft{second}); err != nil { + t.Fatalf("SaveDrafts(second): %v", err) + } + + loaded, err := store.LoadDrafts() + if err != nil { + t.Fatalf("LoadDrafts: %v", err) + } + if len(loaded) != 2 { + t.Fatalf("len(loaded) = %d, want 2", len(loaded)) + } + if loaded[0].WorkspaceID == loaded[1].WorkspaceID { + t.Fatalf("loaded drafts should keep distinct workspace IDs: %+v", loaded) + } +} + +func TestStore_LoadLearningRecordsIgnoresTruncatedTrailingLine(t *testing.T) { + root := t.TempDir() + paths := evolution.NewPaths(root, "") + store := evolution.NewStore(paths) + + record := evolution.LearningRecord{ + ID: "case-1", + Kind: evolution.RecordKindCase, + WorkspaceID: "ws-1", + CreatedAt: time.Unix(1700000000, 0).UTC(), + Summary: "weather task completed", + Status: evolution.RecordStatus("new"), + } + if err := store.AppendLearningRecords([]evolution.LearningRecord{record}); err != nil { + t.Fatalf("AppendLearningRecords: %v", err) + } + + f, err := os.OpenFile(paths.TaskRecords, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + if _, writeErr := f.WriteString("{\"id\":\"broken\""); writeErr != nil { + f.Close() + t.Fatalf("WriteString: %v", writeErr) + } + if closeErr := f.Close(); closeErr != nil { + t.Fatalf("Close: %v", closeErr) + } + + loaded, err := store.LoadLearningRecords() + if err != nil { + t.Fatalf("LoadLearningRecords: %v", err) + } + if len(loaded) != 1 { + t.Fatalf("len(loaded) = %d, want 1", len(loaded)) + } + if loaded[0].ID != "case-1" { + t.Fatalf("loaded[0].ID = %q, want %q", loaded[0].ID, "case-1") + } + + data, err := os.ReadFile(paths.TaskRecords) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !strings.Contains(string(data), "\"broken\"") { + t.Fatalf("expected test fixture to include broken trailing line") + } +} diff --git a/pkg/evolution/success_judge.go b/pkg/evolution/success_judge.go new file mode 100644 index 000000000..b230eb54c --- /dev/null +++ b/pkg/evolution/success_judge.go @@ -0,0 +1,138 @@ +package evolution + +import ( + "context" + "encoding/json" + "strings" + + "github.com/sipeed/picoclaw/pkg/providers" +) + +type TaskSuccessDecision struct { + Success bool + Reason string +} + +type SuccessJudge interface { + JudgeTaskRecord(ctx context.Context, record LearningRecord) (TaskSuccessDecision, error) +} + +type HeuristicSuccessJudge struct{} + +func (j *HeuristicSuccessJudge) JudgeTaskRecord( + _ context.Context, + record LearningRecord, +) (TaskSuccessDecision, error) { + if record.Success == nil || !*record.Success { + return TaskSuccessDecision{Success: false, Reason: "task not completed"}, nil + } + if strings.TrimSpace(record.Summary) == "" { + return TaskSuccessDecision{Success: false, Reason: "missing summary"}, nil + } + if strings.EqualFold(strings.TrimSpace(record.SessionKey), "heartbeat") { + return TaskSuccessDecision{Success: false, Reason: "heartbeat session"}, nil + } + if strings.EqualFold(strings.TrimSpace(record.FinalOutput), "HEARTBEAT_OK") { + return TaskSuccessDecision{Success: false, Reason: "heartbeat output"}, nil + } + if strings.TrimSpace(record.FinalOutput) == "" { + return TaskSuccessDecision{Success: false, Reason: "missing final output"}, nil + } + return TaskSuccessDecision{Success: true, Reason: "heuristic success"}, nil +} + +type LLMTaskSuccessJudge struct { + provider providers.LLMProvider + model string + fallback SuccessJudge +} + +type llmTaskSuccessResponse struct { + Success bool `json:"success"` + Reason string `json:"reason"` +} + +func NewLLMTaskSuccessJudge(provider providers.LLMProvider, model string, fallback SuccessJudge) *LLMTaskSuccessJudge { + if fallback == nil { + fallback = &HeuristicSuccessJudge{} + } + return &LLMTaskSuccessJudge{ + provider: provider, + model: strings.TrimSpace(model), + fallback: fallback, + } +} + +func (j *LLMTaskSuccessJudge) JudgeTaskRecord( + ctx context.Context, + record LearningRecord, +) (TaskSuccessDecision, error) { + if j == nil || j.provider == nil { + return j.fallbackDecision(ctx, record) + } + + model := strings.TrimSpace(j.model) + if model == "" { + model = strings.TrimSpace(j.provider.GetDefaultModel()) + } + if model == "" { + return j.fallbackDecision(ctx, record) + } + + callCtx, cancel := withLLMCallTimeout(ctx, llmTaskSuccessJudgeTimeout) + defer cancel() + resp, err := j.provider.Chat(callCtx, []providers.Message{ + { + Role: "system", + Content: "Return exactly one JSON object with fields success:boolean and reason:string. No markdown fences.", + }, + { + Role: "user", + Content: buildTaskSuccessJudgePrompt(record), + }, + }, nil, model, map[string]any{"temperature": 0}) + if err != nil || resp == nil { + return j.fallbackDecision(ctx, record) + } + + content := strings.TrimSpace(resp.Content) + content = strings.TrimPrefix(content, "```json") + content = strings.TrimPrefix(content, "```") + content = strings.TrimSuffix(content, "```") + content = strings.TrimSpace(content) + if content == "" { + return j.fallbackDecision(ctx, record) + } + + var payload llmTaskSuccessResponse + if err := json.Unmarshal([]byte(content), &payload); err != nil { + return j.fallbackDecision(ctx, record) + } + return TaskSuccessDecision{ + Success: payload.Success, + Reason: strings.TrimSpace(payload.Reason), + }, nil +} + +func (j *LLMTaskSuccessJudge) fallbackDecision( + ctx context.Context, + record LearningRecord, +) (TaskSuccessDecision, error) { + if j == nil || j.fallback == nil { + return TaskSuccessDecision{Success: false, Reason: "no success judge available"}, nil + } + return j.fallback.JudgeTaskRecord(ctx, record) +} + +func buildTaskSuccessJudgePrompt(record LearningRecord) string { + lines := []string{ + "Decide whether this agent task truly achieved the user's goal.", + "Reject tasks that are only partial reasoning, only describe future steps, or obviously did not complete the requested outcome.", + "Accept completed custom workspace skill/theorem tasks when the final output gives a concrete result or concrete completed procedure.", + "", + "Summary: " + fallbackString(record.Summary, "none"), + "Final output: " + fallbackString(record.FinalOutput, "none"), + "Used skills: " + joinOrFallback(record.UsedSkillNames, "none"), + } + return strings.Join(lines, "\n") +} diff --git a/pkg/evolution/types.go b/pkg/evolution/types.go new file mode 100644 index 000000000..0cb6ba792 --- /dev/null +++ b/pkg/evolution/types.go @@ -0,0 +1,153 @@ +package evolution + +import "time" + +type RecordKind string + +const ( + RecordKindTask RecordKind = "task" + RecordKindPattern RecordKind = "pattern" + legacyRecordKindCase RecordKind = "case" + legacyRecordKindRule RecordKind = "rule" + // Deprecated: use RecordKindTask. + RecordKindCase = RecordKindTask + // Deprecated: use RecordKindPattern. + RecordKindRule = RecordKindPattern +) + +type RecordStatus string + +type DraftType string + +const ( + DraftTypeWorkflow DraftType = "workflow" + DraftTypeShortcut DraftType = "shortcut" +) + +type ChangeKind string + +const ( + ChangeKindCreate ChangeKind = "create" + ChangeKindAppend ChangeKind = "append" + ChangeKindReplace ChangeKind = "replace" + ChangeKindMerge ChangeKind = "merge" +) + +type DraftStatus string + +const ( + DraftStatusCandidate DraftStatus = "candidate" + DraftStatusQuarantined DraftStatus = "quarantined" + DraftStatusAccepted DraftStatus = "accepted" +) + +type SkillStatus string + +const ( + SkillStatusActive SkillStatus = "active" + SkillStatusCold SkillStatus = "cold" + SkillStatusArchived SkillStatus = "archived" + SkillStatusDeleted SkillStatus = "deleted" +) + +type AttemptTrail struct { + AttemptedSkills []string `json:"attempted_skills,omitempty"` + FinalSuccessfulPath []string `json:"final_successful_path,omitempty"` + SkillContextSnapshots []SkillContextSnapshot `json:"skill_context_snapshots,omitempty"` +} + +type SkillContextSnapshot struct { + Sequence int `json:"sequence"` + Trigger string `json:"trigger"` + SkillNames []string `json:"skill_names,omitempty"` +} + +type ToolExecutionRecord struct { + Name string `json:"name"` + Success bool `json:"success"` + ErrorSummary string `json:"error_summary,omitempty"` + SkillNames []string `json:"skill_names,omitempty"` +} + +type LearningRecord struct { + ID string `json:"id"` + Kind RecordKind `json:"kind"` + WorkspaceID string `json:"workspace_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + SessionKey string `json:"session_key,omitempty"` + TaskHash string `json:"task_hash,omitempty"` + Summary string `json:"summary"` + UserGoal string `json:"user_goal,omitempty"` + FinalOutput string `json:"final_output,omitempty"` + Source map[string]any `json:"source,omitempty"` + Status RecordStatus `json:"status"` + Success *bool `json:"success,omitempty"` + ToolKinds []string `json:"tool_kinds,omitempty"` + ToolExecutions []ToolExecutionRecord `json:"tool_executions,omitempty"` + InitialSkillNames []string `json:"initial_skill_names,omitempty"` + AddedSkillNames []string `json:"added_skill_names,omitempty"` + UsedSkillNames []string `json:"used_skill_names,omitempty"` + AllLoadedSkillNames []string `json:"all_loaded_skill_names,omitempty"` + ActiveSkillNames []string `json:"active_skill_names,omitempty"` + AttemptTrail *AttemptTrail `json:"attempt_trail,omitempty"` + Signals []string `json:"signals,omitempty"` + SourceRecordIDs []string `json:"source_record_ids,omitempty"` + TaskRecordIDs []string `json:"task_record_ids,omitempty"` + Label string `json:"label,omitempty"` + ClusterReason string `json:"cluster_reason,omitempty"` + EventCount int `json:"event_count,omitempty"` + SuccessRate float64 `json:"success_rate,omitempty"` + MaturityScore float64 `json:"maturity_score,omitempty"` + WinningPath []string `json:"winning_path,omitempty"` + LateAddedSkills []string `json:"late_added_skills,omitempty"` + FinalSnapshotTrigger string `json:"final_snapshot_trigger,omitempty"` + MatchedSkillNames []string `json:"matched_skill_names,omitempty"` +} + +type SkillDraft struct { + ID string `json:"id"` + WorkspaceID string `json:"workspace_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + SourceRecordID string `json:"source_record_id"` + TargetSkillName string `json:"target_skill_name"` + MatchedSkillRefs []string `json:"matched_skill_refs,omitempty"` + DraftType DraftType `json:"draft_type"` + ChangeKind ChangeKind `json:"change_kind"` + HumanSummary string `json:"human_summary"` + IntendedUseCases []string `json:"intended_use_cases,omitempty"` + PreferredEntryPath []string `json:"preferred_entry_path,omitempty"` + AvoidPatterns []string `json:"avoid_patterns,omitempty"` + BodyOrPatch string `json:"body_or_patch"` + Status DraftStatus `json:"status"` + ReviewNotes []string `json:"review_notes,omitempty"` + ScanFindings []string `json:"scan_findings,omitempty"` +} + +type SkillVersionEntry struct { + Version string `json:"version"` + Action string `json:"action"` + Timestamp time.Time `json:"timestamp"` + DraftID string `json:"draft_id,omitempty"` + Summary string `json:"summary"` + Rollback bool `json:"rollback,omitempty"` + RollbackReason string `json:"rollback_reason,omitempty"` +} + +type SkillProfile struct { + SkillName string `json:"skill_name"` + WorkspaceID string `json:"workspace_id"` + CurrentVersion string `json:"current_version"` + Status SkillStatus `json:"status"` + Origin string `json:"origin"` + HumanSummary string `json:"human_summary"` + ChangeReason string `json:"change_reason,omitempty"` + IntendedUseCases []string `json:"intended_use_cases,omitempty"` + PreferredEntryPath []string `json:"preferred_entry_path,omitempty"` + AvoidPatterns []string `json:"avoid_patterns,omitempty"` + LastUsedAt time.Time `json:"last_used_at"` + UseCount int `json:"use_count"` + RetentionScore float64 `json:"retention_score"` + VersionHistory []SkillVersionEntry `json:"version_history"` +} diff --git a/pkg/gateway/events.go b/pkg/gateway/events.go new file mode 100644 index 000000000..0f454ed7d --- /dev/null +++ b/pkg/gateway/events.go @@ -0,0 +1,53 @@ +package gateway + +import ( + "time" + + "github.com/sipeed/picoclaw/pkg/agent" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +type gatewayEventPayload struct { + DurationMS int64 `json:"duration_ms,omitempty"` + Error string `json:"error,omitempty"` +} + +func publishGatewayEvent( + al *agent.AgentLoop, + kind runtimeevents.Kind, + startedAt time.Time, + err error, +) { + if al == nil || al.RuntimeEventBus() == nil { + return + } + + severity := runtimeevents.SeverityInfo + payload := gatewayEventPayload{} + if !startedAt.IsZero() { + payload.DurationMS = time.Since(startedAt).Milliseconds() + } + if err != nil { + severity = runtimeevents.SeverityError + payload.Error = err.Error() + } + + al.RuntimeEventBus().PublishNonBlocking(runtimeevents.Event{ + Kind: kind, + Source: runtimeevents.Source{Component: "gateway"}, + Severity: severity, + Payload: payload, + Attrs: gatewayEventAttrs(payload), + }) +} + +func gatewayEventAttrs(payload gatewayEventPayload) map[string]any { + attrs := map[string]any{} + if payload.DurationMS > 0 { + attrs["duration_ms"] = payload.DurationMS + } + if payload.Error != "" { + attrs["error"] = payload.Error + } + return attrs +} diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index f58590d5b..9b1586f60 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -25,10 +25,12 @@ import ( _ "github.com/sipeed/picoclaw/pkg/channels/irc" _ "github.com/sipeed/picoclaw/pkg/channels/line" _ "github.com/sipeed/picoclaw/pkg/channels/maixcam" + _ "github.com/sipeed/picoclaw/pkg/channels/mqtt" _ "github.com/sipeed/picoclaw/pkg/channels/onebot" _ "github.com/sipeed/picoclaw/pkg/channels/pico" _ "github.com/sipeed/picoclaw/pkg/channels/qq" _ "github.com/sipeed/picoclaw/pkg/channels/slack" + _ "github.com/sipeed/picoclaw/pkg/channels/slack_webhook" _ "github.com/sipeed/picoclaw/pkg/channels/teams_webhook" _ "github.com/sipeed/picoclaw/pkg/channels/telegram" _ "github.com/sipeed/picoclaw/pkg/channels/vk" @@ -39,6 +41,7 @@ import ( "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/cron" "github.com/sipeed/picoclaw/pkg/devices" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/health" "github.com/sipeed/picoclaw/pkg/heartbeat" "github.com/sipeed/picoclaw/pkg/logger" @@ -114,6 +117,7 @@ func (p *startupBlockedProvider) GetDefaultModel() string { // Run starts the gateway runtime using the configuration loaded from configPath. func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) (runErr error) { + startedAt := time.Now() panicPath := filepath.Join(homePath, logPath, panicFile) panicFunc, err := logger.InitPanic(panicPath) if err != nil { @@ -197,6 +201,8 @@ func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) (runEr msgBus := bus.NewMessageBus() agentLoop := agent.NewAgentLoop(cfg, msgBus, provider) + msgBus.SetEventPublisher(agentLoop.RuntimeEventBus()) + publishGatewayEvent(agentLoop, runtimeevents.KindGatewayStart, startedAt, nil) fmt.Println("\n📦 Agent Status:") startupInfo := agentLoop.GetStartupInfo() @@ -216,6 +222,7 @@ func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) (runEr if err != nil { return err } + publishGatewayEvent(agentLoop, runtimeevents.KindGatewayReady, startedAt, nil) closeListeners = false // Setup manual reload channel for /reload endpoint @@ -262,7 +269,7 @@ func Run(debug bool, homePath, configPath string, allowEmptyStartup bool) (runEr select { case <-sigChan: logger.Info("Shutting down...") - shutdownGateway(runningServices, agentLoop, provider, true) + shutdownGateway(runningServices, agentLoop, provider, msgBus, true) return nil case newCfg := <-configReloadChan: if !runningServices.reloading.CompareAndSwap(false, true) { @@ -312,10 +319,20 @@ func executeReload( msgBus *bus.MessageBus, allowEmptyStartup bool, debug bool, -) error { +) (err error) { + startedAt := time.Now() + publishGatewayEvent(agentLoop, runtimeevents.KindGatewayReloadStarted, startedAt, nil) defer runningServices.reloading.Store(false) + defer func() { + if err != nil { + publishGatewayEvent(agentLoop, runtimeevents.KindGatewayReloadFailed, startedAt, err) + return + } + publishGatewayEvent(agentLoop, runtimeevents.KindGatewayReloadCompleted, startedAt, nil) + }() - return handleConfigReload(ctx, agentLoop, newCfg, provider, runningServices, msgBus, allowEmptyStartup, debug) + err = handleConfigReload(ctx, agentLoop, newCfg, provider, runningServices, msgBus, allowEmptyStartup, debug) + return err } func createStartupProvider( @@ -332,7 +349,11 @@ func createStartupProvider( return &startupBlockedProvider{reason: reason}, "", nil } - return providers.CreateProvider(cfg) + provider, modelID, err := providers.CreateProvider(cfg) + if err != nil { + return nil, "", err + } + return provider, modelID, nil } func setupAndStartServices( @@ -383,7 +404,12 @@ func setupAndStartServices( fms.Start() } - runningServices.ChannelManager, err = channels.NewManager(cfg, msgBus, runningServices.MediaStore) + runningServices.ChannelManager, err = channels.NewManager( + cfg, + msgBus, + runningServices.MediaStore, + channels.WithRuntimeEvents(agentLoop.RuntimeEventBus()), + ) if err != nil { if fms, ok := runningServices.MediaStore.(*media.FileMediaStore); ok { fms.Stop() @@ -490,14 +516,21 @@ func shutdownGateway( runningServices *services, agentLoop *agent.AgentLoop, provider providers.LLMProvider, + msgBus *bus.MessageBus, fullShutdown bool, ) { + publishGatewayEvent(agentLoop, runtimeevents.KindGatewayShutdown, time.Time{}, nil) + if cp, ok := provider.(providers.StatefulProvider); ok && fullShutdown { cp.Close() } stopAndCleanupServices(runningServices, gracefulShutdownTimeout, false) + if fullShutdown && msgBus != nil { + msgBus.Close() + } + agentLoop.Stop() agentLoop.Close() @@ -618,6 +651,9 @@ func restartServices( if fms, ok := runningServices.MediaStore.(*media.FileMediaStore); ok { fms.Start() } + if runningServices.ChannelManager != nil { + runningServices.ChannelManager.SetMediaStore(runningServices.MediaStore) + } al.SetMediaStore(runningServices.MediaStore) al.SetChannelManager(runningServices.ChannelManager) diff --git a/pkg/gateway/gateway_test.go b/pkg/gateway/gateway_test.go index 60049337f..ab3833ba6 100644 --- a/pkg/gateway/gateway_test.go +++ b/pkg/gateway/gateway_test.go @@ -1,14 +1,20 @@ package gateway import ( + "context" + "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "testing" + "time" + "github.com/sipeed/picoclaw/pkg/agent" + "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" ) func TestRun_StartupFailuresReturnErrorAndEmitStructuredLog(t *testing.T) { @@ -106,3 +112,100 @@ func TestGatewayRunStartupFailureHelper(t *testing.T) { fmt.Fprintln(os.Stdout, err.Error()) os.Exit(0) } + +func TestPublishGatewayEvent(t *testing.T) { + eventBus := runtimeevents.NewBus() + t.Cleanup(func() { + if err := eventBus.Close(); err != nil { + t.Fatalf("Close runtime event bus: %v", err) + } + }) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + sub, eventsCh, err := eventBus.Channel().OfKind(runtimeevents.KindGatewayStart).SubscribeChan( + ctx, + runtimeevents.SubscribeOptions{Name: "gateway-test", Buffer: 4}, + ) + if err != nil { + t.Fatalf("SubscribeChan() error = %v", err) + } + t.Cleanup(func() { + if err := sub.Close(); err != nil { + t.Fatalf("Close subscription: %v", err) + } + }) + + al := agent.NewAgentLoop( + config.DefaultConfig(), + bus.NewMessageBus(), + &startupBlockedProvider{reason: "not used"}, + agent.WithRuntimeEvents(eventBus), + ) + t.Cleanup(al.Close) + + startedAt := time.Now().Add(-1500 * time.Millisecond) + publishGatewayEvent(al, runtimeevents.KindGatewayStart, startedAt, nil) + + evt := receiveGatewayRuntimeEvent(t, eventsCh) + if evt.Kind != runtimeevents.KindGatewayStart || + evt.Source.Component != "gateway" || + evt.Severity != runtimeevents.SeverityInfo { + t.Fatalf("gateway event = %+v", evt) + } + payload, ok := evt.Payload.(gatewayEventPayload) + if !ok { + t.Fatalf("payload type = %T, want gatewayEventPayload", evt.Payload) + } + if payload.DurationMS <= 0 { + t.Fatalf("DurationMS = %d, want positive", payload.DurationMS) + } + if evt.Attrs["duration_ms"] == nil { + t.Fatalf("gateway event attrs missing duration_ms: %#v", evt.Attrs) + } +} + +func TestShutdownGatewayClosesMessageBus(t *testing.T) { + msgBus := bus.NewMessageBus() + al := agent.NewAgentLoop( + config.DefaultConfig(), + msgBus, + &startupBlockedProvider{reason: "not used"}, + ) + msgBus.SetEventPublisher(al.RuntimeEventBus()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sub, eventsCh, err := al.RuntimeEventBus().Channel().OfKind(runtimeevents.KindBusCloseCompleted).SubscribeChan( + ctx, + runtimeevents.SubscribeOptions{Name: "bus-close-test", Buffer: 4}, + ) + if err != nil { + t.Fatalf("SubscribeChan() error = %v", err) + } + defer func() { + _ = sub.Close() + }() + + shutdownGateway(&services{}, al, &startupBlockedProvider{reason: "not used"}, msgBus, true) + + evt := receiveGatewayRuntimeEvent(t, eventsCh) + if evt.Kind != runtimeevents.KindBusCloseCompleted { + t.Fatalf("shutdown event kind = %q, want %q", evt.Kind, runtimeevents.KindBusCloseCompleted) + } + if err := msgBus.PublishVoiceControl(context.Background(), bus.VoiceControl{}); !errors.Is(err, bus.ErrBusClosed) { + t.Fatalf("PublishVoiceControl after shutdown error = %v, want %v", err, bus.ErrBusClosed) + } +} + +func receiveGatewayRuntimeEvent(t *testing.T, ch <-chan runtimeevents.Event) runtimeevents.Event { + t.Helper() + + select { + case evt := <-ch: + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for gateway runtime event") + return runtimeevents.Event{} + } +} diff --git a/pkg/mcp/events.go b/pkg/mcp/events.go new file mode 100644 index 000000000..3b7f53f96 --- /dev/null +++ b/pkg/mcp/events.go @@ -0,0 +1,92 @@ +package mcp + +import ( + "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" +) + +func (m *Manager) publishServerEvent( + kind runtimeevents.Kind, + serverName string, + cfg config.MCPServerConfig, + toolCount int, + err error, +) { + if m == nil || m.runtimeEvents == nil { + return + } + + severity := runtimeevents.SeverityInfo + if err != nil { + severity = runtimeevents.SeverityError + } + payload := ServerEventPayload{ + Server: serverName, + Type: mcpTransportType(cfg), + URL: cfg.URL, + Command: cfg.Command, + ToolCount: toolCount, + } + if err != nil { + payload.Error = err.Error() + } + + m.runtimeEvents.PublishNonBlocking(runtimeevents.Event{ + Kind: kind, + Source: runtimeevents.Source{Component: "mcp", Name: serverName}, + Severity: severity, + Payload: payload, + Attrs: mcpServerEventAttrs(payload), + }) +} + +func (m *Manager) publishToolDiscovered(serverName string, cfg config.MCPServerConfig, toolName string) { + if m == nil || m.runtimeEvents == nil { + return + } + payload := ServerEventPayload{ + Server: serverName, + Type: mcpTransportType(cfg), + URL: cfg.URL, + Command: cfg.Command, + Tool: toolName, + } + m.runtimeEvents.PublishNonBlocking(runtimeevents.Event{ + Kind: runtimeevents.KindMCPToolDiscovered, + Source: runtimeevents.Source{Component: "mcp", Name: serverName}, + Severity: runtimeevents.SeverityInfo, + Payload: payload, + Attrs: mcpServerEventAttrs(payload), + }) +} + +func mcpServerEventAttrs(payload ServerEventPayload) map[string]any { + attrs := map[string]any{} + setMCPAttrString(attrs, "server", payload.Server) + setMCPAttrString(attrs, "type", payload.Type) + setMCPAttrString(attrs, "tool", payload.Tool) + if payload.ToolCount > 0 { + attrs["tool_count"] = payload.ToolCount + } + setMCPAttrString(attrs, "error", payload.Error) + return attrs +} + +func setMCPAttrString(attrs map[string]any, key, value string) { + if value != "" { + attrs[key] = value + } +} + +func mcpTransportType(cfg config.MCPServerConfig) string { + if cfg.Type != "" { + return cfg.Type + } + if cfg.URL != "" { + return "sse" + } + if cfg.Command != "" { + return "stdio" + } + return "" +} diff --git a/pkg/mcp/manager.go b/pkg/mcp/manager.go index 92ea426a6..958927767 100644 --- a/pkg/mcp/manager.go +++ b/pkg/mcp/manager.go @@ -16,6 +16,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" ) @@ -127,19 +128,47 @@ type ServerConnection struct { // Manager manages multiple MCP server connections type Manager struct { - servers map[string]*ServerConnection - mu sync.RWMutex - closed atomic.Bool // changed from bool to atomic.Bool to avoid TOCTOU race - wg sync.WaitGroup // tracks in-flight CallTool calls + servers map[string]*ServerConnection + runtimeEvents runtimeevents.Bus + mu sync.RWMutex + closed atomic.Bool // changed from bool to atomic.Bool to avoid TOCTOU race + wg sync.WaitGroup // tracks in-flight CallTool calls } var connectServerFunc = connectServer +// ManagerOption configures an MCP manager. +type ManagerOption func(*Manager) + +// WithRuntimeEvents injects the runtime event bus used for MCP observations. +func WithRuntimeEvents(eventBus runtimeevents.Bus) ManagerOption { + return func(m *Manager) { + m.runtimeEvents = eventBus + } +} + +// ServerEventPayload describes MCP server connection events. +type ServerEventPayload struct { + Server string `json:"server"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Command string `json:"command,omitempty"` + Tool string `json:"tool,omitempty"` + ToolCount int `json:"tool_count,omitempty"` + Error string `json:"error,omitempty"` +} + // NewManager creates a new MCP manager -func NewManager() *Manager { - return &Manager{ +func NewManager(opts ...ManagerOption) *Manager { + m := &Manager{ servers: make(map[string]*ServerConnection), } + for _, opt := range opts { + if opt != nil { + opt(m) + } + } + return m } // LoadFromConfig loads MCP servers from configuration @@ -264,8 +293,10 @@ func (m *Manager) ConnectServer( name string, cfg config.MCPServerConfig, ) error { + m.publishServerEvent(runtimeevents.KindMCPServerConnecting, name, cfg, 0, nil) conn, err := connectServerFunc(ctx, name, cfg) if err != nil { + m.publishServerEvent(runtimeevents.KindMCPServerFailed, name, cfg, 0, err) return err } @@ -274,10 +305,19 @@ func (m *Manager) ConnectServer( if m.closed.Load() { _ = conn.Session.Close() + m.publishServerEvent(runtimeevents.KindMCPServerFailed, name, cfg, 0, fmt.Errorf("manager is closed")) return fmt.Errorf("manager is closed") } m.servers[name] = conn + for _, tool := range conn.Tools { + toolName := "" + if tool != nil { + toolName = tool.Name + } + m.publishToolDiscovered(name, cfg, toolName) + } + m.publishServerEvent(runtimeevents.KindMCPServerConnected, name, cfg, len(conn.Tools), nil) return nil } diff --git a/pkg/mcp/manager_test.go b/pkg/mcp/manager_test.go index 682d4c346..5789a37a9 100644 --- a/pkg/mcp/manager_test.go +++ b/pkg/mcp/manager_test.go @@ -10,11 +10,13 @@ import ( "strings" "sync" "testing" + "time" "github.com/modelcontextprotocol/go-sdk/jsonrpc" sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/sipeed/picoclaw/pkg/config" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" ) func TestLoadEnvFile(t *testing.T) { @@ -248,6 +250,95 @@ func TestNewManager_InitialState(t *testing.T) { } } +func TestConnectServerPublishesRuntimeEvents(t *testing.T) { + originalConnectServerFunc := connectServerFunc + t.Cleanup(func() { + connectServerFunc = originalConnectServerFunc + }) + + eventBus := runtimeevents.NewBus() + defer func() { + if err := eventBus.Close(); err != nil { + t.Errorf("event bus close failed: %v", err) + } + }() + + _, eventsCh, err := eventBus.Channel().OfKind( + runtimeevents.KindMCPServerConnected, + runtimeevents.KindMCPServerFailed, + ).SubscribeChan(t.Context(), runtimeevents.SubscribeOptions{Name: "mcp-events", Buffer: 2}) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + connectServerFunc = func( + _ context.Context, + name string, + cfg config.MCPServerConfig, + ) (*ServerConnection, error) { + if name == "bad" { + return nil, fmt.Errorf("connect failed") + } + return &ServerConnection{ + Name: name, + Config: cfg, + Tools: []*sdkmcp.Tool{{Name: "echo"}}, + }, nil + } + + mgr := NewManager(WithRuntimeEvents(eventBus)) + err = mgr.ConnectServer(context.Background(), "good", config.MCPServerConfig{ + Type: "stdio", + Command: "echo", + }) + if err != nil { + t.Fatalf("ConnectServer(good) error = %v", err) + } + connected := receiveMCPRuntimeEvent(t, eventsCh) + if connected.Kind != runtimeevents.KindMCPServerConnected || + connected.Source.Name != "good" || + connected.Severity != runtimeevents.SeverityInfo { + t.Fatalf("connected event = %+v", connected) + } + if connected.Attrs["server"] != "good" || + connected.Attrs["type"] != "stdio" || + connected.Attrs["tool_count"] != 1 { + t.Fatalf("connected attrs = %#v", connected.Attrs) + } + + err = mgr.ConnectServer(context.Background(), "bad", config.MCPServerConfig{ + Type: "stdio", + Command: "echo", + }) + if err == nil { + t.Fatal("expected ConnectServer(bad) to fail") + } + failed := receiveMCPRuntimeEvent(t, eventsCh) + if failed.Kind != runtimeevents.KindMCPServerFailed || + failed.Source.Name != "bad" || + failed.Severity != runtimeevents.SeverityError { + t.Fatalf("failed event = %+v", failed) + } + if failed.Attrs["server"] != "bad" || failed.Attrs["error"] != "connect failed" { + t.Fatalf("failed attrs = %#v", failed.Attrs) + } +} + +func receiveMCPRuntimeEvent(t *testing.T, ch <-chan runtimeevents.Event) runtimeevents.Event { + t.Helper() + + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("runtime event channel closed before expected event") + } + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for runtime event") + return runtimeevents.Event{} + } +} + func TestLoadFromMCPConfig_DisabledOrEmptyServers(t *testing.T) { mgr := NewManager() diff --git a/pkg/providers/bedrock/provider_bedrock.go b/pkg/providers/bedrock/provider_bedrock.go index 3798c5fd8..ee0ac75a0 100644 --- a/pkg/providers/bedrock/provider_bedrock.go +++ b/pkg/providers/bedrock/provider_bedrock.go @@ -135,48 +135,23 @@ func NewProvider(ctx context.Context, opts ...Option) (*Provider, error) { }, nil } -// Chat sends messages to AWS Bedrock using the Converse API. -func (p *Provider) Chat( - ctx context.Context, - messages []Message, - tools []ToolDefinition, - model string, - options map[string]any, -) (*LLMResponse, error) { - // Apply request timeout if context doesn't already have a deadline. - // Use explicit timeout if set, otherwise fall back to common default. - effectiveTimeout := p.requestTimeout - if effectiveTimeout <= 0 { - effectiveTimeout = common.DefaultRequestTimeout - } - if _, hasDeadline := ctx.Deadline(); !hasDeadline { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, effectiveTimeout) - defer cancel() - } +// converseParams holds the shared request parameters for Converse and ConverseStream. +type converseParams struct { + messages []types.Message + system []types.SystemContentBlock + inferenceConfig *types.InferenceConfiguration + toolConfig *types.ToolConfiguration +} - // Build the Converse API input - input := &bedrockruntime.ConverseInput{ - ModelId: aws.String(model), - } - - // Convert messages to Bedrock format +func buildConverseParams(messages []Message, tools []ToolDefinition, options map[string]any) converseParams { bedrockMessages, systemPrompts := convertMessages(messages) - input.Messages = bedrockMessages - // Set system prompts if any - if len(systemPrompts) > 0 { - input.System = systemPrompts - } - - // Set inference configuration only when options are provided var inferenceConfig *types.InferenceConfiguration if maxTokens, ok := common.AsInt(options["max_tokens"]); ok && maxTokens > 0 { if inferenceConfig == nil { inferenceConfig = &types.InferenceConfiguration{} } - // Clamp to int32 range to avoid overflow if maxTokens > math.MaxInt32 { maxTokens = math.MaxInt32 } @@ -190,23 +165,53 @@ func (p *Provider) Chat( inferenceConfig.Temperature = aws.Float32(float32(temp)) } - if inferenceConfig != nil { - input.InferenceConfig = inferenceConfig - } - - // Convert tools to Bedrock format - // Only set ToolConfig if at least one valid tool was produced + var toolConfig *types.ToolConfiguration if len(tools) > 0 { - toolConfig := convertTools(tools) - if len(toolConfig.Tools) > 0 { - input.ToolConfig = toolConfig + tc := convertTools(tools) + if len(tc.Tools) > 0 { + toolConfig = tc } } - // Call Bedrock Converse API + return converseParams{ + messages: bedrockMessages, + system: systemPrompts, + inferenceConfig: inferenceConfig, + toolConfig: toolConfig, + } +} + +// Chat sends messages to AWS Bedrock using the Converse API. +func (p *Provider) Chat( + ctx context.Context, + messages []Message, + tools []ToolDefinition, + model string, + options map[string]any, +) (*LLMResponse, error) { + effectiveTimeout := p.requestTimeout + if effectiveTimeout <= 0 { + effectiveTimeout = common.DefaultRequestTimeout + } + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, effectiveTimeout) + defer cancel() + } + + params := buildConverseParams(messages, tools, options) + input := &bedrockruntime.ConverseInput{ + ModelId: aws.String(model), + Messages: params.messages, + InferenceConfig: params.inferenceConfig, + ToolConfig: params.toolConfig, + } + if len(params.system) > 0 { + input.System = params.system + } + output, err := p.client.Converse(ctx, input) if err != nil { - // Check for SSO token expiration errors and provide actionable guidance if isSSOTokenError(err) { return nil, fmt.Errorf( "bedrock converse: AWS credentials may have expired. If using AWS SSO, run 'aws sso login' to refresh: %w", @@ -216,10 +221,199 @@ func (p *Provider) Chat( return nil, fmt.Errorf("bedrock converse: %w", err) } - // Parse the response return parseResponse(output) } +// ChatStream sends messages to AWS Bedrock using the ConverseStream API. +// It streams the accumulated text so far via the onChunk callback and returns the complete response. +func (p *Provider) ChatStream( + ctx context.Context, + messages []Message, + tools []ToolDefinition, + model string, + options map[string]any, + onChunk func(accumulated string), +) (*LLMResponse, error) { + if p.requestTimeout > 0 { + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, p.requestTimeout) + defer cancel() + } + } + + params := buildConverseParams(messages, tools, options) + input := &bedrockruntime.ConverseStreamInput{ + ModelId: aws.String(model), + Messages: params.messages, + InferenceConfig: params.inferenceConfig, + ToolConfig: params.toolConfig, + } + if len(params.system) > 0 { + input.System = params.system + } + + output, err := p.client.ConverseStream(ctx, input) + if err != nil { + if isSSOTokenError(err) { + return nil, fmt.Errorf( + "bedrock conversestream: AWS credentials may have expired. If using AWS SSO, run 'aws sso login' to refresh: %w", + err, + ) + } + return nil, fmt.Errorf("bedrock conversestream: %w", err) + } + + return parseStreamResponse(ctx, output.GetStream(), onChunk) +} + +// converseStreamReader abstracts the Bedrock event stream so parseStreamResponse +// can be unit-tested with a mock event source. +type converseStreamReader interface { + Events() <-chan types.ConverseStreamOutput + Err() error + Close() error +} + +// parseStreamResponse processes the ConverseStream event stream and accumulates the response. +func parseStreamResponse( + ctx context.Context, + stream converseStreamReader, + onChunk func(accumulated string), +) (resp *LLMResponse, err error) { + if stream == nil { + return nil, fmt.Errorf("bedrock conversestream: nil event stream") + } + defer func() { + if closeErr := stream.Close(); closeErr != nil { + if err == nil { + err = fmt.Errorf("bedrock conversestream: close event stream: %w", closeErr) + } else { + log.Printf("bedrock conversestream: close event stream: %v", closeErr) + } + } + }() + + var textContent strings.Builder + finishReason := "stop" + var usage *UsageInfo + toolCalls := make([]ToolCall, 0) + + // Track active tool use blocks by index + type toolAccum struct { + id string + name string + argsJSON strings.Builder + } + activeTools := map[int]*toolAccum{} + + events := stream.Events() + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case event, ok := <-events: + if !ok { + // Stream closed + goto done + } + + switch e := event.(type) { + case *types.ConverseStreamOutputMemberContentBlockStart: + // New content block starting + if toolUse, ok := e.Value.Start.(*types.ContentBlockStartMemberToolUse); ok { + activeTools[int(aws.ToInt32(e.Value.ContentBlockIndex))] = &toolAccum{ + id: aws.ToString(toolUse.Value.ToolUseId), + name: aws.ToString(toolUse.Value.Name), + } + } + + case *types.ConverseStreamOutputMemberContentBlockDelta: + // Content delta + switch delta := e.Value.Delta.(type) { + case *types.ContentBlockDeltaMemberText: + textContent.WriteString(delta.Value) + if onChunk != nil { + onChunk(textContent.String()) + } + case *types.ContentBlockDeltaMemberToolUse: + idx := int(aws.ToInt32(e.Value.ContentBlockIndex)) + if tool, exists := activeTools[idx]; exists { + tool.argsJSON.WriteString(aws.ToString(delta.Value.Input)) + } + } + + case *types.ConverseStreamOutputMemberContentBlockStop: + // Content block finished - finalize tool if it was a tool use + idx := int(aws.ToInt32(e.Value.ContentBlockIndex)) + if tool, exists := activeTools[idx]; exists { + args := make(map[string]any) + argsStr := tool.argsJSON.String() + if argsStr != "" { + if err := json.Unmarshal([]byte(argsStr), &args); err != nil { + log.Printf("bedrock: stream: failed to parse tool arguments for %q: %v", tool.name, err) + args = map[string]any{"raw": argsStr} + } + } + funcArgs := argsStr + if argsJSON, marshalErr := json.Marshal(args); marshalErr == nil { + funcArgs = string(argsJSON) + } + toolCalls = append(toolCalls, ToolCall{ + ID: tool.id, + Name: tool.name, + Arguments: args, + Function: &FunctionCall{ + Name: tool.name, + Arguments: funcArgs, + }, + }) + delete(activeTools, idx) + } + + case *types.ConverseStreamOutputMemberMessageStop: + // Message complete + switch e.Value.StopReason { + case types.StopReasonToolUse: + finishReason = "tool_calls" + case types.StopReasonMaxTokens: + finishReason = "length" + case types.StopReasonEndTurn: + finishReason = "stop" + case types.StopReasonStopSequence: + finishReason = "stop" + case types.StopReasonContentFiltered: + finishReason = "content_filter" + default: + finishReason = "stop" + } + + case *types.ConverseStreamOutputMemberMetadata: + // Usage metadata + if e.Value.Usage != nil { + usage = &UsageInfo{ + PromptTokens: int(aws.ToInt32(e.Value.Usage.InputTokens)), + CompletionTokens: int(aws.ToInt32(e.Value.Usage.OutputTokens)), + TotalTokens: int(aws.ToInt32(e.Value.Usage.InputTokens)) + int(aws.ToInt32(e.Value.Usage.OutputTokens)), + } + } + } + } + } + +done: + if err := stream.Err(); err != nil { + return nil, fmt.Errorf("bedrock conversestream: %w", err) + } + + return &LLMResponse{ + Content: textContent.String(), + ToolCalls: toolCalls, + FinishReason: finishReason, + Usage: usage, + }, nil +} + // GetDefaultModel returns an empty string as Bedrock models are user-configured. func (p *Provider) GetDefaultModel() string { return "" diff --git a/pkg/providers/bedrock/provider_bedrock_test.go b/pkg/providers/bedrock/provider_bedrock_test.go index 38a5e26da..9d6c747f1 100644 --- a/pkg/providers/bedrock/provider_bedrock_test.go +++ b/pkg/providers/bedrock/provider_bedrock_test.go @@ -8,6 +8,7 @@ package bedrock import ( + "context" "fmt" "testing" @@ -605,3 +606,272 @@ func TestIsSSOTokenError(t *testing.T) { }) } } + +// mockStreamReader implements bedrockruntime.ConverseStreamOutputReader for testing. +type mockStreamReader struct { + ch chan types.ConverseStreamOutput + err error +} + +func (r *mockStreamReader) Events() <-chan types.ConverseStreamOutput { return r.ch } +func (r *mockStreamReader) Close() error { return nil } +func (r *mockStreamReader) Err() error { return r.err } + +func newMockStream(events []types.ConverseStreamOutput) *bedrockruntime.ConverseStreamEventStream { + ch := make(chan types.ConverseStreamOutput, len(events)) + for _, e := range events { + ch <- e + } + close(ch) + + return bedrockruntime.NewConverseStreamEventStream(func(es *bedrockruntime.ConverseStreamEventStream) { + es.Reader = &mockStreamReader{ch: ch} + }) +} + +func TestParseStreamResponse_TextOnly(t *testing.T) { + events := []types.ConverseStreamOutput{ + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + Delta: &types.ContentBlockDeltaMemberText{Value: "Hello "}, + ContentBlockIndex: aws.Int32(0), + }, + }, + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + Delta: &types.ContentBlockDeltaMemberText{Value: "World"}, + ContentBlockIndex: aws.Int32(0), + }, + }, + &types.ConverseStreamOutputMemberMessageStop{ + Value: types.MessageStopEvent{StopReason: types.StopReasonEndTurn}, + }, + &types.ConverseStreamOutputMemberMetadata{ + Value: types.ConverseStreamMetadataEvent{ + Usage: &types.TokenUsage{ + InputTokens: aws.Int32(10), + OutputTokens: aws.Int32(5), + }, + }, + }, + } + + var chunks []string + stream := newMockStream(events) + resp, err := parseStreamResponse(context.Background(), stream, func(accumulated string) { + chunks = append(chunks, accumulated) + }) + + require.NoError(t, err) + assert.Equal(t, "Hello World", resp.Content) + assert.Equal(t, "stop", resp.FinishReason) + assert.Empty(t, resp.ToolCalls) + require.NotNil(t, resp.Usage) + assert.Equal(t, 10, resp.Usage.PromptTokens) + assert.Equal(t, 5, resp.Usage.CompletionTokens) + assert.Equal(t, 15, resp.Usage.TotalTokens) + assert.Equal(t, []string{"Hello ", "Hello World"}, chunks) +} + +func TestParseStreamResponse_ToolCall(t *testing.T) { + events := []types.ConverseStreamOutput{ + &types.ConverseStreamOutputMemberContentBlockStart{ + Value: types.ContentBlockStartEvent{ + ContentBlockIndex: aws.Int32(0), + Start: &types.ContentBlockStartMemberToolUse{ + Value: types.ToolUseBlockStart{ + ToolUseId: aws.String("call_1"), + Name: aws.String("search"), + }, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + ContentBlockIndex: aws.Int32(0), + Delta: &types.ContentBlockDeltaMemberToolUse{ + Value: types.ToolUseBlockDelta{Input: aws.String(`{"q":`)}, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + ContentBlockIndex: aws.Int32(0), + Delta: &types.ContentBlockDeltaMemberToolUse{ + Value: types.ToolUseBlockDelta{Input: aws.String(`"test"}`)}, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockStop{ + Value: types.ContentBlockStopEvent{ContentBlockIndex: aws.Int32(0)}, + }, + &types.ConverseStreamOutputMemberMessageStop{ + Value: types.MessageStopEvent{StopReason: types.StopReasonToolUse}, + }, + } + + stream := newMockStream(events) + resp, err := parseStreamResponse(context.Background(), stream, nil) + + require.NoError(t, err) + assert.Equal(t, "tool_calls", resp.FinishReason) + require.Len(t, resp.ToolCalls, 1) + assert.Equal(t, "call_1", resp.ToolCalls[0].ID) + assert.Equal(t, "search", resp.ToolCalls[0].Name) + assert.Equal(t, map[string]any{"q": "test"}, resp.ToolCalls[0].Arguments) + require.NotNil(t, resp.ToolCalls[0].Function) + assert.Equal(t, "search", resp.ToolCalls[0].Function.Name) + assert.Equal(t, `{"q":"test"}`, resp.ToolCalls[0].Function.Arguments) +} + +func TestParseStreamResponse_TextAndToolCall(t *testing.T) { + events := []types.ConverseStreamOutput{ + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + ContentBlockIndex: aws.Int32(0), + Delta: &types.ContentBlockDeltaMemberText{Value: "Let me search that."}, + }, + }, + &types.ConverseStreamOutputMemberContentBlockStart{ + Value: types.ContentBlockStartEvent{ + ContentBlockIndex: aws.Int32(1), + Start: &types.ContentBlockStartMemberToolUse{ + Value: types.ToolUseBlockStart{ + ToolUseId: aws.String("call_2"), + Name: aws.String("web"), + }, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + ContentBlockIndex: aws.Int32(1), + Delta: &types.ContentBlockDeltaMemberToolUse{ + Value: types.ToolUseBlockDelta{Input: aws.String(`{"url":"https://example.com"}`)}, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockStop{ + Value: types.ContentBlockStopEvent{ContentBlockIndex: aws.Int32(1)}, + }, + &types.ConverseStreamOutputMemberMessageStop{ + Value: types.MessageStopEvent{StopReason: types.StopReasonToolUse}, + }, + } + + var chunks []string + stream := newMockStream(events) + resp, err := parseStreamResponse(context.Background(), stream, func(accumulated string) { + chunks = append(chunks, accumulated) + }) + + require.NoError(t, err) + assert.Equal(t, "Let me search that.", resp.Content) + assert.Equal(t, "tool_calls", resp.FinishReason) + require.Len(t, resp.ToolCalls, 1) + assert.Equal(t, "web", resp.ToolCalls[0].Name) + assert.Equal(t, []string{"Let me search that."}, chunks) +} + +func TestParseStreamResponse_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // Use an unbuffered channel with no events so ctx.Done() is the only ready case. + ch := make(chan types.ConverseStreamOutput) + + stream := bedrockruntime.NewConverseStreamEventStream(func(es *bedrockruntime.ConverseStreamEventStream) { + es.Reader = &mockStreamReader{ch: ch} + }) + + _, err := parseStreamResponse(ctx, stream, nil) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestParseStreamResponse_InvalidToolJSON(t *testing.T) { + events := []types.ConverseStreamOutput{ + &types.ConverseStreamOutputMemberContentBlockStart{ + Value: types.ContentBlockStartEvent{ + ContentBlockIndex: aws.Int32(0), + Start: &types.ContentBlockStartMemberToolUse{ + Value: types.ToolUseBlockStart{ + ToolUseId: aws.String("call_bad"), + Name: aws.String("broken"), + }, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + ContentBlockIndex: aws.Int32(0), + Delta: &types.ContentBlockDeltaMemberToolUse{ + Value: types.ToolUseBlockDelta{Input: aws.String(`{not valid json`)}, + }, + }, + }, + &types.ConverseStreamOutputMemberContentBlockStop{ + Value: types.ContentBlockStopEvent{ContentBlockIndex: aws.Int32(0)}, + }, + &types.ConverseStreamOutputMemberMessageStop{ + Value: types.MessageStopEvent{StopReason: types.StopReasonToolUse}, + }, + } + + stream := newMockStream(events) + resp, err := parseStreamResponse(context.Background(), stream, nil) + + require.NoError(t, err) + require.Len(t, resp.ToolCalls, 1) + assert.Equal(t, map[string]any{"raw": `{not valid json`}, resp.ToolCalls[0].Arguments) + assert.JSONEq(t, `{"raw":"{not valid json"}`, resp.ToolCalls[0].Function.Arguments) +} + +func TestParseStreamResponse_DefaultFinishReason(t *testing.T) { + events := []types.ConverseStreamOutput{ + &types.ConverseStreamOutputMemberContentBlockDelta{ + Value: types.ContentBlockDeltaEvent{ + Delta: &types.ContentBlockDeltaMemberText{Value: "partial"}, + ContentBlockIndex: aws.Int32(0), + }, + }, + } + + stream := newMockStream(events) + resp, err := parseStreamResponse(context.Background(), stream, nil) + + require.NoError(t, err) + assert.Equal(t, "stop", resp.FinishReason) +} + +func TestParseStreamResponse_NilStream(t *testing.T) { + _, err := parseStreamResponse(context.Background(), nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "nil event stream") +} + +func TestParseStreamResponse_StopReasons(t *testing.T) { + tests := []struct { + reason types.StopReason + expected string + }{ + {types.StopReasonEndTurn, "stop"}, + {types.StopReasonMaxTokens, "length"}, + {types.StopReasonToolUse, "tool_calls"}, + {types.StopReasonStopSequence, "stop"}, + {types.StopReasonContentFiltered, "content_filter"}, + } + + for _, tt := range tests { + t.Run(string(tt.reason), func(t *testing.T) { + events := []types.ConverseStreamOutput{ + &types.ConverseStreamOutputMemberMessageStop{ + Value: types.MessageStopEvent{StopReason: tt.reason}, + }, + } + stream := newMockStream(events) + resp, err := parseStreamResponse(context.Background(), stream, nil) + require.NoError(t, err) + assert.Equal(t, tt.expected, resp.FinishReason) + }) + } +} diff --git a/pkg/providers/common/google_schema.go b/pkg/providers/common/google_schema.go new file mode 100644 index 000000000..f7b2a337b --- /dev/null +++ b/pkg/providers/common/google_schema.go @@ -0,0 +1,642 @@ +package common + +import ( + "strconv" + "strings" +) + +const maxGeminiSchemaDepth = 64 + +var geminiSupportedTypes = map[string]bool{ + "array": true, + "boolean": true, + "integer": true, + "number": true, + "object": true, + "string": true, +} + +// SanitizeSchemaForGoogle reduces a JSON Schema to the conservative subset +// accepted by Google/Gemini-style function declarations. It resolves local +// refs, collapses composition keywords like anyOf/oneOf/allOf, and strips +// advanced keywords that Gemini-compatible backends often reject. +func SanitizeSchemaForGoogle(schema map[string]any) map[string]any { + if schema == nil { + return nil + } + + sanitizer := geminiSchemaSanitizer{root: schema} + result := sanitizer.sanitizeNode(schema, nil, 0) + if len(result) == 0 { + return map[string]any{ + "type": "object", + "properties": map[string]any{}, + } + } + if _, hasProps := result["properties"]; hasProps { + result["type"] = "object" + } + return result +} + +// SanitizeSchemaForGemini is kept as a compatibility alias for the original +// Google/Gemini sanitizer name. +func SanitizeSchemaForGemini(schema map[string]any) map[string]any { + return SanitizeSchemaForGoogle(schema) +} + +type geminiSchemaSanitizer struct { + root map[string]any +} + +func (s geminiSchemaSanitizer) sanitizeNode( + node map[string]any, + refTrail map[string]struct{}, + depth int, +) map[string]any { + if node == nil || depth > maxGeminiSchemaDepth { + return map[string]any{} + } + + normalized := s.normalizeNode(node, refTrail, depth) + if len(normalized) == 0 { + return map[string]any{} + } + + result := make(map[string]any) + + if desc, ok := normalized["description"].(string); ok && strings.TrimSpace(desc) != "" { + result["description"] = desc + } + + if schemaType := sanitizeGeminiSchemaType(normalized["type"]); schemaType != "" { + result["type"] = schemaType + } + + if enumValues := sanitizeGeminiEnum(normalized["enum"]); len(enumValues) > 0 { + result["enum"] = enumValues + } + + if propsRaw, ok := normalized["properties"].(map[string]any); ok { + props := make(map[string]any, len(propsRaw)) + for name, rawProp := range propsRaw { + propSchema, ok := rawProp.(map[string]any) + if !ok { + continue + } + sanitizedProp := s.sanitizeNode(propSchema, refTrail, depth+1) + if len(sanitizedProp) == 0 { + sanitizedProp = map[string]any{} + } + props[name] = sanitizedProp + } + result["properties"] = props + result["type"] = "object" + if required := sanitizeGeminiRequired(normalized["required"], props); len(required) > 0 { + result["required"] = required + } + } + + if itemsRaw, ok := normalized["items"].(map[string]any); ok { + items := s.sanitizeNode(itemsRaw, refTrail, depth+1) + if len(items) == 0 { + items = map[string]any{} + } + result["items"] = items + if _, hasType := result["type"]; !hasType { + result["type"] = "array" + } + } + + return result +} + +func (s geminiSchemaSanitizer) normalizeNode( + node map[string]any, + refTrail map[string]struct{}, + depth int, +) map[string]any { + if node == nil || depth > maxGeminiSchemaDepth { + return map[string]any{} + } + + normalized := cloneGeminiSchemaMap(node) + + if ref, ok := normalized["$ref"].(string); ok { + delete(normalized, "$ref") + if _, seen := refTrail[ref]; !seen { + if target, ok := s.resolveLocalSchemaRef(ref); ok { + nextTrail := cloneRefTrail(refTrail) + nextTrail[ref] = struct{}{} + normalized = mergeGeminiSchemaMaps( + s.normalizeNode(target, nextTrail, depth+1), + normalized, + ) + } + } + } + + if rawAllOf, ok := normalized["allOf"]; ok { + delete(normalized, "allOf") + for _, part := range schemaSlice(rawAllOf) { + normalized = mergeGeminiSchemaMaps( + normalized, + s.normalizeNode(part, refTrail, depth+1), + ) + } + } + + if rawAnyOf, ok := normalized["anyOf"]; ok { + delete(normalized, "anyOf") + normalized = mergeGeminiSchemaMaps( + s.mergeUnionBranches(schemaSlice(rawAnyOf), refTrail, depth+1), + normalized, + ) + } + + if rawOneOf, ok := normalized["oneOf"]; ok { + delete(normalized, "oneOf") + normalized = mergeGeminiSchemaMaps( + s.mergeUnionBranches(schemaSlice(rawOneOf), refTrail, depth+1), + normalized, + ) + } + + return normalized +} + +func (s geminiSchemaSanitizer) mergeUnionBranches( + branches []map[string]any, + refTrail map[string]struct{}, + depth int, +) map[string]any { + if len(branches) == 0 { + return map[string]any{} + } + + objectBranches := make([]map[string]any, 0, len(branches)) + arrayBranches := make([]map[string]any, 0, len(branches)) + nonNullBranches := make([]map[string]any, 0, len(branches)) + sameType := "" + sameTypeConsistent := true + + for _, branch := range branches { + normalized := s.normalizeNode(branch, refTrail, depth+1) + if len(normalized) == 0 { + continue + } + + branchType := geminiSchemaBranchType(normalized["type"]) + if branchType == "null" { + continue + } + nonNullBranches = append(nonNullBranches, normalized) + + if sameType == "" { + sameType = branchType + } else if branchType != "" && branchType != sameType { + sameTypeConsistent = false + } + + if branchType == "object" || hasSchemaProperties(normalized) { + objectBranches = append(objectBranches, normalized) + continue + } + if branchType == "array" || hasSchemaItems(normalized) { + arrayBranches = append(arrayBranches, normalized) + } + } + + if len(nonNullBranches) == 0 { + return map[string]any{} + } + if len(objectBranches) > 0 { + return mergeUnionObjectSchemas(objectBranches) + } + if len(arrayBranches) == len(nonNullBranches) && len(arrayBranches) > 0 { + return mergeUnionArraySchemas(arrayBranches) + } + if sameTypeConsistent && sameType != "" { + merged := map[string]any{} + for _, branch := range nonNullBranches { + merged = mergeGeminiSchemaMaps(merged, branch) + } + return merged + } + + best := nonNullBranches[0] + bestScore := geminiUnionBranchScore(best) + for _, branch := range nonNullBranches[1:] { + if score := geminiUnionBranchScore(branch); score > bestScore { + best = branch + bestScore = score + } + } + return cloneGeminiSchemaMap(best) +} + +func (s geminiSchemaSanitizer) resolveLocalSchemaRef(ref string) (map[string]any, bool) { + if ref == "#" { + return s.root, true + } + if !strings.HasPrefix(ref, "#/") { + return nil, false + } + + var current any = s.root + for _, rawToken := range strings.Split(strings.TrimPrefix(ref, "#/"), "/") { + token := strings.ReplaceAll(strings.ReplaceAll(rawToken, "~1", "/"), "~0", "~") + switch value := current.(type) { + case map[string]any: + next, ok := value[token] + if !ok { + return nil, false + } + current = next + case []any: + index, err := strconv.Atoi(token) + if err != nil || index < 0 || index >= len(value) { + return nil, false + } + current = value[index] + default: + return nil, false + } + } + + resolved, ok := current.(map[string]any) + return resolved, ok +} + +func mergeUnionObjectSchemas(branches []map[string]any) map[string]any { + merged := map[string]any{ + "type": "object", + "properties": map[string]any{}, + } + + var commonRequired map[string]struct{} + var requiredOrder []string + + for i, branch := range branches { + merged = mergeGeminiSchemaMaps(merged, branch) + + required := requiredStrings(branch["required"]) + if i == 0 { + commonRequired = make(map[string]struct{}, len(required)) + requiredOrder = append(requiredOrder, required...) + for _, name := range required { + commonRequired[name] = struct{}{} + } + continue + } + + current := make(map[string]struct{}, len(required)) + for _, name := range required { + current[name] = struct{}{} + } + for name := range commonRequired { + if _, ok := current[name]; !ok { + delete(commonRequired, name) + } + } + } + + if len(commonRequired) > 0 { + filtered := make([]string, 0, len(commonRequired)) + for _, name := range requiredOrder { + if _, ok := commonRequired[name]; ok { + filtered = append(filtered, name) + } + } + if len(filtered) > 0 { + merged["required"] = filtered + } + } else { + delete(merged, "required") + } + + return merged +} + +func mergeUnionArraySchemas(branches []map[string]any) map[string]any { + merged := map[string]any{ + "type": "array", + } + for _, branch := range branches { + merged = mergeGeminiSchemaMaps(merged, branch) + } + return merged +} + +func mergeGeminiSchemaMaps(base map[string]any, overlay map[string]any) map[string]any { + if len(base) == 0 { + return cloneGeminiSchemaMap(overlay) + } + if len(overlay) == 0 { + return cloneGeminiSchemaMap(base) + } + + result := cloneGeminiSchemaMap(base) + for key, value := range overlay { + switch key { + case "properties": + overlayProps, ok := value.(map[string]any) + if !ok { + continue + } + existing, _ := result["properties"].(map[string]any) + mergedProps := cloneGeminiSchemaMap(existing) + if mergedProps == nil { + mergedProps = make(map[string]any, len(overlayProps)) + } + for name, rawProp := range overlayProps { + propSchema, ok := rawProp.(map[string]any) + if !ok { + continue + } + if existingProp, ok := mergedProps[name].(map[string]any); ok { + mergedProps[name] = mergeGeminiSchemaMaps(existingProp, propSchema) + } else { + mergedProps[name] = cloneGeminiSchemaMap(propSchema) + } + } + result["properties"] = mergedProps + case "items": + overlayItems, ok := value.(map[string]any) + if !ok { + continue + } + if existingItems, ok := result["items"].(map[string]any); ok { + result["items"] = mergeGeminiSchemaMaps(existingItems, overlayItems) + } else { + result["items"] = cloneGeminiSchemaMap(overlayItems) + } + case "required": + if merged := mergeRequiredLists(result["required"], value); len(merged) > 0 { + result["required"] = merged + } + case "type": + if mergedType := mergeGeminiSchemaTypes(result["type"], value); mergedType != "" { + result["type"] = mergedType + } else { + delete(result, "type") + } + case "description": + desc, ok := value.(string) + if ok && strings.TrimSpace(desc) != "" { + result["description"] = desc + } + default: + result[key] = cloneGeminiSchemaValue(value) + } + } + + return result +} + +func mergeGeminiSchemaTypes(left any, right any) string { + leftType := geminiSchemaBranchType(left) + rightType := geminiSchemaBranchType(right) + + switch { + case leftType == "": + return rightType + case rightType == "": + return leftType + case leftType == rightType: + return leftType + case leftType == "null": + return rightType + case rightType == "null": + return leftType + default: + return "" + } +} + +func sanitizeGeminiSchemaType(raw any) string { + typeName := geminiSchemaBranchType(raw) + if typeName == "null" { + return "" + } + return typeName +} + +func geminiSchemaBranchType(raw any) string { + switch value := raw.(type) { + case string: + if value == "null" { + return value + } + if geminiSupportedTypes[value] { + return value + } + return "" + case []string: + return geminiSchemaBranchType(stringSliceToAny(value)) + case []any: + candidate := "" + sawNull := false + for _, item := range value { + typeName, ok := item.(string) + if !ok { + continue + } + if typeName == "null" { + sawNull = true + continue + } + if !geminiSupportedTypes[typeName] { + continue + } + if candidate == "" { + candidate = typeName + continue + } + if candidate != typeName { + return "" + } + } + if candidate == "" && sawNull { + return "null" + } + return candidate + default: + return "" + } +} + +func sanitizeGeminiEnum(raw any) []any { + values, ok := raw.([]any) + if !ok { + if stringValues, ok := raw.([]string); ok { + return stringSliceToAny(stringValues) + } + return nil + } + + sanitized := make([]any, 0, len(values)) + for _, value := range values { + switch value.(type) { + case string, bool, float64, int, int32, int64: + sanitized = append(sanitized, value) + } + } + if len(sanitized) == 0 { + return nil + } + return sanitized +} + +func sanitizeGeminiRequired(raw any, properties map[string]any) []string { + required := requiredStrings(raw) + if len(required) == 0 { + return nil + } + + filtered := make([]string, 0, len(required)) + seen := make(map[string]struct{}, len(required)) + for _, name := range required { + if _, ok := properties[name]; !ok { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + filtered = append(filtered, name) + } + if len(filtered) == 0 { + return nil + } + return filtered +} + +func requiredStrings(raw any) []string { + switch value := raw.(type) { + case []string: + return append([]string(nil), value...) + case []any: + required := make([]string, 0, len(value)) + for _, item := range value { + name, ok := item.(string) + if ok { + required = append(required, name) + } + } + return required + default: + return nil + } +} + +func mergeRequiredLists(left any, right any) []string { + merged := make([]string, 0) + seen := map[string]struct{}{} + + for _, name := range append(requiredStrings(left), requiredStrings(right)...) { + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + merged = append(merged, name) + } + + return merged +} + +func geminiUnionBranchScore(schema map[string]any) int { + score := 0 + if hasSchemaProperties(schema) { + score += 20 + } + if hasSchemaItems(schema) { + score += 10 + } + if _, ok := schema["enum"]; ok { + score += 5 + } + if _, ok := schema["description"]; ok { + score += 2 + } + score += len(schema) + return score +} + +func hasSchemaProperties(schema map[string]any) bool { + props, ok := schema["properties"].(map[string]any) + return ok && len(props) > 0 +} + +func hasSchemaItems(schema map[string]any) bool { + _, ok := schema["items"].(map[string]any) + return ok +} + +func schemaSlice(raw any) []map[string]any { + switch value := raw.(type) { + case []map[string]any: + return append([]map[string]any(nil), value...) + case []any: + schemas := make([]map[string]any, 0, len(value)) + for _, item := range value { + schema, ok := item.(map[string]any) + if ok { + schemas = append(schemas, schema) + } + } + return schemas + default: + return nil + } +} + +func cloneGeminiSchemaMap(in map[string]any) map[string]any { + if len(in) == 0 { + return nil + } + out := make(map[string]any, len(in)) + for key, value := range in { + out[key] = cloneGeminiSchemaValue(value) + } + return out +} + +func cloneGeminiSchemaValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneGeminiSchemaMap(typed) + case []any: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = cloneGeminiSchemaValue(item) + } + return out + case []string: + return append([]string(nil), typed...) + default: + return typed + } +} + +func cloneRefTrail(in map[string]struct{}) map[string]struct{} { + if len(in) == 0 { + return make(map[string]struct{}) + } + out := make(map[string]struct{}, len(in)) + for key := range in { + out[key] = struct{}{} + } + return out +} + +func stringSliceToAny(values []string) []any { + if len(values) == 0 { + return nil + } + result := make([]any, len(values)) + for i, value := range values { + result[i] = value + } + return result +} diff --git a/pkg/providers/common/google_schema_test.go b/pkg/providers/common/google_schema_test.go new file mode 100644 index 000000000..23aadbf98 --- /dev/null +++ b/pkg/providers/common/google_schema_test.go @@ -0,0 +1,254 @@ +package common + +import "testing" + +func TestSanitizeSchemaForGemini_DereferencesRefsAndFlattensUnions(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "parent": map[string]any{ + "anyOf": []any{ + map[string]any{"$ref": "#/$defs/pageParent"}, + map[string]any{"$ref": "#/$defs/databaseParent"}, + }, + }, + "icon": map[string]any{ + "anyOf": []any{ + map[string]any{"$ref": "#/$defs/emoji"}, + map[string]any{"type": "null"}, + }, + }, + "data": map[string]any{ + "$ref": "#/$defs/dataPayload", + }, + }, + "required": []any{"parent", "icon", "missing"}, + "$defs": map[string]any{ + "pageParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "page_id": map[string]any{ + "type": "string", + }, + }, + "required": []any{"page_id"}, + }, + "databaseParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "database_id": map[string]any{ + "type": "string", + }, + }, + "required": []any{"database_id"}, + }, + "emoji": map[string]any{ + "type": "string", + "pattern": "^:[a-z_]+:$", + }, + "dataPayload": map[string]any{ + "type": "object", + "additionalProperties": false, + "properties": map[string]any{ + "name": map[string]any{ + "type": "string", + "minLength": 1, + }, + "count": map[string]any{ + "type": "integer", + "minimum": 1, + }, + }, + "required": []any{"name"}, + }, + }, + } + + got := SanitizeSchemaForGemini(schema) + assertSchemaKeyAbsent(t, got, "$defs") + assertSchemaKeyAbsent(t, got, "$ref") + assertSchemaKeyAbsent(t, got, "anyOf") + assertSchemaKeyAbsent(t, got, "oneOf") + assertSchemaKeyAbsent(t, got, "allOf") + assertSchemaKeyAbsent(t, got, "additionalProperties") + assertSchemaKeyAbsent(t, got, "pattern") + assertSchemaKeyAbsent(t, got, "minLength") + assertSchemaKeyAbsent(t, got, "minimum") + + if got["type"] != "object" { + t.Fatalf("top-level type = %#v, want object", got["type"]) + } + + props, ok := got["properties"].(map[string]any) + if !ok { + t.Fatalf("properties = %#v, want map", got["properties"]) + } + + parent, ok := props["parent"].(map[string]any) + if !ok { + t.Fatalf("parent schema = %#v, want map", props["parent"]) + } + if parent["type"] != "object" { + t.Fatalf("parent.type = %#v, want object", parent["type"]) + } + parentProps, ok := parent["properties"].(map[string]any) + if !ok { + t.Fatalf("parent.properties = %#v, want map", parent["properties"]) + } + if _, found := parentProps["page_id"]; !found { + t.Fatalf("parent.properties missing page_id: %#v", parentProps) + } + if _, found := parentProps["database_id"]; !found { + t.Fatalf("parent.properties missing database_id: %#v", parentProps) + } + if _, hasRequired := parent["required"]; hasRequired { + t.Fatalf("parent.required = %#v, want omitted for merged anyOf branches", parent["required"]) + } + + icon, ok := props["icon"].(map[string]any) + if !ok { + t.Fatalf("icon schema = %#v, want map", props["icon"]) + } + if icon["type"] != "string" { + t.Fatalf("icon.type = %#v, want string", icon["type"]) + } + + data, ok := props["data"].(map[string]any) + if !ok { + t.Fatalf("data schema = %#v, want map", props["data"]) + } + if data["type"] != "object" { + t.Fatalf("data.type = %#v, want object", data["type"]) + } + dataProps, ok := data["properties"].(map[string]any) + if !ok { + t.Fatalf("data.properties = %#v, want map", data["properties"]) + } + if _, found := dataProps["name"]; !found { + t.Fatalf("data.properties missing name: %#v", dataProps) + } + if _, found := dataProps["count"]; !found { + t.Fatalf("data.properties missing count: %#v", dataProps) + } + + required, ok := got["required"].([]string) + if !ok { + t.Fatalf("required = %#v, want []string", got["required"]) + } + if len(required) != 2 || required[0] != "parent" || required[1] != "icon" { + t.Fatalf("required = %#v, want [parent icon]", required) + } +} + +func TestSanitizeSchemaForGemini_MergesAllOfAndFiltersRequired(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "payload": map[string]any{ + "allOf": []any{ + map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": map[string]any{ + "type": "string", + }, + }, + "required": []any{"id"}, + }, + map[string]any{ + "properties": map[string]any{ + "name": map[string]any{ + "type": "string", + }, + "count": map[string]any{ + "type": "integer", + "minimum": 1, + }, + }, + "required": []any{"name", "missing"}, + }, + }, + }, + }, + } + + got := SanitizeSchemaForGemini(schema) + props := got["properties"].(map[string]any) + payload := props["payload"].(map[string]any) + + if payload["type"] != "object" { + t.Fatalf("payload.type = %#v, want object", payload["type"]) + } + payloadProps, ok := payload["properties"].(map[string]any) + if !ok { + t.Fatalf("payload.properties = %#v, want map", payload["properties"]) + } + for _, key := range []string{"id", "name", "count"} { + if _, found := payloadProps[key]; !found { + t.Fatalf("payload.properties missing %q: %#v", key, payloadProps) + } + } + + required, ok := payload["required"].([]string) + if !ok { + t.Fatalf("payload.required = %#v, want []string", payload["required"]) + } + if len(required) != 2 || required[0] != "id" || required[1] != "name" { + t.Fatalf("payload.required = %#v, want [id name]", required) + } + + assertSchemaKeyAbsent(t, payload, "allOf") + assertSchemaKeyAbsent(t, payload, "minimum") +} + +func TestSanitizeSchemaForGemini_HandlesRecursiveRefs(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "tree": map[string]any{ + "$ref": "#/$defs/node", + }, + }, + "$defs": map[string]any{ + "node": map[string]any{ + "type": "object", + "properties": map[string]any{ + "name": map[string]any{ + "type": "string", + }, + "child": map[string]any{ + "$ref": "#/$defs/node", + }, + }, + }, + }, + } + + got := SanitizeSchemaForGemini(schema) + props := got["properties"].(map[string]any) + tree := props["tree"].(map[string]any) + if tree["type"] != "object" { + t.Fatalf("tree.type = %#v, want object", tree["type"]) + } + assertSchemaKeyAbsent(t, tree, "$ref") +} + +func assertSchemaKeyAbsent(t *testing.T, value any, key string) { + t.Helper() + + switch typed := value.(type) { + case map[string]any: + if _, found := typed[key]; found { + t.Fatalf("schema still contains key %q: %#v", key, typed) + } + for _, nested := range typed { + assertSchemaKeyAbsent(t, nested, key) + } + case []any: + for _, nested := range typed { + assertSchemaKeyAbsent(t, nested, key) + } + case []string: + return + } +} diff --git a/pkg/providers/common/tool_schema_transform.go b/pkg/providers/common/tool_schema_transform.go new file mode 100644 index 000000000..10e96d056 --- /dev/null +++ b/pkg/providers/common/tool_schema_transform.go @@ -0,0 +1,59 @@ +package common + +import ( + "fmt" + "strings" +) + +const ( + ToolSchemaTransformOff = "" + ToolSchemaTransformSimple = "simple" +) + +// NormalizeToolSchemaTransform resolves user-facing aliases to a canonical +// transform mode. Empty values and explicit "off"-style values disable schema +// transformation. +func NormalizeToolSchemaTransform(raw string) (string, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "", "off", "none", "native": + return ToolSchemaTransformOff, nil + case "simple", "basic", "strict", "flat": + return ToolSchemaTransformSimple, nil + default: + return "", fmt.Errorf("unsupported tool_schema_transform %q (supported: off, simple)", raw) + } +} + +// TransformToolDefinitions clones tool definitions and applies the configured +// schema transform to function parameter schemas. When the transform is off, the +// original slice is returned unchanged. +func TransformToolDefinitions(tools []ToolDefinition, transform string) ([]ToolDefinition, error) { + transform, err := NormalizeToolSchemaTransform(transform) + if err != nil { + return nil, err + } + if transform == ToolSchemaTransformOff || len(tools) == 0 { + return tools, nil + } + + out := make([]ToolDefinition, len(tools)) + for i, tool := range tools { + out[i] = tool + if tool.Type != "function" { + continue + } + out[i].Function = tool.Function + out[i].Function.Parameters = transformToolSchema(tool.Function.Parameters, transform) + } + + return out, nil +} + +func transformToolSchema(schema map[string]any, transform string) map[string]any { + switch transform { + case ToolSchemaTransformSimple: + return SanitizeSchemaForGoogle(schema) + default: + return cloneGeminiSchemaMap(schema) + } +} diff --git a/pkg/providers/factory_provider.go b/pkg/providers/factory_provider.go index ce83c6c54..e9e0e6e98 100644 --- a/pkg/providers/factory_provider.go +++ b/pkg/providers/factory_provider.go @@ -15,6 +15,7 @@ import ( anthropicmessages "github.com/sipeed/picoclaw/pkg/providers/anthropic_messages" "github.com/sipeed/picoclaw/pkg/providers/azure" "github.com/sipeed/picoclaw/pkg/providers/bedrock" + "github.com/sipeed/picoclaw/pkg/providers/common" ) type protocolMeta struct { @@ -60,6 +61,8 @@ var protocolMetaByName = map[string]protocolMeta{ "longcat": {defaultAPIBase: "https://api.longcat.chat/openai"}, "modelscope": {defaultAPIBase: "https://api-inference.modelscope.cn/v1"}, "mimo": {defaultAPIBase: "https://api.xiaomimimo.com/v1"}, + "anthropic": {defaultAPIBase: "https://api.anthropic.com/v1"}, + "anthropic-messages": {defaultAPIBase: "https://api.anthropic.com/v1"}, } // createClaudeAuthProvider creates a Claude provider using OAuth credentials from auth store. @@ -110,19 +113,7 @@ func ExtractProtocol(cfg *config.ModelConfig) (protocol, modelID string) { if provider := strings.TrimSpace(cfg.Provider); provider != "" { return NormalizeProvider(provider), model } - if model == "" { - return "", "" - } - - protocol, rest, found := strings.Cut(model, "/") - if !found { - return "openai", model - } - protocol = strings.TrimSpace(protocol) - if protocol == "" { - return "", strings.TrimSpace(rest) - } - return NormalizeProvider(protocol), strings.TrimSpace(rest) + return SplitModelProviderAndID(model, "openai") } // ResolveAPIBase returns the configured API base, or the protocol default when @@ -154,6 +145,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err } protocol, modelID := ExtractProtocol(cfg) + authMethod := strings.ToLower(strings.TrimSpace(cfg.AuthMethod)) userAgent := cfg.UserAgent if userAgent == "" { @@ -163,12 +155,12 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err switch protocol { case "openai": // OpenAI with OAuth/token auth (Codex-style) - if cfg.AuthMethod == "oauth" || cfg.AuthMethod == "token" { + if authMethod == "oauth" || authMethod == "token" { provider, err := createCodexAuthProvider() if err != nil { return nil, "", err } - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) } // OpenAI with API key if cfg.APIKey() == "" && cfg.APIBase == "" { @@ -189,7 +181,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err cfg.CustomHeaders, ) provider.SetProviderName(protocol) - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) case "azure", "azure-openai": // Azure OpenAI uses deployment-based URLs, api-key header auth, @@ -202,13 +194,13 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err "api_base is required for azure protocol (e.g., https://your-resource.openai.azure.com)", ) } - return azure.NewProviderWithTimeout( + return finalizeProviderFromConfig(azure.NewProviderWithTimeout( cfg.APIKey(), cfg.APIBase, cfg.Proxy, userAgent, cfg.RequestTimeout, - ), modelID, nil + ), modelID, cfg) case "bedrock": // AWS Bedrock uses AWS SDK credentials (env vars, profiles, IAM roles, etc.) @@ -244,7 +236,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err if err != nil { return nil, "", fmt.Errorf("creating bedrock provider: %w", err) } - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) case "litellm", "lmstudio", "openrouter", "groq", "zhipu", "nvidia", "venice", "ollama", "moonshot", "shengsuanyun", "deepseek", "cerebras", @@ -270,7 +262,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err cfg.CustomHeaders, ) provider.SetProviderName(protocol) - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) case "gemini": if cfg.APIKey() == "" && cfg.APIBase == "" { @@ -280,7 +272,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err if apiBase == "" { apiBase = getDefaultAPIBase(protocol) } - return NewGeminiProvider( + return finalizeProviderFromConfig(NewGeminiProvider( cfg.APIKey(), apiBase, cfg.Proxy, @@ -288,7 +280,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err cfg.RequestTimeout, cfg.ExtraBody, cfg.CustomHeaders, - ), modelID, nil + ), modelID, cfg) case "minimax": // Minimax requires reasoning_split: true in the request body @@ -317,22 +309,19 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err cfg.CustomHeaders, ) provider.SetProviderName(protocol) - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) case "anthropic": - if cfg.AuthMethod == "oauth" || cfg.AuthMethod == "token" { + if authMethod == "oauth" || authMethod == "token" { // Use OAuth credentials from auth store provider, err := createClaudeAuthProvider() if err != nil { return nil, "", err } - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) } // Use API key with HTTP API - apiBase := cfg.APIBase - if apiBase == "" { - apiBase = "https://api.anthropic.com/v1" - } + apiBase := common.NormalizeBaseURL(cfg.APIBase, "https://api.anthropic.com/v1", true) if cfg.APIKey() == "" { return nil, "", fmt.Errorf("api_key is required for anthropic protocol (model: %s)", cfg.Model) } @@ -347,7 +336,7 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err cfg.CustomHeaders, ) provider.SetProviderName(protocol) - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) case "anthropic-messages": // Anthropic Messages API with native format (HTTP-based, no SDK) @@ -358,12 +347,12 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err if cfg.APIKey() == "" { return nil, "", fmt.Errorf("api_key is required for anthropic-messages protocol (model: %s)", cfg.Model) } - return anthropicmessages.NewProviderWithTimeout( + return finalizeProviderFromConfig(anthropicmessages.NewProviderWithTimeout( cfg.APIKey(), apiBase, userAgent, cfg.RequestTimeout, - ), modelID, nil + ), modelID, cfg) case "coding-plan-anthropic", "alibaba-coding-anthropic": // Alibaba Coding Plan with Anthropic-compatible API @@ -374,29 +363,29 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err if cfg.APIKey() == "" { return nil, "", fmt.Errorf("api_key is required for %q protocol (model: %s)", protocol, cfg.Model) } - return anthropicmessages.NewProviderWithTimeout( + return finalizeProviderFromConfig(anthropicmessages.NewProviderWithTimeout( cfg.APIKey(), apiBase, userAgent, cfg.RequestTimeout, - ), modelID, nil + ), modelID, cfg) case "antigravity": - return NewAntigravityProvider(), modelID, nil + return finalizeProviderFromConfig(NewAntigravityProvider(), modelID, cfg) case "claude-cli", "claudecli": workspace := cfg.Workspace if workspace == "" { workspace = "." } - return NewClaudeCliProvider(workspace), modelID, nil + return finalizeProviderFromConfig(NewClaudeCliProvider(workspace), modelID, cfg) case "codex-cli", "codexcli": workspace := cfg.Workspace if workspace == "" { workspace = "." } - return NewCodexCliProvider(workspace), modelID, nil + return finalizeProviderFromConfig(NewCodexCliProvider(workspace), modelID, cfg) case "github-copilot", "copilot": apiBase := cfg.APIBase @@ -411,15 +400,27 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err if err != nil { return nil, "", err } - return provider, modelID, nil + return finalizeProviderFromConfig(provider, modelID, cfg) default: return nil, "", fmt.Errorf("unknown protocol %q in model %q", protocol, cfg.Model) } } +func finalizeProviderFromConfig( + provider LLMProvider, + modelID string, + cfg *config.ModelConfig, +) (LLMProvider, string, error) { + wrapped, err := wrapProviderWithToolSchemaTransform(provider, cfg.ToolSchemaTransform) + if err != nil { + return nil, "", err + } + return wrapped, modelID, nil +} + func isEmptyAPIKeyAllowed(protocol string) bool { - meta, ok := protocolMetaByName[protocol] + meta, ok := protocolMetaForName(protocol) return ok && meta.emptyAPIKeyAllowed } @@ -439,9 +440,19 @@ func DefaultAPIBaseForProtocol(protocol string) string { // getDefaultAPIBase returns the default API base URL for a given protocol. func getDefaultAPIBase(protocol string) string { - meta, ok := protocolMetaByName[protocol] + meta, ok := protocolMetaForName(protocol) if !ok { return "" } return meta.defaultAPIBase } + +func protocolMetaForName(protocol string) (protocolMeta, bool) { + if meta, ok := protocolMetaByName[protocol]; ok { + return meta, true + } + if meta, ok := attachedModelProviderMetaByName[protocol]; ok { + return meta.protocolMeta, true + } + return protocolMeta{}, false +} diff --git a/pkg/providers/factory_provider_test.go b/pkg/providers/factory_provider_test.go index 3dd1eefb3..eb9b3d600 100644 --- a/pkg/providers/factory_provider_test.go +++ b/pkg/providers/factory_provider_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/sipeed/picoclaw/pkg/auth" "github.com/sipeed/picoclaw/pkg/config" ) @@ -101,6 +102,12 @@ func TestExtractProtocol(t *testing.T) { wantProtocol: "", wantModelID: "gpt-4o", }, + { + name: "unknown prefix falls back to openai", + config: &config.ModelConfig{Model: "meta-llama/Llama-3.1-8B-Instruct"}, + wantProtocol: "openai", + wantModelID: "meta-llama/Llama-3.1-8B-Instruct", + }, { name: "nil config", wantProtocol: "", @@ -605,6 +612,41 @@ func TestCreateProviderFromConfig_CodexCLI(t *testing.T) { } } +func TestCreateProviderFromConfig_OpenAIMixedCaseAuthMethodUsesOAuthBranch(t *testing.T) { + origGetCredential := getCredential + getCredential = func(provider string) (*auth.AuthCredential, error) { + if provider != "openai" { + t.Fatalf("provider = %q, want %q", provider, "openai") + } + return &auth.AuthCredential{ + AccessToken: "test-token", + AccountID: "acct-test", + Provider: "openai", + AuthMethod: "oauth", + }, nil + } + t.Cleanup(func() { + getCredential = origGetCredential + }) + + cfg := &config.ModelConfig{ + ModelName: "test-openai-oauth", + Model: "openai/gpt-5.4", + AuthMethod: "OAuth", + } + + provider, modelID, err := CreateProviderFromConfig(cfg) + if err != nil { + t.Fatalf("CreateProviderFromConfig() error = %v", err) + } + if provider == nil { + t.Fatal("CreateProviderFromConfig() returned nil provider") + } + if modelID != "gpt-5.4" { + t.Errorf("modelID = %q, want %q", modelID, "gpt-5.4") + } +} + func TestCreateProviderFromConfig_MissingAPIKey(t *testing.T) { cfg := &config.ModelConfig{ ModelName: "test-no-key", @@ -619,8 +661,9 @@ func TestCreateProviderFromConfig_MissingAPIKey(t *testing.T) { func TestCreateProviderFromConfig_UnknownProtocol(t *testing.T) { cfg := &config.ModelConfig{ - ModelName: "test-unknown", - Model: "unknown-protocol/model", + ModelName: "test-unknown-provider", + Provider: "unknown-protocol", + Model: "model", } cfg.SetAPIKey("test-key") @@ -630,6 +673,26 @@ func TestCreateProviderFromConfig_UnknownProtocol(t *testing.T) { } } +func TestCreateProviderFromConfig_UnknownModelPrefixDefaultsToOpenAI(t *testing.T) { + cfg := &config.ModelConfig{ + ModelName: "test-unknown-model-prefix", + Model: "meta-llama/Llama-3.1-8B-Instruct", + APIBase: "https://api.example.com/v1", + } + cfg.SetAPIKey("test-key") + + provider, modelID, err := CreateProviderFromConfig(cfg) + if err != nil { + t.Fatalf("CreateProviderFromConfig() error = %v", err) + } + if provider == nil { + t.Fatal("CreateProviderFromConfig() returned nil provider") + } + if modelID != "meta-llama/Llama-3.1-8B-Instruct" { + t.Fatalf("modelID = %q, want full model ID", modelID) + } +} + func TestCreateProviderFromConfig_NilConfig(t *testing.T) { _, _, err := CreateProviderFromConfig(nil) if err == nil { @@ -889,6 +952,71 @@ func TestGetDefaultAPIBase_QwenUSAliases(t *testing.T) { } } +func TestModelProviderOptions(t *testing.T) { + options := ModelProviderOptions() + if len(options) == 0 { + t.Fatal("ModelProviderOptions() returned no options") + } + + seen := make(map[string]ModelProviderOption, len(options)) + for _, option := range options { + seen[option.ID] = option + } + + if _, ok := seen["openai"]; !ok { + t.Fatal("openai option missing") + } + if option, ok := seen["openai"]; ok && !option.CreateAllowed { + t.Fatal("openai should be creatable") + } + if option, ok := seen["lmstudio"]; !ok { + t.Fatal("lmstudio option missing") + } else if !option.EmptyAPIKeyAllowed { + t.Fatal("lmstudio should allow empty API keys") + } + if option, ok := seen["anthropic"]; !ok { + t.Fatal("anthropic option missing") + } else if option.DefaultAPIBase != "https://api.anthropic.com/v1" { + t.Fatalf("anthropic default_api_base = %q, want %q", option.DefaultAPIBase, "https://api.anthropic.com/v1") + } + if _, ok := seen["azure"]; !ok { + t.Fatal("azure option missing") + } + if option, ok := seen["bedrock"]; !ok { + t.Fatal("bedrock option missing") + } else if !option.CreateAllowed { + t.Fatal("bedrock should be creatable and defer credential/build errors to runtime") + } + if option, ok := seen["elevenlabs"]; !ok { + t.Fatal("elevenlabs option missing") + } else { + if option.DefaultAPIBase != "https://api.elevenlabs.io" { + t.Fatalf("elevenlabs default_api_base = %q, want %q", option.DefaultAPIBase, "https://api.elevenlabs.io") + } + if option.DefaultModelAllowed { + t.Fatal("elevenlabs should be ASR-only and therefore not allowed as a default chat model") + } + } + if option, ok := seen["antigravity"]; !ok { + t.Fatal("antigravity option missing") + } else { + if !option.CreateAllowed { + t.Fatal("antigravity should be creatable") + } + if option.DefaultAuthMethod != "oauth" { + t.Fatalf("antigravity default_auth_method = %q, want %q", option.DefaultAuthMethod, "oauth") + } + if !option.AuthMethodLocked { + t.Fatal("antigravity auth method should be locked") + } + } + if option, ok := seen["github-copilot"]; !ok { + t.Fatal("github-copilot option missing") + } else if option.DefaultAPIBase != "localhost:4321" { + t.Fatalf("github-copilot default_api_base = %q, want %q", option.DefaultAPIBase, "localhost:4321") + } +} + func TestCreateProviderFromConfig_MinimaxInjectsReasoningSplit(t *testing.T) { var requestBody map[string]any @@ -1202,3 +1330,42 @@ func TestCreateProviderFromConfig_BedrockWithEndpointURL(t *testing.T) { // Unexpected error - fail the test t.Errorf("unexpected error from bedrock provider: %v", err) } + +func TestCreateProviderFromConfig_ToolSchemaTransformWrapsProvider(t *testing.T) { + cfg := &config.ModelConfig{ + ModelName: "claude-cli-test", + Provider: "claude-cli", + Model: "claude-sonnet-4.6", + Workspace: t.TempDir(), + ToolSchemaTransform: "simple", + } + + provider, modelID, err := CreateProviderFromConfig(cfg) + if err != nil { + t.Fatalf("CreateProviderFromConfig() error = %v", err) + } + if modelID != "claude-sonnet-4.6" { + t.Fatalf("modelID = %q, want %q", modelID, "claude-sonnet-4.6") + } + if _, ok := provider.(*toolSchemaTransformProvider); !ok { + t.Fatalf("provider = %T, want *toolSchemaTransformProvider", provider) + } +} + +func TestCreateProviderFromConfig_InvalidToolSchemaTransform(t *testing.T) { + cfg := &config.ModelConfig{ + ModelName: "claude-cli-test", + Provider: "claude-cli", + Model: "claude-sonnet-4.6", + Workspace: t.TempDir(), + ToolSchemaTransform: "invalid", + } + + _, _, err := CreateProviderFromConfig(cfg) + if err == nil { + t.Fatal("CreateProviderFromConfig() expected error for invalid tool_schema_transform") + } + if !strings.Contains(err.Error(), "tool_schema_transform") { + t.Fatalf("error = %v, want mention tool_schema_transform", err) + } +} diff --git a/pkg/providers/httpapi/gemini_helpers.go b/pkg/providers/httpapi/gemini_helpers.go index a2b2d63c3..87cc4c084 100644 --- a/pkg/providers/httpapi/gemini_helpers.go +++ b/pkg/providers/httpapi/gemini_helpers.go @@ -12,66 +12,6 @@ func extractPartThoughtSignature(thoughtSignature string, thoughtSignatureSnake return "" } -var geminiUnsupportedKeywords = map[string]bool{ - "patternProperties": true, - "additionalProperties": true, - "$schema": true, - "$id": true, - "$ref": true, - "$defs": true, - "definitions": true, - "examples": true, - "minLength": true, - "maxLength": true, - "minimum": true, - "maximum": true, - "multipleOf": true, - "pattern": true, - "format": true, - "minItems": true, - "maxItems": true, - "uniqueItems": true, - "minProperties": true, - "maxProperties": true, -} - -func sanitizeSchemaForGemini(schema map[string]any) map[string]any { - if schema == nil { - return nil - } - - result := make(map[string]any) - for k, v := range schema { - if geminiUnsupportedKeywords[k] { - continue - } - switch val := v.(type) { - case map[string]any: - result[k] = sanitizeSchemaForGemini(val) - case []any: - sanitized := make([]any, len(val)) - for i, item := range val { - if m, ok := item.(map[string]any); ok { - sanitized[i] = sanitizeSchemaForGemini(m) - } else { - sanitized[i] = item - } - } - result[k] = sanitized - default: - result[k] = v - } - } - - if _, hasProps := result["properties"]; hasProps { - if _, hasType := result["type"]; !hasType { - result["type"] = "object" - } - } - - return result -} - func extractProtocol(model string) (protocol, modelID string) { model = strings.TrimSpace(model) protocol, modelID, found := strings.Cut(model, "/") diff --git a/pkg/providers/httpapi/gemini_provider.go b/pkg/providers/httpapi/gemini_provider.go index d1d523757..395c555d1 100644 --- a/pkg/providers/httpapi/gemini_provider.go +++ b/pkg/providers/httpapi/gemini_provider.go @@ -264,7 +264,7 @@ func (p *GeminiProvider) buildRequestBody( funcDecls = append(funcDecls, geminiFunctionDeclaration{ Name: t.Function.Name, Description: t.Function.Description, - Parameters: sanitizeSchemaForGemini(t.Function.Parameters), + Parameters: t.Function.Parameters, }) } if len(funcDecls) > 0 { diff --git a/pkg/providers/httpapi/gemini_provider_test.go b/pkg/providers/httpapi/gemini_provider_test.go index aade90358..b455357c0 100644 --- a/pkg/providers/httpapi/gemini_provider_test.go +++ b/pkg/providers/httpapi/gemini_provider_test.go @@ -259,6 +259,64 @@ func TestGeminiProvider_ChatStreamSkipsEmptyDataFrames(t *testing.T) { } } +func TestGeminiProvider_BuildRequestBody_PreservesComplexToolSchemasByDefault(t *testing.T) { + provider := NewGeminiProvider("test-key", "https://example.com/v1beta", "", "", 0, nil, nil) + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "parent": map[string]any{ + "anyOf": []any{ + map[string]any{"$ref": "#/$defs/pageParent"}, + map[string]any{"$ref": "#/$defs/databaseParent"}, + }, + }, + }, + "$defs": map[string]any{ + "pageParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "page_id": map[string]any{"type": "string"}, + }, + "required": []any{"page_id"}, + }, + "databaseParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "database_id": map[string]any{"type": "string"}, + }, + "required": []any{"database_id"}, + }, + }, + } + + body := provider.buildRequestBody( + []Message{{Role: "user", Content: "hello"}}, + []ToolDefinition{{ + Type: "function", + Function: ToolFunctionDefinition{ + Name: "mcp_notion_create", + Description: "Create a Notion object", + Parameters: schema, + }, + }}, + "gemini-3-flash-preview", + nil, + ) + + tools, ok := body["tools"].([]geminiTool) + if !ok || len(tools) != 1 { + t.Fatalf("tools = %#v, want one geminiTool", body["tools"]) + } + got, ok := tools[0].FunctionDeclarations[0].Parameters.(map[string]any) + if !ok { + t.Fatalf("parameters = %#v, want map", tools[0].FunctionDeclarations[0].Parameters) + } + + if got["$defs"] == nil { + t.Fatalf("parameters = %#v, want raw schema with $defs preserved by default", got) + } +} + func TestGeminiProvider_ChatStreamReturnsErrorOnInvalidDataFrame(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/event-stream") diff --git a/pkg/providers/model_ref.go b/pkg/providers/model_ref.go index be9f63bc6..48e3fb4cb 100644 --- a/pkg/providers/model_ref.go +++ b/pkg/providers/model_ref.go @@ -17,18 +17,13 @@ func ParseModelRef(raw string, defaultProvider string) *ModelRef { return nil } - if idx := strings.Index(raw, "/"); idx > 0 { - provider := NormalizeProvider(raw[:idx]) - model := strings.TrimSpace(raw[idx+1:]) - if model == "" { - return nil - } - return &ModelRef{Provider: provider, Model: model} + provider, model := SplitModelProviderAndID(raw, defaultProvider) + if model == "" { + return nil } - return &ModelRef{ - Provider: NormalizeProvider(defaultProvider), - Model: raw, + Provider: provider, + Model: model, } } @@ -53,6 +48,8 @@ func NormalizeProvider(provider string) string { return "zhipu" case "google": return "gemini" + case "google-antigravity": + return "antigravity" case "alibaba-coding", "qwen-coding": return "coding-plan" case "alibaba-coding-anthropic": @@ -61,6 +58,14 @@ func NormalizeProvider(provider string) string { return "qwen-intl" case "dashscope-us": return "qwen-us" + case "azure-openai": + return "azure" + case "claudecli": + return "claude-cli" + case "codexcli": + return "codex-cli" + case "copilot": + return "github-copilot" } return p diff --git a/pkg/providers/model_ref_test.go b/pkg/providers/model_ref_test.go index 040c511ba..9a164bf48 100644 --- a/pkg/providers/model_ref_test.go +++ b/pkg/providers/model_ref_test.go @@ -72,7 +72,12 @@ func TestNormalizeProvider(t *testing.T) { {"claude", "anthropic"}, {"glm", "zhipu"}, {"google", "gemini"}, + {"google-antigravity", "antigravity"}, {"groq", "groq"}, + {"azure-openai", "azure"}, + {"claudecli", "claude-cli"}, + {"codexcli", "codex-cli"}, + {"copilot", "github-copilot"}, // Alibaba Coding Plan aliases {"alibaba-coding", "coding-plan"}, {"qwen-coding", "coding-plan"}, @@ -131,3 +136,42 @@ func TestParseModelRef_DefaultProviderNormalization(t *testing.T) { t.Errorf("provider = %q, want openai (normalized from GPT)", ref.Provider) } } + +func TestParseModelRef_UnknownPrefixFallsBackToDefaultProvider(t *testing.T) { + ref := ParseModelRef("meta-llama/Llama-3.1-8B-Instruct", "openai") + if ref == nil { + t.Fatal("expected non-nil ref") + } + if ref.Provider != "openai" { + t.Fatalf("provider = %q, want openai", ref.Provider) + } + if ref.Model != "meta-llama/Llama-3.1-8B-Instruct" { + t.Fatalf("model = %q, want full original model ID", ref.Model) + } +} + +func TestParseModelRef_UnknownPrefixPreservesEmptyDefaultProvider(t *testing.T) { + ref := ParseModelRef("meta-llama/Llama-3.1-8B-Instruct", "") + if ref == nil { + t.Fatal("expected non-nil ref") + } + if ref.Provider != "" { + t.Fatalf("provider = %q, want empty", ref.Provider) + } + if ref.Model != "meta-llama/Llama-3.1-8B-Instruct" { + t.Fatalf("model = %q, want full original model ID", ref.Model) + } +} + +func TestParseModelRef_KnownNonSelectableProvider(t *testing.T) { + ref := ParseModelRef("bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", "openai") + if ref == nil { + t.Fatal("expected non-nil ref") + } + if ref.Provider != "bedrock" { + t.Fatalf("provider = %q, want bedrock", ref.Provider) + } + if ref.Model != "us.anthropic.claude-sonnet-4-20250514-v1:0" { + t.Fatalf("model = %q, want preserved bedrock model ID", ref.Model) + } +} diff --git a/pkg/providers/oauth/antigravity_provider.go b/pkg/providers/oauth/antigravity_provider.go index 1ac2d9c7f..abf1e4bd6 100644 --- a/pkg/providers/oauth/antigravity_provider.go +++ b/pkg/providers/oauth/antigravity_provider.go @@ -291,18 +291,17 @@ func (p *AntigravityProvider) buildRequest( } } - // Build tools (sanitize schemas for Gemini compatibility) + // Build tools if len(tools) > 0 { var funcDecls []antigravityFuncDecl for _, t := range tools { if t.Type != "function" { continue } - params := sanitizeSchemaForGemini(t.Function.Parameters) funcDecls = append(funcDecls, antigravityFuncDecl{ Name: t.Function.Name, Description: t.Function.Description, - Parameters: params, + Parameters: t.Function.Parameters, }) } if len(funcDecls) > 0 { @@ -446,71 +445,6 @@ func extractPartThoughtSignature(thoughtSignature string, thoughtSignatureSnake return "" } -// --- Schema sanitization --- - -// Google/Gemini doesn't support many JSON Schema keywords that other providers accept. -var geminiUnsupportedKeywords = map[string]bool{ - "patternProperties": true, - "additionalProperties": true, - "$schema": true, - "$id": true, - "$ref": true, - "$defs": true, - "definitions": true, - "examples": true, - "minLength": true, - "maxLength": true, - "minimum": true, - "maximum": true, - "multipleOf": true, - "pattern": true, - "format": true, - "minItems": true, - "maxItems": true, - "uniqueItems": true, - "minProperties": true, - "maxProperties": true, -} - -func sanitizeSchemaForGemini(schema map[string]any) map[string]any { - if schema == nil { - return nil - } - - result := make(map[string]any) - for k, v := range schema { - if geminiUnsupportedKeywords[k] { - continue - } - // Recursively sanitize nested objects - switch val := v.(type) { - case map[string]any: - result[k] = sanitizeSchemaForGemini(val) - case []any: - sanitized := make([]any, len(val)) - for i, item := range val { - if m, ok := item.(map[string]any); ok { - sanitized[i] = sanitizeSchemaForGemini(m) - } else { - sanitized[i] = item - } - } - result[k] = sanitized - default: - result[k] = v - } - } - - // Ensure top-level has type: "object" if properties are present - if _, hasProps := result["properties"]; hasProps { - if _, hasType := result["type"]; !hasType { - result["type"] = "object" - } - } - - return result -} - // --- Token source --- func createAntigravityTokenSource() func() (string, string, error) { diff --git a/pkg/providers/oauth/antigravity_provider_test.go b/pkg/providers/oauth/antigravity_provider_test.go index 2989f8519..d85e47dfa 100644 --- a/pkg/providers/oauth/antigravity_provider_test.go +++ b/pkg/providers/oauth/antigravity_provider_test.go @@ -1,6 +1,8 @@ package oauthprovider -import "testing" +import ( + "testing" +) func TestBuildRequestUsesFunctionFieldsWhenToolCallNameMissing(t *testing.T) { p := &AntigravityProvider{} @@ -71,3 +73,70 @@ func TestParseSSEResponse_SplitsThoughtAndVisibleContent(t *testing.T) { t.Fatalf("Usage.TotalTokens = %v, want %d", resp.Usage, 216) } } + +func TestBuildRequest_PreservesComplexToolSchemasByDefault(t *testing.T) { + p := &AntigravityProvider{} + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "parent": map[string]any{ + "anyOf": []any{ + map[string]any{"$ref": "#/$defs/pageParent"}, + map[string]any{"$ref": "#/$defs/databaseParent"}, + }, + }, + "icon": map[string]any{ + "anyOf": []any{ + map[string]any{"type": "null"}, + map[string]any{"$ref": "#/$defs/emoji"}, + }, + }, + }, + "$defs": map[string]any{ + "pageParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "page_id": map[string]any{"type": "string"}, + }, + "required": []any{"page_id"}, + }, + "databaseParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "database_id": map[string]any{"type": "string"}, + }, + "required": []any{"database_id"}, + }, + "emoji": map[string]any{ + "type": "string", + "pattern": "^:[a-z_]+:$", + }, + }, + } + + req := p.buildRequest( + []Message{{Role: "user", Content: "hello"}}, + []ToolDefinition{{ + Type: "function", + Function: ToolFunctionDefinition{ + Name: "mcp_notion_create", + Description: "Create a Notion object", + Parameters: schema, + }, + }}, + "gemini-3-flash", + nil, + ) + + if len(req.Tools) != 1 || len(req.Tools[0].FunctionDeclarations) != 1 { + t.Fatalf("request tools = %#v, want one function declaration", req.Tools) + } + + got, ok := req.Tools[0].FunctionDeclarations[0].Parameters.(map[string]any) + if !ok { + t.Fatalf("parameters = %#v, want map", req.Tools[0].FunctionDeclarations[0].Parameters) + } + if got["$defs"] == nil { + t.Fatalf("parameters = %#v, want raw schema with $defs preserved by default", got) + } +} diff --git a/pkg/providers/provider_catalog.go b/pkg/providers/provider_catalog.go new file mode 100644 index 000000000..a9178cb81 --- /dev/null +++ b/pkg/providers/provider_catalog.go @@ -0,0 +1,181 @@ +package providers + +import ( + "sort" + "strings" +) + +// ModelProviderOption describes a canonical provider entry exposed to the Web UI. +type ModelProviderOption struct { + ID string `json:"id"` + DefaultAPIBase string `json:"default_api_base"` + EmptyAPIKeyAllowed bool `json:"empty_api_key_allowed"` + CreateAllowed bool `json:"create_allowed"` + DefaultModelAllowed bool `json:"default_model_allowed"` + DefaultAuthMethod string `json:"default_auth_method,omitempty"` + AuthMethodLocked bool `json:"auth_method_locked,omitempty"` +} + +type attachedModelProviderMeta struct { + protocolMeta + createAllowed bool + defaultModelAllowed bool + defaultAuthMethod string + authMethodLocked bool +} + +// attachedModelProviderMetaByName augments protocolMetaByName for provider +// families that are implemented in CreateProviderFromConfig but intentionally +// kept out of the core HTTP metadata map because they have special auth/runtime +// semantics. +var attachedModelProviderMetaByName = map[string]attachedModelProviderMeta{ + "azure": {createAllowed: true, defaultModelAllowed: true}, + "anthropic": { + protocolMeta: protocolMeta{defaultAPIBase: "https://api.anthropic.com/v1"}, + createAllowed: true, + defaultModelAllowed: true, + }, + "anthropic-messages": { + protocolMeta: protocolMeta{defaultAPIBase: "https://api.anthropic.com/v1"}, + createAllowed: true, + defaultModelAllowed: true, + }, + "bedrock": {createAllowed: true, defaultModelAllowed: true}, + "antigravity": { + createAllowed: true, + defaultModelAllowed: true, + defaultAuthMethod: "oauth", + authMethodLocked: true, + }, + "claude-cli": {createAllowed: true, defaultModelAllowed: true}, + "codex-cli": {createAllowed: true, defaultModelAllowed: true}, + "github-copilot": { + protocolMeta: protocolMeta{defaultAPIBase: "localhost:4321"}, + createAllowed: true, + defaultModelAllowed: true, + }, + // ElevenLabs is intentionally exposed only as an ASR-capable provider. It + // belongs in the shared model catalog because ASR is configured via + // model_list, but it must not be selectable as the default chat model. + "elevenlabs": { + protocolMeta: protocolMeta{defaultAPIBase: "https://api.elevenlabs.io"}, + createAllowed: true, + defaultModelAllowed: false, + }, +} + +// ModelProviderOptions returns the canonical provider catalog exposed to the Web UI. +func ModelProviderOptions() []ModelProviderOption { + optionsByID := make(map[string]ModelProviderOption, len(protocolMetaByName)+len(attachedModelProviderMetaByName)) + for provider := range protocolMetaByName { + if NormalizeProvider(provider) != provider { + continue + } + optionsByID[provider] = ModelProviderOption{ + ID: provider, + DefaultAPIBase: DefaultAPIBaseForProtocol(provider), + EmptyAPIKeyAllowed: IsEmptyAPIKeyAllowedForProtocol(provider), + CreateAllowed: true, + DefaultModelAllowed: true, + } + } + for provider, meta := range attachedModelProviderMetaByName { + if NormalizeProvider(provider) != provider { + continue + } + optionsByID[provider] = ModelProviderOption{ + ID: provider, + DefaultAPIBase: meta.defaultAPIBase, + EmptyAPIKeyAllowed: meta.emptyAPIKeyAllowed, + CreateAllowed: meta.createAllowed, + DefaultModelAllowed: meta.defaultModelAllowed, + DefaultAuthMethod: meta.defaultAuthMethod, + AuthMethodLocked: meta.authMethodLocked, + } + } + + options := make([]ModelProviderOption, 0, len(optionsByID)) + for _, option := range optionsByID { + options = append(options, option) + } + sort.Slice(options, func(i, j int) bool { + return options[i].ID < options[j].ID + }) + return options +} + +// IsSupportedModelProvider reports whether provider resolves to a provider ID +// returned by ModelProviderOptions. +func IsSupportedModelProvider(provider string) bool { + normalized := NormalizeProvider(provider) + if normalized == "" { + return false + } + if _, ok := protocolMetaByName[normalized]; ok { + return true + } + _, ok := attachedModelProviderMetaByName[normalized] + return ok +} + +// IsCreatableModelProvider reports whether provider can be selected for a new +// model entry from the Web UI. +func IsCreatableModelProvider(provider string) bool { + normalized := NormalizeProvider(provider) + if normalized == "" { + return false + } + if _, ok := protocolMetaByName[normalized]; ok { + return true + } + meta, ok := attachedModelProviderMetaByName[normalized] + return ok && meta.createAllowed +} + +// IsDefaultModelProvider reports whether provider can be used as the default +// chat model. Some providers such as ASR-only entries are intentionally +// exposed in model_list management but cannot drive the gateway default model. +func IsDefaultModelProvider(provider string) bool { + normalized := NormalizeProvider(provider) + if normalized == "" { + return false + } + if _, ok := protocolMetaByName[normalized]; ok { + return true + } + meta, ok := attachedModelProviderMetaByName[normalized] + return ok && meta.defaultModelAllowed +} + +// SplitModelProviderAndID separates a legacy "provider/model" string into its +// effective provider and canonical model ID. Unknown prefixes are treated as +// part of the model ID and fall back to defaultProvider. +func SplitModelProviderAndID(model, defaultProvider string) (provider, modelID string) { + model = strings.TrimSpace(model) + if model == "" { + return "", "" + } + + provider, modelID = splitKnownProviderModel(model) + if provider != "" || modelID != "" { + return provider, modelID + } + + return NormalizeProvider(defaultProvider), model +} + +func splitKnownProviderModel(model string) (provider, modelID string) { + provider, modelID, found := strings.Cut(strings.TrimSpace(model), "/") + if !found { + return "", "" + } + provider = strings.TrimSpace(provider) + modelID = strings.TrimSpace(modelID) + if provider == "" { + return "", modelID + } + if !IsSupportedModelProvider(provider) { + return "", "" + } + return NormalizeProvider(provider), modelID +} diff --git a/pkg/providers/tool_schema_transform.go b/pkg/providers/tool_schema_transform.go new file mode 100644 index 000000000..6b6cab7a6 --- /dev/null +++ b/pkg/providers/tool_schema_transform.go @@ -0,0 +1,84 @@ +package providers + +import ( + "context" + + "github.com/sipeed/picoclaw/pkg/providers/common" +) + +type toolSchemaTransformProvider struct { + delegate LLMProvider + transform string +} + +type toolSchemaStreamingProvider struct { + *toolSchemaTransformProvider +} + +func wrapProviderWithToolSchemaTransform(delegate LLMProvider, transform string) (LLMProvider, error) { + transform, err := common.NormalizeToolSchemaTransform(transform) + if err != nil { + return nil, err + } + if transform == common.ToolSchemaTransformOff || delegate == nil { + return delegate, nil + } + base := &toolSchemaTransformProvider{ + delegate: delegate, + transform: transform, + } + if _, ok := delegate.(StreamingProvider); ok { + return &toolSchemaStreamingProvider{toolSchemaTransformProvider: base}, nil + } + return base, nil +} + +func (p *toolSchemaTransformProvider) Chat( + ctx context.Context, + messages []Message, + tools []ToolDefinition, + model string, + options map[string]any, +) (*LLMResponse, error) { + transformed, err := common.TransformToolDefinitions(tools, p.transform) + if err != nil { + return nil, err + } + return p.delegate.Chat(ctx, messages, transformed, model, options) +} + +func (p *toolSchemaTransformProvider) GetDefaultModel() string { + return p.delegate.GetDefaultModel() +} + +func (p *toolSchemaStreamingProvider) ChatStream( + ctx context.Context, + messages []Message, + tools []ToolDefinition, + model string, + options map[string]any, + onChunk func(accumulated string), +) (*LLMResponse, error) { + streaming := p.delegate.(StreamingProvider) + transformed, err := common.TransformToolDefinitions(tools, p.transform) + if err != nil { + return nil, err + } + return streaming.ChatStream(ctx, messages, transformed, model, options, onChunk) +} + +func (p *toolSchemaTransformProvider) SupportsThinking() bool { + tc, ok := p.delegate.(ThinkingCapable) + return ok && tc.SupportsThinking() +} + +func (p *toolSchemaTransformProvider) SupportsNativeSearch() bool { + ns, ok := p.delegate.(NativeSearchCapable) + return ok && ns.SupportsNativeSearch() +} + +func (p *toolSchemaTransformProvider) Close() { + if stateful, ok := p.delegate.(StatefulProvider); ok { + stateful.Close() + } +} diff --git a/pkg/providers/tool_schema_transform_test.go b/pkg/providers/tool_schema_transform_test.go new file mode 100644 index 000000000..a162c3cb4 --- /dev/null +++ b/pkg/providers/tool_schema_transform_test.go @@ -0,0 +1,104 @@ +package providers + +import ( + "context" + "reflect" + "testing" + + providercommon "github.com/sipeed/picoclaw/pkg/providers/common" +) + +type toolCaptureProvider struct { + lastTools []ToolDefinition +} + +func (p *toolCaptureProvider) Chat( + _ context.Context, + _ []Message, + tools []ToolDefinition, + _ string, + _ map[string]any, +) (*LLMResponse, error) { + p.lastTools = tools + return &LLMResponse{Content: "ok"}, nil +} + +func (p *toolCaptureProvider) GetDefaultModel() string { + return "test" +} + +func TestWrapProviderWithToolSchemaTransform_DisabledPassesToolsThrough(t *testing.T) { + capture := &toolCaptureProvider{} + wrapped, err := wrapProviderWithToolSchemaTransform(capture, "") + if err != nil { + t.Fatalf("wrapProviderWithToolSchemaTransform() error = %v", err) + } + + tools := []ToolDefinition{{ + Type: "function", + Function: ToolFunctionDefinition{ + Name: "noop", + Parameters: map[string]any{"type": "object"}, + }, + }} + + _, err = wrapped.Chat(t.Context(), nil, tools, "test", nil) + if err != nil { + t.Fatalf("Chat() error = %v", err) + } + if !reflect.DeepEqual(capture.lastTools, tools) { + t.Fatalf("tools mutated with transform off\n got: %#v\nwant: %#v", capture.lastTools, tools) + } +} + +func TestWrapProviderWithToolSchemaTransform_GoogleSanitizesSchemas(t *testing.T) { + capture := &toolCaptureProvider{} + wrapped, err := wrapProviderWithToolSchemaTransform(capture, "simple") + if err != nil { + t.Fatalf("wrapProviderWithToolSchemaTransform() error = %v", err) + } + + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "parent": map[string]any{ + "anyOf": []any{ + map[string]any{"$ref": "#/$defs/pageParent"}, + map[string]any{"$ref": "#/$defs/databaseParent"}, + }, + }, + }, + "$defs": map[string]any{ + "pageParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "page_id": map[string]any{"type": "string"}, + }, + }, + "databaseParent": map[string]any{ + "type": "object", + "properties": map[string]any{ + "database_id": map[string]any{"type": "string"}, + }, + }, + }, + } + tools := []ToolDefinition{{ + Type: "function", + Function: ToolFunctionDefinition{ + Name: "mcp_notion_create", + Parameters: schema, + }, + }} + + _, err = wrapped.Chat(t.Context(), nil, tools, "test", nil) + if err != nil { + t.Fatalf("Chat() error = %v", err) + } + + want := providercommon.SanitizeSchemaForGoogle(schema) + got := capture.lastTools[0].Function.Parameters + if !reflect.DeepEqual(got, want) { + t.Fatalf("sanitized parameters mismatch\n got: %#v\nwant: %#v", got, want) + } +} diff --git a/pkg/seahorse/short_compaction.go b/pkg/seahorse/short_compaction.go index 30e290926..0dfb1330f 100644 --- a/pkg/seahorse/short_compaction.go +++ b/pkg/seahorse/short_compaction.go @@ -602,8 +602,8 @@ func (e *CompactionEngine) generateLeafSummary( } } - // Check if level 1 succeeded - if content != "" && tokenizer.EstimateMessageTokens(providers.Message{Content: content}) < inputTokens { + // Level 1 only succeeds if it actually reaches the requested target size. + if content != "" && tokenizer.EstimateMessageTokens(providers.Message{Content: content}) <= targetTokens { return content, nil } @@ -627,7 +627,7 @@ func (e *CompactionEngine) generateLeafSummary( return "", err } } - if content != "" && tokenizer.EstimateMessageTokens(providers.Message{Content: content}) < inputTokens { + if content != "" && tokenizer.EstimateMessageTokens(providers.Message{Content: content}) <= aggressiveTarget { return content, nil } diff --git a/pkg/seahorse/short_compaction_test.go b/pkg/seahorse/short_compaction_test.go index ea7dcb52d..da07cdab7 100644 --- a/pkg/seahorse/short_compaction_test.go +++ b/pkg/seahorse/short_compaction_test.go @@ -3,6 +3,7 @@ package seahorse import ( "context" "fmt" + "strings" "sync" "sync/atomic" "testing" @@ -697,6 +698,69 @@ func TestGenerateLeafSummaryEscalationToAggressive(t *testing.T) { } } +func TestGenerateLeafSummaryEscalatesWhenLevel1MissesTarget(t *testing.T) { + var calls []string + normalContent := strings.Repeat("n", 1000) // ~404 tokens: below input, above target + aggressiveContent := strings.Repeat("a", 450) // ~184 tokens: within aggressive target + escalateComplete := func(ctx context.Context, prompt string, opts CompleteOptions) (string, error) { + if contains(prompt, "Aggressive summary policy") { + calls = append(calls, "aggressive") + return aggressiveContent, nil + } + calls = append(calls, "normal") + return normalContent, nil + } + + s := openTestStore(t) + ce, _ := newTestCompactionEngineWithStore(s, escalateComplete) + + msgs := []Message{ + {Role: "user", Content: "hello world", TokenCount: 500}, + {Role: "assistant", Content: "response", TokenCount: 500}, + } + + content, err := ce.generateLeafSummary(context.Background(), msgs, "") + if err != nil { + t.Fatalf("generateLeafSummary: %v", err) + } + if content != aggressiveContent { + t.Fatalf("expected aggressive summary after level 1 missed target") + } + if len(calls) != 2 || calls[0] != "normal" || calls[1] != "aggressive" { + t.Fatalf("expected normal then aggressive calls, got %v", calls) + } +} + +func TestGenerateLeafSummaryAcceptsContentAtTargetBoundary(t *testing.T) { + exactTargetContent := strings.Repeat("x", 488) // (488 + 12) * 2 / 5 = 200 tokens + var aggressiveCalled bool + complete := func(ctx context.Context, prompt string, opts CompleteOptions) (string, error) { + if contains(prompt, "Aggressive summary policy") { + aggressiveCalled = true + } + return exactTargetContent, nil + } + + s := openTestStore(t) + ce, _ := newTestCompactionEngineWithStore(s, complete) + + msgs := []Message{ + {Role: "user", Content: "hello world", TokenCount: 286}, + {Role: "assistant", Content: "response", TokenCount: 286}, + } + + content, err := ce.generateLeafSummary(context.Background(), msgs, "") + if err != nil { + t.Fatalf("generateLeafSummary: %v", err) + } + if content != exactTargetContent { + t.Fatalf("expected level 1 summary at target boundary to be accepted") + } + if aggressiveCalled { + t.Fatal("did not expect aggressive retry when level 1 hit target exactly") + } +} + func TestGenerateLeafSummaryEscalationToTruncation(t *testing.T) { // Both normal and aggressive return empty, should escalate to level 3 truncation emptyComplete := func(ctx context.Context, prompt string, opts CompleteOptions) (string, error) { diff --git a/pkg/skills/loader.go b/pkg/skills/loader.go index f5985a662..e7a82329c 100644 --- a/pkg/skills/loader.go +++ b/pkg/skills/loader.go @@ -42,11 +42,8 @@ func (info SkillInfo) validate() error { if info.Name == "" { errs = errors.Join(errs, errors.New("name is required")) } else { - if len(info.Name) > MaxNameLength { - errs = errors.Join(errs, fmt.Errorf("name exceeds %d characters", MaxNameLength)) - } - if !namePattern.MatchString(info.Name) { - errs = errors.Join(errs, errors.New("name must be alphanumeric with hyphens")) + if err := ValidateSkillName(info.Name); err != nil { + errs = errors.Join(errs, err) } } @@ -148,6 +145,10 @@ func (sl *SkillsLoader) ListSkills() []SkillInfo { } func (sl *SkillsLoader) LoadSkill(name string) (string, bool) { + if err := ValidateSkillName(name); err != nil { + return "", false + } + // 1. load from workspace skills first (project-level) if sl.workspaceSkills != "" { skillFile := filepath.Join(sl.workspaceSkills, name, "SKILL.md") diff --git a/pkg/skills/validation.go b/pkg/skills/validation.go new file mode 100644 index 000000000..504992b4a --- /dev/null +++ b/pkg/skills/validation.go @@ -0,0 +1,29 @@ +package skills + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/sipeed/picoclaw/pkg/utils" +) + +func ValidateSkillName(name string) error { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return fmt.Errorf("skill name is required") + } + if filepath.IsAbs(trimmed) { + return fmt.Errorf("skill name must not be an absolute path") + } + if err := utils.ValidateSkillIdentifier(trimmed); err != nil { + return fmt.Errorf("skill name is invalid: %w", err) + } + if len(trimmed) > MaxNameLength { + return fmt.Errorf("skill name exceeds %d characters", MaxNameLength) + } + if !namePattern.MatchString(trimmed) { + return fmt.Errorf("skill name must be alphanumeric with hyphens") + } + return nil +} diff --git a/pkg/tools/delegate.go b/pkg/tools/delegate.go new file mode 100644 index 000000000..dcde27718 --- /dev/null +++ b/pkg/tools/delegate.go @@ -0,0 +1,104 @@ +package tools + +import ( + "context" + "fmt" + "strings" + + "github.com/sipeed/picoclaw/pkg/routing" +) + +// DelegateTool delegates a task to a specific named agent and waits for +// the result. Unlike spawn (async, fire-and-forget) or subagent (sync but +// generic), delegate targets a named agent and runs the task using that +// agent's own workspace, model, and tools. +type DelegateTool struct { + spawner SubTurnSpawner + allowlistCheck func(targetAgentID string) bool + selfAgentID string +} + +func NewDelegateTool() *DelegateTool { + return &DelegateTool{} +} + +func (t *DelegateTool) SetSpawner(spawner SubTurnSpawner) { + t.spawner = spawner +} + +func (t *DelegateTool) SetAllowlistChecker(check func(targetAgentID string) bool) { + t.allowlistCheck = check +} + +func (t *DelegateTool) SetSelfAgentID(id string) { + t.selfAgentID = id +} + +func (t *DelegateTool) Name() string { + return "delegate" +} + +func (t *DelegateTool) Description() string { + return "Delegate a task to another agent and wait for the result. " + + "Use this when another agent is better suited to handle a specific task " + + "based on their capabilities. The target agent runs with its own workspace, " + + "model, and tools." +} + +func (t *DelegateTool) Parameters() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "agent_id": map[string]any{ + "type": "string", + "description": "The ID of the target agent to delegate the task to", + }, + "task": map[string]any{ + "type": "string", + "description": "Clear description of the task to delegate", + }, + }, + "required": []string{"agent_id", "task"}, + } +} + +func (t *DelegateTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + rawAgentID, _ := args["agent_id"].(string) + if strings.TrimSpace(rawAgentID) == "" { + return ErrorResult("agent_id is required and must be a non-empty string") + } + agentID := routing.NormalizeAgentID(rawAgentID) + + task, _ := args["task"].(string) + if strings.TrimSpace(task) == "" { + return ErrorResult("task is required and must be a non-empty string") + } + + if t.selfAgentID != "" && agentID == t.selfAgentID { + return ErrorResult("cannot delegate to self") + } + + if t.allowlistCheck != nil && !t.allowlistCheck(agentID) { + return ErrorResult(fmt.Sprintf("not allowed to delegate to agent %q", agentID)) + } + + if t.spawner == nil { + return ErrorResult("delegate tool not configured") + } + + result, err := t.spawner.SpawnSubTurn(ctx, SubTurnConfig{ + TargetAgentID: agentID, + SystemPrompt: task, + Async: false, + }) + if err != nil { + return ErrorResult(fmt.Sprintf("delegation to agent %q failed: %v", agentID, err)).WithError(err) + } + if result == nil { + return ErrorResult(fmt.Sprintf("delegation to agent %q returned no result", agentID)) + } + + result.ForLLM = fmt.Sprintf("[Response from agent %q]\n%s", agentID, result.ForLLM) + + return result +} diff --git a/pkg/tools/delegate_test.go b/pkg/tools/delegate_test.go new file mode 100644 index 000000000..729c524a7 --- /dev/null +++ b/pkg/tools/delegate_test.go @@ -0,0 +1,300 @@ +package tools + +import ( + "context" + "fmt" + "strings" + "testing" +) + +// delegateMockSpawner records the config and returns a canned result. +type delegateMockSpawner struct { + lastCfg SubTurnConfig + result *ToolResult + err error +} + +func (m *delegateMockSpawner) SpawnSubTurn(_ context.Context, cfg SubTurnConfig) (*ToolResult, error) { + m.lastCfg = cfg + if m.err != nil { + return nil, m.err + } + if m.result != nil { + return m.result, nil + } + return &ToolResult{ + ForLLM: "completed: " + cfg.SystemPrompt, + ForUser: "completed", + }, nil +} + +func TestDelegateTool_Name(t *testing.T) { + tool := NewDelegateTool() + if tool.Name() != "delegate" { + t.Errorf("Name() = %q, want %q", tool.Name(), "delegate") + } +} + +func TestDelegateTool_Parameters(t *testing.T) { + tool := NewDelegateTool() + params := tool.Parameters() + + props, ok := params["properties"].(map[string]any) + if !ok { + t.Fatal("properties should be a map") + } + _, hasAgentID := props["agent_id"] + if !hasAgentID { + t.Error("agent_id parameter should exist") + } + _, hasTask := props["task"] + if !hasTask { + t.Error("task parameter should exist") + } + + required, ok := params["required"].([]string) + if !ok { + t.Fatal("required should be a string array") + } + if len(required) != 2 { + t.Fatalf("required should have 2 entries, got %d", len(required)) + } +} + +func TestDelegateTool_Execute_Success(t *testing.T) { + spawner := &delegateMockSpawner{} + tool := NewDelegateTool() + tool.SetSpawner(spawner) + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "researcher", + "task": "summarize the logs", + }) + + if result.IsError { + t.Fatalf("expected success, got error: %s", result.ForLLM) + } + if !strings.Contains(result.ForLLM, `[Response from agent "researcher"]`) { + t.Errorf("result should contain attribution, got: %s", result.ForLLM) + } + if !strings.Contains(result.ForLLM, "summarize the logs") { + t.Errorf("result should contain task output, got: %s", result.ForLLM) + } + + // Verify spawner received correct config + if spawner.lastCfg.TargetAgentID != "researcher" { + t.Errorf("TargetAgentID = %q, want %q", spawner.lastCfg.TargetAgentID, "researcher") + } + if spawner.lastCfg.Async { + t.Error("delegate should be synchronous (Async=false)") + } + if spawner.lastCfg.SystemPrompt != "summarize the logs" { + t.Errorf("SystemPrompt = %q, want %q", spawner.lastCfg.SystemPrompt, "summarize the logs") + } +} + +func TestDelegateTool_Execute_EmptyAgentID(t *testing.T) { + tests := []struct { + name string + args map[string]any + }{ + {"missing", map[string]any{"task": "test"}}, + {"empty string", map[string]any{"agent_id": "", "task": "test"}}, + {"whitespace only", map[string]any{"agent_id": " ", "task": "test"}}, + {"wrong type", map[string]any{"agent_id": 123, "task": "test"}}, + } + + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tool.Execute(context.Background(), tt.args) + if !result.IsError { + t.Error("expected error for invalid agent_id") + } + if !strings.Contains(result.ForLLM, "agent_id is required") { + t.Errorf("error should mention agent_id, got: %s", result.ForLLM) + } + }) + } +} + +func TestDelegateTool_Execute_EmptyTask(t *testing.T) { + tests := []struct { + name string + args map[string]any + }{ + {"missing", map[string]any{"agent_id": "a"}}, + {"empty string", map[string]any{"agent_id": "a", "task": ""}}, + {"whitespace only", map[string]any{"agent_id": "a", "task": "\t\n"}}, + } + + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tool.Execute(context.Background(), tt.args) + if !result.IsError { + t.Error("expected error for invalid task") + } + if !strings.Contains(result.ForLLM, "task is required") { + t.Errorf("error should mention task, got: %s", result.ForLLM) + } + }) + } +} + +func TestDelegateTool_Execute_PermissionDenied(t *testing.T) { + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + tool.SetAllowlistChecker(func(targetAgentID string) bool { + return targetAgentID == "allowed-agent" + }) + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "forbidden-agent", + "task": "test", + }) + + if !result.IsError { + t.Error("expected error for denied agent") + } + if !strings.Contains(result.ForLLM, "not allowed to delegate") { + t.Errorf("error should mention permission, got: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_PermissionAllowed(t *testing.T) { + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + tool.SetAllowlistChecker(func(targetAgentID string) bool { + return targetAgentID == "allowed-agent" + }) + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "allowed-agent", + "task": "test", + }) + + if result.IsError { + t.Errorf("expected success for allowed agent, got error: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_NoSpawner(t *testing.T) { + tool := NewDelegateTool() + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "a", + "task": "test", + }) + + if !result.IsError { + t.Error("expected error when spawner is nil") + } + if !strings.Contains(result.ForLLM, "not configured") { + t.Errorf("error should mention not configured, got: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_SpawnerError(t *testing.T) { + spawner := &delegateMockSpawner{ + err: fmt.Errorf("context deadline exceeded"), + } + tool := NewDelegateTool() + tool.SetSpawner(spawner) + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "researcher", + "task": "test", + }) + + if !result.IsError { + t.Error("expected error when spawner fails") + } + if !strings.Contains(result.ForLLM, "delegation to agent") { + t.Errorf("error should mention delegation failure, got: %s", result.ForLLM) + } + if !strings.Contains(result.ForLLM, "context deadline exceeded") { + t.Errorf("error should propagate cause, got: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_NoAllowlistCheck(t *testing.T) { + // When no allowlist checker is set, all agents are allowed + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "any-agent", + "task": "test", + }) + + if result.IsError { + t.Errorf("expected success without allowlist, got error: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_NilResult(t *testing.T) { + tool := NewDelegateTool() + tool.SetSpawner(&nilResultSpawner{}) + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "researcher", + "task": "test", + }) + + if !result.IsError { + t.Error("expected error for nil result") + } + if !strings.Contains(result.ForLLM, "returned no result") { + t.Errorf("error should mention no result, got: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_SelfDelegation(t *testing.T) { + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + tool.SetSelfAgentID("alpha") + + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": "alpha", + "task": "test", + }) + + if !result.IsError { + t.Error("expected error for self-delegation") + } + if !strings.Contains(result.ForLLM, "cannot delegate to self") { + t.Errorf("error should mention self-delegation, got: %s", result.ForLLM) + } +} + +func TestDelegateTool_Execute_SelfDelegation_Normalized(t *testing.T) { + tool := NewDelegateTool() + tool.SetSpawner(&delegateMockSpawner{}) + tool.SetSelfAgentID("alpha") // stored normalized + + // Case-insensitive and whitespace variants should still be caught + variants := []string{"ALPHA", " Alpha ", " alpha "} + for _, v := range variants { + t.Run(v, func(t *testing.T) { + result := tool.Execute(context.Background(), map[string]any{ + "agent_id": v, + "task": "test", + }) + if !result.IsError { + t.Errorf("agent_id=%q should be caught as self-delegation", v) + } + }) + } +} + +// nilResultSpawner always returns (nil, nil). +type nilResultSpawner struct{} + +func (m *nilResultSpawner) SpawnSubTurn(_ context.Context, _ SubTurnConfig) (*ToolResult, error) { + return nil, nil +} diff --git a/pkg/tools/fs/edit.go b/pkg/tools/fs/edit.go index 827ea50c8..7a54a1b01 100644 --- a/pkg/tools/fs/edit.go +++ b/pkg/tools/fs/edit.go @@ -69,10 +69,11 @@ func (t *EditFileTool) Execute(ctx context.Context, args map[string]any) *ToolRe return ErrorResult("new_text is required") } - if err := editFile(t.fs, path, oldText, newText); err != nil { + beforeContent, afterContent, err := editFile(t.fs, path, oldText, newText) + if err != nil { return ErrorResult(err.Error()) } - return SilentResult(fmt.Sprintf("File edited: %s", path)) + return DiffResult(path, beforeContent, afterContent) } type AppendFileTool struct { @@ -131,18 +132,22 @@ func (t *AppendFileTool) Execute(ctx context.Context, args map[string]any) *Tool // editFile reads the file via sysFs, performs the replacement, and writes back. // It uses a fileSystem interface, allowing the same logic for both restricted and unrestricted modes. -func editFile(sysFs fileSystem, path, oldText, newText string) error { +func editFile(sysFs fileSystem, path, oldText, newText string) ([]byte, []byte, error) { content, err := sysFs.ReadFile(path) if err != nil { - return err + return nil, nil, err } newContent, err := replaceEditContent(content, oldText, newText) if err != nil { - return err + return nil, nil, err } - return sysFs.WriteFile(path, newContent) + if err := sysFs.WriteFile(path, newContent); err != nil { + return nil, nil, err + } + + return content, newContent, nil } // appendFile reads the existing content (if any) via sysFs, appends new content, and writes back. diff --git a/pkg/tools/fs/edit_test.go b/pkg/tools/fs/edit_test.go index 4c25322ef..b04c41fff 100644 --- a/pkg/tools/fs/edit_test.go +++ b/pkg/tools/fs/edit_test.go @@ -2,6 +2,7 @@ package fstools import ( "context" + "fmt" "os" "path/filepath" "strings" @@ -31,14 +32,34 @@ func TestEditTool_EditFile_Success(t *testing.T) { t.Errorf("Expected success, got IsError=true: %s", result.ForLLM) } - // Should return SilentResult - if !result.Silent { - t.Errorf("Expected Silent=true for EditFile, got false") + // Successful edits should surface a diff to the user. + if result.Silent { + t.Errorf("Expected Silent=false for EditFile, got true") } - // ForUser should be empty (silent result) - if result.ForUser != "" { - t.Errorf("Expected ForUser to be empty for SilentResult, got: %s", result.ForUser) + if result.ForUser == "" { + t.Fatal("Expected ForUser to contain the diff preview") + } + + if result.ForLLM == result.ForUser { + t.Fatalf("Expected ForLLM to be a compact summary, got identical outputs %q", result.ForLLM) + } + if result.ForLLM != fmt.Sprintf("File edited: %s", testFile) { + t.Fatalf("Expected compact ForLLM summary, got %q", result.ForLLM) + } + + diffPath := strings.TrimLeft(filepath.ToSlash(testFile), "/") + for _, want := range []string{ + fmt.Sprintf("File edited: %s", testFile), + "```diff", + "--- a/" + diffPath, + "+++ b/" + diffPath, + "-Hello World", + "+Hello Universe", + } { + if !strings.Contains(result.ForUser, want) { + t.Fatalf("Expected edit diff to contain %q, got:\n%s", want, result.ForUser) + } } // Verify file was actually edited @@ -412,7 +433,13 @@ func TestEditFileTool_Restricted_InPlaceEdit(t *testing.T) { result := tool.Execute(ctx, args) assert.False(t, result.IsError, "Expected success, got: %s", result.ForLLM) - assert.True(t, result.Silent) + assert.False(t, result.Silent) + assert.Equal(t, "File edited: edit_target.txt", result.ForLLM) + assert.Contains(t, result.ForUser, "```diff") + assert.Contains(t, result.ForUser, "--- a/edit_target.txt") + assert.Contains(t, result.ForUser, "+++ b/edit_target.txt") + assert.Contains(t, result.ForUser, "-Hello World") + assert.Contains(t, result.ForUser, "+Hello Go") data, err := os.ReadFile(filepath.Join(workspace, testFile)) assert.NoError(t, err) diff --git a/pkg/tools/fs/shared.go b/pkg/tools/fs/shared.go index 6d46e692b..acf14169e 100644 --- a/pkg/tools/fs/shared.go +++ b/pkg/tools/fs/shared.go @@ -32,6 +32,10 @@ func SilentResult(forLLM string) *ToolResult { return toolshared.SilentResult(forLLM) } +func DiffResult(path string, before, after []byte) *ToolResult { + return toolshared.DiffResult(path, before, after) +} + func MediaResult(forLLM string, mediaRefs []string) *ToolResult { return toolshared.MediaResult(forLLM, mediaRefs) } diff --git a/pkg/tools/integration/mcp_tool.go b/pkg/tools/integration/mcp_tool.go index 78c348316..8cfc1de5e 100644 --- a/pkg/tools/integration/mcp_tool.go +++ b/pkg/tools/integration/mcp_tool.go @@ -13,6 +13,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/media" toolshared "github.com/sipeed/picoclaw/pkg/tools/shared" @@ -36,6 +37,16 @@ type MCPTool struct { mediaStore media.MediaStore workspace string maxInlineTextRunes int + runtimeEvents runtimeevents.Bus +} + +// MCPToolCallPayload describes MCP tool execution runtime events. +type MCPToolCallPayload struct { + Server string `json:"server"` + Tool string `json:"tool"` + DurationMS int64 `json:"duration_ms,omitempty"` + IsError bool `json:"is_error,omitempty"` + Error string `json:"error,omitempty"` } // NewMCPTool creates a new MCP tool wrapper @@ -62,6 +73,11 @@ func (t *MCPTool) SetMaxInlineTextRunes(limit int) { } } +// SetEventPublisher injects the runtime event bus used for MCP tool observations. +func (t *MCPTool) SetEventPublisher(eventBus runtimeevents.Bus) { + t.runtimeEvents = eventBus +} + const maxMCPInlineTextRunes = 16 * 1024 // sanitizeIdentifierComponent normalizes a string so it can be safely used @@ -237,26 +253,88 @@ func (t *MCPTool) Parameters() map[string]any { // Execute executes the MCP tool func (t *MCPTool) Execute(ctx context.Context, args map[string]any) *ToolResult { + startedAt := time.Now() + t.publishRuntimeEvent(ctx, runtimeevents.KindMCPToolCallStart, startedAt, false, "") + result, err := t.manager.CallTool(ctx, t.serverName, t.tool.Name, args) if err != nil { + t.publishRuntimeEvent(ctx, runtimeevents.KindMCPToolCallEnd, startedAt, true, err.Error()) return ErrorResult(fmt.Sprintf("MCP tool execution failed: %v", err)).WithError(err) } if result == nil { nilErr := fmt.Errorf("MCP tool returned nil result without error") + t.publishRuntimeEvent(ctx, runtimeevents.KindMCPToolCallEnd, startedAt, true, nilErr.Error()) return ErrorResult("MCP tool execution failed: nil result").WithError(nilErr) } // Handle error result from server if result.IsError { errMsg := extractContentText(result.Content) + t.publishRuntimeEvent(ctx, runtimeevents.KindMCPToolCallEnd, startedAt, true, errMsg) return ErrorResult(fmt.Sprintf("MCP tool returned error: %s", errMsg)). WithError(fmt.Errorf("MCP tool error: %s", errMsg)) } + t.publishRuntimeEvent(ctx, runtimeevents.KindMCPToolCallEnd, startedAt, false, "") return t.normalizeResultContent(ctx, result.Content) } +func (t *MCPTool) publishRuntimeEvent( + ctx context.Context, + kind runtimeevents.Kind, + startedAt time.Time, + isError bool, + errMsg string, +) { + if t == nil || t.runtimeEvents == nil { + return + } + + scope := runtimeevents.Scope{ + AgentID: toolshared.ToolAgentID(ctx), + SessionKey: toolshared.ToolSessionKey(ctx), + Channel: toolshared.ToolChannel(ctx), + ChatID: toolshared.ToolChatID(ctx), + MessageID: toolshared.ToolMessageID(ctx), + } + payload := MCPToolCallPayload{ + Server: t.serverName, + Tool: t.tool.Name, + DurationMS: time.Since(startedAt).Milliseconds(), + IsError: isError, + Error: errMsg, + } + severity := runtimeevents.SeverityInfo + if isError { + severity = runtimeevents.SeverityError + } + + t.runtimeEvents.PublishNonBlocking(runtimeevents.Event{ + Kind: kind, + Source: runtimeevents.Source{Component: "mcp", Name: t.serverName}, + Scope: scope, + Severity: severity, + Payload: payload, + Attrs: mcpToolCallEventAttrs(payload), + }) +} + +func mcpToolCallEventAttrs(payload MCPToolCallPayload) map[string]any { + attrs := map[string]any{ + "server": payload.Server, + "tool": payload.Tool, + "duration_ms": payload.DurationMS, + } + if payload.IsError { + attrs["is_error"] = payload.IsError + } + if payload.Error != "" { + attrs["error"] = payload.Error + } + return attrs +} + // extractContentText extracts text from MCP content array func extractContentText(content []mcp.Content) string { var parts []string diff --git a/pkg/tools/integration/mcp_tool_test.go b/pkg/tools/integration/mcp_tool_test.go index 7b0b2cd5a..7c961e1e1 100644 --- a/pkg/tools/integration/mcp_tool_test.go +++ b/pkg/tools/integration/mcp_tool_test.go @@ -7,9 +7,11 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/modelcontextprotocol/go-sdk/mcp" + runtimeevents "github.com/sipeed/picoclaw/pkg/events" "github.com/sipeed/picoclaw/pkg/media" toolshared "github.com/sipeed/picoclaw/pkg/tools/shared" ) @@ -299,6 +301,77 @@ func TestMCPTool_Execute_Success(t *testing.T) { } } +func TestMCPTool_Execute_PublishesRuntimeEvents(t *testing.T) { + eventBus := runtimeevents.NewBus() + defer func() { + if err := eventBus.Close(); err != nil { + t.Errorf("event bus close failed: %v", err) + } + }() + + _, eventsCh, err := eventBus.Channel().OfKind( + runtimeevents.KindMCPToolCallStart, + runtimeevents.KindMCPToolCallEnd, + ).SubscribeChan(t.Context(), runtimeevents.SubscribeOptions{Name: "mcp-tool-events", Buffer: 2}) + if err != nil { + t.Fatalf("SubscribeChan failed: %v", err) + } + + manager := &MockMCPManager{} + mcpTool := NewMCPTool(manager, "github", &mcp.Tool{Name: "search_repos"}) + mcpTool.SetEventPublisher(eventBus) + + ctx := toolshared.WithToolContext(context.Background(), "telegram", "chat-1") + ctx = toolshared.WithToolMessageContext(ctx, "msg-1", "") + ctx = toolshared.WithToolSessionContext(ctx, "main", "session-1", nil) + result := mcpTool.Execute(ctx, map[string]any{"query": "picoclaw"}) + if result == nil || result.IsError { + t.Fatalf("Execute result = %+v", result) + } + + started := receiveMCPToolRuntimeEvent(t, eventsCh) + if started.Kind != runtimeevents.KindMCPToolCallStart || + started.Scope.AgentID != "main" || + started.Scope.SessionKey != "session-1" || + started.Scope.Channel != "telegram" || + started.Scope.ChatID != "chat-1" || + started.Scope.MessageID != "msg-1" { + t.Fatalf("started event = %+v", started) + } + + ended := receiveMCPToolRuntimeEvent(t, eventsCh) + if ended.Kind != runtimeevents.KindMCPToolCallEnd || ended.Severity != runtimeevents.SeverityInfo { + t.Fatalf("ended event = %+v", ended) + } + payload, ok := ended.Payload.(MCPToolCallPayload) + if !ok { + t.Fatalf("ended payload = %T, want MCPToolCallPayload", ended.Payload) + } + if payload.Server != "github" || payload.Tool != "search_repos" || payload.IsError { + t.Fatalf("ended payload = %+v", payload) + } + if ended.Attrs["server"] != "github" || + ended.Attrs["tool"] != "search_repos" || + ended.Attrs["duration_ms"] == nil { + t.Fatalf("ended attrs = %#v", ended.Attrs) + } +} + +func receiveMCPToolRuntimeEvent(t *testing.T, ch <-chan runtimeevents.Event) runtimeevents.Event { + t.Helper() + + select { + case evt, ok := <-ch: + if !ok { + t.Fatal("runtime event channel closed before expected event") + } + return evt + case <-time.After(time.Second): + t.Fatal("timed out waiting for runtime event") + return runtimeevents.Event{} + } +} + // TestMCPTool_Execute_ManagerError tests execution when manager returns error func TestMCPTool_Execute_ManagerError(t *testing.T) { manager := &MockMCPManager{ diff --git a/pkg/tools/integration/web.go b/pkg/tools/integration/web.go index 75821e40d..0568ba8dc 100644 --- a/pkg/tools/integration/web.go +++ b/pkg/tools/integration/web.go @@ -472,6 +472,113 @@ type SogouSearchProvider struct { client *http.Client } +type GeminiSearchProvider struct { + apiKey string + model string + proxy string + client *http.Client +} + +func (p *GeminiSearchProvider) Search( + ctx context.Context, + query string, + count int, + rangeCode string, +) (string, error) { + if strings.TrimSpace(p.apiKey) == "" { + return "", errors.New("no API key provided") + } + model := strings.TrimSpace(p.model) + if model == "" { + model = "gemini-2.5-flash" + } + + payload := map[string]any{ + "contents": []map[string]any{{ + "parts": []map[string]string{{"text": query}}, + }}, + "tools": []map[string]any{{"google_search": map[string]any{}}}, + } + bodyBytes, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("failed to marshal payload: %w", err) + } + + endpoint := fmt.Sprintf( + "https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent", + url.PathEscape(model), + ) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(bodyBytes)) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Goog-Api-Key", p.apiKey) + req.Header.Set("User-Agent", fmt.Sprintf(userAgentHonest, config.Version)) + + resp, err := p.client.Do(req) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return "", fmt.Errorf("failed to read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("gemini search api error (status %d): %s", resp.StatusCode, string(body)) + } + + var searchResp struct { + Candidates []struct { + Content struct { + Parts []struct { + Text string `json:"text"` + } `json:"parts"` + } `json:"content"` + GroundingMetadata struct { + GroundingChunks []struct { + Web struct { + URI string `json:"uri"` + Title string `json:"title"` + } `json:"web"` + } `json:"groundingChunks"` + } `json:"groundingMetadata"` + } `json:"candidates"` + } + if err := json.Unmarshal(body, &searchResp); err != nil { + return "", fmt.Errorf("failed to parse response: %w", err) + } + if len(searchResp.Candidates) == 0 { + return fmt.Sprintf("No results for: %s", query), nil + } + + candidate := searchResp.Candidates[0] + lines := []string{fmt.Sprintf("Results for: %s (via Gemini Google Search)", query)} + for _, part := range candidate.Content.Parts { + if strings.TrimSpace(part.Text) != "" { + lines = append(lines, strings.TrimSpace(part.Text)) + } + } + citationCount := 0 + for _, chunk := range candidate.GroundingMetadata.GroundingChunks { + if strings.TrimSpace(chunk.Web.URI) == "" { + continue + } + citationCount++ + title := strings.TrimSpace(chunk.Web.Title) + if title == "" { + title = chunk.Web.URI + } + lines = append(lines, fmt.Sprintf("%d. %s\n %s", citationCount, title, chunk.Web.URI)) + if citationCount >= count { + break + } + } + return strings.Join(lines, "\n"), nil +} + func (p *SogouSearchProvider) Search( ctx context.Context, query string, @@ -1072,6 +1179,10 @@ type WebSearchToolOptions struct { SogouEnabled bool DuckDuckGoMaxResults int DuckDuckGoEnabled bool + GeminiAPIKey string + GeminiModel string + GeminiMaxResults int + GeminiEnabled bool PerplexityAPIKeys []string PerplexityMaxResults int PerplexityEnabled bool @@ -1104,6 +1215,10 @@ func WebSearchToolOptionsFromConfig(cfg *config.Config) WebSearchToolOptions { SogouEnabled: cfg.Tools.Web.Sogou.Enabled, DuckDuckGoMaxResults: cfg.Tools.Web.DuckDuckGo.MaxResults, DuckDuckGoEnabled: cfg.Tools.Web.DuckDuckGo.Enabled, + GeminiAPIKey: cfg.Tools.Web.Gemini.APIKey.String(), + GeminiModel: cfg.Tools.Web.Gemini.Model, + GeminiMaxResults: cfg.Tools.Web.Gemini.MaxResults, + GeminiEnabled: cfg.Tools.Web.Gemini.Enabled, PerplexityAPIKeys: cfg.Tools.Web.Perplexity.APIKeys.Values(), PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults, PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled, @@ -1135,6 +1250,7 @@ var ( knownWebSearchProviders = []string{ "sogou", "duckduckgo", + "gemini", "brave", "tavily", "perplexity", @@ -1142,7 +1258,7 @@ var ( "glm_search", "baidu_search", } - autoPrimaryWebSearchProviders = []string{"perplexity", "brave", "searxng", "tavily"} + autoPrimaryWebSearchProviders = []string{"perplexity", "brave", "searxng", "tavily", "gemini"} autoFallbackWebSearchProviders = []string{"baidu_search", "glm_search"} ) @@ -1162,6 +1278,8 @@ func (opts WebSearchToolOptions) providerReady(name string) bool { return opts.SogouEnabled case "duckduckgo": return opts.DuckDuckGoEnabled + case "gemini": + return opts.GeminiEnabled && strings.TrimSpace(opts.GeminiAPIKey) != "" case "brave": return opts.BraveEnabled && len(opts.BraveAPIKeys) > 0 case "tavily": @@ -1195,14 +1313,15 @@ func (opts WebSearchToolOptions) resolveProviderName(query string) (string, erro return providerName, nil } + sogouReady := opts.providerReady("sogou") + duckReady := opts.providerReady("duckduckgo") + for _, name := range autoPrimaryWebSearchProviders { if opts.providerReady(name) { return name, nil } } - sogouReady := opts.providerReady("sogou") - duckReady := opts.providerReady("duckduckgo") if sogouReady && duckReady { if prefersDuckDuckGoQuery(query) { return "duckduckgo", nil @@ -1279,6 +1398,24 @@ func (opts WebSearchToolOptions) providerByName(name string) (SearchProvider, in proxy: opts.Proxy, client: client, }, maxResults, nil + case "gemini": + if !opts.providerReady("gemini") { + return nil, 0, nil + } + client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) + if err != nil { + return nil, 0, fmt.Errorf("failed to create HTTP client for Gemini: %w", err) + } + maxResults := 10 + if opts.GeminiMaxResults > 0 { + maxResults = min(opts.GeminiMaxResults, 10) + } + return &GeminiSearchProvider{ + apiKey: opts.GeminiAPIKey, + model: opts.GeminiModel, + proxy: opts.Proxy, + client: client, + }, maxResults, nil case "searxng": if !opts.providerReady("searxng") { return nil, 0, nil diff --git a/pkg/tools/integration/web_test.go b/pkg/tools/integration/web_test.go index ba6b3da45..436413085 100644 --- a/pkg/tools/integration/web_test.go +++ b/pkg/tools/integration/web_test.go @@ -1853,6 +1853,191 @@ func TestWebTool_AutoProviderPrefersConfiguredProvidersBeforeSogou(t *testing.T) } } +func TestWebTool_AutoProviderPrefersConfiguredProvidersBeforeGemini(t *testing.T) { + opts := WebSearchToolOptions{ + GeminiEnabled: true, + GeminiAPIKey: "google-key", + GeminiModel: "gemini-2.5-flash", + GeminiMaxResults: 5, + BraveEnabled: true, + BraveAPIKeys: []string{"brave-key"}, + BraveMaxResults: 5, + SogouEnabled: true, + SogouMaxResults: 5, + DuckDuckGoEnabled: true, + DuckDuckGoMaxResults: 5, + } + + name, err := ResolveWebSearchProviderName(opts, "best robotics companies") + if err != nil { + t.Fatalf("ResolveWebSearchProviderName() error: %v", err) + } + if name != "brave" { + t.Fatalf("provider = %q, want brave", name) + } + + name, err = ResolveWebSearchProviderName(opts, "今天上海天气") + if err != nil { + t.Fatalf("ResolveWebSearchProviderName() error: %v", err) + } + if name != "brave" { + t.Fatalf("provider = %q, want brave", name) + } +} + +func TestWebTool_GeminiRequiresAPIKey(t *testing.T) { + tool, err := NewWebSearchTool(WebSearchToolOptions{ + Provider: "gemini", + GeminiEnabled: true, + SogouEnabled: true, + SogouMaxResults: 5, + }) + if err != nil { + t.Fatalf("NewWebSearchTool() error: %v", err) + } + if _, ok := tool.provider.(*SogouSearchProvider); !ok { + t.Fatalf("expected SogouSearchProvider after missing Gemini API key fallback, got %T", tool.provider) + } +} + +func TestGeminiSearchProvider_SearchSuccess(t *testing.T) { + provider := &GeminiSearchProvider{ + apiKey: "google-key", + model: "gemini-2.5-flash", + client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodPost { + t.Fatalf("method = %s, want POST", req.Method) + } + if got := req.Header.Get("X-Goog-Api-Key"); got != "google-key" { + t.Fatalf("X-Goog-Api-Key = %q, want google-key", got) + } + if !strings.Contains(req.URL.String(), "/models/gemini-2.5-flash:generateContent") { + t.Fatalf("unexpected URL: %s", req.URL.String()) + } + rec := httptest.NewRecorder() + rec.WriteHeader(http.StatusOK) + fmt.Fprint(rec, `{ + "candidates": [ + { + "content": { + "parts": [ + {"text": "Answer paragraph one."}, + {"text": "Answer paragraph two."} + ] + }, + "groundingMetadata": { + "groundingChunks": [ + {"web": {"uri": "https://example.com/a", "title": "Result A"}}, + {"web": {"uri": "https://example.com/b", "title": "Result B"}}, + {"web": {"uri": "https://example.com/c", "title": "Result C"}} + ] + } + } + ] +}`) + return rec.Result(), nil + }), + }, + } + + out, err := provider.Search(context.Background(), "robotics", 2, "") + if err != nil { + t.Fatalf("Search() error: %v", err) + } + if !strings.Contains(out, "Results for: robotics (via Gemini Google Search)") { + t.Fatalf("missing header in output: %s", out) + } + if !strings.Contains(out, "Answer paragraph one.") || !strings.Contains(out, "Answer paragraph two.") { + t.Fatalf("missing response text in output: %s", out) + } + if !strings.Contains(out, "1. Result A") || !strings.Contains(out, "2. Result B") { + t.Fatalf("missing citations in output: %s", out) + } + if strings.Contains(out, "Result C") { + t.Fatalf("expected citations to be limited to count=2, got: %s", out) + } +} + +func TestGeminiSearchProvider_SearchIgnoresRange(t *testing.T) { + provider := &GeminiSearchProvider{ + apiKey: "google-key", + model: "gemini-2.5-flash", + client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + rec := httptest.NewRecorder() + rec.WriteHeader(http.StatusOK) + fmt.Fprint(rec, `{ + "candidates": [ + { + "content": { + "parts": [ + {"text": "Recent robotics result."} + ] + } + } + ] +}`) + return rec.Result(), nil + }), + }, + } + + out, err := provider.Search(context.Background(), "robotics", 2, "d") + if err != nil { + t.Fatalf("Search() error: %v", err) + } + if !strings.Contains(out, "Recent robotics result.") { + t.Fatalf("missing response text in output: %s", out) + } +} + +func TestGeminiSearchProvider_SearchAPIError(t *testing.T) { + provider := &GeminiSearchProvider{ + apiKey: "google-key", + model: "gemini-2.5-flash", + client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + rec := httptest.NewRecorder() + rec.WriteHeader(http.StatusTooManyRequests) + fmt.Fprint(rec, `{"error":"quota exceeded"}`) + return rec.Result(), nil + }), + }, + } + + _, err := provider.Search(context.Background(), "robotics", 2, "") + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "status 429") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestGeminiSearchProvider_SearchEmptyCandidates(t *testing.T) { + provider := &GeminiSearchProvider{ + apiKey: "google-key", + model: "gemini-2.5-flash", + client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + rec := httptest.NewRecorder() + rec.WriteHeader(http.StatusOK) + fmt.Fprint(rec, `{"candidates":[]}`) + return rec.Result(), nil + }), + }, + } + + out, err := provider.Search(context.Background(), "robotics", 2, "") + if err != nil { + t.Fatalf("Search() error: %v", err) + } + if out != "No results for: robotics" { + t.Fatalf("output = %q, want %q", out, "No results for: robotics") + } +} + func TestWebTool_ExplicitProviderFallsBackWhenMissingCredentials(t *testing.T) { tool, err := NewWebSearchTool(WebSearchToolOptions{ Provider: "brave", diff --git a/pkg/tools/integration_facade.go b/pkg/tools/integration_facade.go index 193ecd6f5..dc28fe54b 100644 --- a/pkg/tools/integration_facade.go +++ b/pkg/tools/integration_facade.go @@ -28,6 +28,7 @@ type ( TavilySearchProvider = integrationtools.TavilySearchProvider SogouSearchProvider = integrationtools.SogouSearchProvider DuckDuckGoSearchProvider = integrationtools.DuckDuckGoSearchProvider + GeminiSearchProvider = integrationtools.GeminiSearchProvider PerplexitySearchProvider = integrationtools.PerplexitySearchProvider SearXNGSearchProvider = integrationtools.SearXNGSearchProvider GLMSearchProvider = integrationtools.GLMSearchProvider diff --git a/pkg/tools/registry.go b/pkg/tools/registry.go index 0ff9293a3..e90d683bb 100644 --- a/pkg/tools/registry.go +++ b/pkg/tools/registry.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strings" "sync" "sync/atomic" "time" @@ -24,6 +25,7 @@ type ToolRegistry struct { mu sync.RWMutex version atomic.Uint64 // incremented on Register/RegisterHidden for cache invalidation mediaStore media.MediaStore + allowlist map[string]struct{} } type mediaStoreAware interface { @@ -36,10 +38,40 @@ func NewToolRegistry() *ToolRegistry { } } +// SetAllowlist restricts registrations to the provided runtime tool names. +// A nil slice means "allow all". An empty-but-non-nil slice means "allow none". +func (r *ToolRegistry) SetAllowlist(names []string) { + r.mu.Lock() + defer r.mu.Unlock() + + if names == nil { + r.allowlist = nil + return + } + + allowlist := make(map[string]struct{}, len(names)) + for _, name := range names { + trimmed := strings.ToLower(strings.TrimSpace(name)) + if trimmed == "" { + continue + } + allowlist[trimmed] = struct{}{} + } + r.allowlist = allowlist +} + func (r *ToolRegistry) Register(tool Tool) { r.mu.Lock() defer r.mu.Unlock() name := tool.Name() + if !r.toolAllowedLocked(name) { + logger.DebugCF( + "tools", + "Skipped core tool registration by agent allowlist", + map[string]any{"name": name}, + ) + return + } if _, exists := r.tools[name]; exists { logger.WarnCF("tools", "Tool registration overwrites existing tool", map[string]any{"name": name}) @@ -61,6 +93,14 @@ func (r *ToolRegistry) RegisterHidden(tool Tool) { r.mu.Lock() defer r.mu.Unlock() name := tool.Name() + if !r.toolAllowedLocked(name) { + logger.DebugCF( + "tools", + "Skipped hidden tool registration by agent allowlist", + map[string]any{"name": name}, + ) + return + } if _, exists := r.tools[name]; exists { logger.WarnCF("tools", "Hidden tool registration overwrites existing tool", map[string]any{"name": name}) @@ -128,6 +168,30 @@ func (r *ToolRegistry) Version() uint64 { return r.version.Load() } +func (r *ToolRegistry) toolAllowedLocked(name string) bool { + if r.allowlist == nil { + return true + } + if isToolDiscoveryToolName(name) { + // Discovery tools are part of the MCP control plane: they must remain + // available whenever configured so deferred MCP tools can still be + // unlocked. Per-agent allowlists still apply to the hidden MCP tools + // themselves during RegisterHidden. + return true + } + _, ok := r.allowlist[strings.ToLower(strings.TrimSpace(name))] + return ok +} + +// HasRegistered reports whether a tool name is present in the registry, +// including hidden tools whose TTL is currently zero. +func (r *ToolRegistry) HasRegistered(name string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.tools[name] + return ok +} + // HiddenToolSnapshot holds a consistent snapshot of hidden tools and the // registry version at which it was taken. Used by BM25SearchTool cache. type HiddenToolSnapshot struct { @@ -203,7 +267,9 @@ func (r *ToolRegistry) ExecuteWithContext( map[string]any{ "tool": name, }) - return ErrorResult(fmt.Sprintf("tool %q not found", name)).WithError(fmt.Errorf("tool not found")) + return ErrorResult( + fmt.Sprintf("tool %q not found", name), + ).WithError(fmt.Errorf("tool not found")) } // Validate arguments against the tool's declared schema. @@ -411,6 +477,12 @@ func (r *ToolRegistry) Clone() *ToolRegistry { tools: make(map[string]*ToolEntry, len(r.tools)), mediaStore: r.mediaStore, } + if r.allowlist != nil { + clone.allowlist = make(map[string]struct{}, len(r.allowlist)) + for name := range r.allowlist { + clone.allowlist[name] = struct{}{} + } + } for name, entry := range r.tools { clone.tools[name] = &ToolEntry{ Tool: entry.Tool, @@ -443,7 +515,10 @@ func (r *ToolRegistry) GetSummaries() []string { continue } - summaries = append(summaries, fmt.Sprintf("- `%s` - %s", entry.Tool.Name(), entry.Tool.Description())) + summaries = append( + summaries, + fmt.Sprintf("- `%s` - %s", entry.Tool.Name(), entry.Tool.Description()), + ) } return summaries } diff --git a/pkg/tools/registry_test.go b/pkg/tools/registry_test.go index eac96382f..ee63586ab 100644 --- a/pkg/tools/registry_test.go +++ b/pkg/tools/registry_test.go @@ -53,7 +53,11 @@ type mockAsyncRegistryTool struct { lastCB AsyncCallback } -func (m *mockAsyncRegistryTool) ExecuteAsync(_ context.Context, args map[string]any, cb AsyncCallback) *ToolResult { +func (m *mockAsyncRegistryTool) ExecuteAsync( + _ context.Context, + args map[string]any, + cb AsyncCallback, +) *ToolResult { m.lastCB = cb return m.result } @@ -104,6 +108,69 @@ func TestToolRegistry_RegisterAndGet(t *testing.T) { } } +func TestToolRegistry_AllowlistFiltersRegistrations(t *testing.T) { + r := NewToolRegistry() + r.SetAllowlist([]string{"Allowed_Tool"}) + + r.Register(newMockTool("allowed_tool", "allowed")) + r.Register(newMockTool("blocked_tool", "blocked")) + r.RegisterHidden(newMockTool("hidden_blocked", "hidden blocked")) + + if _, ok := r.Get("allowed_tool"); !ok { + t.Fatal("expected allowed_tool to be registered") + } + if _, ok := r.Get("blocked_tool"); ok { + t.Fatal("blocked_tool should not be registered") + } + if _, ok := r.Get("hidden_blocked"); ok { + t.Fatal("hidden_blocked should not be registered") + } + if got := r.List(); len(got) != 1 || got[0] != "allowed_tool" { + t.Fatalf("registry list = %v, want [allowed_tool]", got) + } +} + +func TestToolRegistry_AllowlistStillAllowsDiscoveryTools(t *testing.T) { + r := NewToolRegistry() + r.SetAllowlist([]string{"mcp_github_search"}) + + r.Register(newMockTool(BM25SearchToolName, "discover hidden tools")) + r.Register(newMockTool(RegexSearchToolName, "discover hidden tools via regex")) + r.Register(newMockTool("blocked_tool", "blocked")) + + if _, ok := r.Get(BM25SearchToolName); !ok { + t.Fatal("expected BM25 discovery tool to bypass allowlist filtering") + } + if _, ok := r.Get(RegexSearchToolName); !ok { + t.Fatal("expected regex discovery tool to bypass allowlist filtering") + } + if _, ok := r.Get("blocked_tool"); ok { + t.Fatal("blocked_tool should not be registered") + } +} + +func TestToolRegistry_HasRegisteredIncludesHiddenTools(t *testing.T) { + r := NewToolRegistry() + r.SetAllowlist([]string{"visible", "hidden"}) + + r.Register(newMockTool("visible", "visible")) + r.RegisterHidden(newMockTool("hidden", "hidden")) + r.RegisterHidden(newMockTool("blocked", "blocked")) + + if !r.HasRegistered("visible") { + t.Fatal("expected visible tool to be registered") + } + if !r.HasRegistered("hidden") { + t.Fatal("expected hidden tool to be reported as registered") + } + if r.HasRegistered("blocked") { + t.Fatal("blocked tool should not be registered") + } + if _, ok := r.Get("hidden"); ok { + t.Fatal("hidden tool with zero TTL should not be callable through Get") + } +} + func TestToolRegistry_Get_NotFound(t *testing.T) { r := NewToolRegistry() _, ok := r.Get("nonexistent") @@ -305,7 +372,11 @@ func TestToolRegistry_ToProviderDefs(t *testing.T) { t.Errorf("Name: want %q, got %q", want.Function.Name, got.Function.Name) } if got.Function.Description != want.Function.Description { - t.Errorf("Description: want %q, got %q", want.Function.Description, got.Function.Description) + t.Errorf( + "Description: want %q, got %q", + want.Function.Description, + got.Function.Description, + ) } } @@ -449,7 +520,10 @@ func TestToolRegistry_Clone(t *testing.T) { t.Errorf("expected parent to have 4 tools, got %d", r.Count()) } if clone.Count() != 3 { - t.Errorf("expected clone to still have 3 tools after parent mutation, got %d", clone.Count()) + t.Errorf( + "expected clone to still have 3 tools after parent mutation, got %d", + clone.Count(), + ) } if _, ok := clone.Get("spawn"); ok { t.Error("expected clone NOT to have 'spawn' tool registered on parent after cloning") @@ -745,7 +819,14 @@ func TestToolRegistry_ExecuteWithContext_SanitizesLargeBase64Payload(t *testing. result: SilentResult(payload), }) - result := r.ExecuteWithContext(context.Background(), "base64_tool", nil, "telegram", "chat-1", nil) + result := r.ExecuteWithContext( + context.Background(), + "base64_tool", + nil, + "telegram", + "chat-1", + nil, + ) if result.ForLLM != largeBase64OmittedMessage { t.Fatalf("expected sanitized payload, got %q", result.ForLLM) @@ -765,7 +846,14 @@ func TestToolRegistry_ExecuteWithContext_ExtractsInlineMediaDataURL(t *testing.T result: SilentResult(payload), }) - result := r.ExecuteWithContext(context.Background(), "inline_media_tool", nil, "telegram", "chat-42", nil) + result := r.ExecuteWithContext( + context.Background(), + "inline_media_tool", + nil, + "telegram", + "chat-42", + nil, + ) if len(result.Media) != 1 { t.Fatalf("expected 1 media ref, got %d", len(result.Media)) @@ -800,7 +888,14 @@ func TestToolRegistry_ExecuteWithContext_SanitizesInlineMediaWithoutStore(t *tes result: SilentResult(payload), }) - result := r.ExecuteWithContext(context.Background(), "inline_media_no_store", nil, "telegram", "chat-42", nil) + result := r.ExecuteWithContext( + context.Background(), + "inline_media_no_store", + nil, + "telegram", + "chat-42", + nil, + ) if strings.Contains(result.ForLLM, "data:image/png;base64") { t.Fatalf("expected inline data URL to be removed from ForLLM, got %q", result.ForLLM) diff --git a/pkg/tools/result_test.go b/pkg/tools/result_test.go index 5f08cb4fa..3e7848687 100644 --- a/pkg/tools/result_test.go +++ b/pkg/tools/result_test.go @@ -41,6 +41,53 @@ func TestSilentResult(t *testing.T) { } } +func TestDiffResult(t *testing.T) { + result := DiffResult("pkg/tools/fs/edit.go", []byte("hello world\n"), []byte("hello universe\n")) + + if result.Silent { + t.Error("Expected Silent to be false") + } + if result.IsError { + t.Error("Expected IsError to be false") + } + if result.Async { + t.Error("Expected Async to be false") + } + if result.ForLLM == result.ForUser { + t.Fatalf("Expected ForLLM to omit the full diff, got %q", result.ForLLM) + } + if len(result.ForLLM) >= len(result.ForUser) { + t.Fatalf("Expected ForLLM to stay smaller than ForUser, got %d vs %d", len(result.ForLLM), len(result.ForUser)) + } + + for _, want := range []string{ + "File edited: pkg/tools/fs/edit.go", + "```diff", + "--- a/pkg/tools/fs/edit.go", + "+++ b/pkg/tools/fs/edit.go", + "-hello world", + "+hello universe", + } { + if !strings.Contains(result.ForUser, want) { + t.Fatalf("DiffResult output missing %q:\n%s", want, result.ForUser) + } + } +} + +func TestDiffResult_NormalizesAbsolutePathsAndHandlesNoOpChanges(t *testing.T) { + result := DiffResult("/tmp/test.txt", []byte("same\n"), []byte("same\n")) + + if !strings.Contains(result.ForUser, "File edited: /tmp/test.txt") { + t.Fatalf("Expected original path in output, got %q", result.ForUser) + } + if !strings.Contains(result.ForUser, "(no content change)") { + t.Fatalf("Expected no-content-change marker, got %q", result.ForUser) + } + if !strings.Contains(result.ForLLM, "(no content change)") { + t.Fatalf("Expected compact no-op summary in ForLLM, got %q", result.ForLLM) + } +} + func TestAsyncResult(t *testing.T) { result := AsyncResult("async task started") diff --git a/pkg/tools/search_tool.go b/pkg/tools/search_tool.go index c5884c9de..511b81a03 100644 --- a/pkg/tools/search_tool.go +++ b/pkg/tools/search_tool.go @@ -14,6 +14,8 @@ import ( const ( MaxRegexPatternLength = 200 + RegexSearchToolName = "tool_search_tool_regex" + BM25SearchToolName = "tool_search_tool_bm25" ) type RegexSearchTool struct { @@ -27,7 +29,7 @@ func NewRegexSearchTool(r *ToolRegistry, ttl int, maxSearchResults int) *RegexSe } func (t *RegexSearchTool) Name() string { - return "tool_search_tool_regex" + return RegexSearchToolName } func (t *RegexSearchTool) Description() string { @@ -96,7 +98,7 @@ func NewBM25SearchTool(r *ToolRegistry, ttl int, maxSearchResults int) *BM25Sear } func (t *BM25SearchTool) Name() string { - return "tool_search_tool_bm25" + return BM25SearchToolName } func (t *BM25SearchTool) Description() string { @@ -294,6 +296,15 @@ func (t *BM25SearchTool) getOrBuildEngine() *bm25CachedEngine { return cached } +func isToolDiscoveryToolName(name string) bool { + switch strings.ToLower(strings.TrimSpace(name)) { + case BM25SearchToolName, RegexSearchToolName: + return true + default: + return false + } +} + // SearchBM25 ranks hidden tools against query using BM25 via utils.BM25Engine. // This non-cached variant rebuilds the engine on every call. Used by tests // and any code that doesn't hold a BM25SearchTool instance. diff --git a/pkg/tools/shared/diff_result.go b/pkg/tools/shared/diff_result.go new file mode 100644 index 000000000..3ed7bdda1 --- /dev/null +++ b/pkg/tools/shared/diff_result.go @@ -0,0 +1,162 @@ +package toolshared + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" + "unicode/utf8" + + "github.com/pmezard/go-difflib/difflib" +) + +const ( + noContentChangeDiffMessage = "(no content change)" + noNewlineAtEOFMarker = `\ No newline at end of file` + diffPreviewSkippedMessage = "[diff preview skipped: file too large for inline preview]" + diffPreviewTruncatedNote = "[diff preview truncated; call read_file for the full edited contents]" + maxDiffInputBytes = 64 * 1024 + maxDiffInputLines = 2000 + maxUserDiffPreviewBytes = 16 * 1024 +) + +// DiffResult creates a user-visible tool result containing a unified diff for +// a successful file edit. The diff is included for both the LLM and the user so +// the follow-up assistant response can reason about the resulting change set, +// including EOF newline transitions. +func DiffResult(path string, before, after []byte) *ToolResult { + summary := fmt.Sprintf("File edited: %s", path) + if exceedsDiffPreviewLimits(before, after) { + return SilentResult(summary + "\n" + diffPreviewSkippedMessage) + } + + diff, err := buildUnifiedDiff(path, before, after) + if err != nil { + return UserResult(fmt.Sprintf("%s\n[diff unavailable: %v]", summary, err)) + } + + userDiff, truncated := truncateDiffPreview(diff, maxUserDiffPreviewBytes) + userContent := fmt.Sprintf("%s\n```diff\n%s\n```", summary, userDiff) + if truncated { + userContent += "\n" + diffPreviewTruncatedNote + } + + llmContent := summary + if diff == noContentChangeDiffMessage { + llmContent = summary + "\n" + noContentChangeDiffMessage + } else if truncated { + llmContent = summary + "\n" + diffPreviewTruncatedNote + } + + return &ToolResult{ + ForLLM: llmContent, + ForUser: userContent, + Silent: false, + IsError: false, + Async: false, + } +} + +func buildUnifiedDiff(path string, before, after []byte) (string, error) { + diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: splitDiffLinesPreservingEOF(before), + B: splitDiffLinesPreservingEOF(after), + FromFile: "a/" + diffDisplayPath(path), + ToFile: "b/" + diffDisplayPath(path), + Context: 3, + }) + if err != nil { + return "", err + } + + diff = strings.TrimRight(diff, "\n") + if diff == "" { + return noContentChangeDiffMessage, nil + } + + return diff, nil +} + +func splitDiffLinesPreservingEOF(content []byte) []string { + if len(content) == 0 { + return nil + } + + lines := make([]string, 0, bytes.Count(content, []byte{'\n'})+1) + lineStart := 0 + for i, b := range content { + if b != '\n' { + continue + } + lines = append(lines, string(content[lineStart:i+1])) + lineStart = i + 1 + } + if lineStart < len(content) { + lines = append(lines, string(content[lineStart:])) + } + + if lacksTrailingNewline(content) { + lines[len(lines)-1] += "\n" + lines = append(lines, noNewlineAtEOFMarker+"\n") + } + + return lines +} + +func lacksTrailingNewline(content []byte) bool { + return len(content) > 0 && !bytes.HasSuffix(content, []byte("\n")) +} + +func exceedsDiffPreviewLimits(before, after []byte) bool { + return len(before) > maxDiffInputBytes || + len(after) > maxDiffInputBytes || + countDiffLines(before) > maxDiffInputLines || + countDiffLines(after) > maxDiffInputLines +} + +func countDiffLines(content []byte) int { + if len(content) == 0 { + return 0 + } + + lines := bytes.Count(content, []byte{'\n'}) + if !bytes.HasSuffix(content, []byte("\n")) { + lines++ + } + return lines +} + +func truncateDiffPreview(diff string, maxBytes int) (string, bool) { + if maxBytes <= 0 || len(diff) <= maxBytes { + return diff, false + } + + truncated := diff[:maxBytes] + for len(truncated) > 0 && !utf8.ValidString(truncated) { + truncated = truncated[:len(truncated)-1] + } + + lastNewline := strings.LastIndexByte(truncated, '\n') + if lastNewline > 0 { + truncated = truncated[:lastNewline] + } + + truncated = strings.TrimRight(truncated, "\n") + if truncated == "" { + truncated = diff[:maxBytes] + for len(truncated) > 0 && !utf8.ValidString(truncated) { + truncated = truncated[:len(truncated)-1] + } + truncated = strings.TrimRight(truncated, "\n") + } + + return truncated, true +} + +func diffDisplayPath(path string) string { + displayPath := strings.TrimLeft(filepath.ToSlash(path), "/") + if displayPath == "" { + return "file" + } + return displayPath +} diff --git a/pkg/tools/shared/diff_result_test.go b/pkg/tools/shared/diff_result_test.go new file mode 100644 index 000000000..9d4f38ea5 --- /dev/null +++ b/pkg/tools/shared/diff_result_test.go @@ -0,0 +1,177 @@ +package toolshared + +import ( + "bytes" + "strings" + "testing" +) + +func TestDiffResult_UserVisibleUnifiedDiff(t *testing.T) { + result := DiffResult("/tmp/example.txt", []byte("alpha\nbeta\ngamma\n"), []byte("alpha\nbeta 2\ngamma\n")) + + if result == nil { + t.Fatal("DiffResult() returned nil") + } + if result.Silent { + t.Fatal("expected DiffResult to be user-visible") + } + if result.IsError { + t.Fatal("expected DiffResult to be successful") + } + if result.ForLLM == result.ForUser { + t.Fatal("expected compact model context instead of duplicating the full diff") + } + if len(result.ForLLM) >= len(result.ForUser) { + t.Fatalf("expected ForLLM to stay smaller than ForUser, got %d vs %d", len(result.ForLLM), len(result.ForUser)) + } + if result.ForLLM != "File edited: /tmp/example.txt" { + t.Fatalf("expected compact summary in ForLLM, got %q", result.ForLLM) + } + + for _, want := range []string{ + "File edited: /tmp/example.txt", + "```diff", + "--- a/tmp/example.txt", + "+++ b/tmp/example.txt", + "@@ -1,3 +1,3 @@", + " alpha", + "-beta", + "+beta 2", + " gamma", + } { + if !strings.Contains(result.ForUser, want) { + t.Fatalf("DiffResult output missing %q:\n%s", want, result.ForUser) + } + } +} + +func TestBuildUnifiedDiff_NoContentChange(t *testing.T) { + diff, err := buildUnifiedDiff("test.txt", []byte("same\n"), []byte("same\n")) + if err != nil { + t.Fatalf("buildUnifiedDiff() error = %v", err) + } + if diff != noContentChangeDiffMessage { + t.Fatalf("buildUnifiedDiff() = %q, want %q", diff, noContentChangeDiffMessage) + } +} + +func TestBuildUnifiedDiff_PreservesTrailingNewlineRemoval(t *testing.T) { + diff, err := buildUnifiedDiff("test.txt", []byte("same\n"), []byte("same")) + if err != nil { + t.Fatalf("buildUnifiedDiff() error = %v", err) + } + + for _, want := range []string{ + "--- a/test.txt", + "+++ b/test.txt", + " same", + "+" + noNewlineAtEOFMarker, + } { + if !strings.Contains(diff, want) { + t.Fatalf("buildUnifiedDiff() missing %q:\n%s", want, diff) + } + } +} + +func TestBuildUnifiedDiff_PreservesTrailingNewlineAddition(t *testing.T) { + diff, err := buildUnifiedDiff("test.txt", []byte("same"), []byte("same\n")) + if err != nil { + t.Fatalf("buildUnifiedDiff() error = %v", err) + } + + for _, want := range []string{ + "--- a/test.txt", + "+++ b/test.txt", + " same", + "-" + noNewlineAtEOFMarker, + } { + if !strings.Contains(diff, want) { + t.Fatalf("buildUnifiedDiff() missing %q:\n%s", want, diff) + } + } +} + +func TestBuildUnifiedDiff_UsesNormalizedDisplayPaths(t *testing.T) { + diff, err := buildUnifiedDiff("/tmp/nested/example.txt", []byte("before\n"), []byte("after\n")) + if err != nil { + t.Fatalf("buildUnifiedDiff() error = %v", err) + } + + for _, want := range []string{ + "--- a/tmp/nested/example.txt", + "+++ b/tmp/nested/example.txt", + } { + if !strings.Contains(diff, want) { + t.Fatalf("buildUnifiedDiff() missing %q:\n%s", want, diff) + } + } +} + +func TestDiffResult_SkipsPreviewForLargeFiles(t *testing.T) { + before := bytes.Repeat([]byte("a"), maxDiffInputBytes+1) + after := bytes.Repeat([]byte("b"), maxDiffInputBytes+1) + + result := DiffResult("big.txt", before, after) + + if !result.Silent { + t.Fatal("expected large diff previews to be skipped silently") + } + if result.ForUser != "" { + t.Fatalf("expected no user-facing preview when skipped, got %q", result.ForUser) + } + if !strings.Contains(result.ForLLM, diffPreviewSkippedMessage) { + t.Fatalf("expected skipped-preview note, got %q", result.ForLLM) + } +} + +func TestDiffResult_TruncatesLargeUserPreview(t *testing.T) { + after := []byte(strings.Repeat("abcd", maxUserDiffPreviewBytes/4) + "\n") + + result := DiffResult("preview.txt", []byte("before\n"), after) + + if result.Silent { + t.Fatal("expected preview to remain user-visible below the input caps") + } + if !strings.Contains(result.ForUser, diffPreviewTruncatedNote) { + t.Fatalf("expected truncated preview note, got %q", result.ForUser) + } + if !strings.Contains(result.ForLLM, diffPreviewTruncatedNote) { + t.Fatalf("expected model summary to mention truncation, got %q", result.ForLLM) + } + if len(result.ForLLM) >= len(result.ForUser) { + t.Fatalf("expected ForLLM to remain smaller than ForUser, "+ + "got %d vs %d", len(result.ForLLM), len(result.ForUser)) + } +} + +func TestDiffDisplayPath(t *testing.T) { + tests := []struct { + name string + path string + want string + }{ + { + name: "absolute path", + path: "/tmp/example.txt", + want: "tmp/example.txt", + }, + { + name: "relative path", + path: "pkg/tools/fs/edit.go", + want: "pkg/tools/fs/edit.go", + }, + { + name: "empty path", + path: "", + want: "file", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := diffDisplayPath(tt.path); got != tt.want { + t.Fatalf("diffDisplayPath(%q) = %q, want %q", tt.path, got, tt.want) + } + }) + } +} diff --git a/pkg/tools/shared_facade.go b/pkg/tools/shared_facade.go index 8409ea060..85bac140a 100644 --- a/pkg/tools/shared_facade.go +++ b/pkg/tools/shared_facade.go @@ -101,6 +101,10 @@ func SilentResult(forLLM string) *ToolResult { return toolshared.SilentResult(forLLM) } +func DiffResult(path string, before, after []byte) *ToolResult { + return toolshared.DiffResult(path, before, after) +} + func AsyncResult(forLLM string) *ToolResult { return toolshared.AsyncResult(forLLM) } diff --git a/pkg/tools/spawn.go b/pkg/tools/spawn.go index d019d511a..a9a373856 100644 --- a/pkg/tools/spawn.go +++ b/pkg/tools/spawn.go @@ -92,11 +92,12 @@ func (t *SpawnTool) execute( label, _ := args["label"].(string) agentID, _ := args["agent_id"].(string) + targetAgentID := strings.TrimSpace(agentID) // Check allowlist if targeting a specific agent - if agentID != "" && t.allowlistCheck != nil { - if !t.allowlistCheck(agentID) { - return ErrorResult(fmt.Sprintf("not allowed to spawn agent '%s'", agentID)) + if targetAgentID != "" && t.allowlistCheck != nil { + if !t.allowlistCheck(targetAgentID) { + return ErrorResult(fmt.Sprintf("not allowed to spawn agent '%s'", targetAgentID)) } } @@ -123,12 +124,14 @@ Task: %s`, // Launch async sub-turn in goroutine go func() { result, err := t.spawner.SpawnSubTurn(ctx, SubTurnConfig{ - Model: t.defaultModel, - Tools: nil, // Will inherit from parent via context - SystemPrompt: systemPrompt, - MaxTokens: t.maxTokens, - Temperature: t.temperature, - Async: true, // Async execution + Model: t.defaultModel, + Tools: nil, // Will inherit from parent via context + SystemPrompt: systemPrompt, + MaxTokens: t.maxTokens, + Temperature: t.temperature, + Async: true, // Async execution + Critical: true, // Background spawn should survive parent turn completion + TargetAgentID: targetAgentID, }) if err != nil { result = ErrorResult(fmt.Sprintf("Spawn failed: %v", err)).WithError(err) diff --git a/pkg/tools/spawn_test.go b/pkg/tools/spawn_test.go index fda6bbd89..c91c79578 100644 --- a/pkg/tools/spawn_test.go +++ b/pkg/tools/spawn_test.go @@ -6,10 +6,18 @@ import ( "testing" ) -// mockSpawner implements SubTurnSpawner for testing -type mockSpawner struct{} +// mockSpawner implements SubTurnSpawner for testing. +type mockSpawner struct { + lastConfig SubTurnConfig + done chan struct{} +} func (m *mockSpawner) SpawnSubTurn(ctx context.Context, cfg SubTurnConfig) (*ToolResult, error) { + m.lastConfig = cfg + if m.done != nil { + close(m.done) + } + // Extract task from system prompt for response task := cfg.SystemPrompt if strings.Contains(task, "Task: ") { @@ -62,12 +70,14 @@ func TestSpawnTool_Execute_ValidTask(t *testing.T) { provider := &MockLLMProvider{} manager := NewSubagentManager(provider, "test-model", "/tmp/test") tool := NewSpawnTool(manager) - tool.SetSpawner(&mockSpawner{}) + spawner := &mockSpawner{done: make(chan struct{})} + tool.SetSpawner(spawner) ctx := context.Background() args := map[string]any{ - "task": "Write a haiku about coding", - "label": "haiku-task", + "task": "Write a haiku about coding", + "label": "haiku-task", + "agent_id": "research", } result := tool.Execute(ctx, args) @@ -80,6 +90,13 @@ func TestSpawnTool_Execute_ValidTask(t *testing.T) { if !result.Async { t.Error("SpawnTool should return async result") } + <-spawner.done + if spawner.lastConfig.TargetAgentID != "research" { + t.Errorf("TargetAgentID = %q, want research", spawner.lastConfig.TargetAgentID) + } + if !spawner.lastConfig.Critical { + t.Error("SpawnTool should mark background subturns as critical") + } } func TestSpawnTool_Execute_NilManager(t *testing.T) { diff --git a/pkg/tools/subagent.go b/pkg/tools/subagent.go index ada89efb7..feeabe536 100644 --- a/pkg/tools/subagent.go +++ b/pkg/tools/subagent.go @@ -30,6 +30,7 @@ type SubTurnConfig struct { ActualSystemPrompt string InitialMessages []providers.Message InitialTokenBudget *atomic.Int64 // Shared token budget for team members; nil if no budget + TargetAgentID string // If set, run as this agent (its workspace, model, tools) } type SubagentTask struct { diff --git a/pkg/utils/tool_feedback.go b/pkg/utils/tool_feedback.go index de7cb467e..1834d7f78 100644 --- a/pkg/utils/tool_feedback.go +++ b/pkg/utils/tool_feedback.go @@ -1,12 +1,35 @@ package utils import ( + "bytes" + "encoding/json" "fmt" "strings" ) const ToolFeedbackContinuationHint = "Continuing the current task." +func FormatArgsJSON(args map[string]any, prettyPrint, disableEscapeHTML bool) string { + // Normalize nil to empty map for consistent output + if args == nil { + args = map[string]any{} + } + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + if prettyPrint { + enc.SetIndent("", " ") + } + if disableEscapeHTML { + enc.SetEscapeHTML(false) + } + if err := enc.Encode(args); err != nil { + // Fallback to fmt.Sprintf to preserve visibility of problematic args + return fmt.Sprintf("%v", args) + } + return strings.TrimSpace(buf.String()) +} + // FormatToolFeedbackMessage renders a tool feedback message for chat channels. // It keeps the tool name on the first line for animation and can include both // a human explanation and the serialized tool arguments in the body. diff --git a/pkg/utils/tool_feedback_test.go b/pkg/utils/tool_feedback_test.go index c30f53827..da4accce4 100644 --- a/pkg/utils/tool_feedback_test.go +++ b/pkg/utils/tool_feedback_test.go @@ -1,6 +1,9 @@ package utils -import "testing" +import ( + "encoding/json" + "testing" +) func TestFormatToolFeedbackMessage(t *testing.T) { got := FormatToolFeedbackMessage( @@ -56,3 +59,98 @@ func TestFitToolFeedbackMessage_TruncatesSingleLineMessage(t *testing.T) { t.Fatalf("FitToolFeedbackMessage() = %q, want %q", got, want) } } + +func TestFormatArgsJSON_Defaults(t *testing.T) { + args := map[string]any{"path": "README.md", "line": 42} + got := FormatArgsJSON(args, false, false) + var gotVal, wantVal any + if err := json.Unmarshal([]byte(got), &gotVal); err != nil { + t.Fatalf("FormatArgsJSON() returned invalid JSON: %v", err) + } + want := `{"path":"README.md","line":42}` + if err := json.Unmarshal([]byte(want), &wantVal); err != nil { + t.Fatalf("invalid test want JSON: %v", err) + } + if !jsonValEq(gotVal, wantVal) { + t.Fatalf("FormatArgsJSON() = %q, want %q", got, want) + } +} + +func TestFormatArgsJSON_PrettyPrint(t *testing.T) { + args := map[string]any{"path": "README.md", "line": 42} + got := FormatArgsJSON(args, true, false) + var gotVal any + if err := json.Unmarshal([]byte(got), &gotVal); err != nil { + t.Fatalf("FormatArgsJSON() returned invalid JSON: %v", err) + } + want := `{"path":"README.md","line":42}` + var wantVal any + if err := json.Unmarshal([]byte(want), &wantVal); err != nil { + t.Fatalf("invalid test want JSON: %v", err) + } + if !jsonValEq(gotVal, wantVal) { + t.Fatalf("FormatArgsJSON() prettyPrint = %q, want structure %q", got, want) + } +} + +func TestFormatArgsJSON_DisableEscapeHTML(t *testing.T) { + args := map[string]any{"msg": "a < b && c > d"} + got := FormatArgsJSON(args, false, true) + var gotVal, wantVal any + want := `{"msg":"a < b && c > d"}` + if err := json.Unmarshal([]byte(got), &gotVal); err != nil { + t.Fatalf("FormatArgsJSON() returned invalid JSON: %v", err) + } + if err := json.Unmarshal([]byte(want), &wantVal); err != nil { + t.Fatalf("invalid test want JSON: %v", err) + } + if !jsonValEq(gotVal, wantVal) { + t.Fatalf("FormatArgsJSON() disableEscapeHTML = %q, want %q", got, want) + } +} + +func TestFormatArgsJSON_PrettyPrintAndDisableEscapeHTML(t *testing.T) { + args := map[string]any{"msg": "a < b && c > d"} + got := FormatArgsJSON(args, true, true) + var gotVal, wantVal any + want := `{"msg":"a < b && c > d"}` + if err := json.Unmarshal([]byte(got), &gotVal); err != nil { + t.Fatalf("FormatArgsJSON() returned invalid JSON: %v", err) + } + if err := json.Unmarshal([]byte(want), &wantVal); err != nil { + t.Fatalf("invalid test want JSON: %v", err) + } + if !jsonValEq(gotVal, wantVal) { + t.Fatalf("FormatArgsJSON() combined = %q, want %q", got, want) + } +} + +func TestFormatArgsJSON_EscapeHTMLByDefault(t *testing.T) { + args := map[string]any{"msg": "a < b && c > d"} + got := FormatArgsJSON(args, false, false) + var gotVal, wantVal any + want := `{"msg":"a \u003c b \u0026\u0026 c \u003e d"}` + if err := json.Unmarshal([]byte(got), &gotVal); err != nil { + t.Fatalf("FormatArgsJSON() returned invalid JSON: %v", err) + } + if err := json.Unmarshal([]byte(want), &wantVal); err != nil { + t.Fatalf("invalid test want JSON: %v", err) + } + if !jsonValEq(gotVal, wantVal) { + t.Fatalf("FormatArgsJSON() default escape = %q, want %q", got, want) + } +} + +func TestFormatArgsJSON_NilArgs(t *testing.T) { + got := FormatArgsJSON(nil, false, false) + want := `{}` + if got != want { + t.Fatalf("FormatArgsJSON() nil = %q, want %q", got, want) + } +} + +func jsonValEq(a, b any) bool { + aJSON, _ := json.Marshal(a) + bJSON, _ := json.Marshal(b) + return string(aJSON) == string(bJSON) +} diff --git a/web/README.md b/web/README.md index 2a57524e0..774ad8f5d 100644 --- a/web/README.md +++ b/web/README.md @@ -47,7 +47,7 @@ The current frontend exposes these major pages and flows: - Current built-in flows: OpenAI, Anthropic, and Google Antigravity. - `/channels/*` - Configure supported channels from a shared catalog. - - Current catalog: `weixin`, `telegram`, `discord`, `slack`, `feishu`, `dingtalk`, `line`, `qq`, `onebot`, `wecom`, `whatsapp`, `whatsapp_native`, `pico`, `maixcam`, `matrix`, `irc`. + - Current catalog: `weixin`, `telegram`, `discord`, `slack`, `feishu`, `dingtalk`, `line`, `qq`, `onebot`, `wecom`, `whatsapp`, `whatsapp_native`, `pico`, `maixcam`, `matrix`, `irc`, `mqtt`. - Includes QR-based binding helpers for WeChat and WeCom. - `/agent/skills` - Browse built-in, global, and workspace skills. @@ -55,7 +55,7 @@ The current frontend exposes these major pages and flows: - `/agent/tools` - View tool availability and enable or disable tool switches through config-backed APIs. - `/config` - - Edit agent defaults, exec controls, cron controls, heartbeat, device monitoring, launcher networking, and launch-at-login settings. + - Edit agent defaults, self-evolution, exec controls, cron controls, heartbeat, device monitoring, launcher networking, and launch-at-login settings. - `/logs` - View the in-memory gateway log buffer and clear it. diff --git a/web/backend/api/channels.go b/web/backend/api/channels.go index 82cd54b72..e77b11f8b 100644 --- a/web/backend/api/channels.go +++ b/web/backend/api/channels.go @@ -30,6 +30,7 @@ var channelCatalog = []channelCatalogItem{ {Name: "maixcam", ConfigKey: "maixcam"}, {Name: "matrix", ConfigKey: "matrix"}, {Name: "irc", ConfigKey: "irc"}, + {Name: "mqtt", ConfigKey: "mqtt"}, } type channelConfigResponse struct { @@ -106,6 +107,7 @@ var channelSecretFieldMap = map[string][]string{ "whatsapp": {}, "whatsapp_native": {}, "maixcam": {}, + "mqtt": {"username", "password"}, } func buildChannelConfigResponse(cfg *config.Config, item channelCatalogItem) channelConfigResponse { diff --git a/web/backend/api/gateway.go b/web/backend/api/gateway.go index 67b055236..45f7e6912 100644 --- a/web/backend/api/gateway.go +++ b/web/backend/api/gateway.go @@ -382,6 +382,9 @@ func (h *Handler) gatewayStartReady() (bool, string, error) { if modelCfg == nil { return false, fmt.Sprintf("default model %q is invalid", modelName), nil } + if !defaultModelAllowedForModelConfig(modelCfg) { + return false, fmt.Sprintf("default model %q is not usable for chat", modelName), nil + } if !hasModelConfiguration(modelCfg) { return false, fmt.Sprintf("default model %q has no credentials configured", modelName), nil diff --git a/web/backend/api/gateway_test.go b/web/backend/api/gateway_test.go index 1d9352972..f383089a6 100644 --- a/web/backend/api/gateway_test.go +++ b/web/backend/api/gateway_test.go @@ -357,6 +357,44 @@ func TestGatewayStartReady_NoDefaultModel(t *testing.T) { } } +func TestGatewayStartReady_RejectsASROnlyDefaultModel(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Provider: "elevenlabs", + Model: "scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + cfg.Agents.Defaults.ModelName = "elevenlabs-asr" + + err = config.SaveConfig(configPath, cfg) + if err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + ready, reason, err := h.gatewayStartReady() + if err != nil { + t.Fatalf("gatewayStartReady() error = %v", err) + } + if ready { + t.Fatal("gatewayStartReady() ready = true, want false") + } + if reason != `default model "elevenlabs-asr" is not usable for chat` { + t.Fatalf( + "gatewayStartReady() reason = %q, want %q", + reason, + `default model "elevenlabs-asr" is not usable for chat`, + ) + } +} + func TestLooksLikeGatewayCommandLine(t *testing.T) { cases := []struct { name string diff --git a/web/backend/api/model_status.go b/web/backend/api/model_status.go index d262cf124..6cfda501d 100644 --- a/web/backend/api/model_status.go +++ b/web/backend/api/model_status.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/url" + "os/exec" "strconv" "strings" "sync" @@ -47,6 +48,7 @@ var ( probeTCPServiceFunc = probeTCPService probeOllamaModelFunc = probeOllamaModel probeOpenAICompatibleModelFunc = probeOpenAICompatibleModel + probeCommandAvailableFunc = probeCommandAvailable modelProbeNowFunc = time.Now modelProbeState = newModelProbeCacheState() ) @@ -83,17 +85,23 @@ func (s *modelProbeCacheState) resetForTest() { } func hasModelConfiguration(m *config.ModelConfig) bool { + protocol := modelProtocol(m) authMethod := strings.ToLower(strings.TrimSpace(m.AuthMethod)) apiKey := strings.TrimSpace(m.APIKey()) if authMethod == "oauth" || authMethod == "token" { - if provider, ok := oauthProviderForModel(m); ok { - cred, err := oauthGetCredential(provider) - if err != nil || cred == nil { - return false - } - return strings.TrimSpace(cred.AccessToken) != "" || strings.TrimSpace(cred.RefreshToken) != "" + if configured, checked := hasStoredOAuthCredential(m); checked { + return configured } + } + + if authMethod == "" && providerUsesImplicitOAuth(protocol) { + if configured, checked := hasStoredOAuthCredential(m); checked { + return configured + } + } + + if providerUsesAmbientCredentials(protocol) { return true } @@ -104,6 +112,40 @@ func hasModelConfiguration(m *config.ModelConfig) bool { return apiKey != "" } +func hasStoredOAuthCredential(m *config.ModelConfig) (bool, bool) { + provider, ok := oauthProviderForModel(m) + if !ok { + return false, false + } + cred, err := oauthGetCredential(provider) + if err != nil || cred == nil { + return false, true + } + return strings.TrimSpace(cred.AccessToken) != "" || strings.TrimSpace(cred.RefreshToken) != "", true +} + +func providerUsesImplicitOAuth(protocol string) bool { + switch protocol { + case "antigravity", "google-antigravity": + return true + default: + return false + } +} + +func providerUsesAmbientCredentials(protocol string) bool { + switch protocol { + case "bedrock": + // Bedrock relies on the AWS SDK credential chain instead of an explicit + // API key stored in ModelConfig. We cannot reliably preflight every AWS + // credential source here, so avoid misclassifying valid environments as + // "unconfigured" and defer concrete credential failures to runtime. + return true + default: + return false + } +} + func modelConfigurationStatus(m *config.ModelConfig) modelConfigurationSummary { if !hasModelConfiguration(m) { return modelConfigurationSummary{Available: false, Status: modelStatusUnconfigured} @@ -180,8 +222,10 @@ func runLocalModelProbe(m *config.ModelConfig) bool { return probeOpenAICompatibleModelFunc(apiBase, modelID, m.APIKey()) case "github-copilot", "copilot": return probeTCPServiceFunc(apiBase) - case "claude-cli", "claudecli", "codex-cli", "codexcli": - return true + case "claude-cli", "claudecli": + return probeCommandAvailableFunc("claude") + case "codex-cli", "codexcli": + return probeCommandAvailableFunc("codex") default: if hasLocalAPIBase(apiBase) { return probeOpenAICompatibleModelFunc(apiBase, modelID, m.APIKey()) @@ -190,6 +234,11 @@ func runLocalModelProbe(m *config.ModelConfig) bool { } } +func probeCommandAvailable(command string) bool { + _, err := exec.LookPath(command) + return err == nil +} + func modelProbeCacheKey(m *config.ModelConfig) string { protocol, modelID := splitModel(m) @@ -385,8 +434,11 @@ func modelProbeAPIBase(m *config.ModelConfig) string { } protocol := modelProtocol(m) - if providers.IsEmptyAPIKeyAllowedForProtocol(protocol) { - return providers.DefaultAPIBaseForProtocol(protocol) + + // Resolve the default API base for any known protocol so that probes + // work even when the config stores only a provider without an explicit api_base. + if defaultBase := providers.DefaultAPIBaseForProtocol(protocol); defaultBase != "" { + return normalizeModelProbeAPIBase(defaultBase) } switch protocol { diff --git a/web/backend/api/models.go b/web/backend/api/models.go index cf903ce4c..8a66918f9 100644 --- a/web/backend/api/models.go +++ b/web/backend/api/models.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/sipeed/picoclaw/pkg/audio/asr" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/providers" @@ -35,20 +36,194 @@ type modelResponse struct { Proxy string `json:"proxy,omitempty"` AuthMethod string `json:"auth_method,omitempty"` // Advanced fields - ConnectMode string `json:"connect_mode,omitempty"` - Workspace string `json:"workspace,omitempty"` - RPM int `json:"rpm,omitempty"` - MaxTokensField string `json:"max_tokens_field,omitempty"` - RequestTimeout int `json:"request_timeout,omitempty"` - ThinkingLevel string `json:"thinking_level,omitempty"` - ExtraBody map[string]any `json:"extra_body,omitempty"` - CustomHeaders map[string]string `json:"custom_headers,omitempty"` + ConnectMode string `json:"connect_mode,omitempty"` + Workspace string `json:"workspace,omitempty"` + RPM int `json:"rpm,omitempty"` + MaxTokensField string `json:"max_tokens_field,omitempty"` + RequestTimeout int `json:"request_timeout,omitempty"` + ThinkingLevel string `json:"thinking_level,omitempty"` + ToolSchemaTransform string `json:"tool_schema_transform,omitempty"` + ExtraBody map[string]any `json:"extra_body,omitempty"` + CustomHeaders map[string]string `json:"custom_headers,omitempty"` // Meta - Enabled bool `json:"enabled"` - Available bool `json:"available"` - Status string `json:"status"` - IsDefault bool `json:"is_default"` - IsVirtual bool `json:"is_virtual"` + Enabled bool `json:"enabled"` + Available bool `json:"available"` + Status string `json:"status"` + IsDefault bool `json:"is_default"` + IsVirtual bool `json:"is_virtual"` + DefaultModelAllowed bool `json:"default_model_allowed"` +} + +func normalizeStoredModelConfig(mc *config.ModelConfig) bool { + if mc == nil { + return false + } + + changed := false + model := strings.TrimSpace(mc.Model) + if model != mc.Model { + mc.Model = model + changed = true + } + provider := strings.TrimSpace(mc.Provider) + if provider != mc.Provider { + mc.Provider = provider + changed = true + } + authMethod := strings.ToLower(strings.TrimSpace(mc.AuthMethod)) + if authMethod != mc.AuthMethod { + mc.AuthMethod = authMethod + changed = true + } + + if provider != "" { + normalizedProvider := providers.NormalizeProvider(provider) + if providers.IsSupportedModelProvider(normalizedProvider) && normalizedProvider != provider { + mc.Provider = normalizedProvider + changed = true + } + if mc.Provider == "elevenlabs" { + if _, strippedModel, found := strings.Cut( + model, + "/", + ); found && + providers.NormalizeProvider(strings.TrimSpace(provider)) == "elevenlabs" { + strippedModel = strings.TrimSpace(strippedModel) + if strippedModel != "" && strippedModel != mc.Model { + mc.Model = strippedModel + changed = true + } + } + if strings.TrimSpace(mc.Model) != asr.ElevenLabsSupportedModelID() { + mc.Model = asr.ElevenLabsSupportedModelID() + changed = true + } + } + return changed + } + + effectiveProvider, modelID := providers.SplitModelProviderAndID(model, "openai") + if effectiveProvider == "" { + return changed + } + if mc.Provider != effectiveProvider { + mc.Provider = effectiveProvider + changed = true + } + if mc.Model != modelID { + mc.Model = modelID + changed = true + } + return changed +} + +func normalizeIncomingModelConfig(mc *config.ModelConfig) { + if mc == nil { + return + } + + mc.Model = strings.TrimSpace(mc.Model) + mc.Provider = strings.TrimSpace(mc.Provider) + mc.AuthMethod = strings.ToLower(strings.TrimSpace(mc.AuthMethod)) + if mc.Provider == "" { + mc.Provider, mc.Model = providers.SplitModelProviderAndID(mc.Model, "openai") + } else { + mc.Provider = providers.NormalizeProvider(mc.Provider) + if mc.Provider == "elevenlabs" { + if _, strippedModel, found := strings.Cut(mc.Model, "/"); found { + strippedModel = strings.TrimSpace(strippedModel) + if strippedModel != "" { + mc.Model = strippedModel + } + } + } + } + if mc.Provider == "antigravity" && mc.AuthMethod == "" { + mc.AuthMethod = "oauth" + } +} + +func createAllowedForProvider(provider string) bool { + normalized := providers.NormalizeProvider(provider) + switch normalized { + case "bedrock": + // Bedrock currently authenticates through the AWS SDK credential chain + // (env vars, shared profiles, IAM roles, etc.), and this Web layer does + // not yet have a reliable preflight check for those credential sources. + // Keep it creatable in the catalog and let provider construction/runtime + // return the concrete AWS error when the environment is incomplete. + return true + case "claude-cli", "codex-cli": + return cliProviderCreateAllowedFromCurrentStatus(normalized) + default: + return providers.IsCreatableModelProvider(normalized) + } +} + +// cliProviderCreateAllowedFromCurrentStatus intentionally reuses the existing +// local model status pipeline so provider catalog gating follows the same CLI +// executable probe used by launcher readiness. +func cliProviderCreateAllowedFromCurrentStatus(provider string) bool { + status := modelConfigurationStatus(&config.ModelConfig{ + Provider: provider, + Model: provider, + }) + return status.Available +} + +func modelProviderOptionsForResponse() []providers.ModelProviderOption { + options := providers.ModelProviderOptions() + for i := range options { + options[i].CreateAllowed = createAllowedForProvider(options[i].ID) + } + return options +} + +func defaultModelAllowedForModelConfig(mc *config.ModelConfig) bool { + provider, _ := providers.ExtractProtocol(mc) + return providers.IsDefaultModelProvider(provider) +} + +func validateIncomingModelConfig(mc *config.ModelConfig, existing *config.ModelConfig) error { + if mc == nil { + return fmt.Errorf("model config is required") + } + if err := mc.Validate(); err != nil { + return err + } + if strings.TrimSpace(mc.Provider) == "" { + return fmt.Errorf("provider is required") + } + if !providers.IsSupportedModelProvider(mc.Provider) { + return fmt.Errorf("provider %q is not supported", mc.Provider) + } + if mc.Provider == "elevenlabs" && strings.TrimSpace(mc.Model) != asr.ElevenLabsSupportedModelID() { + return fmt.Errorf("provider %q only supports model %q", mc.Provider, asr.ElevenLabsSupportedModelID()) + } + if !createAllowedForProvider(mc.Provider) { + if existing == nil { + return fmt.Errorf("provider %q is not available for new models", mc.Provider) + } + existingProvider, _ := providers.ExtractProtocol(existing) + if providers.NormalizeProvider(existingProvider) != mc.Provider { + return fmt.Errorf("provider %q is not available for selection", mc.Provider) + } + } + return nil +} + +func normalizeStoredModelProviders(cfg *config.Config) bool { + if cfg == nil { + return false + } + + changed := false + for _, model := range cfg.ModelList { + if normalizeStoredModelConfig(model) { + changed = true + } + } + return changed } // handleListModels returns all model_list entries with masked API keys. @@ -61,6 +236,10 @@ func (h *Handler) handleListModels(w http.ResponseWriter, r *http.Request) { return } + // Normalize legacy provider/model storage in memory so GET can round-trip + // through the current API shape without mutating the on-disk config. + normalizeStoredModelProviders(cfg) + defaultModel := cfg.Agents.Defaults.GetModelName() modelStatuses := make([]modelConfigurationSummary, len(cfg.ModelList)) @@ -78,35 +257,38 @@ func (h *Handler) handleListModels(w http.ResponseWriter, r *http.Request) { for i, m := range cfg.ModelList { provider, modelID := providers.ExtractProtocol(m) models = append(models, modelResponse{ - Index: i, - ModelName: m.ModelName, - Provider: provider, - Model: modelID, - APIBase: m.APIBase, - APIKey: maskAPIKey(m.APIKey()), - Proxy: m.Proxy, - AuthMethod: m.AuthMethod, - ConnectMode: m.ConnectMode, - Workspace: m.Workspace, - RPM: m.RPM, - MaxTokensField: m.MaxTokensField, - RequestTimeout: m.RequestTimeout, - ThinkingLevel: m.ThinkingLevel, - ExtraBody: m.ExtraBody, - CustomHeaders: m.CustomHeaders, - Enabled: m.Enabled, - Available: modelStatuses[i].Available, - Status: modelStatuses[i].Status, - IsDefault: m.ModelName == defaultModel, - IsVirtual: m.IsVirtual(), + Index: i, + ModelName: m.ModelName, + Provider: provider, + Model: modelID, + APIBase: m.APIBase, + APIKey: maskAPIKey(m.APIKey()), + Proxy: m.Proxy, + AuthMethod: m.AuthMethod, + ConnectMode: m.ConnectMode, + Workspace: m.Workspace, + RPM: m.RPM, + MaxTokensField: m.MaxTokensField, + RequestTimeout: m.RequestTimeout, + ThinkingLevel: m.ThinkingLevel, + ToolSchemaTransform: m.ToolSchemaTransform, + ExtraBody: m.ExtraBody, + CustomHeaders: m.CustomHeaders, + Enabled: m.Enabled, + Available: modelStatuses[i].Available, + Status: modelStatuses[i].Status, + IsDefault: m.ModelName == defaultModel, + IsVirtual: m.IsVirtual(), + DefaultModelAllowed: defaultModelAllowedForModelConfig(m), }) } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{ - "models": models, - "total": len(models), - "default_model": defaultModel, + "models": models, + "total": len(models), + "default_model": defaultModel, + "provider_options": modelProviderOptionsForResponse(), }) } @@ -132,7 +314,9 @@ func (h *Handler) handleAddModel(w http.ResponseWriter, r *http.Request) { return } - if err = mc.Validate(); err != nil { + normalizeIncomingModelConfig(&mc.ModelConfig) + + if err = validateIncomingModelConfig(&mc.ModelConfig, nil); err != nil { http.Error(w, fmt.Sprintf("Validation error: %v", err), http.StatusBadRequest) return } @@ -148,6 +332,7 @@ func (h *Handler) handleAddModel(w http.ResponseWriter, r *http.Request) { } cfg.ModelList = append(cfg.ModelList, &mc.ModelConfig) + normalizeStoredModelProviders(cfg) if err := config.SaveConfig(h.configPath, cfg); err != nil { http.Error(w, fmt.Sprintf("Failed to save config: %v", err), http.StatusInternalServerError) @@ -198,11 +383,6 @@ func (h *Handler) handleUpdateModel(w http.ResponseWriter, r *http.Request) { return } - if err = mc.Validate(); err != nil { - http.Error(w, fmt.Sprintf("Validation error: %v", err), http.StatusBadRequest) - return - } - cfg, err := config.LoadConfig(h.configPath) if err != nil { http.Error(w, fmt.Sprintf("Failed to load config: %v", err), http.StatusInternalServerError) @@ -237,6 +417,9 @@ func (h *Handler) handleUpdateModel(w http.ResponseWriter, r *http.Request) { } else if len(mc.CustomHeaders) == 0 { mc.CustomHeaders = nil } + if _, ok := rawFields["tool_schema_transform"]; !ok { + mc.ToolSchemaTransform = cfg.ModelList[idx].ToolSchemaTransform + } // Preserve the existing Provider when the caller omits it. This keeps the // update API backward-compatible for clients that haven't started sending // the new field yet, while still allowing explicit clearing via "". @@ -248,9 +431,9 @@ func (h *Handler) handleUpdateModel(w http.ResponseWriter, r *http.Request) { // This keeps provider-omitted updates backward-compatible even when an // older client edits the visible model ID. if strings.TrimSpace(cfg.ModelList[idx].Provider) == "" { - existingProtocol, existingModelID := providers.ExtractProtocol(cfg.ModelList[idx]) existingRawModel := strings.TrimSpace(cfg.ModelList[idx].Model) incomingModel := strings.TrimSpace(mc.Model) + existingProtocol, existingModelID := providers.ExtractProtocol(cfg.ModelList[idx]) if existingRawModel != "" && existingRawModel != existingModelID && incomingModel != "" { if incomingModel == existingModelID { mc.Model = existingRawModel @@ -267,7 +450,20 @@ func (h *Handler) handleUpdateModel(w http.ResponseWriter, r *http.Request) { } } + normalizeIncomingModelConfig(&mc.ModelConfig) + if err = validateIncomingModelConfig(&mc.ModelConfig, cfg.ModelList[idx]); err != nil { + http.Error(w, fmt.Sprintf("Validation error: %v", err), http.StatusBadRequest) + return + } + if cfg.Agents.Defaults.ModelName == cfg.ModelList[idx].ModelName && + !defaultModelAllowedForModelConfig(&mc.ModelConfig) { + // Allow users to recover from legacy/invalid defaults by saving the model + // and clearing the default chat model reference in the same write. + cfg.Agents.Defaults.ModelName = "" + } + cfg.ModelList[idx] = &mc.ModelConfig + normalizeStoredModelProviders(cfg) logger.Debugf("update model config: %#v", mc.ModelConfig) @@ -367,6 +563,19 @@ func (h *Handler) handleSetDefaultModel(w http.ResponseWriter, r *http.Request) http.Error(w, fmt.Sprintf("Cannot set virtual model %q as default", req.ModelName), http.StatusBadRequest) return } + for _, m := range cfg.ModelList { + if m.ModelName == req.ModelName { + if !defaultModelAllowedForModelConfig(m) { + http.Error( + w, + fmt.Sprintf("Model %q cannot be used as the default chat model", req.ModelName), + http.StatusBadRequest, + ) + return + } + break + } + } cfg.Agents.Defaults.ModelName = req.ModelName diff --git a/web/backend/api/models_test.go b/web/backend/api/models_test.go index f374ac15b..0b1f04848 100644 --- a/web/backend/api/models_test.go +++ b/web/backend/api/models_test.go @@ -12,6 +12,7 @@ import ( "github.com/sipeed/picoclaw/pkg/auth" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/providers" ) func resetModelProbeHooks(t *testing.T) { @@ -20,17 +21,46 @@ func resetModelProbeHooks(t *testing.T) { origTCPProbe := probeTCPServiceFunc origOllamaProbe := probeOllamaModelFunc origOpenAIProbe := probeOpenAICompatibleModelFunc + origCommandProbe := probeCommandAvailableFunc origNow := modelProbeNowFunc resetModelProbeCache() t.Cleanup(func() { probeTCPServiceFunc = origTCPProbe probeOllamaModelFunc = origOllamaProbe probeOpenAICompatibleModelFunc = origOpenAIProbe + probeCommandAvailableFunc = origCommandProbe modelProbeNowFunc = origNow resetModelProbeCache() }) } +func addModelAndLoadLatest(t *testing.T, configPath string, body string) *config.ModelConfig { + t.Helper() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if len(cfg.ModelList) == 0 { + t.Fatal("model_list should contain the newly added model") + } + + return cfg.ModelList[len(cfg.ModelList)-1] +} + func TestHandleListModels_AvailabilityUsesRuntimeProbesForLocalModels(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -94,7 +124,8 @@ func TestHandleListModels_AvailabilityUsesRuntimeProbesForLocalModels(t *testing }, } cfg.Agents.Defaults.ModelName = "openai-oauth" - if err := config.SaveConfig(configPath, cfg); err != nil { + err = config.SaveConfig(configPath, cfg) + if err != nil { t.Fatalf("SaveConfig() error = %v", err) } @@ -113,7 +144,8 @@ func TestHandleListModels_AvailabilityUsesRuntimeProbesForLocalModels(t *testing var resp struct { Models []modelResponse `json:"models"` } - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + err = json.Unmarshal(rec.Body.Bytes(), &resp) + if err != nil { t.Fatalf("Unmarshal() error = %v", err) } @@ -181,14 +213,91 @@ func TestHandleListModels_AvailabilityForOAuthModelWithCredential(t *testing.T) AuthMethod: "oauth", }} cfg.Agents.Defaults.ModelName = "claude-oauth" - if err := config.SaveConfig(configPath, cfg); err != nil { + err = config.SaveConfig(configPath, cfg) + if err != nil { t.Fatalf("SaveConfig() error = %v", err) } - if err := auth.SetCredential(oauthProviderAnthropic, &auth.AuthCredential{ + if setCredentialErr := auth.SetCredential(oauthProviderAnthropic, &auth.AuthCredential{ AccessToken: "anthropic-token", Provider: oauthProviderAnthropic, AuthMethod: "oauth", + }); setCredentialErr != nil { + t.Fatalf("SetCredential() error = %v", setCredentialErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp struct { + Models []modelResponse `json:"models"` + } + err = json.Unmarshal(rec.Body.Bytes(), &resp) + if err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(resp.Models)) + } + if !resp.Models[0].Available { + t.Fatalf("oauth model available = false, want true with stored credential") + } +} + +func TestHasModelConfiguration_OAuthWithoutMappedCredentialFallsBackToAPIKey(t *testing.T) { + noKey := &config.ModelConfig{ + Provider: "gemini", + Model: "gemini-2.5-flash", + AuthMethod: "oauth", + } + if hasModelConfiguration(noKey) { + t.Fatal("oauth model without credential mapping and api key should be unconfigured") + } + + withKey := &config.ModelConfig{ + Provider: "gemini", + Model: "gemini-2.5-flash", + AuthMethod: "oauth", + APIKeys: config.SimpleSecureStrings("gemini-key"), + } + if !hasModelConfiguration(withKey) { + t.Fatal("oauth model without credential mapping should fall back to api key configuration") + } +} + +func TestHandleListModels_AntigravityImplicitOAuthAvailability(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + resetOAuthHooks(t) + resetModelProbeHooks(t) + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "gemini-flash", + Provider: "antigravity", + Model: "gemini-3-flash", + }} + err = config.SaveConfig(configPath, cfg) + if err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + if err := auth.SetCredential(oauthProviderGoogleAntigravity, &auth.AuthCredential{ + AccessToken: "antigravity-token", + Provider: oauthProviderGoogleAntigravity, + AuthMethod: "oauth", }); err != nil { t.Fatalf("SetCredential() error = %v", err) } @@ -208,14 +317,158 @@ func TestHandleListModels_AvailabilityForOAuthModelWithCredential(t *testing.T) var resp struct { Models []modelResponse `json:"models"` } - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { - t.Fatalf("Unmarshal() error = %v", err) + if unmarshalErr := json.Unmarshal(rec.Body.Bytes(), &resp); unmarshalErr != nil { + t.Fatalf("Unmarshal() error = %v", unmarshalErr) } if len(resp.Models) != 1 { t.Fatalf("len(models) = %d, want 1", len(resp.Models)) } if !resp.Models[0].Available { - t.Fatalf("oauth model available = false, want true with stored credential") + t.Fatal("antigravity model available = false, want true with stored credential even without auth_method") + } +} + +func TestHandleListModels_BedrockUsesAmbientCredentialStatus(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + resetOAuthHooks(t) + resetModelProbeHooks(t) + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "bedrock-claude", + Provider: "bedrock", + Model: "us.anthropic.claude-sonnet-4-20250514-v1:0", + }} + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp struct { + Models []modelResponse `json:"models"` + } + if unmarshalErr := json.Unmarshal(rec.Body.Bytes(), &resp); unmarshalErr != nil { + t.Fatalf("Unmarshal() error = %v", unmarshalErr) + } + if len(resp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(resp.Models)) + } + if !resp.Models[0].Available { + t.Fatal("bedrock model available = false, want true because Bedrock uses ambient AWS credentials") + } + if resp.Models[0].Status != modelStatusAvailable { + t.Fatalf("bedrock model status = %q, want %q", resp.Models[0].Status, modelStatusAvailable) + } +} + +func TestHandleListModels_CLIProvidersRequireInstalledCommands(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + resetOAuthHooks(t) + resetModelProbeHooks(t) + + probeCommandAvailableFunc = func(command string) bool { + switch command { + case "claude": + return false + case "codex": + return true + default: + return false + } + } + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{ + { + ModelName: "claude-cli-model", + Provider: "claude-cli", + Model: "claude-cli", + }, + { + ModelName: "codex-cli-model", + Provider: "codex-cli", + Model: "codex-cli", + }, + } + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp struct { + Models []modelResponse `json:"models"` + ProviderOptions []providers.ModelProviderOption `json:"provider_options"` + } + if unmarshalErr := json.Unmarshal(rec.Body.Bytes(), &resp); unmarshalErr != nil { + t.Fatalf("Unmarshal() error = %v", unmarshalErr) + } + + modelsByName := make(map[string]modelResponse, len(resp.Models)) + for _, model := range resp.Models { + modelsByName[model.ModelName] = model + } + if model := modelsByName["claude-cli-model"]; model.Available || model.Status != modelStatusUnreachable { + t.Fatalf( + "claude-cli status = (%t, %q), want (%t, %q)", + model.Available, + model.Status, + false, + modelStatusUnreachable, + ) + } + if model := modelsByName["codex-cli-model"]; !model.Available || model.Status != modelStatusAvailable { + t.Fatalf( + "codex-cli status = (%t, %q), want (%t, %q)", + model.Available, + model.Status, + true, + modelStatusAvailable, + ) + } + + optionsByID := make(map[string]providers.ModelProviderOption, len(resp.ProviderOptions)) + for _, option := range resp.ProviderOptions { + optionsByID[option.ID] = option + } + if option, ok := optionsByID["claude-cli"]; !ok { + t.Fatal("claude-cli provider option missing") + } else if option.CreateAllowed { + t.Fatal("claude-cli should not be creatable when the claude command is missing") + } + if option, ok := optionsByID["codex-cli"]; !ok { + t.Fatal("codex-cli provider option missing") + } else if !option.CreateAllowed { + t.Fatal("codex-cli should be creatable when the codex command is available") } } @@ -321,8 +574,8 @@ func TestHandleListModels_NormalizesWildcardLocalAPIBaseForProbe(t *testing.T) { var resp struct { Models []modelResponse `json:"models"` } - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { - t.Fatalf("Unmarshal() error = %v", err) + if unmarshalErr := json.Unmarshal(rec.Body.Bytes(), &resp); unmarshalErr != nil { + t.Fatalf("Unmarshal() error = %v", unmarshalErr) } if len(resp.Models) != 1 { t.Fatalf("len(models) = %d, want 1", len(resp.Models)) @@ -508,6 +761,223 @@ func TestHandleAddModel_PersistsProvider(t *testing.T) { } } +func TestHandleAddModel_RejectsUnsupportedProvider(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"bad-provider", + "provider":"not-supported", + "model":"gpt-4o-mini" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadRequest, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `provider "not-supported" is not supported`) { + t.Fatalf("body = %q, want unsupported provider error", rec.Body.String()) + } +} + +func TestHandleAddModel_AllowsBedrockProvider(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"bedrock-claude", + "provider":"bedrock", + "model":"us.anthropic.claude-sonnet-4-20250514-v1:0" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + added := cfg.ModelList[len(cfg.ModelList)-1] + if got := added.Provider; got != "bedrock" { + t.Fatalf("provider = %q, want %q", got, "bedrock") + } + if got := added.Model; got != "us.anthropic.claude-sonnet-4-20250514-v1:0" { + t.Fatalf("model = %q, want bedrock model ID", got) + } +} + +func TestHandleAddModel_NormalizesLegacyElevenLabsASRConfig(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Model: "elevenlabs/scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + if err = config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"new-model", + "provider":"openai", + "model":"gpt-4o-mini", + "api_key":"sk-new-model-key" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if len(updated.ModelList) != 2 { + t.Fatalf("len(model_list) = %d, want 2", len(updated.ModelList)) + } + if got := updated.ModelList[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q after normalization", got, "elevenlabs") + } + if got := updated.ModelList[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q after normalization", got, "scribe_v1") + } +} + +func TestHandleAddModel_NormalizesExplicitElevenLabsUnsupportedModelID(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Provider: "elevenlabs", + Model: "scribe_v2", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + if err = config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"new-model", + "provider":"openai", + "model":"gpt-4o-mini", + "api_key":"sk-new-model-key" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if got := updated.ModelList[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q after normalization", got, "elevenlabs") + } + if got := updated.ModelList[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q after normalization", got, "scribe_v1") + } +} + +func TestHandleAddModel_RejectsMissingCLIProviderCommand(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + resetOAuthHooks(t) + resetModelProbeHooks(t) + + probeCommandAvailableFunc = func(command string) bool { + return false + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"claude-cli-model", + "provider":"claude-cli", + "model":"claude-cli" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadRequest, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `provider "claude-cli" is not available for new models`) { + t.Fatalf("body = %q, want missing cli command error", rec.Body.String()) + } +} + +func TestHandleAddModel_DefaultsAntigravityToOAuth(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + added := addModelAndLoadLatest(t, configPath, `{ + "model_name":"gemini-flash", + "provider":"antigravity", + "model":"gemini-3-flash" + }`) + if got := added.AuthMethod; got != "oauth" { + t.Fatalf("auth_method = %q, want %q", got, "oauth") + } +} + +func TestHandleAddModel_NormalizesMixedCaseAuthMethod(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + added := addModelAndLoadLatest(t, configPath, `{ + "model_name":"openai-oauth", + "provider":"openai", + "model":"gpt-5.4", + "auth_method":"OAuth" + }`) + if got := added.AuthMethod; got != "oauth" { + t.Fatalf("auth_method = %q, want %q", got, "oauth") + } +} + func TestHandleAddModel_PreservesExplicitProviderPrefixedModel(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -584,6 +1054,37 @@ func TestHandleAddModel_PersistsCustomHeaders(t *testing.T) { } } +func TestHandleAddModel_PersistsToolSchemaTransform(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"new-model-transform", + "model":"openai/gpt-4o-mini", + "tool_schema_transform":"simple" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + added := cfg.ModelList[len(cfg.ModelList)-1] + if got := added.ToolSchemaTransform; got != "simple" { + t.Fatalf("tool_schema_transform = %q, want %q", got, "simple") + } +} + func TestHandleUpdateModel_CustomHeadersPreserveAndClear(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -649,6 +1150,69 @@ func TestHandleUpdateModel_CustomHeadersPreserveAndClear(t *testing.T) { } } +func TestHandleUpdateModel_ToolSchemaTransformPreserveAndClear(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "editable", + Model: "openai/gpt-4o-mini", + APIKeys: config.SimpleSecureStrings("sk-existing"), + ToolSchemaTransform: "simple", + }} + err = config.SaveConfig(configPath, cfg) + if err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + recPreserve := httptest.NewRecorder() + reqPreserve := httptest.NewRequest(http.MethodPut, "/api/models/0", bytes.NewBufferString(`{ + "model_name":"editable", + "model":"openai/gpt-4o-mini" + }`)) + reqPreserve.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(recPreserve, reqPreserve) + if recPreserve.Code != http.StatusOK { + t.Fatalf("preserve status = %d, want %d, body=%s", recPreserve.Code, http.StatusOK, recPreserve.Body.String()) + } + + afterPreserve, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() after preserve error = %v", err) + } + if got := afterPreserve.ModelList[0].ToolSchemaTransform; got != "simple" { + t.Fatalf("preserved tool_schema_transform = %q, want %q", got, "simple") + } + + recClear := httptest.NewRecorder() + reqClear := httptest.NewRequest(http.MethodPut, "/api/models/0", bytes.NewBufferString(`{ + "model_name":"editable", + "model":"openai/gpt-4o-mini", + "tool_schema_transform":"" + }`)) + reqClear.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(recClear, reqClear) + if recClear.Code != http.StatusOK { + t.Fatalf("clear status = %d, want %d, body=%s", recClear.Code, http.StatusOK, recClear.Body.String()) + } + + afterClear, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() after clear error = %v", err) + } + if afterClear.ModelList[0].ToolSchemaTransform != "" { + t.Fatalf("tool_schema_transform = %q, want empty", afterClear.ModelList[0].ToolSchemaTransform) + } +} + func TestHandleUpdateModel_PersistsProvider(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -751,7 +1315,8 @@ func TestHandleListModels_PreservesExplicitProviderPrefixedModel(t *testing.T) { Provider: "openrouter", Model: "openrouter/auto", }} - if err := config.SaveConfig(configPath, cfg); err != nil { + err = config.SaveConfig(configPath, cfg) + if err != nil { t.Fatalf("SaveConfig() error = %v", err) } @@ -770,7 +1335,8 @@ func TestHandleListModels_PreservesExplicitProviderPrefixedModel(t *testing.T) { var resp struct { Models []modelResponse `json:"models"` } - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + err = json.Unmarshal(rec.Body.Bytes(), &resp) + if err != nil { t.Fatalf("Unmarshal() error = %v", err) } if len(resp.Models) != 1 { @@ -784,6 +1350,55 @@ func TestHandleListModels_PreservesExplicitProviderPrefixedModel(t *testing.T) { } } +func TestHandleListModels_ExposesElevenLabsASRProvider(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Model: "elevenlabs/scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + if err = config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp struct { + Models []modelResponse `json:"models"` + } + if err = json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(resp.Models)) + } + if got := resp.Models[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q", got, "elevenlabs") + } + if got := resp.Models[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q", got, "scribe_v1") + } + if resp.Models[0].DefaultModelAllowed { + t.Fatal("elevenlabs ASR model should not be allowed as the default chat model") + } +} + func TestHandleUpdateModel_PreservesLegacyModelPrefixWhenProviderOmitted(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -846,11 +1461,230 @@ func TestHandleUpdateModel_PreservesLegacyModelPrefixWhenProviderOmitted(t *test if err != nil { t.Fatalf("LoadConfig() error = %v", err) } - if got := updated.ModelList[0].Provider; got != "" { - t.Fatalf("provider = %q, want empty", got) + if got := updated.ModelList[0].Provider; got != "openrouter" { + t.Fatalf("provider = %q, want %q", got, "openrouter") } - if got := updated.ModelList[0].Model; got != "openrouter/openai/gpt-5.4" { - t.Fatalf("model = %q, want %q", got, "openrouter/openai/gpt-5.4") + if got := updated.ModelList[0].Model; got != "openai/gpt-5.4" { + t.Fatalf("model = %q, want %q", got, "openai/gpt-5.4") + } +} + +func TestHandleUpdateModel_MigratesLegacyElevenLabsASRWhenProviderOmitted(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Model: "elevenlabs/scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + if err = config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + recList := httptest.NewRecorder() + reqList := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(recList, reqList) + + if recList.Code != http.StatusOK { + t.Fatalf("list status = %d, want %d, body=%s", recList.Code, http.StatusOK, recList.Body.String()) + } + + var listResp struct { + Models []modelResponse `json:"models"` + } + if err = json.Unmarshal(recList.Body.Bytes(), &listResp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(listResp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(listResp.Models)) + } + if got := listResp.Models[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q", got, "elevenlabs") + } + if got := listResp.Models[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q", got, "scribe_v1") + } + + recUpdate := httptest.NewRecorder() + reqUpdate := httptest.NewRequest(http.MethodPut, "/api/models/0", bytes.NewBufferString(`{ + "model_name":"elevenlabs-asr", + "model":"scribe_v1", + "api_base":"https://api.elevenlabs.io" + }`)) + reqUpdate.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(recUpdate, reqUpdate) + + if recUpdate.Code != http.StatusOK { + t.Fatalf("update status = %d, want %d, body=%s", recUpdate.Code, http.StatusOK, recUpdate.Body.String()) + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if got := updated.ModelList[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q", got, "elevenlabs") + } + if got := updated.ModelList[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q", got, "scribe_v1") + } + if got := updated.ModelList[0].APIBase; got != "https://api.elevenlabs.io" { + t.Fatalf("api_base = %q, want %q", got, "https://api.elevenlabs.io") + } +} + +func TestHandleUpdateModel_RoundTripsExplicitLegacyElevenLabsModelID(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Provider: "elevenlabs", + Model: "scribe_v2", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + if err = config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + recList := httptest.NewRecorder() + reqList := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(recList, reqList) + + if recList.Code != http.StatusOK { + t.Fatalf("list status = %d, want %d, body=%s", recList.Code, http.StatusOK, recList.Body.String()) + } + + var listResp struct { + Models []modelResponse `json:"models"` + } + if err = json.Unmarshal(recList.Body.Bytes(), &listResp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(listResp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(listResp.Models)) + } + if got := listResp.Models[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q", got, "elevenlabs") + } + if got := listResp.Models[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q after GET normalization", got, "scribe_v1") + } + + recUpdate := httptest.NewRecorder() + reqUpdate := httptest.NewRequest(http.MethodPut, "/api/models/0", bytes.NewBufferString(`{ + "model_name":"elevenlabs-asr", + "provider":"elevenlabs", + "model":"scribe_v1", + "api_base":"https://api.elevenlabs.io" + }`)) + reqUpdate.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(recUpdate, reqUpdate) + + if recUpdate.Code != http.StatusOK { + t.Fatalf("update status = %d, want %d, body=%s", recUpdate.Code, http.StatusOK, recUpdate.Body.String()) + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if got := updated.ModelList[0].Provider; got != "elevenlabs" { + t.Fatalf("provider = %q, want %q", got, "elevenlabs") + } + if got := updated.ModelList[0].Model; got != "scribe_v1" { + t.Fatalf("model = %q, want %q", got, "scribe_v1") + } + if got := updated.ModelList[0].APIBase; got != "https://api.elevenlabs.io" { + t.Fatalf("api_base = %q, want %q", got, "https://api.elevenlabs.io") + } +} + +func TestHandleUpdateModel_ClearsDefaultWhenSavingASROnlyModel(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "elevenlabs-asr", + Provider: "elevenlabs", + Model: "scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }} + cfg.Agents.Defaults.ModelName = "elevenlabs-asr" + if err = config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPut, "/api/models/0", bytes.NewBufferString(`{ + "model_name":"elevenlabs-asr", + "provider":"elevenlabs", + "model":"scribe_v1", + "api_base":"https://api.elevenlabs.io" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if got := updated.Agents.Defaults.ModelName; got != "" { + t.Fatalf("default model = %q, want cleared default", got) + } +} + +func TestHandleAddModel_RejectsUnsupportedElevenLabsModelID(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models", bytes.NewBufferString(`{ + "model_name":"elevenlabs-asr", + "provider":"elevenlabs", + "model":"scribe_v2" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadRequest, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `provider "elevenlabs" only supports model "scribe_v1"`) { + t.Fatalf("body = %q, want elevenlabs model validation error", rec.Body.String()) } } @@ -890,11 +1724,125 @@ func TestHandleUpdateModel_PreservesLegacyModelPrefixWhenProviderOmittedAndModel if err != nil { t.Fatalf("LoadConfig() error = %v", err) } - if got := updated.ModelList[0].Provider; got != "" { - t.Fatalf("provider = %q, want empty", got) + if got := updated.ModelList[0].Provider; got != "openrouter" { + t.Fatalf("provider = %q, want %q", got, "openrouter") } - if got := updated.ModelList[0].Model; got != "openrouter/openai/gpt-5.5" { - t.Fatalf("model = %q, want %q", got, "openrouter/openai/gpt-5.5") + if got := updated.ModelList[0].Model; got != "openai/gpt-5.5" { + t.Fatalf("model = %q, want %q", got, "openai/gpt-5.5") + } +} + +func TestHandleListModels_ReturnsProviderOptionsWithoutPersistingLegacyMigration(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "legacy-openrouter", + Model: "openrouter/openai/gpt-5.4", + }} + err = config.SaveConfig(configPath, cfg) + if err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp struct { + Models []modelResponse `json:"models"` + ProviderOptions []providers.ModelProviderOption `json:"provider_options"` + } + if unmarshalErr := json.Unmarshal(rec.Body.Bytes(), &resp); unmarshalErr != nil { + t.Fatalf("Unmarshal() error = %v", unmarshalErr) + } + if len(resp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(resp.Models)) + } + if got := resp.Models[0].Provider; got != "openrouter" { + t.Fatalf("provider = %q, want %q", got, "openrouter") + } + if got := resp.Models[0].Model; got != "openai/gpt-5.4" { + t.Fatalf("model = %q, want %q", got, "openai/gpt-5.4") + } + + optionsByID := make(map[string]providers.ModelProviderOption, len(resp.ProviderOptions)) + for _, option := range resp.ProviderOptions { + optionsByID[option.ID] = option + } + if len(optionsByID) == 0 { + t.Fatal("provider_options should not be empty") + } + if option, ok := optionsByID["openai"]; !ok { + t.Fatal("openai provider option missing") + } else if option.DefaultAPIBase != "https://api.openai.com/v1" { + t.Fatalf("openai default_api_base = %q, want %q", option.DefaultAPIBase, "https://api.openai.com/v1") + } + if option, ok := optionsByID["anthropic"]; !ok { + t.Fatal("anthropic provider option missing") + } else if option.DefaultAPIBase != "https://api.anthropic.com/v1" { + t.Fatalf("anthropic default_api_base = %q, want %q", option.DefaultAPIBase, "https://api.anthropic.com/v1") + } + if _, ok := optionsByID["azure"]; !ok { + t.Fatal("azure provider option missing") + } + if option, ok := optionsByID["github-copilot"]; !ok { + t.Fatal("github-copilot provider option missing") + } else if option.DefaultAPIBase != "localhost:4321" { + t.Fatalf("github-copilot default_api_base = %q, want %q", option.DefaultAPIBase, "localhost:4321") + } + if option, ok := optionsByID["elevenlabs"]; !ok { + t.Fatal("elevenlabs provider option missing") + } else { + if option.DefaultAPIBase != "https://api.elevenlabs.io" { + t.Fatalf("elevenlabs default_api_base = %q, want %q", option.DefaultAPIBase, "https://api.elevenlabs.io") + } + if option.DefaultModelAllowed { + t.Fatal("elevenlabs should be marked as not allowed for default chat model selection") + } + } + if option, ok := optionsByID["lmstudio"]; !ok { + t.Fatal("lmstudio provider option missing") + } else if !option.EmptyAPIKeyAllowed { + t.Fatal("lmstudio should allow empty api keys") + } + if option, ok := optionsByID["bedrock"]; !ok { + t.Fatal("bedrock provider option missing") + } else if !option.CreateAllowed { + t.Fatal("bedrock should stay creatable and defer AWS credential failures to runtime") + } + if option, ok := optionsByID["antigravity"]; !ok { + t.Fatal("antigravity provider option missing") + } else { + if option.DefaultAuthMethod != "oauth" { + t.Fatalf("antigravity default_auth_method = %q, want %q", option.DefaultAuthMethod, "oauth") + } + if !option.AuthMethodLocked { + t.Fatal("antigravity auth method should be locked") + } + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if got := updated.ModelList[0].Provider; got != "" { + t.Fatalf("persisted provider = %q, want unchanged empty provider", got) + } + if got := updated.ModelList[0].Model; got != "openrouter/openai/gpt-5.4" { + t.Fatalf("persisted model = %q, want unchanged legacy model", got) } } @@ -942,6 +1890,115 @@ func TestHandleListModels_ReturnsProviderField(t *testing.T) { } } +func TestHandleListModels_PreservesKnownProviderInCatalog(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "bedrock-claude", + Model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + }} + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/models", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp struct { + Models []modelResponse `json:"models"` + ProviderOptions []providers.ModelProviderOption `json:"provider_options"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Models) != 1 { + t.Fatalf("len(models) = %d, want 1", len(resp.Models)) + } + if got := resp.Models[0].Provider; got != "bedrock" { + t.Fatalf("provider = %q, want %q", got, "bedrock") + } + if got := resp.Models[0].Model; got != "us.anthropic.claude-sonnet-4-20250514-v1:0" { + t.Fatalf("model = %q, want %q", got, "us.anthropic.claude-sonnet-4-20250514-v1:0") + } + foundBedrock := false + for _, option := range resp.ProviderOptions { + if option.ID == "bedrock" { + foundBedrock = true + if !option.CreateAllowed { + t.Fatal("bedrock should stay creatable in provider_options") + } + } + } + if !foundBedrock { + t.Fatal("bedrock should be included in provider_options for compatibility") + } +} + +func TestHandleUpdateModel_AllowsExistingBedrockProvider(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{{ + ModelName: "bedrock-claude", + Provider: "bedrock", + Model: "us.anthropic.claude-sonnet-4-20250514-v1:0", + APIBase: "us-west-2", + }} + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPut, "/api/models/0", bytes.NewBufferString(`{ + "model_name":"bedrock-claude", + "provider":"bedrock", + "model":"us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "api_base":"us-east-1" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + updated, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + if got := updated.ModelList[0].Provider; got != "bedrock" { + t.Fatalf("provider = %q, want %q", got, "bedrock") + } + if got := updated.ModelList[0].Model; got != "us.anthropic.claude-3-7-sonnet-20250219-v1:0" { + t.Fatalf("model = %q, want updated bedrock model", got) + } + if got := updated.ModelList[0].APIBase; got != "us-east-1" { + t.Fatalf("api_base = %q, want %q", got, "us-east-1") + } +} + func TestHandleListModels_ReturnsEffectiveProviderField(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -1053,6 +2110,45 @@ func TestHandleSetDefaultModel_RejectsNonexistentModel(t *testing.T) { } } +func TestHandleSetDefaultModel_RejectsElevenLabsASRProvider(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + cfg.ModelList = []*config.ModelConfig{ + { + ModelName: "elevenlabs-asr", + Provider: "elevenlabs", + Model: "scribe_v1", + APIKeys: config.SimpleSecureStrings("sk_elevenlabs_test"), + }, + } + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/models/default", bytes.NewBufferString(`{ + "model_name": "elevenlabs-asr" + }`)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadRequest, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "cannot be used as the default chat model") { + t.Fatalf("body = %q, want default chat model rejection", rec.Body.String()) + } +} + func TestMaskAPIKey(t *testing.T) { tests := []struct { name string diff --git a/web/backend/api/tools.go b/web/backend/api/tools.go index 3476e3c53..c6a9d4254 100644 --- a/web/backend/api/tools.go +++ b/web/backend/api/tools.go @@ -49,6 +49,7 @@ type webSearchProviderConfig struct { BaseURL string `json:"base_url,omitempty"` APIKey string `json:"api_key,omitempty"` APIKeys []string `json:"api_keys,omitempty"` + Model string `json:"model,omitempty"` APIKeySet bool `json:"api_key_set,omitempty"` } @@ -446,6 +447,14 @@ func (h *Handler) handleUpdateWebSearchConfig(w http.ResponseWriter, r *http.Req cfg.Tools.Web.DuckDuckGo.Enabled = settings.Enabled cfg.Tools.Web.DuckDuckGo.MaxResults = settings.MaxResults } + if settings, ok := req.Settings["gemini"]; ok { + cfg.Tools.Web.Gemini.Enabled = settings.Enabled + cfg.Tools.Web.Gemini.MaxResults = settings.MaxResults + cfg.Tools.Web.Gemini.Model = strings.TrimSpace(settings.Model) + if key := strings.TrimSpace(settings.APIKey); key != "" { + cfg.Tools.Web.Gemini.APIKey = *config.NewSecureString(key) + } + } if settings, ok := req.Settings["brave"]; ok { cfg.Tools.Web.Brave.Enabled = settings.Enabled cfg.Tools.Web.Brave.MaxResults = settings.MaxResults @@ -505,7 +514,7 @@ func normalizeWebSearchProvider(provider string) string { switch strings.ToLower(strings.TrimSpace(provider)) { case "", "auto": return "auto" - case "sogou", "brave", "tavily", "duckduckgo", "perplexity", "searxng", "glm_search", "baidu_search": + case "sogou", "brave", "tavily", "duckduckgo", "gemini", "perplexity", "searxng", "glm_search", "baidu_search": return strings.ToLower(strings.TrimSpace(provider)) default: return "" @@ -549,6 +558,12 @@ func buildWebSearchConfigResponse(cfg *config.Config) webSearchConfigResponse { Enabled: cfg.Tools.Web.DuckDuckGo.Enabled, MaxResults: cfg.Tools.Web.DuckDuckGo.MaxResults, }, + "gemini": { + Enabled: cfg.Tools.Web.Gemini.Enabled, + MaxResults: cfg.Tools.Web.Gemini.MaxResults, + Model: cfg.Tools.Web.Gemini.Model, + APIKeySet: cfg.Tools.Web.Gemini.APIKey.String() != "", + }, "brave": { Enabled: cfg.Tools.Web.Brave.Enabled, MaxResults: cfg.Tools.Web.Brave.MaxResults, @@ -604,6 +619,13 @@ func buildWebSearchConfigResponse(cfg *config.Config) webSearchConfigResponse { Configured: picotools.WebSearchProviderReady(opts, "duckduckgo"), Current: current == "duckduckgo", }, + { + ID: "gemini", + Label: "Gemini (Google Search)", + Configured: picotools.WebSearchProviderReady(opts, "gemini"), + Current: current == "gemini", + RequiresAuth: true, + }, { ID: "brave", Label: "Brave Search", diff --git a/web/backend/api/tools_test.go b/web/backend/api/tools_test.go index a09a49fd6..520847bed 100644 --- a/web/backend/api/tools_test.go +++ b/web/backend/api/tools_test.go @@ -540,7 +540,7 @@ func TestHandleUpdateWebSearchConfig_PreservesAndReplacesMultiKeys(t *testing.T) } } -func TestResolveCurrentWebSearchProvider_PrefersConfiguredProvidersBeforeSogou(t *testing.T) { +func TestResolveCurrentWebSearchProvider_PrefersConfiguredProvidersInAutoMode(t *testing.T) { cfg := config.DefaultConfig() cfg.Tools.Web.Provider = "auto" cfg.Tools.Web.Sogou.Enabled = true diff --git a/web/frontend/package.json b/web/frontend/package.json index ab07b40a2..8e53fb850 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -18,32 +18,34 @@ }, "dependencies": { "@fontsource-variable/inter": "^5.2.8", - "@tabler/icons-react": "^3.40.0", - "@tailwindcss/vite": "^4.2.2", + "@radix-ui/react-popover": "^1.1.15", + "@tabler/icons-react": "^3.43.0", + "@tailwindcss/vite": "^4.2.4", "@tanstack/react-query": "^5.99.0", - "@tanstack/react-router": "^1.168.23", + "@tanstack/react-router": "^1.169.2", "@tanstack/react-router-devtools": "^1.166.13", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", + "cmdk": "^1.1.1", "dayjs": "^1.11.20", "highlight.js": "^11.11.1", - "i18next": "^26.0.7", + "i18next": "^26.0.10", "i18next-browser-languagedetector": "^8.2.1", "jotai": "^2.19.1", "radix-ui": "^1.4.3", "react": "19.2.5", "react-dom": "19.2.5", - "react-i18next": "^17.0.4", + "react-i18next": "^17.0.6", "react-markdown": "^10.1.0", "react-textarea-autosize": "^8.5.9", "rehype-highlight": "^7.0.2", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "remark-gfm": "^4.0.1", - "shadcn": "^4.3.0", + "shadcn": "^4.7.0", "sonner": "^2.0.7", "tailwind-merge": "^3.5.0", - "tailwindcss": "^4.2.2", + "tailwindcss": "^4.2.4", "tw-animate-css": "^1.4.0", "wrap-ansi": "^10.0.0" }, @@ -61,11 +63,11 @@ "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.5.2", - "globals": "^17.5.0", + "globals": "^17.6.0", "prettier": "^3.8.3", "prettier-plugin-tailwindcss": "^0.7.2", "typescript": "~5.9.3", - "typescript-eslint": "^8.59.0", + "typescript-eslint": "^8.59.1", "vite": "^8.0.10" } } diff --git a/web/frontend/pnpm-lock.yaml b/web/frontend/pnpm-lock.yaml index cb5ca18de..3ff74a4be 100644 --- a/web/frontend/pnpm-lock.yaml +++ b/web/frontend/pnpm-lock.yaml @@ -11,27 +11,33 @@ importers: '@fontsource-variable/inter': specifier: ^5.2.8 version: 5.2.8 + '@radix-ui/react-popover': + specifier: ^1.1.15 + version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.5(react@19.2.5))(react@19.2.5) '@tabler/icons-react': - specifier: ^3.40.0 - version: 3.41.1(react@19.2.5) + specifier: ^3.43.0 + version: 3.43.0(react@19.2.5) '@tailwindcss/vite': - specifier: ^4.2.2 - version: 4.2.2(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0)) + specifier: ^4.2.4 + version: 4.2.4(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0)) '@tanstack/react-query': specifier: ^5.99.0 version: 5.99.0(react@19.2.5) '@tanstack/react-router': - specifier: ^1.168.23 - version: 1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + specifier: ^1.169.2 + version: 1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5) '@tanstack/react-router-devtools': specifier: ^1.166.13 - version: 1.166.13(@tanstack/react-router@1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(@tanstack/router-core@1.168.15)(csstype@3.2.3)(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + version: 1.166.13(@tanstack/react-router@1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(@tanstack/router-core@1.169.2)(csstype@3.2.3)(react-dom@19.2.5(react@19.2.5))(react@19.2.5) class-variance-authority: specifier: ^0.7.1 version: 0.7.1 clsx: specifier: ^2.1.1 version: 2.1.1 + cmdk: + specifier: ^1.1.1 + version: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.5(react@19.2.5))(react@19.2.5) dayjs: specifier: ^1.11.20 version: 1.11.20 @@ -39,8 +45,8 @@ importers: specifier: ^11.11.1 version: 11.11.1 i18next: - specifier: ^26.0.7 - version: 26.0.7(typescript@5.9.3) + specifier: ^26.0.10 + version: 26.0.10(typescript@5.9.3) i18next-browser-languagedetector: specifier: ^8.2.1 version: 8.2.1 @@ -57,8 +63,8 @@ importers: specifier: 19.2.5 version: 19.2.5(react@19.2.5) react-i18next: - specifier: ^17.0.4 - version: 17.0.4(i18next@26.0.7(typescript@5.9.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(typescript@5.9.3) + specifier: ^17.0.6 + version: 17.0.6(i18next@26.0.10(typescript@5.9.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(typescript@5.9.3) react-markdown: specifier: ^10.1.0 version: 10.1.0(@types/react@19.2.14)(react@19.2.5) @@ -78,8 +84,8 @@ importers: specifier: ^4.0.1 version: 4.0.1 shadcn: - specifier: ^4.3.0 - version: 4.3.0(@types/node@25.6.0)(typescript@5.9.3) + specifier: ^4.7.0 + version: 4.7.0(@types/node@25.6.0)(typescript@5.9.3) sonner: specifier: ^2.0.7 version: 2.0.7(react-dom@19.2.5(react@19.2.5))(react@19.2.5) @@ -87,8 +93,8 @@ importers: specifier: ^3.5.0 version: 3.5.0 tailwindcss: - specifier: ^4.2.2 - version: 4.2.2 + specifier: ^4.2.4 + version: 4.2.4 tw-animate-css: specifier: ^1.4.0 version: 1.4.0 @@ -98,13 +104,13 @@ importers: devDependencies: '@eslint/js': specifier: ^10.0.1 - version: 10.0.1(eslint@10.2.1(jiti@2.6.1)) + version: 10.0.1(eslint@10.2.1(jiti@2.7.0)) '@tailwindcss/typography': specifier: ^0.5.19 - version: 0.5.19(tailwindcss@4.2.2) + version: 0.5.19(tailwindcss@4.2.4) '@tanstack/router-plugin': specifier: ^1.164.0 - version: 1.167.9(@tanstack/react-router@1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0)) + version: 1.167.9(@tanstack/react-router@1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0)) '@trivago/prettier-plugin-sort-imports': specifier: ^6.0.2 version: 6.0.2(prettier@3.8.3) @@ -119,25 +125,25 @@ importers: version: 19.2.3(@types/react@19.2.14) '@typescript-eslint/eslint-plugin': specifier: ^8.58.2 - version: 8.58.2(@typescript-eslint/parser@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3))(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) + version: 8.58.2(@typescript-eslint/parser@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3))(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) '@vitejs/plugin-react': specifier: ^6.0.1 - version: 6.0.1(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0)) + version: 6.0.1(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0)) eslint: specifier: ^10.2.1 - version: 10.2.1(jiti@2.6.1) + version: 10.2.1(jiti@2.7.0) eslint-config-prettier: specifier: ^10.1.8 - version: 10.1.8(eslint@10.2.1(jiti@2.6.1)) + version: 10.1.8(eslint@10.2.1(jiti@2.7.0)) eslint-plugin-react-hooks: specifier: ^7.1.1 - version: 7.1.1(eslint@10.2.1(jiti@2.6.1)) + version: 7.1.1(eslint@10.2.1(jiti@2.7.0)) eslint-plugin-react-refresh: specifier: ^0.5.2 - version: 0.5.2(eslint@10.2.1(jiti@2.6.1)) + version: 0.5.2(eslint@10.2.1(jiti@2.7.0)) globals: - specifier: ^17.5.0 - version: 17.5.0 + specifier: ^17.6.0 + version: 17.6.0 prettier: specifier: ^3.8.3 version: 3.8.3 @@ -148,11 +154,11 @@ importers: specifier: ~5.9.3 version: 5.9.3 typescript-eslint: - specifier: ^8.59.0 - version: 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) + specifier: ^8.59.1 + version: 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) vite: specifier: ^8.0.10 - version: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0) + version: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0) packages: @@ -160,8 +166,8 @@ packages: resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} engines: {node: '>=6.9.0'} - '@babel/compat-data@7.29.0': - resolution: {integrity: sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==} + '@babel/compat-data@7.29.3': + resolution: {integrity: sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg==} engines: {node: '>=6.9.0'} '@babel/core@7.29.0': @@ -180,8 +186,8 @@ packages: resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} engines: {node: '>=6.9.0'} - '@babel/helper-create-class-features-plugin@7.28.6': - resolution: {integrity: sha512-dTOdvsjnG3xNT9Y0AUg1wAl38y+4Rl4sf9caSQZOXdNqVn+H+HbbJ4IyyHaIqNR6SW9oJpA/RuRjsjCw2IdIow==} + '@babel/helper-create-class-features-plugin@7.29.3': + resolution: {integrity: sha512-RpLYy2sb51oNLjuu1iD3bwBqCBWUzjO0ocp+iaCP/lJtb2CPLcnC2Fftw+4sAzaMELGeWTgExSKADbdo0GFVzA==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 @@ -243,6 +249,11 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + '@babel/parser@7.29.3': + resolution: {integrity: sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==} + engines: {node: '>=6.0.0'} + hasBin: true + '@babel/plugin-syntax-jsx@7.28.6': resolution: {integrity: sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==} engines: {node: '>=6.9.0'} @@ -289,8 +300,8 @@ packages: resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} engines: {node: '>=6.9.0'} - '@dotenvx/dotenvx@1.61.0': - resolution: {integrity: sha512-utL3cpZoFzflyqUkjYbxYujI6STBTmO5LFn4bbin/NZnRWN6wQ7eErhr3/Vpa5h/jicPFC6kTa42r940mQftJQ==} + '@dotenvx/dotenvx@1.65.0': + resolution: {integrity: sha512-v4FA/Lw3pTEloLxBqTOaYDX6MNo0Jo7lGBsPZhwnJBqRJp0AzQg1ZZNxrFsh6HVC6QWeWrfIKLn0y2eyIXaVDg==} hasBin: true '@ecies/ciphers@0.2.6': @@ -547,8 +558,8 @@ packages: resolution: {integrity: sha512-doc2sWgJpbFQ64UflSVd17ibMGDuxO1yKgOgLMwavzESnXjFWJqUeG8saYosqKpHp4kWiM5x1nXvEjbpx90gzw==} engines: {node: '>=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0'} - '@inquirer/confirm@6.0.11': - resolution: {integrity: sha512-pTpHjg0iEIRMYV/7oCZUMf27/383E6Wyhfc/MY+AVQGEoUobffIYWOK9YLP2XFRGz/9i6WlTQh1CkFVIo2Y7XA==} + '@inquirer/confirm@6.0.12': + resolution: {integrity: sha512-h9FgGun3QwVYNj5TWIZZ+slii73bMoBFjPfVIGtnFuL4t8gBiNDV9PcSfIzkuxvgquJKt9nr1QzszpBzTbH8Og==} engines: {node: '>=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0'} peerDependencies: '@types/node': '>=18' @@ -556,8 +567,8 @@ packages: '@types/node': optional: true - '@inquirer/core@11.1.8': - resolution: {integrity: sha512-/u+yJk2pOKNDOh1ZgdUH2RQaRx6OOH4I0uwL95qPvTFTIL38YBsuSC4r1yXBB3Q6JvNqFFc202gk0Ew79rrcjA==} + '@inquirer/core@11.1.9': + resolution: {integrity: sha512-BDE4fG22uYh1bGSifcj7JSx119TVYNViMhMu85usp4Fswrzh6M0DV3yld64jA98uOAa2GSQ4Bg4bZRm2d2cwSg==} engines: {node: '>=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0'} peerDependencies: '@types/node': '>=18' @@ -604,8 +615,8 @@ packages: '@cfworker/json-schema': optional: true - '@mswjs/interceptors@0.41.3': - resolution: {integrity: sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==} + '@mswjs/interceptors@0.41.8': + resolution: {integrity: sha512-pRLMNKTSGRoLq+KnEB/7OY5vijw1XmcheAAOiv6pj7W1FG32kAGqj1C/RK/cqxRGr1Fh+zBi8sDur8kj3EQv6A==} engines: {node: '>=18'} '@napi-rs/wasm-runtime@1.1.4': @@ -1451,77 +1462,77 @@ packages: resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} - '@tabler/icons-react@3.41.1': - resolution: {integrity: sha512-kUgweE+DJtAlMZVIns1FTDdcbpRVnkK7ZpUOXmoxy3JAF0rSHj0TcP4VHF14+gMJGnF+psH2Zt26BLT6owetBA==} + '@tabler/icons-react@3.43.0': + resolution: {integrity: sha512-rXUuCQEeRbEk3lJxs3gwzdtaaITSwc/JUbp+AkqsGff5uBpzZw7eKPDk53xKoKLyjrbj82Ai4GuVG0kO89Jf5g==} peerDependencies: react: '>= 16' - '@tabler/icons@3.41.1': - resolution: {integrity: sha512-OaRnVbRmH2nHtFeg+RmMJ/7m2oBIF9XCJAUD5gQnMrpK9f05ydj8MZrAf3NZQqOXyxGN1UBL0D5IKLLEUfr74Q==} + '@tabler/icons@3.43.0': + resolution: {integrity: sha512-qXwS17Op9jqr3Asvu31fejyw8+OnRDKH7oR8nQXyUgW1pI44ET8OKG9kssy+XIvvAIyej6gZdGmviNUn1VMfPw==} - '@tailwindcss/node@4.2.2': - resolution: {integrity: sha512-pXS+wJ2gZpVXqFaUEjojq7jzMpTGf8rU6ipJz5ovJV6PUGmlJ+jvIwGrzdHdQ80Sg+wmQxUFuoW1UAAwHNEdFA==} + '@tailwindcss/node@4.2.4': + resolution: {integrity: sha512-Ai7+yQPxz3ddrDQzFfBKdHEVBg0w3Zl83jnjuwxnZOsnH9pGn93QHQtpU0p/8rYWxvbFZHneni6p1BSLK4DkGA==} - '@tailwindcss/oxide-android-arm64@4.2.2': - resolution: {integrity: sha512-dXGR1n+P3B6748jZO/SvHZq7qBOqqzQ+yFrXpoOWWALWndF9MoSKAT3Q0fYgAzYzGhxNYOoysRvYlpixRBBoDg==} + '@tailwindcss/oxide-android-arm64@4.2.4': + resolution: {integrity: sha512-e7MOr1SAn9U8KlZzPi1ZXGZHeC5anY36qjNwmZv9pOJ8E4Q6jmD1vyEHkQFmNOIN7twGPEMXRHmitN4zCMN03g==} engines: {node: '>= 20'} cpu: [arm64] os: [android] - '@tailwindcss/oxide-darwin-arm64@4.2.2': - resolution: {integrity: sha512-iq9Qjr6knfMpZHj55/37ouZeykwbDqF21gPFtfnhCCKGDcPI/21FKC9XdMO/XyBM7qKORx6UIhGgg6jLl7BZlg==} + '@tailwindcss/oxide-darwin-arm64@4.2.4': + resolution: {integrity: sha512-tSC/Kbqpz/5/o/C2sG7QvOxAKqyd10bq+ypZNf+9Fi2TvbVbv1zNpcEptcsU7DPROaSbVgUXmrzKhurFvo5eDg==} engines: {node: '>= 20'} cpu: [arm64] os: [darwin] - '@tailwindcss/oxide-darwin-x64@4.2.2': - resolution: {integrity: sha512-BlR+2c3nzc8f2G639LpL89YY4bdcIdUmiOOkv2GQv4/4M0vJlpXEa0JXNHhCHU7VWOKWT/CjqHdTP8aUuDJkuw==} + '@tailwindcss/oxide-darwin-x64@4.2.4': + resolution: {integrity: sha512-yPyUXn3yO/ufR6+Kzv0t4fCg2qNr90jxXc5QqBpjlPNd0NqyDXcmQb/6weunH/MEDXW5dhyEi+agTDiqa3WsGg==} engines: {node: '>= 20'} cpu: [x64] os: [darwin] - '@tailwindcss/oxide-freebsd-x64@4.2.2': - resolution: {integrity: sha512-YUqUgrGMSu2CDO82hzlQ5qSb5xmx3RUrke/QgnoEx7KvmRJHQuZHZmZTLSuuHwFf0DJPybFMXMYf+WJdxHy/nQ==} + '@tailwindcss/oxide-freebsd-x64@4.2.4': + resolution: {integrity: sha512-BoMIB4vMQtZsXdGLVc2z+P9DbETkiopogfWZKbWwM8b/1Vinbs4YcUwo+kM/KeLkX3Ygrf4/PsRndKaYhS8Eiw==} engines: {node: '>= 20'} cpu: [x64] os: [freebsd] - '@tailwindcss/oxide-linux-arm-gnueabihf@4.2.2': - resolution: {integrity: sha512-FPdhvsW6g06T9BWT0qTwiVZYE2WIFo2dY5aCSpjG/S/u1tby+wXoslXS0kl3/KXnULlLr1E3NPRRw0g7t2kgaQ==} + '@tailwindcss/oxide-linux-arm-gnueabihf@4.2.4': + resolution: {integrity: sha512-7pIHBLTHYRAlS7V22JNuTh33yLH4VElwKtB3bwchK/UaKUPpQ0lPQiOWcbm4V3WP2I6fNIJ23vABIvoy2izdwA==} engines: {node: '>= 20'} cpu: [arm] os: [linux] - '@tailwindcss/oxide-linux-arm64-gnu@4.2.2': - resolution: {integrity: sha512-4og1V+ftEPXGttOO7eCmW7VICmzzJWgMx+QXAJRAhjrSjumCwWqMfkDrNu1LXEQzNAwz28NCUpucgQPrR4S2yw==} + '@tailwindcss/oxide-linux-arm64-gnu@4.2.4': + resolution: {integrity: sha512-+E4wxJ0ZGOzSH325reXTWB48l42i93kQqMvDyz5gqfRzRZ7faNhnmvlV4EPGJU3QJM/3Ab5jhJ5pCRUsKn6OQw==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] libc: [glibc] - '@tailwindcss/oxide-linux-arm64-musl@4.2.2': - resolution: {integrity: sha512-oCfG/mS+/+XRlwNjnsNLVwnMWYH7tn/kYPsNPh+JSOMlnt93mYNCKHYzylRhI51X+TbR+ufNhhKKzm6QkqX8ag==} + '@tailwindcss/oxide-linux-arm64-musl@4.2.4': + resolution: {integrity: sha512-bBADEGAbo4ASnppIziaQJelekCxdMaxisrk+fB7Thit72IBnALp9K6ffA2G4ruj90G9XRS2VQ6q2bCKbfFV82g==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] libc: [musl] - '@tailwindcss/oxide-linux-x64-gnu@4.2.2': - resolution: {integrity: sha512-rTAGAkDgqbXHNp/xW0iugLVmX62wOp2PoE39BTCGKjv3Iocf6AFbRP/wZT/kuCxC9QBh9Pu8XPkv/zCZB2mcMg==} + '@tailwindcss/oxide-linux-x64-gnu@4.2.4': + resolution: {integrity: sha512-7Mx25E4WTfnht0TVRTyC00j3i0M+EeFe7wguMDTlX4mRxafznw0CA8WJkFjWYH5BlgELd1kSjuU2JiPnNZbJDA==} engines: {node: '>= 20'} cpu: [x64] os: [linux] libc: [glibc] - '@tailwindcss/oxide-linux-x64-musl@4.2.2': - resolution: {integrity: sha512-XW3t3qwbIwiSyRCggeO2zxe3KWaEbM0/kW9e8+0XpBgyKU4ATYzcVSMKteZJ1iukJ3HgHBjbg9P5YPRCVUxlnQ==} + '@tailwindcss/oxide-linux-x64-musl@4.2.4': + resolution: {integrity: sha512-2wwJRF7nyhOR0hhHoChc04xngV3iS+akccHTGtz965FwF0up4b2lOdo6kI1EbDaEXKgvcrFBYcYQQ/rrnWFVfA==} engines: {node: '>= 20'} cpu: [x64] os: [linux] libc: [musl] - '@tailwindcss/oxide-wasm32-wasi@4.2.2': - resolution: {integrity: sha512-eKSztKsmEsn1O5lJ4ZAfyn41NfG7vzCg496YiGtMDV86jz1q/irhms5O0VrY6ZwTUkFy/EKG3RfWgxSI3VbZ8Q==} + '@tailwindcss/oxide-wasm32-wasi@4.2.4': + resolution: {integrity: sha512-FQsqApeor8Fo6gUEklzmaa9994orJZZDBAlQpK2Mq+DslRKFJeD6AjHpBQ0kZFQohVr8o85PPh8eOy86VlSCmw==} engines: {node: '>=14.0.0'} cpu: [wasm32] bundledDependencies: @@ -1532,20 +1543,20 @@ packages: - '@emnapi/wasi-threads' - tslib - '@tailwindcss/oxide-win32-arm64-msvc@4.2.2': - resolution: {integrity: sha512-qPmaQM4iKu5mxpsrWZMOZRgZv1tOZpUm+zdhhQP0VhJfyGGO3aUKdbh3gDZc/dPLQwW4eSqWGrrcWNBZWUWaXQ==} + '@tailwindcss/oxide-win32-arm64-msvc@4.2.4': + resolution: {integrity: sha512-L9BXqxC4ToVgwMFqj3pmZRqyHEztulpUJzCxUtLjobMCzTPsGt1Fa9enKbOpY2iIyVtaHNeNvAK8ERP/64sqGQ==} engines: {node: '>= 20'} cpu: [arm64] os: [win32] - '@tailwindcss/oxide-win32-x64-msvc@4.2.2': - resolution: {integrity: sha512-1T/37VvI7WyH66b+vqHj/cLwnCxt7Qt3WFu5Q8hk65aOvlwAhs7rAp1VkulBJw/N4tMirXjVnylTR72uI0HGcA==} + '@tailwindcss/oxide-win32-x64-msvc@4.2.4': + resolution: {integrity: sha512-ESlKG0EpVJQwRjXDDa9rLvhEAh0mhP1sF7sap9dNZT0yyl9SAG6T7gdP09EH0vIv0UNTlo6jPWyujD6559fZvw==} engines: {node: '>= 20'} cpu: [x64] os: [win32] - '@tailwindcss/oxide@4.2.2': - resolution: {integrity: sha512-qEUA07+E5kehxYp9BVMpq9E8vnJuBHfJEC0vPC5e7iL/hw7HR61aDKoVoKzrG+QKp56vhNZe4qwkRmMC0zDLvg==} + '@tailwindcss/oxide@4.2.4': + resolution: {integrity: sha512-9El/iI069DKDSXwTvB9J4BwdO5JhRrOweGaK25taBAvBXyXqJAX+Jqdvs8r8gKpsI/1m0LeJLyQYTf/WLrBT1Q==} engines: {node: '>= 20'} '@tailwindcss/typography@0.5.19': @@ -1553,8 +1564,8 @@ packages: peerDependencies: tailwindcss: '>=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1' - '@tailwindcss/vite@4.2.2': - resolution: {integrity: sha512-mEiF5HO1QqCLXoNEfXVA1Tzo+cYsrqV7w9Juj2wdUFyW07JRenqMG225MvPwr3ZD9N1bFQj46X7r33iHxLUW0w==} + '@tailwindcss/vite@4.2.4': + resolution: {integrity: sha512-pCvohwOCspk3ZFn6eJzrrX3g4n2JY73H6MmYC87XfGPyTty4YsCjYTMArRZm/zOI8dIt3+EcrLHAFPe5A4bgtw==} peerDependencies: vite: ^5.2.0 || ^6 || ^7 || ^8 @@ -1582,8 +1593,8 @@ packages: '@tanstack/router-core': optional: true - '@tanstack/react-router@1.168.23': - resolution: {integrity: sha512-+GblieDnutG6oipJJPNtRJjrWF8QTZEG/l0532+BngFkVK48oHNOcvIkSoAFYftK1egAwM7KBxXsb0Ou+X6/MQ==} + '@tanstack/react-router@1.169.2': + resolution: {integrity: sha512-OJM7Kguc7ERnweaNRWsyWgIKcl3z23rD1B4jaxjzd9RGdnzpt2HfrWa9rggbT0Hfzhfo4D2ZmsfoTme035tniQ==} engines: {node: '>=20.19'} peerDependencies: react: '>=18.0.0 || >=19.0.0' @@ -1595,16 +1606,15 @@ packages: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - '@tanstack/router-core@1.168.15': - resolution: {integrity: sha512-Wr0424NDtD8fT/uALobMZ9DdcfsTyXtW5IPR++7zvW8/7RaIOeaqXpVDId8ywaGtqPWLWOfaUg2zUtYtukoXYA==} - engines: {node: '>=20.19'} - hasBin: true - '@tanstack/router-core@1.168.7': resolution: {integrity: sha512-z4UEdlzMrFaKBsG4OIxlZEm+wsYBtEp//fnX6kW18jhQpETNcM6u2SXNdX+bcIYp6AaR7ERS3SBENzjC/xxwQQ==} engines: {node: '>=20.19'} hasBin: true + '@tanstack/router-core@1.169.2': + resolution: {integrity: sha512-5sm0DJF1A7Mz+9gy4Gz/lLovNailK3yot4vYvz9MkBUPw26uLnhQiR8hSCYxucjE0wD6Mdlc5l+Z0/XTlZ7xHw==} + engines: {node: '>=20.19'} + '@tanstack/router-devtools-core@1.167.3': resolution: {integrity: sha512-fJ1VMhyQgnoashTrP763c2HRc9kofgF61L7Jb3F6eTHAmCKtGVx8BRtiFt37sr3U0P0jmaaiiSPGP6nT5JtVNg==} engines: {node: '>=20.19'} @@ -1736,16 +1746,16 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/eslint-plugin@8.59.0': - resolution: {integrity: sha512-HyAZtpdkgZwpq8Sz3FSUvCR4c+ScbuWa9AksK2Jweub7w4M3yTz4O11AqVJzLYjy/B9ZWPyc81I+mOdJU/bDQw==} + '@typescript-eslint/eslint-plugin@8.59.1': + resolution: {integrity: sha512-BOziFIfE+6osHO9FoJG4zjoHUcvI7fTNBSpdAwrNH0/TLvzjsk2oo8XSSOT2HhqUyhZPfHv4UOffoJ9oEEQ7Ag==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: - '@typescript-eslint/parser': ^8.59.0 + '@typescript-eslint/parser': ^8.59.1 eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/parser@8.59.0': - resolution: {integrity: sha512-TI1XGwKbDpo9tRW8UDIXCOeLk55qe9ZFGs8MTKU6/M08HWTw52DD/IYhfQtOEhEdPhLMT26Ka/x7p70nd3dzDg==} + '@typescript-eslint/parser@8.59.1': + resolution: {integrity: sha512-HDQH9O/47Dxi1ceDhBXdaldtf/WV9yRYMjbjCuNk3qnaTD564qwv61Y7+gTxwxRKzSrgO5uhtw584igXVuuZkA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 @@ -1757,8 +1767,8 @@ packages: peerDependencies: typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/project-service@8.59.0': - resolution: {integrity: sha512-Lw5ITrR5s5TbC19YSvlr63ZfLaJoU6vtKTHyB0GQOpX0W7d5/Ir6vUahWi/8Sps/nOukZQ0IB3SmlxZnjaKVnw==} + '@typescript-eslint/project-service@8.59.1': + resolution: {integrity: sha512-+MuHQlHiEr00Of/IQbE/MmEoi44znZHbR/Pz7Opq4HryUOlRi+/44dro9Ycy8Fyo+/024IWtw8m4JUMCGTYxDg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.1.0' @@ -1767,8 +1777,8 @@ packages: resolution: {integrity: sha512-SgmyvDPexWETQek+qzZnrG6844IaO02UVyOLhI4wpo82dpZJY9+6YZCKAMFzXb7qhx37mFK1QcPQ18tud+vo6Q==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/scope-manager@8.59.0': - resolution: {integrity: sha512-UzR16Ut8IpA3Mc4DbgAShlPPkVm8xXMWafXxB0BocaVRHs8ZGakAxGRskF7FId3sdk9lgGD73GSFaWmWFDE4dg==} + '@typescript-eslint/scope-manager@8.59.1': + resolution: {integrity: sha512-LwuHQI4pDOYVKvmH2dkaJo6YZCSgouVgnS/z7yBPKBMvgtBvyLqiLy9Z6b7+m/TRcX1NFYUqZetI5Y+aT4GEfg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@typescript-eslint/tsconfig-utils@8.58.2': @@ -1777,8 +1787,8 @@ packages: peerDependencies: typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/tsconfig-utils@8.59.0': - resolution: {integrity: sha512-91Sbl3s4Kb3SybliIY6muFBmHVv+pYXfybC4Oolp3dvk8BvIE3wOPc+403CWIT7mJNkfQRGtdqghzs2+Z91Tqg==} + '@typescript-eslint/tsconfig-utils@8.59.1': + resolution: {integrity: sha512-/0nEyPbX7gRsk0Uwfe4ALwwgxuA66d/l2mhRDNlAvaj4U3juhUtJNq0DsY8M2AYwwb9rEq2hrC3IcIcEt++iJA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.1.0' @@ -1790,8 +1800,8 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/type-utils@8.59.0': - resolution: {integrity: sha512-3TRiZaQSltGqGeNrJzzr1+8YcEobKH9rHnqIp/1psfKFmhRQDNMGP5hBufanYTGznwShzVLs3Mz+gDN7HkWfXg==} + '@typescript-eslint/type-utils@8.59.1': + resolution: {integrity: sha512-klWPBR2ciQHS3f++ug/mVnWKPjBUo7icEL3FAO1lhAR1Z1i5NQYZ1EannMSRYcq5qCv5wNALlXr6fksRHyYl7w==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 @@ -1801,8 +1811,8 @@ packages: resolution: {integrity: sha512-9TukXyATBQf/Jq9AMQXfvurk+G5R2MwfqQGDR2GzGz28HvY/lXNKGhkY+6IOubwcquikWk5cjlgPvD2uAA7htQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/types@8.59.0': - resolution: {integrity: sha512-nLzdsT1gdOgFxxxwrlNVUBzSNBEEHJ86bblmk4QAS6stfig7rcJzWKqCyxFy3YRRHXDWEkb2NralA1nOYkkm/A==} + '@typescript-eslint/types@8.59.1': + resolution: {integrity: sha512-ZDCjgccSdYPw5Bxh+my4Z0lJU96ZDN7jbBzvmEn0FZx3RtU1C7VWl6NbDx94bwY3V5YsgwRzJPOgeY2Q/nLG8A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@typescript-eslint/typescript-estree@8.58.2': @@ -1811,8 +1821,8 @@ packages: peerDependencies: typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/typescript-estree@8.59.0': - resolution: {integrity: sha512-O9Re9P1BmBLFJyikRbQpLku/QA3/AueZNO9WePLBwQrvkixTmDe8u76B6CYUAITRl/rHawggEqUGn5QIkVRLMw==} + '@typescript-eslint/typescript-estree@8.59.1': + resolution: {integrity: sha512-OUd+vJS05sSkOip+BkZ/2NS8RMxrAAJemsC6vU3kmfLyeaJT0TftHkV9mcx2107MmsBVXXexhVu4F0TZXyMl4g==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.1.0' @@ -1824,8 +1834,8 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.1.0' - '@typescript-eslint/utils@8.59.0': - resolution: {integrity: sha512-I1R/K7V07XsMJ12Oaxg/O9GfrysGTmCRhvZJBv0RE0NcULMzjqVpR5kRRQjHsz3J/bElU7HwCO7zkqL+MSUz+g==} + '@typescript-eslint/utils@8.59.1': + resolution: {integrity: sha512-3pIeoXhCeYH9FSCBI8P3iNwJlGuzPlYKkTlen2O9T1DSeeg8UG8jstq6BLk+Mda0qup7mgk4z4XL4OzRaxZ8LA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 @@ -1835,12 +1845,13 @@ packages: resolution: {integrity: sha512-f1WO2Lx8a9t8DARmcWAUPJbu0G20bJlj8L4z72K00TMeJAoyLr/tHhI/pzYBLrR4dXWkcxO1cWYZEOX8DKHTqA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/visitor-keys@8.59.0': - resolution: {integrity: sha512-/uejZt4dSere1bx12WLlPfv8GktzcaDtuJ7s42/HEZ5zGj9oxRaD4bj7qwSunXkf+pbAhFt2zjpHYUiT5lHf0Q==} + '@typescript-eslint/visitor-keys@8.59.1': + resolution: {integrity: sha512-LdDNl6C5iJExcM0Yh0PwAIBb9PrSiCsWamF/JyEZawm3kFDnRoaq3LGE4bpyRao/fWeGKKyw7icx0YxrLFC5Cg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@vitejs/plugin-react@6.0.1': resolution: {integrity: sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ==} @@ -1884,8 +1895,8 @@ packages: ajv@6.14.0: resolution: {integrity: sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==} - ajv@8.18.0: - resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} @@ -1935,8 +1946,8 @@ packages: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} - baseline-browser-mapping@2.10.17: - resolution: {integrity: sha512-HdrkN8eVG2CXxeifv/VdJ4A4RSra1DTW8dc/hdxzhGHN8QePs6gKaWM9pHPcpCoxYZJuOZ8drHmbdpLHjCYjLA==} + baseline-browser-mapping@2.10.27: + resolution: {integrity: sha512-zEs/ufmZoUd7WftKpKyXaT6RFxpQ5Qm9xytKRHvJfxFV9DFJkZph9RvJ1LcOUi0Z1ZVijMte65JbILeV+8QQEA==} engines: {node: '>=6.0.0'} hasBin: true @@ -1984,8 +1995,8 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - caniuse-lite@1.0.30001787: - resolution: {integrity: sha512-mNcrMN9KeI68u7muanUpEejSLghOKlVhRqS/Za2IeyGllJ9I9otGpR9g3nsw7n4W378TE/LyIteA0+/FOZm4Kg==} + caniuse-lite@1.0.30001792: + resolution: {integrity: sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw==} ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -2033,6 +2044,12 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cmdk@1.1.1: + resolution: {integrity: sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + react-dom: ^18 || ^19 || ^19.0.0-rc + code-block-writer@13.0.3: resolution: {integrity: sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==} @@ -2191,8 +2208,8 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.334: - resolution: {integrity: sha512-mgjZAz7Jyx1SRCwEpy9wefDS7GvNPazLthHg8eQMJ76wBdGQQDW33TCrUTvQ4wzpmOrv2zrFoD3oNufMdyMpog==} + electron-to-chromium@1.5.352: + resolution: {integrity: sha512-9wHk8x6dyuimoe18EdiDPWKExNdxYqo4fn4FwOVVper6RxT3cmpBwBkWWfSOCYJjQdIco/nPhJhNLmn4Ufg1Yg==} emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} @@ -2204,8 +2221,8 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} - enhanced-resolve@5.20.1: - resolution: {integrity: sha512-Qohcme7V1inbAfvjItgw0EaxVX5q2rdVEZHRBrEQdRZTssLDGsL8Lwrznl8oQ/6kuTJONLaDcGjkNP247XEhcA==} + enhanced-resolve@5.21.0: + resolution: {integrity: sha512-otxSQPw4lkOZWkHpB3zaEQs6gWYEsmX4xQF68ElXC/TWvGxGMSGOvoNbaLXm6/cS/fSfHtsEdw90y20PCd+sCA==} engines: {node: '>=10.13.0'} entities@6.0.1: @@ -2322,8 +2339,8 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} - eventsource-parser@3.0.6: - resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} + eventsource-parser@3.0.8: + resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==} engines: {node: '>=18.0.0'} eventsource@3.0.7: @@ -2338,8 +2355,8 @@ packages: resolution: {integrity: sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==} engines: {node: ^18.19.0 || >=20.5.0} - express-rate-limit@8.3.2: - resolution: {integrity: sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg==} + express-rate-limit@8.5.1: + resolution: {integrity: sha512-5O6KYmyJEpuPJV5hNTXKbAHWRqrzyu+OI3vUnSd2kXFubIVpG7ezpgxQy76Zo5GQZtrQBg86hF+CM/NX+cioiQ==} engines: {node: '>= 16'} peerDependencies: express: '>= 4.11' @@ -2370,8 +2387,8 @@ packages: fast-string-width@3.0.2: resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} - fast-uri@3.1.0: - resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + fast-uri@3.1.2: + resolution: {integrity: sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==} fast-wrap-ansi@0.2.0: resolution: {integrity: sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==} @@ -2431,8 +2448,8 @@ packages: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} - fs-extra@11.3.4: - resolution: {integrity: sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==} + fs-extra@11.3.5: + resolution: {integrity: sha512-eKpRKAovdpZtR1WopLHxlBWvAgPny3c4gX1G5Jhwmmw4XJj0ifSD5qB5TOo8hmA0wlRKDAOAhEE1yVPgs6Fgcg==} engines: {node: '>=14.14'} fsevents@2.3.3: @@ -2493,8 +2510,8 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} - globals@17.5.0: - resolution: {integrity: sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==} + globals@17.6.0: + resolution: {integrity: sha512-sepffkT8stwnIYbsMBpoCHJuJM5l98FUF2AnE07hfvE0m/qp3R586hw4jF4uadbhvg1ooIdzuu7CsfD2jzCaNA==} engines: {node: '>=18'} goober@2.1.18: @@ -2509,16 +2526,16 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - graphql@16.13.2: - resolution: {integrity: sha512-5bJ+nf/UCpAjHM8i06fl7eLyVC9iuNAjm9qzkiu2ZGhM0VscSvS6WDPfAwkdkBuoXGM9FJSbKl6wylMwP9Ktig==} + graphql@16.14.0: + resolution: {integrity: sha512-BBvQ/406p+4CZbTpCbVPSxfzrZrbnuWSP1ELYgyS6B+hNeKzgrdB4JczCa5VZUBQrDa9hUngm0KnexY6pJRN5Q==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} - hasown@2.0.2: - resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + hasown@2.0.3: + resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==} engines: {node: '>= 0.4'} hast-util-from-parse5@8.0.3: @@ -2564,8 +2581,8 @@ packages: resolution: {integrity: sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w==} engines: {node: '>=12.0.0'} - hono@4.12.14: - resolution: {integrity: sha512-am5zfg3yu6sqn5yjKBNqhnTX7Cv+m00ox+7jbaKkrLMRJ4rAdldd1xPd/JzbBWspqaQv6RSTrgFN95EsfhC+7w==} + hono@4.12.18: + resolution: {integrity: sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==} engines: {node: '>=16.9.0'} html-parse-stringify@3.0.1: @@ -2596,8 +2613,8 @@ packages: i18next-browser-languagedetector@8.2.1: resolution: {integrity: sha512-bZg8+4bdmaOiApD7N7BPT9W8MLZG+nPTOFlLiJiT8uzKXFjhxw4v2ierCXOwB5sFDMtuA5G4kgYZ0AznZxQ/cw==} - i18next@26.0.7: - resolution: {integrity: sha512-f7tL/iw0VQsx4nC5oNxBM2RjM8alNys5KzyiQTU6A9TI5TI89py4/Ez1cKFvHiLWsvzOXvuGUES+Kk/A2WiANQ==} + i18next@26.0.10: + resolution: {integrity: sha512-k3yGPAlWR2RdMYoVXJoDZDT87qeHIWKH7gVksdZMpRty7QX/D9QZeYGvN08KGbKHke9wn01eYT+EEsrqX/YTlw==} peerDependencies: typescript: ^5 || ^6 peerDependenciesMeta: @@ -2630,8 +2647,8 @@ packages: inline-style-parser@0.2.7: resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==} - ip-address@10.1.0: - resolution: {integrity: sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==} + ip-address@10.2.0: + resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} engines: {node: '>= 12'} ipaddr.js@1.9.1: @@ -2729,8 +2746,8 @@ packages: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} - isbot@5.1.39: - resolution: {integrity: sha512-obH0yYahGXdzNxo+djmHhBYThUKDkz565cxkIlt2L9hXfv1NlaLKoDBHo6KxXsYrIXx2RK3x5vY36CfZcobxEw==} + isbot@5.1.40: + resolution: {integrity: sha512-yNeeynhhtIVRBk12tBV4eHNxwB42HzR4Q3Ea7vCOiJhImGaAIdIMrbJtacQlBizGLjUPw+akkFI5Dn9T70XoVQ==} engines: {node: '>=18'} isexe@2.0.0: @@ -2743,12 +2760,12 @@ packages: javascript-natural-sort@0.7.1: resolution: {integrity: sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==} - jiti@2.6.1: - resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} + jiti@2.7.0: + resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true - jose@6.2.2: - resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==} + jose@6.2.3: + resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} jotai@2.19.1: resolution: {integrity: sha512-sqm9lVZiqBHZH8aSRk32DSiZDHY3yUIlulXYn9GQj7/LvoUdYXSMti7ZPJGo+6zjzKFt5a25k/I6iBCi43PJcw==} @@ -2803,8 +2820,8 @@ packages: engines: {node: '>=6'} hasBin: true - jsonfile@6.2.0: - resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==} + jsonfile@6.2.1: + resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -3106,8 +3123,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - msw@2.13.4: - resolution: {integrity: sha512-fPlKBeFe+8rpcyR3umUmmHuNwu6gc6T3STvkgEa9WDX/HEgal9wDeflpCUAIRtmvaLZM2igfI5y1bZ9G5J26KA==} + msw@2.14.4: + resolution: {integrity: sha512-HVPZJ9Rx4nDCWhjNQ57lKQGSE+0zDHw0xWE2IN2rLOUTLkagEBWNlvWuKYNwG2pQWq96TMd8NiSK/6vO1udnWQ==} engines: {node: '>=18'} hasBin: true peerDependencies: @@ -3125,6 +3142,11 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanoid@3.3.12: + resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -3141,8 +3163,8 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - node-releases@2.0.37: - resolution: {integrity: sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==} + node-releases@2.0.38: + resolution: {integrity: sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==} normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} @@ -3285,6 +3307,10 @@ packages: resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==} engines: {node: ^10 || ^12 || >=14} + postcss@8.5.14: + resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} + engines: {node: ^10 || ^12 || >=14} + powershell-utils@0.1.0: resolution: {integrity: sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==} engines: {node: '>=20'} @@ -3405,8 +3431,8 @@ packages: peerDependencies: react: ^19.2.5 - react-i18next@17.0.4: - resolution: {integrity: sha512-hQipmK4EF0y6RO6tt6WuqnmWpWYEXmQUUzecmMBuNsIgYd3smXcG4GtYPWhvgxn0pqMOItKlEO8H24HCs5hc3g==} + react-i18next@17.0.6: + resolution: {integrity: sha512-WzJ6SMKF+GTD7JZZqxSR1AKKmXjaSu39sClUrNlwxS4Tl7a99O+ltFy6yhPMO+wgZuxpQjJ2PZkfrQKmAqrLhw==} peerDependencies: i18next: '>= 26.0.1' react: '>= 16.8.0' @@ -3515,8 +3541,8 @@ packages: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} - rettime@0.11.7: - resolution: {integrity: sha512-DoAm1WjR1eH7z8sHPtvvUMIZh4/CSKkGCz6CxPqOrEAnOGtOuHSnSE9OC+razqxKuf4ub7pAYyl/vZV0vGs5tg==} + rettime@0.11.11: + resolution: {integrity: sha512-ILJRqVWBCTlg9r42fFgwVZx1gnFAcQF8mRoMkbgQfIrjEDf9nbBFDFx00oloOa+Q869FUtaYDXZvEfnecQSCoQ==} reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} @@ -3563,8 +3589,8 @@ packages: peerDependencies: seroval: ^1.0 - seroval-plugins@1.5.2: - resolution: {integrity: sha512-qpY0Cl+fKYFn4GOf3cMiq6l72CpuVaawb6ILjubOQ+diJ54LfOWaSSPsaswN8DRPIPW4Yq+tE1k5aKd7ILyaFg==} + seroval-plugins@1.5.4: + resolution: {integrity: sha512-S0xQPhUTefAhNvNWFg0c1J8qJArHt5KdtJ/cFAofo06KD1MVSeFWyl4iiu+ApDIuw0WhjpOfCdgConOfAnLgkw==} engines: {node: '>=10'} peerDependencies: seroval: ^1.0 @@ -3573,8 +3599,8 @@ packages: resolution: {integrity: sha512-OwrZRZAfhHww0WEnKHDY8OM0U/Qs8OTfIDWhUD4BLpNJUfXK4cGmjiagGze086m+mhI+V2nD0gfbHEnJjb9STA==} engines: {node: '>=10'} - seroval@1.5.2: - resolution: {integrity: sha512-xcRN39BdsnO9Tf+VzsE7b3JyTJASItIV1FVFewJKCFcW4s4haIKS3e6vj8PGB9qBwC7tnuOywQMdv5N4qkzi7Q==} + seroval@1.5.4: + resolution: {integrity: sha512-46uFvgrXTVxZcUorgSSRZ4y+ieqLLQRMlG4bnCZKW3qI6BZm7Rg4ntMW4p1mILEEBZWrFlcpp0AyIIlM6jD9iw==} engines: {node: '>=10'} serve-static@2.2.1: @@ -3587,8 +3613,8 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - shadcn@4.3.0: - resolution: {integrity: sha512-7vhnBh2LVLyxOd1ZQWwXv7OATCnQcxdqc8FbZdNigZriNOwDsHklQmPpvPt1jcrFK5mzMI+cyuAYv8WzERx2Og==} + shadcn@4.7.0: + resolution: {integrity: sha512-70fwnesNrY1GgeD7Kdzn+3SsYeyfibm8immsA5L68+OusoPTvYF01oWExl8/latKpMpvVXcbgdbbE6VFBJQ38w==} hasBin: true shebang-command@2.0.0: @@ -3709,11 +3735,11 @@ packages: tailwind-merge@3.5.0: resolution: {integrity: sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A==} - tailwindcss@4.2.2: - resolution: {integrity: sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==} + tailwindcss@4.2.4: + resolution: {integrity: sha512-HhKppgO81FQof5m6TEnuBWCZGgfRAWbaeOaGT00KOy/Pf/j6oUihdvBpA7ltCeAvZpFhW3j0PTclkxsd4IXYDA==} - tapable@2.3.2: - resolution: {integrity: sha512-1MOpMXuhGzGL5TTCZFItxCc0AARf1EZFQkGqMm7ERKj8+Hgr5oLvJOVFcC+lRmR8hCe2S3jC4T5D7Vg/d7/fhA==} + tapable@2.3.3: + resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} tiny-invariant@1.3.3: @@ -3723,11 +3749,11 @@ packages: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} - tldts-core@7.0.28: - resolution: {integrity: sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ==} + tldts-core@7.0.30: + resolution: {integrity: sha512-uiHN8PIB1VmWyS98eZYja4xzlYqeFZVjb4OuYlJQnZAuJhMw4PbKQOKgHKhBdJR3FE/t5mUQ1Kd80++B+qhD1Q==} - tldts@7.0.28: - resolution: {integrity: sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw==} + tldts@7.0.30: + resolution: {integrity: sha512-ELrFxuqsDdHUwoh0XxDbxuLD3Wnz49Z57IFvTtvWy1hJdcMZjXLIuonjilCiWHlT2GbE4Wlv1wKVTzDFnXH1aw==} hasBin: true to-regex-range@5.0.1: @@ -3776,16 +3802,16 @@ packages: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} - type-fest@5.5.0: - resolution: {integrity: sha512-PlBfpQwiUvGViBNX84Yxwjsdhd1TUlXr6zjX7eoirtCPIr08NAmxwa+fcYBTeRQxHo9YC9wwF3m9i700sHma8g==} + type-fest@5.6.0: + resolution: {integrity: sha512-8ZiHFm91orbSAe2PSAiSVBVko18pbhbiB3U9GglSzF/zCGkR+rxpHx6sEMCUm4kxY4LjDIUGgCfUMtwfZfjfUA==} engines: {node: '>=20'} type-is@2.0.1: resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} engines: {node: '>= 0.6'} - typescript-eslint@8.59.0: - resolution: {integrity: sha512-BU3ONW9X+v90EcCH9ZS6LMackcVtxRLlI3XrYyqZIwVSHIk7Qf7bFw1z0M9Q0IUxhTMZCf8piY9hTYaNEIASrw==} + typescript-eslint@8.59.1: + resolution: {integrity: sha512-xqDcFVBmlrltH64lklOVp1wYxgJr6LVdg3NamBgH2OOQDLFdTKfIZXF5PfghrnXQKXZGTQs8tr1vL7fJvq8CTQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 @@ -4025,8 +4051,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - yocto-spinner@1.1.0: - resolution: {integrity: sha512-/BY0AUXnS7IKO354uLLA2eRcWiqDifEbd6unXCsOxkFDAkhgUL3PH9X2bFoaU0YchnDXsF+iKleeTLJGckbXfA==} + yocto-spinner@1.2.0: + resolution: {integrity: sha512-Yw0hUB6UA3o4YUgKy3oSe9a4cxoaZ9sBfYDw+JSxo6Id0KoJGoxzPA24qqUXYKBWABs/zDSGTz9kww7t3F0XGw==} engines: {node: '>=18.19'} yoctocolors@2.1.2: @@ -4061,7 +4087,7 @@ snapshots: js-tokens: 4.0.0 picocolors: 1.1.1 - '@babel/compat-data@7.29.0': {} + '@babel/compat-data@7.29.3': {} '@babel/core@7.29.0': dependencies: @@ -4070,7 +4096,7 @@ snapshots: '@babel/helper-compilation-targets': 7.28.6 '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) '@babel/helpers': 7.29.2 - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/template': 7.28.6 '@babel/traverse': 7.29.0 '@babel/types': 7.29.0 @@ -4085,7 +4111,7 @@ snapshots: '@babel/generator@7.29.1': dependencies: - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/types': 7.29.0 '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 @@ -4097,13 +4123,13 @@ snapshots: '@babel/helper-compilation-targets@7.28.6': dependencies: - '@babel/compat-data': 7.29.0 + '@babel/compat-data': 7.29.3 '@babel/helper-validator-option': 7.27.1 browserslist: 4.28.2 lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.28.6(@babel/core@7.29.0)': + '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 '@babel/helper-annotate-as-pure': 7.27.3 @@ -4178,6 +4204,10 @@ snapshots: dependencies: '@babel/types': 7.29.0 + '@babel/parser@7.29.3': + dependencies: + '@babel/types': 7.29.0 + '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 @@ -4200,7 +4230,7 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-create-class-features-plugin': 7.28.6(@babel/core@7.29.0) + '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0) '@babel/helper-plugin-utils': 7.28.6 '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0) @@ -4223,7 +4253,7 @@ snapshots: '@babel/template@7.28.6': dependencies: '@babel/code-frame': 7.29.0 - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/types': 7.29.0 '@babel/traverse@7.29.0': @@ -4231,7 +4261,7 @@ snapshots: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 '@babel/helper-globals': 7.28.0 - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/template': 7.28.6 '@babel/types': 7.29.0 debug: 4.4.3 @@ -4243,7 +4273,7 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 - '@dotenvx/dotenvx@1.61.0': + '@dotenvx/dotenvx@1.65.0': dependencies: commander: 11.1.0 dotenv: 17.4.2 @@ -4254,7 +4284,7 @@ snapshots: object-treeify: 1.1.33 picomatch: 4.0.4 which: 4.0.0 - yocto-spinner: 1.1.0 + yocto-spinner: 1.2.0 '@ecies/ciphers@0.2.6(@noble/ciphers@1.3.0)': dependencies: @@ -4354,9 +4384,9 @@ snapshots: '@esbuild/win32-x64@0.27.4': optional: true - '@eslint-community/eslint-utils@4.9.1(eslint@10.2.1(jiti@2.6.1))': + '@eslint-community/eslint-utils@4.9.1(eslint@10.2.1(jiti@2.7.0))': dependencies: - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} @@ -4377,9 +4407,9 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/js@10.0.1(eslint@10.2.1(jiti@2.6.1))': + '@eslint/js@10.0.1(eslint@10.2.1(jiti@2.7.0))': optionalDependencies: - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) '@eslint/object-schema@3.0.5': {} @@ -4407,9 +4437,9 @@ snapshots: '@fontsource-variable/inter@5.2.8': {} - '@hono/node-server@1.19.14(hono@4.12.14)': + '@hono/node-server@1.19.14(hono@4.12.18)': dependencies: - hono: 4.12.14 + hono: 4.12.18 '@humanfs/core@0.19.1': {} @@ -4424,14 +4454,14 @@ snapshots: '@inquirer/ansi@2.0.5': {} - '@inquirer/confirm@6.0.11(@types/node@25.6.0)': + '@inquirer/confirm@6.0.12(@types/node@25.6.0)': dependencies: - '@inquirer/core': 11.1.8(@types/node@25.6.0) + '@inquirer/core': 11.1.9(@types/node@25.6.0) '@inquirer/type': 4.0.5(@types/node@25.6.0) optionalDependencies: '@types/node': 25.6.0 - '@inquirer/core@11.1.8(@types/node@25.6.0)': + '@inquirer/core@11.1.9(@types/node@25.6.0)': dependencies: '@inquirer/ansi': 2.0.5 '@inquirer/figures': 2.0.5 @@ -4470,18 +4500,18 @@ snapshots: '@modelcontextprotocol/sdk@1.29.0(zod@3.25.76)': dependencies: - '@hono/node-server': 1.19.14(hono@4.12.14) - ajv: 8.18.0 - ajv-formats: 3.0.1(ajv@8.18.0) + '@hono/node-server': 1.19.14(hono@4.12.18) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 cors: 2.8.6 cross-spawn: 7.0.6 eventsource: 3.0.7 - eventsource-parser: 3.0.6 + eventsource-parser: 3.0.8 express: 5.2.1 - express-rate-limit: 8.3.2(express@5.2.1) - hono: 4.12.14 - jose: 6.2.2 + express-rate-limit: 8.5.1(express@5.2.1) + hono: 4.12.18 + jose: 6.2.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 raw-body: 3.0.2 @@ -4490,7 +4520,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@mswjs/interceptors@0.41.3': + '@mswjs/interceptors@0.41.8': dependencies: '@open-draft/deferred-promise': 2.2.0 '@open-draft/logger': 0.3.0 @@ -5343,85 +5373,85 @@ snapshots: '@sindresorhus/merge-streams@4.0.0': {} - '@tabler/icons-react@3.41.1(react@19.2.5)': + '@tabler/icons-react@3.43.0(react@19.2.5)': dependencies: - '@tabler/icons': 3.41.1 + '@tabler/icons': 3.43.0 react: 19.2.5 - '@tabler/icons@3.41.1': {} + '@tabler/icons@3.43.0': {} - '@tailwindcss/node@4.2.2': + '@tailwindcss/node@4.2.4': dependencies: '@jridgewell/remapping': 2.3.5 - enhanced-resolve: 5.20.1 - jiti: 2.6.1 + enhanced-resolve: 5.21.0 + jiti: 2.7.0 lightningcss: 1.32.0 magic-string: 0.30.21 source-map-js: 1.2.1 - tailwindcss: 4.2.2 + tailwindcss: 4.2.4 - '@tailwindcss/oxide-android-arm64@4.2.2': + '@tailwindcss/oxide-android-arm64@4.2.4': optional: true - '@tailwindcss/oxide-darwin-arm64@4.2.2': + '@tailwindcss/oxide-darwin-arm64@4.2.4': optional: true - '@tailwindcss/oxide-darwin-x64@4.2.2': + '@tailwindcss/oxide-darwin-x64@4.2.4': optional: true - '@tailwindcss/oxide-freebsd-x64@4.2.2': + '@tailwindcss/oxide-freebsd-x64@4.2.4': optional: true - '@tailwindcss/oxide-linux-arm-gnueabihf@4.2.2': + '@tailwindcss/oxide-linux-arm-gnueabihf@4.2.4': optional: true - '@tailwindcss/oxide-linux-arm64-gnu@4.2.2': + '@tailwindcss/oxide-linux-arm64-gnu@4.2.4': optional: true - '@tailwindcss/oxide-linux-arm64-musl@4.2.2': + '@tailwindcss/oxide-linux-arm64-musl@4.2.4': optional: true - '@tailwindcss/oxide-linux-x64-gnu@4.2.2': + '@tailwindcss/oxide-linux-x64-gnu@4.2.4': optional: true - '@tailwindcss/oxide-linux-x64-musl@4.2.2': + '@tailwindcss/oxide-linux-x64-musl@4.2.4': optional: true - '@tailwindcss/oxide-wasm32-wasi@4.2.2': + '@tailwindcss/oxide-wasm32-wasi@4.2.4': optional: true - '@tailwindcss/oxide-win32-arm64-msvc@4.2.2': + '@tailwindcss/oxide-win32-arm64-msvc@4.2.4': optional: true - '@tailwindcss/oxide-win32-x64-msvc@4.2.2': + '@tailwindcss/oxide-win32-x64-msvc@4.2.4': optional: true - '@tailwindcss/oxide@4.2.2': + '@tailwindcss/oxide@4.2.4': optionalDependencies: - '@tailwindcss/oxide-android-arm64': 4.2.2 - '@tailwindcss/oxide-darwin-arm64': 4.2.2 - '@tailwindcss/oxide-darwin-x64': 4.2.2 - '@tailwindcss/oxide-freebsd-x64': 4.2.2 - '@tailwindcss/oxide-linux-arm-gnueabihf': 4.2.2 - '@tailwindcss/oxide-linux-arm64-gnu': 4.2.2 - '@tailwindcss/oxide-linux-arm64-musl': 4.2.2 - '@tailwindcss/oxide-linux-x64-gnu': 4.2.2 - '@tailwindcss/oxide-linux-x64-musl': 4.2.2 - '@tailwindcss/oxide-wasm32-wasi': 4.2.2 - '@tailwindcss/oxide-win32-arm64-msvc': 4.2.2 - '@tailwindcss/oxide-win32-x64-msvc': 4.2.2 + '@tailwindcss/oxide-android-arm64': 4.2.4 + '@tailwindcss/oxide-darwin-arm64': 4.2.4 + '@tailwindcss/oxide-darwin-x64': 4.2.4 + '@tailwindcss/oxide-freebsd-x64': 4.2.4 + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.2.4 + '@tailwindcss/oxide-linux-arm64-gnu': 4.2.4 + '@tailwindcss/oxide-linux-arm64-musl': 4.2.4 + '@tailwindcss/oxide-linux-x64-gnu': 4.2.4 + '@tailwindcss/oxide-linux-x64-musl': 4.2.4 + '@tailwindcss/oxide-wasm32-wasi': 4.2.4 + '@tailwindcss/oxide-win32-arm64-msvc': 4.2.4 + '@tailwindcss/oxide-win32-x64-msvc': 4.2.4 - '@tailwindcss/typography@0.5.19(tailwindcss@4.2.2)': + '@tailwindcss/typography@0.5.19(tailwindcss@4.2.4)': dependencies: postcss-selector-parser: 6.0.10 - tailwindcss: 4.2.2 + tailwindcss: 4.2.4 - '@tailwindcss/vite@4.2.2(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0))': + '@tailwindcss/vite@4.2.4(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0))': dependencies: - '@tailwindcss/node': 4.2.2 - '@tailwindcss/oxide': 4.2.2 - tailwindcss: 4.2.2 - vite: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0) + '@tailwindcss/node': 4.2.4 + '@tailwindcss/oxide': 4.2.4 + tailwindcss: 4.2.4 + vite: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0) '@tanstack/history@1.161.6': {} @@ -5432,23 +5462,23 @@ snapshots: '@tanstack/query-core': 5.99.0 react: 19.2.5 - '@tanstack/react-router-devtools@1.166.13(@tanstack/react-router@1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(@tanstack/router-core@1.168.15)(csstype@3.2.3)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': + '@tanstack/react-router-devtools@1.166.13(@tanstack/react-router@1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(@tanstack/router-core@1.169.2)(csstype@3.2.3)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': dependencies: - '@tanstack/react-router': 1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5) - '@tanstack/router-devtools-core': 1.167.3(@tanstack/router-core@1.168.15)(csstype@3.2.3) + '@tanstack/react-router': 1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + '@tanstack/router-devtools-core': 1.167.3(@tanstack/router-core@1.169.2)(csstype@3.2.3) react: 19.2.5 react-dom: 19.2.5(react@19.2.5) optionalDependencies: - '@tanstack/router-core': 1.168.15 + '@tanstack/router-core': 1.169.2 transitivePeerDependencies: - csstype - '@tanstack/react-router@1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': + '@tanstack/react-router@1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': dependencies: '@tanstack/history': 1.161.6 '@tanstack/react-store': 0.9.3(react-dom@19.2.5(react@19.2.5))(react@19.2.5) - '@tanstack/router-core': 1.168.15 - isbot: 5.1.39 + '@tanstack/router-core': 1.169.2 + isbot: 5.1.40 react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -5459,13 +5489,6 @@ snapshots: react-dom: 19.2.5(react@19.2.5) use-sync-external-store: 1.6.0(react@19.2.5) - '@tanstack/router-core@1.168.15': - dependencies: - '@tanstack/history': 1.161.6 - cookie-es: 3.1.1 - seroval: 1.5.2 - seroval-plugins: 1.5.2(seroval@1.5.2) - '@tanstack/router-core@1.168.7': dependencies: '@tanstack/history': 1.161.6 @@ -5473,9 +5496,16 @@ snapshots: seroval: 1.5.1 seroval-plugins: 1.5.1(seroval@1.5.1) - '@tanstack/router-devtools-core@1.167.3(@tanstack/router-core@1.168.15)(csstype@3.2.3)': + '@tanstack/router-core@1.169.2': dependencies: - '@tanstack/router-core': 1.168.15 + '@tanstack/history': 1.161.6 + cookie-es: 3.1.1 + seroval: 1.5.4 + seroval-plugins: 1.5.4(seroval@1.5.4) + + '@tanstack/router-devtools-core@1.167.3(@tanstack/router-core@1.169.2)(csstype@3.2.3)': + dependencies: + '@tanstack/router-core': 1.169.2 clsx: 2.1.1 goober: 2.1.18(csstype@3.2.3) optionalDependencies: @@ -5494,7 +5524,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@tanstack/router-plugin@1.167.9(@tanstack/react-router@1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0))': + '@tanstack/router-plugin@1.167.9(@tanstack/react-router@1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5))(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0))': dependencies: '@babel/core': 7.29.0 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) @@ -5510,8 +5540,8 @@ snapshots: unplugin: 2.3.11 zod: 3.25.76 optionalDependencies: - '@tanstack/react-router': 1.168.23(react-dom@19.2.5(react@19.2.5))(react@19.2.5) - vite: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0) + '@tanstack/react-router': 1.169.2(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + vite: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0) transitivePeerDependencies: - supports-color @@ -5519,7 +5549,7 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/generator': 7.29.1 - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/types': 7.29.0 ansis: 4.2.0 babel-dead-code-elimination: 1.0.12 @@ -5606,15 +5636,15 @@ snapshots: '@types/validate-npm-package-name@4.0.2': {} - '@typescript-eslint/eslint-plugin@8.58.2(@typescript-eslint/parser@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3))(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/eslint-plugin@8.58.2(@typescript-eslint/parser@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3))(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/parser': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) '@typescript-eslint/scope-manager': 8.58.2 - '@typescript-eslint/type-utils': 8.58.2(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - '@typescript-eslint/utils': 8.58.2(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/type-utils': 8.58.2(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/utils': 8.58.2(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.58.2 - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -5622,15 +5652,15 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/eslint-plugin@8.59.0(@typescript-eslint/parser@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3))(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/eslint-plugin@8.59.1(@typescript-eslint/parser@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3))(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - '@typescript-eslint/scope-manager': 8.59.0 - '@typescript-eslint/type-utils': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - '@typescript-eslint/utils': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - '@typescript-eslint/visitor-keys': 8.59.0 - eslint: 10.2.1(jiti@2.6.1) + '@typescript-eslint/parser': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.59.1 + '@typescript-eslint/type-utils': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/utils': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.59.1 + eslint: 10.2.1(jiti@2.7.0) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -5638,31 +5668,31 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/parser@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: - '@typescript-eslint/scope-manager': 8.59.0 - '@typescript-eslint/types': 8.59.0 - '@typescript-eslint/typescript-estree': 8.59.0(typescript@5.9.3) - '@typescript-eslint/visitor-keys': 8.59.0 + '@typescript-eslint/scope-manager': 8.59.1 + '@typescript-eslint/types': 8.59.1 + '@typescript-eslint/typescript-estree': 8.59.1(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.59.1 debug: 4.4.3 - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color '@typescript-eslint/project-service@8.58.2(typescript@5.9.3)': dependencies: - '@typescript-eslint/tsconfig-utils': 8.59.0(typescript@5.9.3) - '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/tsconfig-utils': 8.59.1(typescript@5.9.3) + '@typescript-eslint/types': 8.59.1 debug: 4.4.3 typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.59.0(typescript@5.9.3)': + '@typescript-eslint/project-service@8.59.1(typescript@5.9.3)': dependencies: - '@typescript-eslint/tsconfig-utils': 8.59.0(typescript@5.9.3) - '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/tsconfig-utils': 8.59.1(typescript@5.9.3) + '@typescript-eslint/types': 8.59.1 debug: 4.4.3 typescript: 5.9.3 transitivePeerDependencies: @@ -5673,38 +5703,38 @@ snapshots: '@typescript-eslint/types': 8.58.2 '@typescript-eslint/visitor-keys': 8.58.2 - '@typescript-eslint/scope-manager@8.59.0': + '@typescript-eslint/scope-manager@8.59.1': dependencies: - '@typescript-eslint/types': 8.59.0 - '@typescript-eslint/visitor-keys': 8.59.0 + '@typescript-eslint/types': 8.59.1 + '@typescript-eslint/visitor-keys': 8.59.1 '@typescript-eslint/tsconfig-utils@8.58.2(typescript@5.9.3)': dependencies: typescript: 5.9.3 - '@typescript-eslint/tsconfig-utils@8.59.0(typescript@5.9.3)': + '@typescript-eslint/tsconfig-utils@8.59.1(typescript@5.9.3)': dependencies: typescript: 5.9.3 - '@typescript-eslint/type-utils@8.58.2(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/type-utils@8.58.2(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: '@typescript-eslint/types': 8.58.2 '@typescript-eslint/typescript-estree': 8.58.2(typescript@5.9.3) - '@typescript-eslint/utils': 8.58.2(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/utils': 8.58.2(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) debug: 4.4.3 - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/type-utils@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/type-utils@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: - '@typescript-eslint/types': 8.59.0 - '@typescript-eslint/typescript-estree': 8.59.0(typescript@5.9.3) - '@typescript-eslint/utils': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/types': 8.59.1 + '@typescript-eslint/typescript-estree': 8.59.1(typescript@5.9.3) + '@typescript-eslint/utils': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) debug: 4.4.3 - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 transitivePeerDependencies: @@ -5712,7 +5742,7 @@ snapshots: '@typescript-eslint/types@8.58.2': {} - '@typescript-eslint/types@8.59.0': {} + '@typescript-eslint/types@8.59.1': {} '@typescript-eslint/typescript-estree@8.58.2(typescript@5.9.3)': dependencies: @@ -5729,12 +5759,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/typescript-estree@8.59.0(typescript@5.9.3)': + '@typescript-eslint/typescript-estree@8.59.1(typescript@5.9.3)': dependencies: - '@typescript-eslint/project-service': 8.59.0(typescript@5.9.3) - '@typescript-eslint/tsconfig-utils': 8.59.0(typescript@5.9.3) - '@typescript-eslint/types': 8.59.0 - '@typescript-eslint/visitor-keys': 8.59.0 + '@typescript-eslint/project-service': 8.59.1(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.59.1(typescript@5.9.3) + '@typescript-eslint/types': 8.59.1 + '@typescript-eslint/visitor-keys': 8.59.1 debug: 4.4.3 minimatch: 10.2.5 semver: 7.7.4 @@ -5744,24 +5774,24 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.58.2(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/utils@8.58.2(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@10.2.1(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@10.2.1(jiti@2.7.0)) '@typescript-eslint/scope-manager': 8.58.2 '@typescript-eslint/types': 8.58.2 '@typescript-eslint/typescript-estree': 8.58.2(typescript@5.9.3) - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3)': + '@typescript-eslint/utils@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@10.2.1(jiti@2.6.1)) - '@typescript-eslint/scope-manager': 8.59.0 - '@typescript-eslint/types': 8.59.0 - '@typescript-eslint/typescript-estree': 8.59.0(typescript@5.9.3) - eslint: 10.2.1(jiti@2.6.1) + '@eslint-community/eslint-utils': 4.9.1(eslint@10.2.1(jiti@2.7.0)) + '@typescript-eslint/scope-manager': 8.59.1 + '@typescript-eslint/types': 8.59.1 + '@typescript-eslint/typescript-estree': 8.59.1(typescript@5.9.3) + eslint: 10.2.1(jiti@2.7.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -5771,17 +5801,17 @@ snapshots: '@typescript-eslint/types': 8.58.2 eslint-visitor-keys: 5.0.1 - '@typescript-eslint/visitor-keys@8.59.0': + '@typescript-eslint/visitor-keys@8.59.1': dependencies: - '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/types': 8.59.1 eslint-visitor-keys: 5.0.1 '@ungap/structured-clone@1.3.0': {} - '@vitejs/plugin-react@6.0.1(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0))': + '@vitejs/plugin-react@6.0.1(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0))': dependencies: '@rolldown/pluginutils': 1.0.0-rc.7 - vite: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0) + vite: 8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0) accepts@2.0.0: dependencies: @@ -5796,9 +5826,9 @@ snapshots: agent-base@7.1.4: {} - ajv-formats@3.0.1(ajv@8.18.0): + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: - ajv: 8.18.0 + ajv: 8.20.0 ajv@6.14.0: dependencies: @@ -5807,10 +5837,10 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 - ajv@8.18.0: + ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 - fast-uri: 3.1.0 + fast-uri: 3.1.2 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 @@ -5844,7 +5874,7 @@ snapshots: babel-dead-code-elimination@1.0.12: dependencies: '@babel/core': 7.29.0 - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/traverse': 7.29.0 '@babel/types': 7.29.0 transitivePeerDependencies: @@ -5856,7 +5886,7 @@ snapshots: balanced-match@4.0.4: {} - baseline-browser-mapping@2.10.17: {} + baseline-browser-mapping@2.10.27: {} binary-extensions@2.3.0: {} @@ -5888,10 +5918,10 @@ snapshots: browserslist@4.28.2: dependencies: - baseline-browser-mapping: 2.10.17 - caniuse-lite: 1.0.30001787 - electron-to-chromium: 1.5.334 - node-releases: 2.0.37 + baseline-browser-mapping: 2.10.27 + caniuse-lite: 1.0.30001792 + electron-to-chromium: 1.5.352 + node-releases: 2.0.38 update-browserslist-db: 1.2.3(browserslist@4.28.2) bundle-name@4.1.0: @@ -5912,7 +5942,7 @@ snapshots: callsites@3.1.0: {} - caniuse-lite@1.0.30001787: {} + caniuse-lite@1.0.30001792: {} ccount@2.0.1: {} @@ -5958,6 +5988,18 @@ snapshots: clsx@2.1.1: {} + cmdk@1.1.1(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.5(react@19.2.5))(react@19.2.5): + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.5) + '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.5) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + react: 19.2.5 + react-dom: 19.2.5(react@19.2.5) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + code-block-writer@13.0.3: {} color-convert@2.0.1: @@ -6070,7 +6112,7 @@ snapshots: ee-first@1.1.1: {} - electron-to-chromium@1.5.334: {} + electron-to-chromium@1.5.352: {} emoji-regex@10.6.0: {} @@ -6078,10 +6120,10 @@ snapshots: encodeurl@2.0.0: {} - enhanced-resolve@5.20.1: + enhanced-resolve@5.21.0: dependencies: graceful-fs: 4.2.11 - tapable: 2.3.2 + tapable: 2.3.3 entities@6.0.1: {} @@ -6136,24 +6178,24 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-config-prettier@10.1.8(eslint@10.2.1(jiti@2.6.1)): + eslint-config-prettier@10.1.8(eslint@10.2.1(jiti@2.7.0)): dependencies: - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) - eslint-plugin-react-hooks@7.1.1(eslint@10.2.1(jiti@2.6.1)): + eslint-plugin-react-hooks@7.1.1(eslint@10.2.1(jiti@2.7.0)): dependencies: '@babel/core': 7.29.0 '@babel/parser': 7.29.2 - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) hermes-parser: 0.25.1 zod: 4.3.6 zod-validation-error: 4.0.2(zod@4.3.6) transitivePeerDependencies: - supports-color - eslint-plugin-react-refresh@0.5.2(eslint@10.2.1(jiti@2.6.1)): + eslint-plugin-react-refresh@0.5.2(eslint@10.2.1(jiti@2.7.0)): dependencies: - eslint: 10.2.1(jiti@2.6.1) + eslint: 10.2.1(jiti@2.7.0) eslint-scope@9.1.2: dependencies: @@ -6166,9 +6208,9 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@10.2.1(jiti@2.6.1): + eslint@10.2.1(jiti@2.7.0): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@10.2.1(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@10.2.1(jiti@2.7.0)) '@eslint-community/regexpp': 4.12.2 '@eslint/config-array': 0.23.5 '@eslint/config-helpers': 0.5.5 @@ -6199,7 +6241,7 @@ snapshots: natural-compare: 1.4.0 optionator: 0.9.4 optionalDependencies: - jiti: 2.6.1 + jiti: 2.7.0 transitivePeerDependencies: - supports-color @@ -6227,11 +6269,11 @@ snapshots: etag@1.8.1: {} - eventsource-parser@3.0.6: {} + eventsource-parser@3.0.8: {} eventsource@3.0.7: dependencies: - eventsource-parser: 3.0.6 + eventsource-parser: 3.0.8 execa@5.1.1: dependencies: @@ -6260,10 +6302,10 @@ snapshots: strip-final-newline: 4.0.0 yoctocolors: 2.1.2 - express-rate-limit@8.3.2(express@5.2.1): + express-rate-limit@8.5.1(express@5.2.1): dependencies: express: 5.2.1 - ip-address: 10.1.0 + ip-address: 10.2.0 express@5.2.1: dependencies: @@ -6320,7 +6362,7 @@ snapshots: dependencies: fast-string-truncated-width: 3.0.3 - fast-uri@3.1.0: {} + fast-uri@3.1.2: {} fast-wrap-ansi@0.2.0: dependencies: @@ -6382,10 +6424,10 @@ snapshots: fresh@2.0.0: {} - fs-extra@11.3.4: + fs-extra@11.3.5: dependencies: graceful-fs: 4.2.11 - jsonfile: 6.2.0 + jsonfile: 6.2.1 universalify: 2.0.1 fsevents@2.3.3: @@ -6411,7 +6453,7 @@ snapshots: get-proto: 1.0.1 gopd: 1.2.0 has-symbols: 1.1.0 - hasown: 2.0.2 + hasown: 2.0.3 math-intrinsics: 1.1.0 get-nonce@1.0.1: {} @@ -6442,7 +6484,7 @@ snapshots: dependencies: is-glob: 4.0.3 - globals@17.5.0: {} + globals@17.6.0: {} goober@2.1.18(csstype@3.2.3): dependencies: @@ -6452,11 +6494,11 @@ snapshots: graceful-fs@4.2.11: {} - graphql@16.13.2: {} + graphql@16.14.0: {} has-symbols@1.1.0: {} - hasown@2.0.2: + hasown@2.0.3: dependencies: function-bind: 1.1.2 @@ -6563,7 +6605,7 @@ snapshots: highlight.js@11.11.1: {} - hono@4.12.14: {} + hono@4.12.18: {} html-parse-stringify@3.0.1: dependencies: @@ -6596,7 +6638,7 @@ snapshots: dependencies: '@babel/runtime': 7.29.2 - i18next@26.0.7(typescript@5.9.3): + i18next@26.0.10(typescript@5.9.3): optionalDependencies: typescript: 5.9.3 @@ -6619,7 +6661,7 @@ snapshots: inline-style-parser@0.2.7: {} - ip-address@10.1.0: {} + ip-address@10.2.0: {} ipaddr.js@1.9.1: {} @@ -6682,7 +6724,7 @@ snapshots: dependencies: is-inside-container: 1.0.0 - isbot@5.1.39: {} + isbot@5.1.40: {} isexe@2.0.0: {} @@ -6690,9 +6732,9 @@ snapshots: javascript-natural-sort@0.7.1: {} - jiti@2.6.1: {} + jiti@2.7.0: {} - jose@6.2.2: {} + jose@6.2.3: {} jotai@2.19.1(@babel/core@7.29.0)(@babel/template@7.28.6)(@types/react@19.2.14)(react@19.2.5): optionalDependencies: @@ -6723,7 +6765,7 @@ snapshots: json5@2.2.3: {} - jsonfile@6.2.0: + jsonfile@6.2.1: dependencies: universalify: 2.0.1 optionalDependencies: @@ -7203,24 +7245,24 @@ snapshots: ms@2.1.3: {} - msw@2.13.4(@types/node@25.6.0)(typescript@5.9.3): + msw@2.14.4(@types/node@25.6.0)(typescript@5.9.3): dependencies: - '@inquirer/confirm': 6.0.11(@types/node@25.6.0) - '@mswjs/interceptors': 0.41.3 + '@inquirer/confirm': 6.0.12(@types/node@25.6.0) + '@mswjs/interceptors': 0.41.8 '@open-draft/deferred-promise': 3.0.0 '@types/statuses': 2.0.6 cookie: 1.1.1 - graphql: 16.13.2 + graphql: 16.14.0 headers-polyfill: 5.0.1 is-node-process: 1.2.0 outvariant: 1.4.3 path-to-regexp: 6.3.0 picocolors: 1.1.1 - rettime: 0.11.7 + rettime: 0.11.11 statuses: 2.0.2 strict-event-emitter: 0.5.1 tough-cookie: 6.0.1 - type-fest: 5.5.0 + type-fest: 5.6.0 until-async: 3.0.2 yargs: 17.7.2 optionalDependencies: @@ -7232,6 +7274,8 @@ snapshots: nanoid@3.3.11: {} + nanoid@3.3.12: {} + natural-compare@1.4.0: {} negotiator@1.0.0: {} @@ -7244,7 +7288,7 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - node-releases@2.0.37: {} + node-releases@2.0.38: {} normalize-path@3.0.0: {} @@ -7392,6 +7436,12 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postcss@8.5.14: + dependencies: + nanoid: 3.3.12 + picocolors: 1.1.1 + source-map-js: 1.2.1 + powershell-utils@0.1.0: {} prelude-ls@1.2.1: {} @@ -7505,11 +7555,11 @@ snapshots: react: 19.2.5 scheduler: 0.27.0 - react-i18next@17.0.4(i18next@26.0.7(typescript@5.9.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(typescript@5.9.3): + react-i18next@17.0.6(i18next@26.0.10(typescript@5.9.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(typescript@5.9.3): dependencies: '@babel/runtime': 7.29.2 html-parse-stringify: 3.0.1 - i18next: 26.0.7(typescript@5.9.3) + i18next: 26.0.10(typescript@5.9.3) react: 19.2.5 use-sync-external-store: 1.6.0(react@19.2.5) optionalDependencies: @@ -7650,7 +7700,7 @@ snapshots: onetime: 7.0.0 signal-exit: 4.1.0 - rettime@0.11.7: {} + rettime@0.11.11: {} reusify@1.1.0: {} @@ -7719,13 +7769,13 @@ snapshots: dependencies: seroval: 1.5.1 - seroval-plugins@1.5.2(seroval@1.5.2): + seroval-plugins@1.5.4(seroval@1.5.4): dependencies: - seroval: 1.5.2 + seroval: 1.5.4 seroval@1.5.1: {} - seroval@1.5.2: {} + seroval@1.5.4: {} serve-static@2.2.1: dependencies: @@ -7740,13 +7790,13 @@ snapshots: setprototypeof@1.2.0: {} - shadcn@4.3.0(@types/node@25.6.0)(typescript@5.9.3): + shadcn@4.7.0(@types/node@25.6.0)(typescript@5.9.3): dependencies: '@babel/core': 7.29.0 - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.3 '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0) - '@dotenvx/dotenvx': 1.61.0 + '@dotenvx/dotenvx': 1.65.0 '@modelcontextprotocol/sdk': 1.29.0(zod@3.25.76) '@types/validate-npm-package-name': 4.0.2 browserslist: 4.28.2 @@ -7757,15 +7807,15 @@ snapshots: diff: 8.0.4 execa: 9.6.1 fast-glob: 3.3.3 - fs-extra: 11.3.4 + fs-extra: 11.3.5 fuzzysort: 3.1.0 https-proxy-agent: 7.0.6 kleur: 4.1.5 - msw: 2.13.4(@types/node@25.6.0)(typescript@5.9.3) + msw: 2.14.4(@types/node@25.6.0)(typescript@5.9.3) node-fetch: 3.3.2 open: 11.0.0 ora: 8.2.0 - postcss: 8.5.10 + postcss: 8.5.14 postcss-selector-parser: 7.1.1 prompts: 2.4.2 recast: 0.23.11 @@ -7896,9 +7946,9 @@ snapshots: tailwind-merge@3.5.0: {} - tailwindcss@4.2.2: {} + tailwindcss@4.2.4: {} - tapable@2.3.2: {} + tapable@2.3.3: {} tiny-invariant@1.3.3: {} @@ -7907,11 +7957,11 @@ snapshots: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - tldts-core@7.0.28: {} + tldts-core@7.0.30: {} - tldts@7.0.28: + tldts@7.0.30: dependencies: - tldts-core: 7.0.28 + tldts-core: 7.0.30 to-regex-range@5.0.1: dependencies: @@ -7921,7 +7971,7 @@ snapshots: tough-cookie@6.0.1: dependencies: - tldts: 7.0.28 + tldts: 7.0.30 trim-lines@3.0.1: {} @@ -7957,7 +8007,7 @@ snapshots: dependencies: prelude-ls: 1.2.1 - type-fest@5.5.0: + type-fest@5.6.0: dependencies: tagged-tag: 1.0.0 @@ -7967,13 +8017,13 @@ snapshots: media-typer: 1.1.0 mime-types: 3.0.2 - typescript-eslint@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3): + typescript-eslint@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.59.0(@typescript-eslint/parser@8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3))(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - '@typescript-eslint/parser': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - '@typescript-eslint/typescript-estree': 8.59.0(typescript@5.9.3) - '@typescript-eslint/utils': 8.59.0(eslint@10.2.1(jiti@2.6.1))(typescript@5.9.3) - eslint: 10.2.1(jiti@2.6.1) + '@typescript-eslint/eslint-plugin': 8.59.1(@typescript-eslint/parser@8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3))(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/parser': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.59.1(typescript@5.9.3) + '@typescript-eslint/utils': 8.59.1(eslint@10.2.1(jiti@2.7.0))(typescript@5.9.3) + eslint: 10.2.1(jiti@2.7.0) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -8104,7 +8154,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.6.1)(tsx@4.21.0): + vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.4)(jiti@2.7.0)(tsx@4.21.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -8115,7 +8165,7 @@ snapshots: '@types/node': 25.6.0 esbuild: 0.27.4 fsevents: 2.3.3 - jiti: 2.6.1 + jiti: 2.7.0 tsx: 4.21.0 void-elements@3.1.0: {} @@ -8173,7 +8223,7 @@ snapshots: yocto-queue@0.1.0: {} - yocto-spinner@1.1.0: + yocto-spinner@1.2.0: dependencies: yoctocolors: 2.1.2 diff --git a/web/frontend/src/api/models.ts b/web/frontend/src/api/models.ts index d2d2dca88..9fd29e0fd 100644 --- a/web/frontend/src/api/models.ts +++ b/web/frontend/src/api/models.ts @@ -19,19 +19,33 @@ export interface ModelInfo { max_tokens_field?: string request_timeout?: number thinking_level?: string + tool_schema_transform?: string extra_body?: Record custom_headers?: Record // Meta + enabled: boolean available: boolean status: "available" | "unconfigured" | "unreachable" is_default: boolean is_virtual: boolean + default_model_allowed?: boolean +} + +export interface ModelProviderOption { + id: string + default_api_base: string + empty_api_key_allowed: boolean + create_allowed: boolean + default_model_allowed: boolean + default_auth_method?: string + auth_method_locked?: boolean } interface ModelsListResponse { models: ModelInfo[] total: number default_model: string + provider_options: ModelProviderOption[] } interface ModelActionResponse { @@ -45,7 +59,13 @@ const BASE_URL = "" async function request(path: string, options?: RequestInit): Promise { const res = await launcherFetch(`${BASE_URL}${path}`, options) if (!res.ok) { - throw new Error(`API error: ${res.status} ${res.statusText}`) + let detail = "" + try { + detail = await res.text() + } catch { + // ignore + } + throw new Error(detail || `API error: ${res.status} ${res.statusText}`) } return res.json() as Promise } @@ -94,4 +114,97 @@ export async function setDefaultModel( return response } +export interface TestModelResponse { + success: boolean + latency_ms: number + status: string + error?: string +} + +export async function testModel(index: number): Promise { + return request(`/api/models/${index}/test`, { + method: "POST", + }) +} + +export interface TestModelInlineRequest { + provider: string + model: string + api_base?: string + api_key?: string + auth_method?: string + model_index?: number +} + +export async function testModelInline( + params: TestModelInlineRequest, +): Promise { + return request("/api/models/test-inline", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(params), + }) +} + +export interface UpstreamModel { + id: string + owned_by?: string +} + +export interface FetchModelsRequest { + provider: string + api_key?: string + api_base?: string +} + +export interface FetchModelsResponse { + models: UpstreamModel[] + total: number +} + +export async function fetchUpstreamModels( + req: FetchModelsRequest, +): Promise { + return request("/api/models/fetch", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(req), + }) +} + +// --- Model Catalog API --- + +export interface CatalogModel { + id: string + owned_by?: string + extra?: Record +} + +export interface CatalogEntry { + id: string + provider: string + api_base: string + api_key_mask: string + models: CatalogModel[] + fetched_at: string +} + +interface CatalogListResponse { + entries: CatalogEntry[] + total: number +} + +export async function getCatalogs(): Promise { + return request("/api/models/catalog") +} + +export async function deleteCatalog(id: string): Promise { + await request>( + `/api/models/catalog/${encodeURIComponent(id)}`, + { + method: "DELETE", + }, + ) +} + export type { ModelsListResponse, ModelActionResponse } diff --git a/web/frontend/src/api/tools.ts b/web/frontend/src/api/tools.ts index a77f3ba80..c7501b188 100644 --- a/web/frontend/src/api/tools.ts +++ b/web/frontend/src/api/tools.ts @@ -30,6 +30,7 @@ export interface WebSearchProviderConfig { max_results: number base_url?: string api_key?: string + model?: string api_key_set?: boolean } diff --git a/web/frontend/src/components/agent/tools/web-search-general-settings.tsx b/web/frontend/src/components/agent/tools/web-search-general-settings.tsx index f3c8004b5..e8c81408e 100644 --- a/web/frontend/src/components/agent/tools/web-search-general-settings.tsx +++ b/web/frontend/src/components/agent/tools/web-search-general-settings.tsx @@ -66,10 +66,7 @@ export function WebSearchGeneralSettings({
)} + + {modelProviders.has(providerId) && ( + + + updateSettings((current) => ({ + ...current, + model: event.target.value, + })) + } + placeholder={t( + "pages.agent.tools.web_search.model_placeholder", + "Optional model override", + )} + className="bg-muted/40 hover:bg-muted/60 focus:bg-background focus:ring-primary/20 h-10 rounded-xl border-transparent shadow-none transition-colors" + /> + + )}
)} diff --git a/web/frontend/src/components/app-header.tsx b/web/frontend/src/components/app-header.tsx index 465d218be..700cc21e0 100644 --- a/web/frontend/src/components/app-header.tsx +++ b/web/frontend/src/components/app-header.tsx @@ -288,6 +288,9 @@ export function AppHeader() { i18n.changeLanguage("en")}> English + i18n.changeLanguage("pt-BR")}> + Português (Brasil) + i18n.changeLanguage("zh")}> 简体中文 diff --git a/web/frontend/src/components/channels/channel-config-fields.ts b/web/frontend/src/components/channels/channel-config-fields.ts index 35356954b..cf8f50adf 100644 --- a/web/frontend/src/components/channels/channel-config-fields.ts +++ b/web/frontend/src/components/channels/channel-config-fields.ts @@ -14,6 +14,7 @@ export const SECRET_FIELD_MAP = { encrypt_key: "_encrypt_key", verification_token: "_verification_token", secret: "_secret", + username: "_username", password: "_password", nickserv_password: "_nickserv_password", sasl_password: "_sasl_password", @@ -33,6 +34,7 @@ const CHANNEL_SECRET_FIELDS: Record = { pico: ["token"], matrix: ["access_token"], irc: ["password", "nickserv_password", "sasl_password"], + mqtt: ["username", "password"], } const SECRET_FIELD_SET = new Set(Object.keys(SECRET_FIELD_MAP)) diff --git a/web/frontend/src/components/channels/channel-config-page.tsx b/web/frontend/src/components/channels/channel-config-page.tsx index d253980f8..8a8300d08 100644 --- a/web/frontend/src/components/channels/channel-config-page.tsx +++ b/web/frontend/src/components/channels/channel-config-page.tsx @@ -24,6 +24,7 @@ import { getChannelDisplayName } from "@/components/channels/channel-display-nam import { DiscordForm } from "@/components/channels/channel-forms/discord-form" import { FeishuForm } from "@/components/channels/channel-forms/feishu-form" import { GenericForm } from "@/components/channels/channel-forms/generic-form" +import { MqttForm } from "@/components/channels/channel-forms/mqtt-form" import { SlackForm } from "@/components/channels/channel-forms/slack-form" import { TelegramForm } from "@/components/channels/channel-forms/telegram-form" import { WecomForm } from "@/components/channels/channel-forms/wecom-form" @@ -215,6 +216,8 @@ function isConfigured( ) case "irc": return hasValue("server") + case "mqtt": + return hasValue("broker") && hasValue("agent_id") default: return false } @@ -250,6 +253,8 @@ function getRequiredFieldKeys(channelName: string): string[] { return ["homeserver", "user_id", "access_token"] case "irc": return ["server"] + case "mqtt": + return ["broker", "agent_id"] default: return [] } @@ -279,6 +284,7 @@ const CHANNELS_WITHOUT_DOCS = new Set([ "irc", "whatsapp", "whatsapp_native", + "mqtt", ]) export function ChannelConfigPage({ channelName }: ChannelConfigPageProps) { @@ -618,6 +624,15 @@ export function ChannelConfigPage({ channelName }: ChannelConfigPageProps) { arrayFieldResetVersion={arrayFieldResetVersion} /> ) + case "mqtt": + return ( + + ) case "weixin": return ( void + configuredSecrets: string[] + fieldErrors?: Record +} + +function asString(value: unknown): string { + return typeof value === "string" ? value : "" +} + +function asNumber(value: unknown): string { + if (typeof value === "number") return String(value) + if (typeof value === "string" && value !== "") return value + return "" +} + +function CodeLine({ children }: { children: string }) { + return ( + + {children} + + ) +} + +export function MqttForm({ + config, + onChange, + configuredSecrets, + fieldErrors = {}, +}: MqttFormProps) { + const { t } = useTranslation() + const prefix = asString(config.topic_prefix) || "/picoclaw" + const agentID = asString(config.agent_id) || "{agent_id}" + const topicBase = `${prefix}/${agentID}/{client_id}` + + return ( +
+ + + + onChange("broker", e.target.value)} + placeholder="mqtt://broker.example.com:1883" + /> + + + + onChange("agent_id", e.target.value)} + placeholder="my-agent" + /> + + + + onChange("topic_prefix", e.target.value)} + placeholder="/picoclaw" + /> + + + + + + + + onChange("_username", v)} + placeholder={getSecretInputPlaceholder( + configuredSecrets, + "username", + t("channels.mqtt.secretSet"), + t("channels.mqtt.secretEmpty"), + )} + /> + + + + onChange("_password", v)} + placeholder={getSecretInputPlaceholder( + configuredSecrets, + "password", + t("channels.mqtt.secretSet"), + t("channels.mqtt.secretEmpty"), + )} + /> + + + + + + + + onChange("client_id", e.target.value)} + placeholder={t("channels.mqtt.clientIdPlaceholder")} + /> + + + + onChange("keep_alive", Number(e.target.value))} + placeholder="60" + /> + + + + onChange("qos", Number(e.target.value))} + placeholder="0" + /> + + + + + + + + {t("channels.mqtt.protocolTitle")} + + + {t("channels.mqtt.protocolDesc")} + + + +
+

+ {t("channels.mqtt.uplink")} +

+ {`${topicBase}/request`} + 
+              {`{\n  "text": "your message"\n}`}
+
+
+

+ + {t("channels.mqtt.fieldText")} + + {" — "} + {t("channels.mqtt.uplinkTextDesc")} +

+
+
+ +
+

+ {t("channels.mqtt.downlink")} +

+ {`${topicBase}/response`} + 
+              {`{\n  "text": "agent response"\n}`}
+
+
+

+ + {t("channels.mqtt.fieldText")} + + {" — "} + {t("channels.mqtt.downlinkTextDesc")} +

+
+
+ +
+

+ {t("channels.mqtt.topicParams")} +

+
+

+ + {prefix} + + {" — "} + {t("channels.mqtt.topicPrefixDesc")} +

+

+ + {agentID} + + {" — "} + {t("channels.mqtt.agentIdDesc")} +

+

+ + {"{client_id}"} + + {" — "} + {t("channels.mqtt.clientIdDesc")} +

+
+
+ + +
+ ) +} diff --git a/web/frontend/src/components/chat/assistant-message.tsx b/web/frontend/src/components/chat/assistant-message.tsx index f3969ea37..a5ad9ce20 100644 --- a/web/frontend/src/components/chat/assistant-message.tsx +++ b/web/frontend/src/components/chat/assistant-message.tsx @@ -56,11 +56,38 @@ export function AssistantMessage({ const formattedTimestamp = timestamp !== "" ? formatMessageTime(timestamp) : "" - const handleCopy = () => { - navigator.clipboard.writeText(content).then(() => { + const handleCopy = async () => { + const markCopied = () => { setIsCopied(true) setTimeout(() => setIsCopied(false), 2000) - }) + } + + try { + if (navigator.clipboard?.writeText) { + await navigator.clipboard.writeText(content) + markCopied() + return + } + } catch { + // HTTP 或受限环境下可能不支持 Clipboard API，继续走降级方案 + } + + const textArea = document.createElement("textarea") + textArea.value = content + textArea.setAttribute("readonly", "") + textArea.style.position = "fixed" + textArea.style.left = "-9999px" + document.body.appendChild(textArea) + textArea.select() + + try { + const copied = document.execCommand("copy") + if (copied) { + markCopied() + } + } finally { + document.body.removeChild(textArea) + } } const collapsedLabel = isThought diff --git a/web/frontend/src/components/chat/chat-composer.tsx b/web/frontend/src/components/chat/chat-composer.tsx index b3354cc33..569ed21e4 100644 --- a/web/frontend/src/components/chat/chat-composer.tsx +++ b/web/frontend/src/components/chat/chat-composer.tsx @@ -128,7 +128,10 @@ export function ChatComposer({
{contextUsage && ( - + )} {canInput ? ( diff --git a/web/frontend/src/components/chat/context-usage-ring.tsx b/web/frontend/src/components/chat/context-usage-ring.tsx index 4a32e617b..037a20cef 100644 --- a/web/frontend/src/components/chat/context-usage-ring.tsx +++ b/web/frontend/src/components/chat/context-usage-ring.tsx @@ -127,7 +127,7 @@ export function ContextUsageRing({ : "pointer-events-none scale-95 opacity-0" }`} > -
+
diff --git a/web/frontend/src/components/config/config-page.tsx b/web/frontend/src/components/config/config-page.tsx index cf9b61dad..893cad6e7 100644 --- a/web/frontend/src/components/config/config-page.tsx +++ b/web/frontend/src/components/config/config-page.tsx @@ -20,8 +20,10 @@ import { AgentDefaultsSection, CronSection, DevicesSection, + EvolutionSection, ExecSection, LauncherSection, + MCPSection, RuntimeSection, } from "@/components/config/config-sections" import { @@ -29,9 +31,12 @@ import { EMPTY_FORM, EMPTY_LAUNCHER_FORM, type LauncherForm, + type MCPServerForm, buildFormFromConfig, parseCIDRText, + parseFloatField, parseIntField, + parseJSONObjectField, parseMultilineList, } from "@/components/config/form-model" import { PageHeader } from "@/components/page-header" @@ -40,6 +45,21 @@ import { Button } from "@/components/ui/button" import { showSaveSuccessOrRestartToast } from "@/lib/restart-required" import { refreshGatewayState } from "@/store/gateway" +function buildStringMapMergePatch( + next: Record, + previous: Record, +): Record { + const patch: Record = { ...next } + + for (const key of Object.keys(previous)) { + if (!(key in next)) { + patch[key] = null + } + } + + return patch +} + export function ConfigPage() { const { t } = useTranslation() const queryClient = useQueryClient() @@ -143,6 +163,44 @@ export function ConfigPage() { setLauncherForm((prev) => ({ ...prev, [key]: value })) } + const handleMCPServerAdd = () => { + const nextIndex = form.mcpServers.length + 1 + const server: MCPServerForm = { + id: `mcp-${Date.now()}-${nextIndex}`, + name: "", + enabled: true, + deferredOverride: null, + type: "stdio", + url: "", + command: "", + argsText: "", + envText: "{}", + envFile: "", + headersText: "{}", + } + updateField("mcpServers", [...form.mcpServers, server]) + } + + const handleMCPServerRemove = (id: string) => { + updateField( + "mcpServers", + form.mcpServers.filter((server) => server.id !== id), + ) + } + + const handleMCPServerFieldChange = ( + id: string, + key: K, + value: MCPServerForm[K], + ) => { + updateField( + "mcpServers", + form.mcpServers.map((server) => + server.id === id ? { ...server, [key]: value } : server, + ), + ) + } + const handleReset = () => { setForm(baseline) setLauncherForm(launcherBaseline) @@ -178,6 +236,17 @@ export function ConfigPage() { throw new Error("Session scope is required.") } + if ( + form.mcpEnabled && + form.mcpDiscoveryEnabled && + !form.mcpDiscoveryUseBM25 && + !form.mcpDiscoveryUseRegex + ) { + throw new Error( + "MCP discovery requires at least one search method (BM25 or regex).", + ) + } + const maxTokens = parseIntField(form.maxTokens, "Max tokens", { min: 1, }) @@ -214,10 +283,195 @@ export function ConfigPage() { "Cron exec timeout", { min: 0 }, ) + const evolutionMinTaskCount = parseIntField( + form.evolutionMinTaskCount, + "Evolution minimum task count", + { min: 1 }, + ) + const evolutionMinSuccessRatio = parseFloatField( + form.evolutionMinSuccessRatio, + "Evolution minimum success ratio", + { min: 0.01, max: 1 }, + ) + const mcpDiscoveryValidationEnabled = + form.mcpEnabled && form.mcpDiscoveryEnabled + const mcpDiscoveryPatch: Record = { + enabled: form.mcpDiscoveryEnabled, + use_bm25: form.mcpDiscoveryUseBM25, + use_regex: form.mcpDiscoveryUseRegex, + } + + if (mcpDiscoveryValidationEnabled) { + mcpDiscoveryPatch.ttl = parseIntField( + form.mcpDiscoveryTTL, + "MCP discovery ttl", + { + min: 1, + }, + ) + mcpDiscoveryPatch.max_search_results = parseIntField( + form.mcpDiscoveryMaxSearchResults, + "MCP discovery max search results", + { min: 1 }, + ) + } const execConfigPatch: Record = { enabled: form.execEnabled, } + let mcpServersPatch: Record | null> = {} + if (form.mcpEnabled) { + const baselineServerNames = new Set( + baseline.mcpServers + .map((server) => server.name.trim()) + .filter((name) => name !== ""), + ) + + const normalizedServers = form.mcpServers + .map((server) => ({ + ...server, + name: server.name.trim(), + url: server.url.trim(), + command: server.command.trim(), + envFile: server.envFile.trim(), + })) + .filter((server) => server.name !== "") + + const serverNameCounts = new Map() + for (const server of normalizedServers) { + serverNameCounts.set( + server.name, + (serverNameCounts.get(server.name) ?? 0) + 1, + ) + } + + const duplicateNames = Array.from(serverNameCounts.entries()) + .filter(([, count]) => count > 1) + .map(([name]) => name) + .sort((a, b) => a.localeCompare(b)) + + if (duplicateNames.length > 0) { + throw new Error( + `MCP server names must be unique. Duplicates: ${duplicateNames.join(", ")}.`, + ) + } + + const currentServerNames = new Set( + normalizedServers.map((server) => server.name), + ) + + const removedServerEntries = Array.from(baselineServerNames) + .filter((name) => !currentServerNames.has(name)) + .map((name) => [name, null] as const) + + const baselineServersByName = new Map( + baseline.mcpServers + .map((server) => ({ + ...server, + name: server.name.trim(), + })) + .filter((server) => server.name !== "") + .map((server) => [server.name, server] as const), + ) + + const upsertServerEntries = normalizedServers.map((server) => { + const deferredPatch = { deferred: server.deferredOverride } + const baselineServer = baselineServersByName.get(server.name) + const shouldValidateServer = server.enabled + + if (server.type !== "stdio") { + if (shouldValidateServer && server.url === "") { + throw new Error(`MCP server ${server.name} requires a URL.`) + } + + if (shouldValidateServer) { + try { + const parsedURL = new URL(server.url) + if ( + parsedURL.protocol !== "http:" && + parsedURL.protocol !== "https:" + ) { + throw new Error("invalid protocol") + } + } catch { + throw new Error( + `MCP server ${server.name} requires a valid HTTP(S) URL.`, + ) + } + } + + const baselineHeaders = baselineServer + ? parseJSONObjectField( + baselineServer.headersText, + `Saved MCP server ${server.name} headers`, + ) + : {} + + return [ + server.name, + { + ...deferredPatch, + enabled: server.enabled, + type: server.type, + url: server.url, + headers: buildStringMapMergePatch( + shouldValidateServer + ? parseJSONObjectField( + server.headersText, + `MCP server ${server.name} headers`, + ) + : baselineHeaders, + baselineHeaders, + ), + command: null, + args: null, + env: null, + env_file: null, + }, + ] as const + } + + if (shouldValidateServer && server.command === "") { + throw new Error(`MCP server ${server.name} requires a command.`) + } + + const baselineEnv = baselineServer + ? parseJSONObjectField( + baselineServer.envText, + `Saved MCP server ${server.name} env`, + ) + : {} + + return [ + server.name, + { + ...deferredPatch, + enabled: server.enabled, + type: "stdio", + command: server.command, + args: parseMultilineList(server.argsText), + env: buildStringMapMergePatch( + shouldValidateServer + ? parseJSONObjectField( + server.envText, + `MCP server ${server.name} env`, + ) + : baselineEnv, + baselineEnv, + ), + env_file: server.envFile === "" ? null : server.envFile, + url: null, + headers: null, + }, + ] as const + }) + + mcpServersPatch = Object.fromEntries([ + ...upsertServerEntries, + ...removedServerEntries, + ]) + } + if (form.execEnabled) { execConfigPatch.allow_remote = form.allowRemote execConfigPatch.enable_deny_patterns = form.enableDenyPatterns @@ -259,12 +513,31 @@ export function ConfigPage() { session: { dm_scope: dmScope, }, + evolution: { + enabled: form.evolutionEnabled, + mode: form.evolutionMode, + state_dir: + form.evolutionStateDir.trim() === "" + ? null + : form.evolutionStateDir.trim(), + min_task_count: evolutionMinTaskCount, + min_success_ratio: evolutionMinSuccessRatio, + cold_path_trigger: form.evolutionColdPathTrigger, + cold_path_times: parseMultilineList( + form.evolutionColdPathTimesText, + ), + }, tools: { cron: { allow_command: form.allowCommand, exec_timeout_minutes: cronExecTimeoutMinutes, }, exec: execConfigPatch, + mcp: { + enabled: form.mcpEnabled, + discovery: mcpDiscoveryPatch, + servers: mcpServersPatch, + }, }, heartbeat: { enabled: form.heartbeatEnabled, @@ -415,6 +688,16 @@ export function ConfigPage() { + + + + diff --git a/web/frontend/src/components/config/config-sections.tsx b/web/frontend/src/components/config/config-sections.tsx index 2b0ee2029..185a2dc60 100644 --- a/web/frontend/src/components/config/config-sections.tsx +++ b/web/frontend/src/components/config/config-sections.tsx @@ -1,3 +1,4 @@ +import { IconPlus, IconTrash } from "@tabler/icons-react" import { useState } from "react" import type { ReactNode } from "react" import { useTranslation } from "react-i18next" @@ -6,6 +7,8 @@ import { type CoreConfigForm, DM_SCOPE_OPTIONS, type LauncherForm, + type MCPServerForm, + type MCPServerType, } from "@/components/config/form-model" import { Field, SwitchCardField } from "@/components/shared-form" import { Button } from "@/components/ui/button" @@ -231,6 +234,489 @@ interface ExecSectionProps { onFieldChange: UpdateCoreField } +interface MCPSectionProps { + form: CoreConfigForm + onFieldChange: UpdateCoreField + onAddServer: () => void + onRemoveServer: (id: string) => void + onServerFieldChange: ( + id: string, + key: K, + value: MCPServerForm[K], + ) => void +} + +interface EvolutionSectionProps { + form: CoreConfigForm + onFieldChange: UpdateCoreField +} + +export function EvolutionSection({ + form, + onFieldChange, +}: EvolutionSectionProps) { + const { t } = useTranslation() + + return ( + + + onFieldChange("evolutionEnabled", checked) + } + /> + + + + + + + onFieldChange("evolutionStateDir", e.target.value)} + placeholder="e.g. /var/lib/picoclaw/evolution" + /> + + + + + onFieldChange("evolutionMinTaskCount", e.target.value) + } + /> + + + + + onFieldChange("evolutionMinSuccessRatio", e.target.value) + } + /> + + + + + + + {form.evolutionColdPathTrigger === "scheduled" && ( + +