21b5e6b0d4
- Introduced a new design document for research backend integration, outlining scope, architecture, and API specifications. - Added frontend API service functions for managing research agents, nodes, and reports. - Extended existing Go backend packages to support new research functionalities. feat: Implement PicoClaw context window improvements - Created a design document detailing seven context-window improvements to enhance performance and prevent errors. - Improvements include token estimation fixes, structured error handling, and proactive context management strategies. fix: Document known issues and resolutions - Added a known issues document detailing a critical compile error related to invalid Unicode escapes in Go, including the fix applied. docs: Add context management skill documentation - Introduced a new skill for context management, detailing usage scenarios, project mapping, state persistence, and integration with PicoClaw.
102 lines
3.1 KiB
Go
102 lines
3.1 KiB
Go
package tokenizer
|
|
|
|
import (
|
|
"encoding/json"
|
|
"unicode/utf8"
|
|
|
|
"github.com/sipeed/picoclaw/pkg/providers"
|
|
)
|
|
|
|
// EstimateMessageTokens estimates the token count for a single message,
|
|
// including Content, ReasoningContent, ToolCalls arguments, ToolCallID
|
|
// metadata, and Media items. Uses a heuristic of 2.5 characters per token.
|
|
func EstimateMessageTokens(msg providers.Message) int {
|
|
contentChars := utf8.RuneCountInString(msg.Content)
|
|
|
|
// SystemParts are structured system blocks used for cache-aware adapters.
|
|
// They carry the same content as Content, but in multiple blocks.
|
|
// We estimate them as an alternative representation, not additive.
|
|
systemPartsChars := 0
|
|
if len(msg.SystemParts) > 0 {
|
|
for _, part := range msg.SystemParts {
|
|
systemPartsChars += utf8.RuneCountInString(part.Text)
|
|
}
|
|
// Per-part overhead for JSON structure (type, text, cache_control).
|
|
const perPartOverhead = 20
|
|
systemPartsChars += len(msg.SystemParts) * perPartOverhead
|
|
}
|
|
|
|
// Use the larger of the two representations to stay conservative.
|
|
chars := contentChars
|
|
if systemPartsChars > chars {
|
|
chars = systemPartsChars
|
|
}
|
|
|
|
chars += utf8.RuneCountInString(msg.ReasoningContent)
|
|
|
|
for _, tc := range msg.ToolCalls {
|
|
chars += len(tc.ID) + len(tc.Type)
|
|
if tc.Function != nil {
|
|
// Count function name + arguments (the wire format for most providers).
|
|
// tc.Name mirrors tc.Function.Name — count only once to avoid double-counting.
|
|
chars += len(tc.Function.Name) + len(tc.Function.Arguments)
|
|
} else {
|
|
// Fallback: some provider formats use top-level Name without Function.
|
|
chars += len(tc.Name)
|
|
}
|
|
}
|
|
|
|
if msg.ToolCallID != "" {
|
|
chars += len(msg.ToolCallID)
|
|
}
|
|
|
|
// Per-message overhead for role label, JSON structure, separators.
|
|
const messageOverhead = 12
|
|
chars += messageOverhead
|
|
|
|
// Use 4 characters per token (conservative estimate, matching OpenCode's approach).
|
|
// The previous 2.5 chars/token was too optimistic and led to context
|
|
// window overflow errors. English text typically uses ~4 chars/token.
|
|
tokens := chars / 4
|
|
if chars%4 != 0 {
|
|
tokens++ // Round up
|
|
}
|
|
|
|
// Media items (images, files) are serialized by provider adapters into
|
|
// multipart or image_url payloads. Add a fixed per-item token estimate
|
|
// directly (not through the chars heuristic) since actual cost depends
|
|
// on resolution and provider-specific image tokenization.
|
|
const mediaTokensPerItem = 256
|
|
tokens += len(msg.Media) * mediaTokensPerItem
|
|
|
|
return tokens
|
|
}
|
|
|
|
// EstimateToolDefsTokens estimates the total token cost of tool definitions
|
|
// as they appear in the LLM request.
|
|
func EstimateToolDefsTokens(defs []providers.ToolDefinition) int {
|
|
if len(defs) == 0 {
|
|
return 0
|
|
}
|
|
|
|
totalChars := 0
|
|
for _, d := range defs {
|
|
totalChars += len(d.Function.Name) + len(d.Function.Description)
|
|
|
|
if d.Function.Parameters != nil {
|
|
if paramJSON, err := json.Marshal(d.Function.Parameters); err == nil {
|
|
totalChars += len(paramJSON)
|
|
}
|
|
}
|
|
|
|
// Per-tool overhead: type field, JSON structure, separators.
|
|
totalChars += 20
|
|
}
|
|
|
|
// Use 4 characters per token (conservative estimate, matching OpenCode)
|
|
result := totalChars / 4
|
|
if totalChars%4 != 0 {
|
|
result++ // Round up
|
|
}
|
|
return result
|
|
}
|