Merge pull request #1466 from trheyi/main

Enhance tool inspection and remove studio configuration
This commit is contained in:
Max 2026-02-14 21:59:24 +08:00 committed by GitHub
commit 6c04ef9b89
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 419 additions and 78 deletions

1
.gitignore vendored
View file

@ -65,3 +65,4 @@ sandbox/proxy/claude-proxy-linux-*
release/*
sandbox/TODO-VNC.md
sandbox/docker/chrome/PLAN.md
sandbox/DESIGN-REMOTE.md

View file

@ -125,6 +125,7 @@ func ToMap(caps *openai.Capabilities) map[string]interface{} {
}
result["audio"] = caps.Audio
result["stt"] = caps.STT
result["tool_calls"] = caps.ToolCalls
result["reasoning"] = caps.Reasoning
result["streaming"] = caps.Streaming
@ -144,6 +145,7 @@ func convertAnthropicCaps(caps *anthropic.Capabilities) *openai.Capabilities {
return &openai.Capabilities{
Vision: caps.Vision,
Audio: caps.Audio,
STT: caps.STT,
ToolCalls: caps.ToolCalls,
Reasoning: caps.Reasoning,
Streaming: caps.Streaming,

220
agent/llm/process.go Normal file
View file

@ -0,0 +1,220 @@
package llm
import (
"context"
"encoding/json"
"fmt"
"github.com/yaoapp/gou/connector"
gouHTTP "github.com/yaoapp/gou/http"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/gou/runtime/v8/bridge"
agentContext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/output/message"
"github.com/yaoapp/yao/openapi/oauth/authorized"
)
func init() {
process.Register("llm.ChatCompletions", ProcessChatCompletions)
}
// ProcessChatCompletions implements the llm.ChatCompletions Process.
// A universal replacement for openai.chat.Completions that auto-detects
// connector type (openai, anthropic, etc.) and routes accordingly.
//
// Usage:
//
// Process("llm.ChatCompletions", connector, messages)
// Process("llm.ChatCompletions", connector, messages, opts)
// Process("llm.ChatCompletions", connector, messages, opts, callback)
//
// Args:
// - connector (string): Connector ID, any type (openai / anthropic / ...)
// - messages ([]map): Message array, supports multimodal content (image_url, etc.)
// - opts (map): Optional. temperature, max_tokens, etc.
// - callback (func): Optional. Streaming callback func(data []byte) int
//
// Returns: OpenAI-compatible format { choices: [{ message: { role, content } }], ... }
func ProcessChatCompletions(p *process.Process) interface{} {
p.ValidateArgNums(2)
// 1. Parse connector ID
connectorID := p.ArgsString(0)
if connectorID == "" {
return newErrorResponse("llm.ChatCompletions: connector is required")
}
// 2. Parse messages
rawMessages := p.ArgsArray(1)
messages := make([]map[string]interface{}, 0, len(rawMessages))
for i, v := range rawMessages {
msg, ok := v.(map[string]interface{})
if !ok {
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: message %d is not an object", i))
}
messages = append(messages, msg)
}
// 3. Parse optional opts
var opts map[string]interface{}
if p.NumOfArgs() > 2 && p.Args[2] != nil {
if o, ok := p.Args[2].(map[string]interface{}); ok {
opts = o
}
}
// 4. Parse optional callback (for streaming)
var callback func(data []byte) int
if p.NumOfArgs() > 3 && p.Args[3] != nil {
switch cb := p.Args[3].(type) {
case func(data []byte) int:
callback = cb
case bridge.FunctionT:
callback = func(data []byte) int {
v, err := cb.Call(string(data))
if err != nil {
return gouHTTP.HandlerReturnError
}
ret, ok := v.(int)
if !ok {
return gouHTTP.HandlerReturnError
}
return ret
}
}
}
// 5. Select connector
conn, err := connector.Select(connectorID)
if err != nil {
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: connector %s not found: %v", connectorID, err))
}
// 6. Build completion options (reuse jsapi.go logic)
completionOptions := buildCompletionOptions(conn, opts)
// 7. Create LLM instance (auto-selects openai/anthropic provider)
llmInstance, err := New(conn, completionOptions)
if err != nil {
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: failed to create LLM: %v", err))
}
// 8. Parse messages to context.Message format (reuse jsapi.go logic)
interfaceMessages := make([]interface{}, len(messages))
for i, m := range messages {
interfaceMessages[i] = m
}
ctxMessages, err := parseMessages(interfaceMessages)
if err != nil {
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: invalid messages: %v", err))
}
// 8.1 Normalize multimodal content: convert []interface{} maps to []ContentPart
// so that providers (especially Anthropic) can type-assert correctly.
for i := range ctxMessages {
if parts, ok := ctxMessages[i].Content.([]interface{}); ok {
ctxMessages[i].Content = normalizeContentParts(parts)
}
}
// 9. Build a minimal headless context for LLM call
parent := p.Context
if parent == nil {
parent = context.Background()
}
authInfo := authorized.ProcessAuthInfo(p)
chatID := agentContext.GenChatID()
ctx := agentContext.New(parent, authInfo, chatID)
defer ctx.Release()
// 10. Create stream handler
var streamHandler message.StreamFunc
if callback != nil {
// With callback: forward raw chunks to caller
streamHandler = func(chunkType message.StreamChunkType, data []byte) int {
if chunkType == message.ChunkText || chunkType == message.ChunkThinking {
return callback(data)
}
return 0
}
} else {
// No callback: no-op handler, just collect final response
streamHandler = func(chunkType message.StreamChunkType, data []byte) int {
return 0
}
}
// 11. Execute LLM stream call
response, err := llmInstance.Stream(ctx, ctxMessages, completionOptions, streamHandler)
if err != nil {
return newErrorResponse(fmt.Sprintf("llm.ChatCompletions: LLM call failed: %v", err))
}
// 12. Convert CompletionResponse to OpenAI-compatible format
// { choices: [{ message: { role, content } }], id, model, ... }
return toOpenAIFormat(response)
}
// toOpenAIFormat converts CompletionResponse to OpenAI chat.completions format
// for backward compatibility with code that consumed openai.chat.Completions.
func toOpenAIFormat(resp *agentContext.CompletionResponse) map[string]interface{} {
if resp == nil {
return map[string]interface{}{
"choices": []interface{}{},
}
}
msgMap := map[string]interface{}{
"role": resp.Role,
"content": resp.Content,
}
if len(resp.ToolCalls) > 0 {
msgMap["tool_calls"] = resp.ToolCalls
}
choice := map[string]interface{}{
"index": 0,
"message": msgMap,
"finish_reason": "stop",
}
result := map[string]interface{}{
"id": resp.ID,
"object": "chat.completion",
"created": resp.Created,
"model": resp.Model,
"choices": []interface{}{choice},
}
if resp.Usage != nil {
result["usage"] = resp.Usage
}
return result
}
// newErrorResponse creates an error response in OpenAI-compatible format
func newErrorResponse(errMsg string) map[string]interface{} {
return map[string]interface{}{
"error": map[string]interface{}{
"message": errMsg,
"type": "invalid_request_error",
},
}
}
// normalizeContentParts converts []interface{} (raw maps from Process args)
// to []agentContext.ContentPart (strongly typed) via JSON round-trip.
// This is essential for providers (e.g. Anthropic) that type-assert on
// []ContentPart to apply format-specific conversions (image_url → image).
func normalizeContentParts(parts []interface{}) []agentContext.ContentPart {
raw, err := json.Marshal(parts)
if err != nil {
return nil
}
var typed []agentContext.ContentPart
if err := json.Unmarshal(raw, &typed); err != nil {
return nil
}
return typed
}

View file

@ -5,6 +5,7 @@ import (
"github.com/yaoapp/kun/maps"
"github.com/yaoapp/kun/utils"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/engine"
"github.com/yaoapp/yao/share"
)
@ -14,10 +15,14 @@ var inspectCmd = &cobra.Command{
Long: L("Show app configure"),
Run: func(cmd *cobra.Command, args []string) {
Boot()
engine.InspectExtTools()
res := maps.Map{
"version": share.VERSION,
"config": config.Conf,
}
if share.Tools != nil {
res["tools"] = share.Tools
}
utils.Dump(res)
},
}

View file

@ -104,18 +104,6 @@ var rootCmd = &cobra.Command{
},
}
var studioCmd = &cobra.Command{
Use: "studio",
Short: "Yao Studio CLI",
Long: `Yao Studio CLI`,
CompletionOptions: cobra.CompletionOptions{
DisableDefaultCmd: true,
},
Run: func(cmd *cobra.Command, args []string) {
cmd.Help()
},
}
var suiCmd = &cobra.Command{
Use: "sui",
Short: L("SUI Template Engine"),
@ -143,8 +131,6 @@ var agentCmd = &cobra.Command{
// Command initialize
func init() {
// studioCmd.AddCommand(studio.RunCmd)
// Sui
suiCmd.AddCommand(sui.WatchCmd)
suiCmd.AddCommand(sui.BuildCmd)
@ -166,7 +152,6 @@ func init() {
// socketCmd,
// websocketCmd,
// packCmd,
// studioCmd,
suiCmd,
agentCmd,
// upgradeCmd,

View file

@ -169,7 +169,6 @@ var startCmd = &cobra.Command{
printConnectors(false)
printStores(false)
printMCPs(false)
// printStudio(false, host)
}
@ -398,29 +397,6 @@ func printStores(silent bool) {
}
}
func printStudio(silent bool, host string) {
if silent {
log.Info("[Studio] http://%s:%d", host, config.Conf.Studio.Port)
if config.Conf.Studio.Auto {
log.Info("[Studio] Secret: %s", config.Conf.Studio.Secret)
}
return
}
fmt.Println(color.WhiteString("\n---------------------------------"))
fmt.Println(color.WhiteString(L("Yao Studio Server")))
fmt.Println(color.WhiteString("---------------------------------"))
fmt.Print(color.CyanString("HOST : "))
fmt.Print(color.WhiteString(" %s\n", config.Conf.Host))
fmt.Print(color.CyanString("PORT : "))
fmt.Print(color.WhiteString(" %d\n", config.Conf.Studio.Port))
if config.Conf.Studio.Auto {
fmt.Print(color.CyanString("SECRET: "))
fmt.Print(color.WhiteString(" %s\n", config.Conf.Studio.Secret))
}
}
func printSchedules(silent bool) {
if len(schedule.Schedules) == 0 {

View file

@ -5,15 +5,12 @@ import (
"io"
"os"
"path/filepath"
"strings"
"github.com/caarlos0/env/v6"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/joho/godotenv"
"github.com/yaoapp/kun/exception"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/crypto"
"gopkg.in/natefinch/lumberjack.v2"
)
@ -138,16 +135,6 @@ func LoadWithRoot(root string) Config {
cfg.AppSource = cfg.Root
}
// Studio Secret
if cfg.Studio.Secret == "" {
v, err := crypto.Hash(crypto.HashTypes["SHA256"], uuid.New().String())
if err != nil {
exception.New("Can't gengrate studio secret %s", 500, err.Error()).Throw()
}
cfg.Studio.Secret = strings.ToUpper(v)
cfg.Studio.Auto = true
}
// DataRoot
if cfg.DataRoot == "" {
cfg.DataRoot = filepath.Join(cfg.Root, "data")

View file

@ -23,18 +23,10 @@ type Config struct {
DB Database `json:"db,omitempty"` // The database config
AllowFrom []string `json:"allowfrom,omitempty" envSeparator:"|" env:"YAO_ALLOW_FROM"` // Domain list the separator is |
Session Session `json:"session,omitempty"` // Session Config
Studio Studio `json:"studio,omitempty"` // Studio config
Runtime Runtime `json:"runtime,omitempty"` // Runtime config
Trace Trace `json:"trace,omitempty"` // Trace config
}
// Studio the studio config
type Studio struct {
Port int `json:"studio_port,omitempty" env:"YAO_STUDIO_PORT" envDefault:"5077"` // Studio port
Secret string `json:"studio_secret,omitempty" env:"YAO_STUDIO_SECRET"` // Studio Secret, if does not set, auto-generate a secret
Auto bool `json:"-"`
}
// Database 数据库配置
type Database struct {
Driver string `json:"driver,omitempty" env:"YAO_DB_DRIVER" envDefault:"sqlite3"` // 数据库驱动 sqlite3| mysql| postgres

View file

@ -4,12 +4,15 @@ import (
"fmt"
"log"
"os"
"os/exec"
"regexp"
"strings"
"time"
"github.com/fatih/color"
"github.com/yaoapp/gou/application"
"github.com/yaoapp/gou/ffmpeg"
"github.com/yaoapp/gou/pdf"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/kun/exception"
"github.com/yaoapp/yao/agent"
@ -139,6 +142,12 @@ func Load(cfg config.Config, options LoadOption, progressCallback ...func(string
warnings = append(warnings, Warning{Widget: "Connector", Error: err})
}
// Inspect external tools (silent, non-fatal)
loadStep("ExtTools", func() error {
InspectExtTools()
return nil
}, callback)
// Load FileSystem
err = loadStep("FileSystem", func() error {
return fs.Load(cfg)
@ -755,6 +764,146 @@ func loadApp(root string) error {
return nil
}
// InspectExtTools performs silent detection of external tools (ffmpeg, pdf converters, docker)
// and stores the results in share.Tools for later access via Inspect / settings API.
// Exported so it can be called from both engine.Load() and cmd/inspect.go.
func InspectExtTools() {
tools := &share.ExtTools{}
// Inspect ffmpeg/ffprobe
ffmpegStatus := ffmpeg.Inspect()
if s, ok := ffmpegStatus["ffmpeg"]; ok {
tools.FFmpeg = &share.ExtToolInfo{
Name: s.Name,
Available: s.Available,
Path: s.Path,
Version: s.Version,
EnvVar: s.EnvVar,
Error: s.Error,
}
}
if s, ok := ffmpegStatus["ffprobe"]; ok {
tools.FFprobe = &share.ExtToolInfo{
Name: s.Name,
Available: s.Available,
Path: s.Path,
Version: s.Version,
EnvVar: s.EnvVar,
Error: s.Error,
}
}
// Inspect PDF tools
pdfStatus := pdf.Inspect()
if s, ok := pdfStatus["pdftoppm"]; ok {
tools.Pdftoppm = &share.ExtToolInfo{
Name: s.Name,
Available: s.Available,
Path: s.Path,
Version: s.Version,
EnvVar: s.EnvVar,
Error: s.Error,
}
}
if s, ok := pdfStatus["mutool"]; ok {
tools.Mutool = &share.ExtToolInfo{
Name: s.Name,
Available: s.Available,
Path: s.Path,
Version: s.Version,
EnvVar: s.EnvVar,
Error: s.Error,
}
}
if s, ok := pdfStatus["imagemagick"]; ok {
tools.ImageMagick = &share.ExtToolInfo{
Name: s.Name,
Available: s.Available,
Path: s.Path,
Version: s.Version,
EnvVar: s.EnvVar,
Error: s.Error,
}
}
// Inspect Docker
tools.Docker = inspectDocker()
share.Tools = tools
}
// inspectDocker silently detects Docker availability, version, and host configuration.
// Returns real runtime info: what Docker is actually connected to, not just env vars.
func inspectDocker() *share.DockerInfo {
sandboxHost := os.Getenv("YAO_SANDBOX_HOST")
sandboxMode := os.Getenv("YAO_SANDBOX_MODE")
info := &share.DockerInfo{}
// Try to find docker CLI path
dockerPath, err := exec.LookPath("docker")
if err != nil {
info.Available = false
info.Error = "docker not found in PATH"
return info
}
info.Path = dockerPath
// Get real Docker context info: actual daemon address from docker info
cmd := exec.Command(dockerPath, "info", "--format", "{{.ClientInfo.Context}}")
ctxOutput, _ := cmd.Output()
dockerContext := strings.TrimSpace(string(ctxOutput))
// Get the actual daemon host from docker context inspect
var actualHost string
if dockerContext != "" {
cmd = exec.Command(dockerPath, "context", "inspect", dockerContext, "--format", "{{.Endpoints.docker.Host}}")
hostOutput, err := cmd.Output()
if err == nil {
actualHost = strings.TrimSpace(string(hostOutput))
}
}
// Fallback: DOCKER_HOST env or let docker figure it out
if actualHost == "" {
actualHost = os.Getenv("DOCKER_HOST")
}
info.Host = actualHost
// Get docker server version (also verifies daemon is reachable)
cmd = exec.Command(dockerPath, "version", "--format", "{{.Server.Version}}")
output, err := cmd.Output()
if err != nil {
info.Available = false
if actualHost != "" {
info.Error = fmt.Sprintf("docker daemon not reachable at %s", actualHost)
} else {
info.Error = "docker daemon not reachable"
}
return info
}
info.Available = true
info.Version = strings.TrimSpace(string(output))
// Determine sandbox mode from real host
if sandboxMode != "" {
info.Mode = sandboxMode
} else if actualHost == "" || strings.HasPrefix(actualHost, "unix://") || strings.HasPrefix(actualHost, "ssh://") || strings.HasPrefix(actualHost, "npipe://") {
info.Mode = "local"
} else if strings.HasPrefix(actualHost, "tcp://") {
info.Mode = "remote"
} else {
info.Mode = "local"
}
// Yao sandbox env vars
info.EnvVars = map[string]string{
"YAO_SANDBOX_HOST": sandboxHost,
"YAO_SANDBOX_MODE": sandboxMode,
}
return info
}
func printErr(mode, widget string, err error) {
message := fmt.Sprintf("[%s] %s", widget, err.Error())
if !strings.Contains(message, "does not exists") && !strings.Contains(message, "no such file or directory") && mode == "development" {

View file

@ -38,12 +38,16 @@ func processPing(process *process.Process) interface{} {
// processInspect 返回系统信息
func processInspect(process *process.Process) interface{} {
return map[string]interface{}{
result := map[string]interface{}{
"VERSION": fmt.Sprintf("%s %s", share.VERSION, share.PRVERSION),
"CUI": fmt.Sprintf("%s %s", share.CUI, share.PRCUI),
"BUILDNAME": share.BUILDNAME,
"CONFIG": config.Conf,
}
if share.Tools != nil {
result["TOOLS"] = share.Tools
}
return result
}
// processFavicon 运行模型 MustCreate

View file

@ -126,7 +126,8 @@ func getCapabilitiesWithModels(conn connector.Connector, modelCapabilities map[s
// matchesFilters checks if capabilities match all requested filters
// Filters are matched case-insensitively and support the following capability keys:
// - vision: true or string value like "openai", "claude"
// - audio: bool
// - audio: bool (LLM supports audio input/understanding)
// - stt: bool (Speech-to-Text / audio transcription model, e.g. Whisper)
// - tool_calls: bool
// - reasoning: bool
// - streaming: bool

View file

@ -151,3 +151,38 @@ type AppRoot struct {
Screens string
Data string
}
// ExtToolInfo represents the detection result for a single external tool.
type ExtToolInfo struct {
Name string `json:"name"` // Tool name (e.g. "ffmpeg", "pdftoppm")
Available bool `json:"available"` // Whether the tool is available
Path string `json:"path,omitempty"` // Resolved executable path
Version string `json:"version,omitempty"` // Version string
EnvVar string `json:"env_var,omitempty"` // Environment variable name for custom path override
Error string `json:"error,omitempty"` // Error message if not available
}
// DockerInfo represents the detection result for Docker.
type DockerInfo struct {
Available bool `json:"available"` // Whether Docker daemon is reachable
Path string `json:"path,omitempty"` // Docker CLI path
Version string `json:"version,omitempty"` // Docker server version
Mode string `json:"mode,omitempty"` // "local" or "remote"
Host string `json:"host,omitempty"` // DOCKER_HOST value (empty = local socket)
Error string `json:"error,omitempty"` // Error message if not available
EnvVars map[string]string `json:"env_vars,omitempty"` // Related Yao environment variables
}
// ExtTools holds the detection results for all external tools.
// Populated during engine.Load() and exposed via utils.app.Inspect.
type ExtTools struct {
FFmpeg *ExtToolInfo `json:"ffmpeg"`
FFprobe *ExtToolInfo `json:"ffprobe"`
Pdftoppm *ExtToolInfo `json:"pdftoppm"`
Mutool *ExtToolInfo `json:"mutool"`
ImageMagick *ExtToolInfo `json:"imagemagick"`
Docker *DockerInfo `json:"docker"`
}
// Tools holds the global external tool detection results.
var Tools *ExtTools

View file

@ -10,7 +10,6 @@ import (
"github.com/yaoapp/kun/exception"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/kun/maps"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/helper"
"golang.org/x/crypto/bcrypt"
)
@ -113,20 +112,6 @@ func auth(field string, value string, password string, sid string) maps.Map {
session.Global().Expire(time.Duration(token.ExpiresAt)*time.Second).ID(sid).Set("user", row)
session.Global().Expire(time.Duration(token.ExpiresAt)*time.Second).ID(sid).Set("issuer", "yao")
studio := map[string]interface{}{}
if config.Conf.Mode == "development" {
studioToken := helper.JwtMake(id, map[string]interface{}{}, map[string]interface{}{
"expires_at": expiresAt,
"sid": sid,
"issuer": "yao",
}, []byte(config.Conf.Studio.Secret))
studio["port"] = config.Conf.Studio.Port
studio["token"] = studioToken.Token
studio["expires_at"] = studioToken.ExpiresAt
}
// Get user menus
menus := process.New("yao.app.menu").WithSID(sid).Run()
return maps.Map{
@ -134,6 +119,5 @@ func auth(field string, value string, password string, sid string) maps.Map {
"token": token.Token,
"user": row,
"menus": menus,
"studio": studio,
}
}