opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(image): add image generation and enhanced reading capabilities

Browse source 
- Introduced `image_generate` tool for generating images from text prompts, with options for specifying output file paths and image dimensions.
- Updated `image_read` functionality to allow optional provider specification for enhanced image analysis.
- Implemented new `GenerateImage` method in the LLM API for seamless integration of image generation capabilities.
- Enhanced documentation to include detailed usage examples for both image reading and generation tools.
- Updated tests to validate new image generation features and ensure robust functionality across image tools.
This commit is contained in:
Max 2026-05-06 10:59:35 +08:00
parent 20fc9c24df
commit af0d4edd74
23 changed files with 1066 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
agent/content/image/image.go
View file

@ -12,7 +12,7 @@ import (
"github.com/yaoapp/yao/agent/output/message"
searchTypes "github.com/yaoapp/yao/agent/search/types"
"github.com/yaoapp/yao/attachment"
"github.com/yaoapp/yao/tools/vision"
toolsImage "github.com/yaoapp/yao/tools/image"
)
// Image handles image content
@ -375,7 +375,7 @@ func (h *Image) readImageWithTools(ctx *agentContext.Context, content agentConte
loadingID := h.sendLoading(ctx, i18n.T(ctx.Locale, "content.image.analyzing"))
resp, err := vision.ReadImage(ctx.Context, src, "Please describe this image in detail.", 1080, ctx.Authorized)
resp, err := toolsImage.ReadImage(ctx.Context, src, "Please describe this image in detail.", 1080, ctx.Authorized, "")
h.sendLoadingDone(ctx, loadingID)

54
agent/context/jsapi_llm.go
View file

@ -14,6 +14,9 @@ type LlmAPI interface {
// Returns *llm.Result or error information
Stream(connector string, messages []interface{}, opts map[string]interface{}) interface{}
// GenerateImage generates an image from a text prompt using an image generation model
GenerateImage(connector string, prompt string, opts map[string]interface{}) interface{}
// Parallel LLM call methods - inspired by JavaScript Promise
// All waits for all LLM calls to complete (like Promise.all)
All(requests []interface{}) []interface{}
@ -68,6 +71,9 @@ func (ctx *Context) newLlmObject(iso *v8go.Isolate) *v8go.ObjectTemplate {
// Single LLM call method
llmObj.Set("Stream", ctx.llmStreamMethod(iso))
// Image generation method
llmObj.Set("GenerateImage", ctx.llmGenerateImageMethod(iso))
// Parallel LLM call methods - inspired by JavaScript Promise
llmObj.Set("All", ctx.llmAllMethod(iso))
llmObj.Set("Any", ctx.llmAnyMethod(iso))
@ -163,6 +169,54 @@ func (ctx *Context) llmStreamMethod(iso *v8go.Isolate) *v8go.FunctionTemplate {
})
}
// llmGenerateImageMethod implements ctx.llm.GenerateImage(connector, prompt, options?)
// Usage: const result = ctx.llm.GenerateImage("dall-e-3", "A sunset over mountains", { size: "1024x1024" })
// Returns: { connector, image (base64), format, error }
func (ctx *Context) llmGenerateImageMethod(iso *v8go.Isolate) *v8go.FunctionTemplate {
return v8go.NewFunctionTemplate(iso, func(info *v8go.FunctionCallbackInfo) *v8go.Value {
v8ctx := info.Context()
args := info.Args()
if len(args) < 2 {
return bridge.JsException(v8ctx, "GenerateImage requires connector and prompt parameters")
}
if !args[0].IsString() {
return bridge.JsException(v8ctx, "connector must be a string")
}
connectorID := args[0].String()
if !args[1].IsString() {
return bridge.JsException(v8ctx, "prompt must be a string")
}
prompt := args[1].String()
var opts map[string]interface{}
if len(args) >= 3 && !args[2].IsUndefined() && !args[2].IsNull() {
goVal, err := bridge.GoValue(args[2], v8ctx)
if err == nil {
if optsMap, ok := goVal.(map[string]interface{}); ok {
opts = optsMap
}
}
}
llmAPI := ctx.Llm()
if llmAPI == nil {
return bridge.JsException(v8ctx, "LLM API not available")
}
result := llmAPI.GenerateImage(connectorID, prompt, opts)
jsVal, err := bridge.JsValue(v8ctx, result)
if err != nil {
return bridge.JsException(v8ctx, "failed to convert result: "+err.Error())
}
return jsVal
})
}
// llmAllMethod implements ctx.llm.All(requests, options?)
// Usage: const results = ctx.llm.All([
//

19
agent/llm/doc.yml
View file

@ -23,3 +23,22 @@ entries:
return:
type: object
desc: "OpenAI-compatible response: { id, object, created, model, choices: [{ index, message: { role, content, tool_calls? }, finish_reason }], usage? }"
- name: ImageGeneration
desc: Generate an image from a text prompt using an image generation model
args:
- name: connector
type: string
required: true
desc: Connector ID for an image generation model (e.g. dall-e-3)
- name: prompt
type: string
required: true
desc: Text description of the image to generate
- name: opts
type: object
required: false
desc: "Generation options: size (1024x1024), quality, style, n, etc."
return:
type: object
desc: "Image generation result: { image (base64), format (png) }"

183
agent/llm/image.go Normal file
View file

@ -0,0 +1,183 @@
package llm
import (
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/yaoapp/gou/connector"
gouhttp "github.com/yaoapp/gou/http"
goullm "github.com/yaoapp/gou/llm"
)
// ImageGenResponse holds the result of an image generation call.
// Image is always base64 encoded; if the provider returns a URL, it is downloaded and converted.
type ImageGenResponse struct {
Image string `json:"image"` // base64 encoded image data
Format string `json:"format"` // image format, e.g. "png", "jpeg"
}
// GenerateImage calls the /images/generations endpoint through the connector.
// options may include: size, n, quality, style, model, etc.
func GenerateImage(conn connector.Connector, prompt string, options map[string]interface{}) (*ImageGenResponse, error) {
host, key, authMode := resolveConnSettings(conn)
if host == "" {
return nil, fmt.Errorf("no host found in connector settings")
}
if key == "" {
return nil, fmt.Errorf("API key is not set")
}
if options == nil {
options = map[string]interface{}{}
}
options["prompt"] = prompt
if _, ok := options["model"]; !ok {
if lc, ok := conn.(goullm.LLMConnector); ok {
if m := lc.GetModel(); m != "" {
options["model"] = m
}
}
}
url := connector.BuildAPIURL(host, "/images/generations")
req := gouhttp.New(url)
req.SetHeader("Content-Type", "application/json")
setImageAuthHeaders(req, authMode, key)
resp := req.Post(options)
if resp.Status != 200 {
errMsg := extractAPIError(resp.Data)
return nil, fmt.Errorf("image generation failed (status %d, url %s): %s", resp.Status, url, errMsg)
}
return extractImageFromResponse(resp.Data)
}
func resolveConnSettings(conn connector.Connector) (host, key string, authMode goullm.AuthMode) {
authMode = goullm.AuthBearer
if lc, ok := conn.(goullm.LLMConnector); ok {
host = lc.GetURL()
key = lc.GetKey()
authMode = lc.GetAuthMode()
}
if host == "" || key == "" {
setting := conn.Setting()
if host == "" {
host, _ = setting["host"].(string)
}
if key == "" {
key, _ = setting["key"].(string)
}
}
return
}
func setImageAuthHeaders(req *gouhttp.Request, authMode goullm.AuthMode, key string) {
switch authMode {
case goullm.AuthAPIKey:
req.SetHeader("api-key", key)
case goullm.AuthXAPIKey:
req.SetHeader("x-api-key", key)
default:
req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", key))
}
}
func extractImageFromResponse(data interface{}) (*ImageGenResponse, error) {
raw, err := json.Marshal(data)
if err != nil {
return nil, fmt.Errorf("marshal response: %w", err)
}
var parsed struct {
Data []struct {
B64JSON *string `json:"b64_json"`
URL *string `json:"url"`
} `json:"data"`
}
if err := json.Unmarshal(raw, &parsed); err != nil {
return nil, fmt.Errorf("unmarshal response: %w", err)
}
if len(parsed.Data) == 0 {
return nil, fmt.Errorf("provider returned empty data array, no image was generated")
}
item := parsed.Data[0]
if item.B64JSON != nil && *item.B64JSON != "" {
return &ImageGenResponse{Image: *item.B64JSON, Format: "png"}, nil
}
if item.URL != nil && *item.URL != "" {
b64, format, err := downloadImageAsBase64(*item.URL)
if err != nil {
return nil, fmt.Errorf("provider returned url but download failed: %w", err)
}
return &ImageGenResponse{Image: b64, Format: format}, nil
}
return nil, fmt.Errorf("provider returned data but neither b64_json nor url field is present, the model may not support image generation")
}
func downloadImageAsBase64(imageURL string) (b64 string, format string, err error) {
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Get(imageURL)
if err != nil {
return "", "", fmt.Errorf("http get: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", "", fmt.Errorf("download returned status %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", "", fmt.Errorf("read body: %w", err)
}
if len(body) == 0 {
return "", "", fmt.Errorf("downloaded image is empty")
}
format = "png"
ct := resp.Header.Get("Content-Type")
switch {
case strings.Contains(ct, "jpeg") || strings.Contains(ct, "jpg"):
format = "jpeg"
case strings.Contains(ct, "webp"):
format = "webp"
case strings.Contains(ct, "gif"):
format = "gif"
default:
if strings.Contains(imageURL, ".jpeg") || strings.Contains(imageURL, ".jpg") {
format = "jpeg"
} else if strings.Contains(imageURL, ".webp") {
format = "webp"
}
}
b64 = base64.StdEncoding.EncodeToString(body)
return b64, format, nil
}
func extractAPIError(data interface{}) string {
raw, err := json.Marshal(data)
if err != nil {
return fmt.Sprintf("%v", data)
}
var parsed struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
if err := json.Unmarshal(raw, &parsed); err == nil && parsed.Error.Message != "" {
return parsed.Error.Message
}
return string(raw)
}

197
agent/llm/image_test.go Normal file
View file

@ -0,0 +1,197 @@
package llm
import (
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestExtractImageFromResponse_B64(t *testing.T) {
data := map[string]interface{}{
"data": []interface{}{
map[string]interface{}{
"b64_json": "iVBORw0KGgoAAAANS...",
},
},
}
resp, err := extractImageFromResponse(data)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if resp.Image != "iVBORw0KGgoAAAANS..." {
t.Errorf("got Image=%q, want %q", resp.Image, "iVBORw0KGgoAAAANS...")
}
if resp.Format != "png" {
t.Errorf("got Format=%q, want %q", resp.Format, "png")
}
}
func TestExtractImageFromResponse_URL(t *testing.T) {
fakeImage := []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10} // fake JPEG header bytes
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/jpeg")
w.Write(fakeImage)
}))
defer srv.Close()
data := map[string]interface{}{
"data": []interface{}{
map[string]interface{}{
"b64_json": nil,
"url": srv.URL + "/image_0.jpeg",
},
},
}
resp, err := extractImageFromResponse(data)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expected := base64.StdEncoding.EncodeToString(fakeImage)
if resp.Image != expected {
t.Errorf("got Image=%q, want %q", resp.Image, expected)
}
if resp.Format != "jpeg" {
t.Errorf("got Format=%q, want %q", resp.Format, "jpeg")
}
}
func TestExtractImageFromResponse_URLPng(t *testing.T) {
fakeImage := []byte{0x89, 0x50, 0x4E, 0x47} // PNG magic bytes
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
w.Write(fakeImage)
}))
defer srv.Close()
data := map[string]interface{}{
"data": []interface{}{
map[string]interface{}{
"url": srv.URL + "/output.png",
},
},
}
resp, err := extractImageFromResponse(data)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if resp.Format != "png" {
t.Errorf("got Format=%q, want %q", resp.Format, "png")
}
if resp.Image == "" {
t.Error("expected non-empty base64 Image")
}
}
func TestExtractImageFromResponse_URLDownloadFail(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
data := map[string]interface{}{
"data": []interface{}{
map[string]interface{}{
"url": srv.URL + "/missing.png",
},
},
}
_, err := extractImageFromResponse(data)
if err == nil {
t.Error("expected error for failed download")
}
}
func TestExtractImageFromResponse_Empty(t *testing.T) {
data := map[string]interface{}{
"data": []interface{}{},
}
_, err := extractImageFromResponse(data)
if err == nil {
t.Error("expected error for empty data array")
}
}
func TestExtractImageFromResponse_NoData(t *testing.T) {
data := map[string]interface{}{}
_, err := extractImageFromResponse(data)
if err == nil {
t.Error("expected error for missing data field")
}
}
func TestExtractImageFromResponse_NullBoth(t *testing.T) {
data := map[string]interface{}{
"data": []interface{}{
map[string]interface{}{
"b64_json": nil,
"url": nil,
},
},
}
_, err := extractImageFromResponse(data)
if err == nil {
t.Error("expected error when both b64_json and url are null")
}
}
func TestDownloadImageAsBase64(t *testing.T) {
payload := []byte("fake-png-data")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
w.Write(payload)
}))
defer srv.Close()
b64, format, err := downloadImageAsBase64(srv.URL + "/test.png")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if format != "png" {
t.Errorf("got format=%q, want %q", format, "png")
}
decoded, _ := base64.StdEncoding.DecodeString(b64)
if string(decoded) != string(payload) {
t.Errorf("decoded content mismatch")
}
}
func TestDownloadImageAsBase64_FormatFromURL(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
w.Write([]byte("data"))
}))
defer srv.Close()
_, format, err := downloadImageAsBase64(srv.URL + "/image.webp")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if format != "webp" {
t.Errorf("got format=%q, want %q (from URL fallback)", format, "webp")
}
}
func TestExtractAPIError_WithMessage(t *testing.T) {
data := map[string]interface{}{
"error": map[string]interface{}{
"message": "insufficient quota",
},
}
msg := extractAPIError(data)
if msg != "insufficient quota" {
t.Errorf("got %q, want %q", msg, "insufficient quota")
}
}
func TestExtractAPIError_NoMessage(t *testing.T) {
data := map[string]interface{}{
"something": "else",
}
msg := extractAPIError(data)
raw, _ := json.Marshal(data)
if msg != string(raw) {
t.Errorf("got %q, want raw JSON fallback", msg)
}
}

31
agent/llm/jsapi.go
View file

@ -31,6 +31,37 @@ func SetJSAPIFactory() {
}
}
// GenerateImage implements LlmAPI.GenerateImage - generates an image from a text prompt
func (api *JSAPI) GenerateImage(connectorID string, prompt string, opts map[string]interface{}) interface{} {
result := &ImageGenResult{
Connector: connectorID,
}
conn, err := connector.Select(connectorID)
if err != nil {
result.Error = fmt.Sprintf("failed to select connector %s: %v", connectorID, err)
return result
}
resp, err := GenerateImage(conn, prompt, opts)
if err != nil {
result.Error = fmt.Sprintf("image generation failed: %v", err)
return result
}
result.Image = resp.Image
result.Format = resp.Format
return result
}
// ImageGenResult is the return type for GenerateImage JSAPI
type ImageGenResult struct {
Connector string `json:"connector"`
Image string `json:"image,omitempty"`
Format string `json:"format,omitempty"`
Error string `json:"error,omitempty"`
}
// Stream implements LlmAPI.Stream - calls LLM with streaming output to ctx.Writer
func (api *JSAPI) Stream(connectorID string, messages []interface{}, opts map[string]interface{}) interface{} {
return api.StreamWithHandler(connectorID, messages, opts, nil)

50
agent/llm/process.go
View file

@ -16,6 +16,7 @@ import (
func init() {
process.Register("llm.ChatCompletions", ProcessChatCompletions)
process.Register("llm.ImageGeneration", ProcessImageGeneration)
}
// ProcessChatCompletions implements the llm.ChatCompletions Process.
@ -155,6 +156,55 @@ func ProcessChatCompletions(p *process.Process) interface{} {
return toOpenAIFormat(response)
}
// ProcessImageGeneration implements the llm.ImageGeneration Process.
//
// Usage:
//
// Process("llm.ImageGeneration", connectorID, prompt)
// Process("llm.ImageGeneration", connectorID, prompt, opts)
//
// Args:
// - connectorID (string): Connector ID for an image generation model
// - prompt (string): Text description of the image to generate
// - opts (map): Optional. size, quality, style, n, etc.
//
// Returns: { image (base64), format (png) }
func ProcessImageGeneration(p *process.Process) interface{} {
p.ValidateArgNums(2)
connectorID := p.ArgsString(0)
if connectorID == "" {
return newErrorResponse("llm.ImageGeneration: connector is required")
}
prompt := p.ArgsString(1)
if prompt == "" {
return newErrorResponse("llm.ImageGeneration: prompt is required")
}
var opts map[string]interface{}
if p.NumOfArgs() > 2 && p.Args[2] != nil {
if o, ok := p.Args[2].(map[string]interface{}); ok {
opts = o
}
}
conn, _, err := selectWithCapabilities(connectorID)
if err != nil {
return newErrorResponse(fmt.Sprintf("llm.ImageGeneration: connector %s not found: %v", connectorID, err))
}
resp, err := GenerateImage(conn, prompt, opts)
if err != nil {
return newErrorResponse(fmt.Sprintf("llm.ImageGeneration: %v", err))
}
return map[string]interface{}{
"image": resp.Image,
"format": resp.Format,
}
}
// toOpenAIFormat converts CompletionResponse to OpenAI chat.completions format
// for backward compatibility with code that consumed openai.chat.Completions.
func toOpenAIFormat(resp *agentContext.CompletionResponse) map[string]interface{} {

2
agent/sandbox/v2/claude/command.go
View file

@ -388,7 +388,7 @@ func buildModelCapabilityPrompt(req *types.StreamRequest) string {
if !primaryHasVision {
if _, hasVisionRole := req.Roles["vision"]; hasVisionRole {
guidance = append(guidance,
"**Image/Vision**: Your current model cannot process images directly. Use the `image_read` system tool (`tai tool image_read`) to analyze images — see the yao-vision skill for details",
"**Image/Vision**: Your current model cannot process images directly. Use the `image_read` system tool (`tai tool image_read`) to analyze images — see the yao-image skill for details",
)
}
}

65
llmprovider/presets.yml
View file

@ -87,7 +87,7 @@
enabled: false
- id: tencent-hy3-preview
model: tencent/hy3-preview:free
name: 混元 3 Preview (Free)
name: Hunyuan 3 Preview (Free)
max_input_tokens: 262144
max_output_tokens: 262144
capabilities: [tool_calls, streaming, json]
@ -97,7 +97,7 @@
enabled: false
- id: tencent-hy3-preview-reasoning-high
model: tencent/hy3-preview:free
name: 混元 3 Preview Reasoning (Free)
name: Hunyuan 3 Preview Reasoning (Free)
max_input_tokens: 262144
max_output_tokens: 262144
capabilities: [tool_calls, streaming, json, reasoning]
@ -193,7 +193,7 @@
enabled: false
# ─── OpenAI ─────────────────────────────────────────────
# reasoning_effort 档位: low | medium | high (5.4-mini/o4-mini)
# reasoning_effort levels: low | medium | high (5.4-mini/o4-mini)
# low | medium | high | xhigh (5.5/5.4/codex/5)
- key: openai
name: OpenAI
@ -471,6 +471,23 @@
name: TTS-1 HD
capabilities: [audio]
enabled: false
# --- Image Generation ---
- id: gpt-image-2
name: GPT Image 2
capabilities: [image_generation]
enabled: false
- id: gpt-image-1.5
name: GPT Image 1.5
capabilities: [image_generation]
enabled: false
- id: gpt-image-1
name: GPT Image 1
capabilities: [image_generation]
enabled: false
- id: gpt-image-1-mini
name: GPT Image 1 Mini
capabilities: [image_generation]
enabled: false
# ─── DeepSeek (OpenAI) ──────────────────────────────────
- key: deepseek
@ -856,6 +873,22 @@
max_output_tokens: 65536
capabilities: [vision, tool_calls, streaming, json]
enabled: false
# --- Image Generation ---
- id: imagen-4
model: models/imagen-4.0-generate-001
name: Imagen 4
capabilities: [image_generation]
enabled: false
- id: imagen-4-ultra
model: models/imagen-4.0-ultra-generate-001
name: Imagen 4 Ultra
capabilities: [image_generation]
enabled: false
- id: imagen-4-fast
model: models/imagen-4.0-fast-generate-001
name: Imagen 4 Fast
capabilities: [image_generation]
enabled: false
# ─── xAI (Grok) ────────────────────────────────────────
# grok-4.x: reasoning.enabled true|false
@ -921,6 +954,19 @@
options:
reasoning_effort: high
enabled: false
# --- Image Generation ---
- id: grok-imagine-image
name: Grok Imagine
capabilities: [image_generation]
enabled: false
- id: grok-imagine-image-pro
name: Grok Imagine Pro
capabilities: [image_generation]
enabled: false
- id: grok-imagine-image-quality
name: Grok Imagine Quality
capabilities: [image_generation]
enabled: false
# ─── MiniMax (International) ────────────────────────────
- key: minimax_intl
@ -1390,6 +1436,19 @@
max_input_tokens: 4096
capabilities: [embedding]
enabled: false
# --- 图片生成 ---
- id: doubao-seedream-5-0-260128
name: 豆包 Seedream 5.0
capabilities: [image_generation]
enabled: false
- id: doubao-seedream-4-5-251128
name: 豆包 Seedream 4.5
capabilities: [image_generation]
enabled: false
- id: doubao-seedream-4-0-250828
name: 豆包 Seedream 4.0
capabilities: [image_generation]
enabled: false
# ─── 腾讯混元 MaaS ─────────────────────────────────────
# hy3-preview: reasoning.level disabled | low | high

54
tools/image/generate.go Normal file
View file

@ -0,0 +1,54 @@
package image
import (
_ "embed"
"fmt"
"github.com/yaoapp/gou/process"
agentLLM "github.com/yaoapp/yao/agent/llm"
"github.com/yaoapp/yao/openapi/oauth/authorized"
)
//go:embed generate_schema.json
var GenerateSchemaJSON []byte
// GenerateHandler is the tools.image_generate process handler.
func GenerateHandler(proc *process.Process) interface{} {
prompt := proc.ArgsString(0)
if prompt == "" {
return map[string]interface{}{"error": "prompt is required"}
}
provider := proc.ArgsString(1)
size := proc.ArgsString(2, "1024x1024")
authInfo := authorized.ProcessAuthInfo(proc)
if authInfo == nil {
return map[string]interface{}{"error": "unauthorized: no auth info in request"}
}
connectorID := provider
if connectorID == "" {
connectorID = findFirstImageGenConnector(authInfo)
if connectorID == "" {
return map[string]interface{}{"error": "no image generation provider available; configure one or specify a provider"}
}
}
conn, _, err := agentLLM.ResolveConnector(connectorID, authInfo)
if err != nil {
return map[string]interface{}{"error": fmt.Sprintf("resolve connector: %v", err)}
}
options := map[string]interface{}{"size": size}
resp, err := agentLLM.GenerateImage(conn, prompt, options)
if err != nil {
return map[string]interface{}{"error": fmt.Sprintf("image generation failed: %v", err)}
}
return map[string]interface{}{
"image": resp.Image,
"format": resp.Format,
"size": size,
}
}

25
tools/image/generate_schema.json Normal file
View file

@ -0,0 +1,25 @@
{
"name": "image_generate",
"description": "Generate an image from a text prompt using an image generation model (e.g. DALL-E, Seedream).",
"process": "tools.image_generate",
"inputSchema": {
"type": "object",
"properties": {
"prompt": {
"type": "string",
"description": "Text description of the image to generate."
},
"provider": {
"type": "string",
"description": "Provider connector ID (e.g. 'llm.my-openai:dall-e-3'). Use image_providers to list available options. If omitted, the first available image generation provider is used."
},
"size": {
"type": "string",
"description": "Image dimensions (default: 1024x1024). Common values: 1024x1024, 1024x1792, 1792x1024.",
"default": "1024x1024"
}
},
"required": ["prompt"]
},
"x-process-args": ["$args.prompt", "$args.provider", "$args.size"]
}

35
tools/image/generate_test.go Normal file
View file

@ -0,0 +1,35 @@
package image
import (
"testing"
"github.com/yaoapp/gou/process"
)
func TestGenerateHandler_NoPrompt(t *testing.T) {
proc := &process.Process{
Args: []interface{}{""},
}
result := GenerateHandler(proc)
m, ok := result.(map[string]interface{})
if !ok {
t.Fatal("expected map result")
}
if errMsg, _ := m["error"].(string); errMsg != "prompt is required" {
t.Errorf("expected 'prompt is required', got %q", errMsg)
}
}
func TestGenerateHandler_NoAuth(t *testing.T) {
proc := &process.Process{
Args: []interface{}{"A sunset", "", "1024x1024"},
}
result := GenerateHandler(proc)
m, ok := result.(map[string]interface{})
if !ok {
t.Fatal("expected map result")
}
if _, hasErr := m["error"]; !hasErr {
t.Error("expected error when no auth info")
}
}

142
tools/image/providers.go Normal file
View file

@ -0,0 +1,142 @@
package image
import (
_ "embed"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/yao/llmprovider"
"github.com/yaoapp/yao/openapi/oauth/authorized"
oauthTypes "github.com/yaoapp/yao/openapi/oauth/types"
)
//go:embed providers_schema.json
var ProvidersSchemaJSON []byte
type providerResult struct {
Key string `json:"key"`
Name string `json:"name"`
Models []modelResult `json:"models"`
}
type modelResult struct {
ID string `json:"id"`
Name string `json:"name"`
ConnectorID string `json:"connector_id"`
}
// ProvidersHandler is the tools.image_providers process handler.
func ProvidersHandler(proc *process.Process) interface{} {
capability := proc.ArgsString(0, "image_generation")
authInfo := authorized.ProcessAuthInfo(proc)
if authInfo == nil {
return map[string]interface{}{"error": "unauthorized: no auth info in request"}
}
if llmprovider.Global == nil {
return map[string]interface{}{"error": "llmprovider registry not initialized"}
}
providers, err := listProvidersByCapability(capability, authInfo)
if err != nil {
return map[string]interface{}{"error": err.Error()}
}
return map[string]interface{}{
"capability": capability,
"providers": providers,
}
}
// listProvidersByCapability returns providers matching the given capability,
// filtered by owner scope (builtin always included, dynamic filtered by auth).
func listProvidersByCapability(capability string, authInfo *oauthTypes.AuthorizedInfo) ([]providerResult, error) {
enabled := true
filter := &llmprovider.ProviderFilter{
Capabilities: []string{capability},
Enabled: &enabled,
Source: llmprovider.ProviderSourceAll,
}
allProviders, err := llmprovider.Global.List(filter)
if err != nil {
return nil, err
}
var results []providerResult
for _, p := range allProviders {
if p.Source == llmprovider.ProviderSourceDynamic && authInfo != nil {
if !dynamicOwnerMatch(&p.Owner, authInfo) {
continue
}
}
var models []modelResult
for _, m := range p.Models {
if !m.Enabled {
continue
}
if !modelHasCapability(m.Capabilities, capability) {
continue
}
cid := p.ConnectorID
if p.Source == llmprovider.ProviderSourceDynamic && m.ID != "" {
cid = p.ConnectorID + ":" + m.ID
}
name := m.Name
if name == "" {
name = m.ID
}
models = append(models, modelResult{
ID: m.ID,
Name: name,
ConnectorID: cid,
})
}
if len(models) == 0 {
continue
}
results = append(results, providerResult{
Key: p.Key,
Name: p.Name,
Models: models,
})
}
return results, nil
}
// findFirstImageGenConnector returns the connector ID of the first available
// image generation provider, or empty string if none found.
func findFirstImageGenConnector(authInfo *oauthTypes.AuthorizedInfo) string {
if llmprovider.Global == nil {
return ""
}
providers, err := listProvidersByCapability("image_generation", authInfo)
if err != nil || len(providers) == 0 {
return ""
}
if len(providers[0].Models) == 0 {
return ""
}
return providers[0].Models[0].ConnectorID
}
func dynamicOwnerMatch(owner *llmprovider.ProviderOwner, authInfo *oauthTypes.AuthorizedInfo) bool {
if authInfo.GetTeamID() != "" {
return owner.Type == "team" && owner.TeamID == authInfo.GetTeamID()
}
return owner.Type == "user" && owner.UserID == authInfo.GetUserID()
}
func modelHasCapability(caps []string, target string) bool {
for _, c := range caps {
if c == target {
return true
}
}
return false
}

17
tools/image/providers_schema.json Normal file
View file

@ -0,0 +1,17 @@
{
"name": "image_providers",
"description": "List available image providers. Default lists image generation providers; pass capability='vision' to list vision (image reading) providers.",
"process": "tools.image_providers",
"inputSchema": {
"type": "object",
"properties": {
"capability": {
"type": "string",
"description": "Filter by capability: 'image_generation' (default) or 'vision'.",
"default": "image_generation",
"enum": ["image_generation", "vision"]
}
}
},
"x-process-args": ["$args.capability"]
}

54
tools/image/providers_test.go Normal file
View file

@ -0,0 +1,54 @@
package image
import (
"testing"
"github.com/yaoapp/gou/process"
)
func TestModelHasCapability_Found(t *testing.T) {
caps := []string{"chat", "image_generation", "vision"}
if !modelHasCapability(caps, "image_generation") {
t.Error("expected true for image_generation")
}
if !modelHasCapability(caps, "vision") {
t.Error("expected true for vision")
}
}
func TestModelHasCapability_NotFound(t *testing.T) {
caps := []string{"chat", "embedding"}
if modelHasCapability(caps, "image_generation") {
t.Error("expected false for image_generation")
}
}
func TestModelHasCapability_Empty(t *testing.T) {
if modelHasCapability(nil, "image_generation") {
t.Error("expected false for nil caps")
}
if modelHasCapability([]string{}, "image_generation") {
t.Error("expected false for empty caps")
}
}
func TestProvidersHandler_NoAuth(t *testing.T) {
proc := &process.Process{
Args: []interface{}{"image_generation"},
}
result := ProvidersHandler(proc)
m, ok := result.(map[string]interface{})
if !ok {
t.Fatal("expected map result")
}
if _, hasErr := m["error"]; !hasErr {
t.Error("expected error when no auth info")
}
}
func TestFindFirstImageGenConnector_NoGlobal(t *testing.T) {
result := findFirstImageGenConnector(nil)
if result != "" {
t.Errorf("expected empty string, got %q", result)
}
}

26
tools/image/read.go
View file

@ -1,4 +1,4 @@
package vision
package image
import (
"bytes"
@ -6,7 +6,7 @@ import (
_ "embed"
"encoding/base64"
"fmt"
"image"
stdimage "image"
_ "image/gif"
"image/jpeg"
_ "image/png"
@ -28,8 +28,8 @@ import (
ws "github.com/yaoapp/yao/workspace"
)
//go:embed schema.json
var SchemaJSON []byte
//go:embed read_schema.json
var ReadSchemaJSON []byte
// ImageReadResponse is the return type for image_read.
type ImageReadResponse struct {
@ -40,7 +40,8 @@ type ImageReadResponse struct {
// ReadImage reads and analyzes an image using the vision model.
// It resolves the image from src, finds a vision-capable connector via llmprovider,
// and returns the model's text description.
func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, authInfo *oauthTypes.AuthorizedInfo) (*ImageReadResponse, error) {
// provider is optional; when non-empty it overrides the default "use::vision" role.
func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, authInfo *oauthTypes.AuthorizedInfo, provider string) (*ImageReadResponse, error) {
if prompt == "" {
prompt = "Please describe this image in detail."
}
@ -53,7 +54,11 @@ func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, au
return nil, fmt.Errorf("resolve image: %w", err)
}
conn, caps, err := agentLLM.ResolveConnector("use::vision", authInfo)
connectorRole := "use::vision"
if provider != "" {
connectorRole = provider
}
conn, caps, err := agentLLM.ResolveConnector(connectorRole, authInfo)
if err != nil {
return nil, fmt.Errorf("resolve vision connector: %w", err)
}
@ -87,8 +92,8 @@ func ReadImage(goCtx context.Context, src string, prompt string, maxSize int, au
}, nil
}
// Handler is the tools.image_read process handler.
func Handler(proc *process.Process) interface{} {
// ReadHandler is the tools.image_read process handler.
func ReadHandler(proc *process.Process) interface{} {
src := proc.ArgsString(0)
if src == "" {
return map[string]interface{}{"error": "image_path is required: provide a file path, URL, or URI"}
@ -103,6 +108,7 @@ func Handler(proc *process.Process) interface{} {
proc.ArgsString(1, "Please describe this image in detail."),
proc.ArgsInt(2, 1080),
authInfo,
proc.ArgsString(3),
)
if err != nil {
return map[string]interface{}{"error": err.Error()}
@ -188,7 +194,7 @@ func httpGet(rawURL string) ([]byte, error) {
// resizeImage decodes, resizes (longest edge <= maxSize), re-encodes as JPEG.
// Returns original bytes unchanged when already small enough or if decode fails.
func resizeImage(data []byte, maxSize int) ([]byte, string) {
img, _, err := image.Decode(bytes.NewReader(data))
img, _, err := stdimage.Decode(bytes.NewReader(data))
if err != nil {
return data, http.DetectContentType(data)
}
@ -205,7 +211,7 @@ func resizeImage(data []byte, maxSize int) ([]byte, string) {
if newH < 1 {
newH = 1
}
dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
dst := stdimage.NewRGBA(stdimage.Rect(0, 0, newW, newH))
draw.BiLinear.Scale(dst, dst.Bounds(), img, bounds, draw.Over, nil)
var buf bytes.Buffer
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); err != nil {

6
tools/image/read_schema.json
View file

@ -18,9 +18,13 @@
"type": "integer",
"description": "Max dimension in pixels for the longest edge. Image is resized (preserving aspect ratio) before sending to the vision model. Default 1080.",
"default": 1080
},
"provider": {
"type": "string",
"description": "Vision provider connector ID (e.g. from image_providers with capability='vision'). If omitted, uses the default vision model."
}
},
"required": ["image_path"]
},
"x-process-args": ["$args.image_path", "$args.prompt", "$args.max_size"]
"x-process-args": ["$args.image_path", "$args.prompt", "$args.max_size", "$args.provider"]
}

8
tools/image/read_test.go
View file

@ -1,8 +1,8 @@
package vision
package image
import (
"encoding/base64"
"image"
stdimage "image"
"image/color"
_ "image/jpeg"
"image/png"
@ -15,7 +15,7 @@ import (
)
func makePNG(w, h int) []byte {
img := image.NewRGBA(image.Rect(0, 0, w, h))
img := stdimage.NewRGBA(stdimage.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.Set(x, y, color.RGBA{R: 255, G: 0, B: 0, A: 255})
@ -45,7 +45,7 @@ func TestResizeImage_LargeImage(t *testing.T) {
if mime != "image/jpeg" {
t.Errorf("mime = %q, want image/jpeg", mime)
}
img, _, err := image.Decode(strings.NewReader(string(data)))
img, _, err := stdimage.Decode(strings.NewReader(string(data)))
if err != nil {
t.Fatalf("failed to decode resized image: %v", err)
}

8
tools/mcps/image.json
View file

@ -1,8 +1,10 @@
{
"name": "yao-vision",
"name": "yao-image",
"transport": "process",
"description": "Vision tools for image understanding",
"description": "Image tools for reading, generating, and managing images",
"tools": {
"image_read": "tools.image_read"
"image_read": "tools.image_read",
"image_generate": "tools.image_generate",
"image_providers": "tools.image_providers"
}
}

65
tools/prompts/system-tools.md
View file

@ -4,13 +4,13 @@
These environment variables are set by the Yao sandbox. **Always use these variables — never hardcode paths.**
| Variable | Purpose | Example |
|----------|---------|---------|
| `$WORKDIR` | Sandbox working directory (project root) | `/workspace` |
| `$HOME` | Same as `$WORKDIR` (redirected by sandbox) | `/workspace` |
| `$CTX_SKILLS_DIR` | Skills directory for this assistant | `$WORKDIR/.yao/assistants/<id>/skills` |
| `$CTX_ASSISTANT_ID` | Current assistant ID | `yao.agent-smith` |
| `$CTX_WORKSPACE_ID` | Current workspace ID | `ws-abc123` |
| Variable | Purpose | Example |
| ------------------- | ------------------------------------------ | -------------------------------------- |
| `$WORKDIR` | Sandbox working directory (project root) | `/workspace` |
| `$HOME` | Same as `$WORKDIR` (redirected by sandbox) | `/workspace` |
| `$CTX_SKILLS_DIR` | Skills directory for this assistant | `$WORKDIR/.yao/assistants/<id>/skills` |
| `$CTX_ASSISTANT_ID` | Current assistant ID | `yao.agent-smith` |
| `$CTX_WORKSPACE_ID` | Current workspace ID | `ws-abc123` |
### Path Rules
@ -21,6 +21,33 @@ These environment variables are set by the Yao sandbox. **Always use these varia
Resolve first: `echo "$WORKDIR"`, then use the printed value.
- On Windows, use `$env:WORKDIR` / `$env:CTX_SKILLS_DIR` syntax instead.
### Workspace Path in Replies
When replying to users, **never expose raw `/workspace/...` paths**. Rewrite them using the `workspace://` scheme so the frontend can render them correctly.
**Format**: `workspace://<workspace-id>/relative/path`
**Step 1 — resolve the real workspace ID** (do this once per session):
```bash
echo "$CTX_WORKSPACE_ID"
```
Use the **actual printed value** (e.g. `ws-bfc4c2de-b53...`) in all subsequent replies.
**Step 2 — rewrite paths in replies**:
- `/workspace/output/result.png` → `workspace://<real-id>/output/result.png`
**Common mistakes** (all wrong):
- `workspace://$CTX_WORKSPACE_ID/...` ← shell variable literally in reply
- `workspace://ws-bfc4c2de-b53/...` ← example/placeholder ID instead of the real one
- `workspace://<workspace-id>/...` ← template placeholder instead of the real one
- `/workspace/output/...` ← raw path without `workspace://` scheme
You **must** run `echo "$CTX_WORKSPACE_ID"` and use the exact output.
### Attachments
User-uploaded files are placed in `$WORKDIR/.attachments/{chatID}/`.
@ -40,15 +67,17 @@ You have access to Yao system tools via the `tai` command in bash.
**Calling convention**: `tai tool <name> '<json_args>'`
| Tool | Skill (auto-loaded) | Description |
|------|---------------------|-------------|
| `web_search` | yao-web | Search the web for real-time information |
| `web_fetch` | yao-web | Fetch and read a web page by URL |
| `process_call` | yao-process | Execute a Yao Process (server-side function) |
| `process_allowed` | yao-process | Check which processes are allowed |
| `doc_list` | yao-doc | Search/list available process documentation |
| `doc_inspect` | yao-doc | Get detailed docs for a specific process |
| `doc_validate` | yao-doc | Validate a process name and get suggestions |
| `image_read` | yao-vision | Read and analyze images using a vision model |
| Tool | Skill (auto-loaded) | Description |
| ----------------- | ------------------- | --------------------------------------------------- |
| `web_search` | yao-web | Search the web for real-time information |
| `web_fetch` | yao-web | Fetch and read a web page by URL |
| `process_call` | yao-process | Execute a Yao Process (server-side function) |
| `process_allowed` | yao-process | Check which processes are allowed |
| `doc_list` | yao-doc | Search/list available process documentation |
| `doc_inspect` | yao-doc | Get detailed docs for a specific process |
| `doc_validate` | yao-doc | Validate a process name and get suggestions |
| `image_read` | yao-image | Read and analyze images using a vision model |
| `image_generate` | yao-image | Generate images from text prompts |
| `image_providers` | yao-image | List available image generation or vision providers |
The system skills (`yao-web`, `yao-process`, `yao-doc`, `yao-vision`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description.
The system skills (`yao-web`, `yao-process`, `yao-doc`, `yao-image`) in `$HOME/.claude/skills/` are **auto-discovered** — they contain detailed parameter docs and workflow guidance. You do not need to manually read them; they are loaded automatically when your task matches their description.

58
tools/skills/yao-image/SKILL.md
View file

@ -1,11 +1,11 @@
---
name: yao-vision
description: Image understanding expert. ALWAYS invoke this skill when you need to read, analyze, or describe an image that you cannot process directly. Use for screenshots, photos, charts, diagrams, or any visual content.
name: yao-image
description: Image expert. ALWAYS invoke this skill when you need to read, analyze, describe, or generate images. Use for screenshots, photos, charts, diagrams, AI-generated images, or any visual content.
---
# Vision Tools
# Image Tools
Use when you encounter images you cannot read natively (e.g., as a text-only model).
Use these tools when you encounter images you cannot read natively, or when you need to generate new images.
## image_read
@ -31,9 +31,9 @@ tai tool image_read '{"image_path": "workspace://ws-id/path/to/image.png", "prom
tai tool image_read '{"image_path": "attach://__yao.attachment/file-id-123", "prompt": "Describe"}'
```
### Yao data file:
### With a specific vision provider:
```bash
tai tool image_read '{"image_path": "yao://uploads/photo.jpg", "prompt": "Describe"}'
tai tool image_read '{"image_path": "/path/to/image.png", "prompt": "Describe", "provider": "llm.my-openai:gpt-4o"}'
```
| Parameter | Type | Required | Description |
@ -41,6 +41,52 @@ tai tool image_read '{"image_path": "yao://uploads/photo.jpg", "prompt": "Descri
| image_path | string | yes | File path, URL, workspace://, attach://, or yao:// URI |
| prompt | string | no | Analysis instruction (default: describe in detail) |
| max_size | integer | no | Max dimension in pixels for longest edge (default: 1080) |
| provider | string | no | Vision provider connector ID. If omitted, uses default vision model |
Images are automatically resized (preserving aspect ratio) before sending to the vision model.
Supported formats: PNG, JPEG, GIF, WebP.
## image_generate
Generate an image from a text prompt and save it to a file.
### Basic usage (always specify output):
```bash
tai tool image_generate '{"prompt": "A serene mountain landscape at sunset", "output": "landscape.png"}'
```
### With specific provider and size:
```bash
tai tool image_generate '{"prompt": "A futuristic city skyline", "provider": "llm.my-openai:dall-e-3", "size": "1792x1024", "output": "output/city.png"}'
```
| Parameter | Type | Required | Description |
| --------- | ------ | -------- | ----------------------------------------------------------------- |
| prompt | string | yes | Text description of the image to generate |
| output | string | yes | File path to save the generated image (parent dirs created automatically) |
| provider | string | no | Provider connector ID (use `image_providers` to list). Auto-selects if omitted |
| size | string | no | Image dimensions (default: 1024x1024). Common: 1024x1024, 1024x1792, 1792x1024 |
**Important**: Always pass `output`. The tool saves the image directly and returns only the file path and size. Without `output`, the raw base64 data is returned which may exceed output limits.
Use relative paths (e.g. `"output": "fox.png"`) — they resolve relative to the current working directory (`$WORKDIR`). No need to prepend `$WORKDIR` manually.
## image_providers
List available image providers filtered by capability.
### List image generation providers (default):
```bash
tai tool image_providers '{}'
```
### List vision (image reading) providers:
```bash
tai tool image_providers '{"capability": "vision"}'
```
| Parameter | Type | Required | Description |
| ---------- | ------ | -------- | ----------------------------------------------------------- |
| capability | string | no | `image_generation` (default) or `vision` |
Returns a list of providers with their available models and connector IDs that can be passed to `image_generate` or `image_read`.

4
tools/skills_test.go
View file

@ -6,11 +6,12 @@ import (
"testing"
)
func TestSkillsFS_ContainsThreeSkills(t *testing.T) {
func TestSkillsFS_ContainsAllSkills(t *testing.T) {
expected := map[string]bool{
"skills/yao-web/SKILL.md": false,
"skills/yao-process/SKILL.md": false,
"skills/yao-doc/SKILL.md": false,
"skills/yao-image/SKILL.md": false,
}
err := fs.WalkDir(SkillsFS, "skills", func(path string, d fs.DirEntry, err error) error {
@ -41,6 +42,7 @@ func TestSkillsFS_FrontmatterFields(t *testing.T) {
{"skills/yao-web/SKILL.md", "yao-web"},
{"skills/yao-process/SKILL.md", "yao-process"},
{"skills/yao-doc/SKILL.md", "yao-doc"},
{"skills/yao-image/SKILL.md", "yao-image"},
}
for _, s := range skills {

14
tools/tools.go
View file

@ -9,8 +9,8 @@ import (
"github.com/yaoapp/gou/process"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/tools/docs"
"github.com/yaoapp/yao/tools/image"
"github.com/yaoapp/yao/tools/proc"
"github.com/yaoapp/yao/tools/vision"
"github.com/yaoapp/yao/tools/webfetch"
"github.com/yaoapp/yao/tools/websearch"
)
@ -24,8 +24,8 @@ var mcpProcessDSL []byte
//go:embed mcps/doc.json
var mcpDocDSL []byte
//go:embed mcps/vision.json
var mcpVisionDSL []byte
//go:embed mcps/image.json
var mcpImageDSL []byte
func init() {
process.RegisterGroup("tools", map[string]process.Handler{
@ -36,7 +36,9 @@ func init() {
"doc_list": docs.ListHandler,
"doc_inspect": docs.InspectHandler,
"doc_validate": docs.ValidateHandler,
"image_read": vision.Handler,
"image_read": image.ReadHandler,
"image_generate": image.GenerateHandler,
"image_providers": image.ProvidersHandler,
})
registerMCPServer(mcpWebDSL, "yao-web",
@ -45,8 +47,8 @@ func init() {
proc.SchemaJSON, proc.AllowedSchemaJSON)
registerMCPServer(mcpDocDSL, "yao-doc",
docs.ListSchemaJSON, docs.InspectSchemaJSON, docs.ValidateSchemaJSON)
registerMCPServer(mcpVisionDSL, "yao-vision",
vision.SchemaJSON)
registerMCPServer(mcpImageDSL, "yao-image",
image.ReadSchemaJSON, image.GenerateSchemaJSON, image.ProvidersSchemaJSON)
}
func registerMCPServer(dsl []byte, id string, schemas ...[]byte) {