Merge pull request #1514 from trheyi/main

feat(system): add Vision and Voice capabilities to System configuration

agent/assistant/load_system.go

@@ -27,6 +27,8 @@ var systemAgents = []string{
 	"robot_prompt",
 	"needsearch",
 	"entity",
+	"vision",
+	"fetch",
 }
 
 // SystemConfig holds the system agents connector configuration
@@ -40,6 +42,8 @@ type SystemConfig struct {
 	RobotPrompt string // Connector for __yao.robot_prompt agent
 	NeedSearch  string // Connector for __yao.needsearch agent
 	Entity      string // Connector for __yao.entity agent
+	Vision      string // Connector for vision capabilities
+	Voice       string // Connector for voice/STT capabilities
 }
 
 // systemConfig holds the system agents configuration (global variable like others in load.go)
@@ -237,6 +241,14 @@ func resolveSystemConnector(agentID string) string {
 			if systemConfig.Entity != "" {
 				return systemConfig.Entity
 			}
+		case "__yao.vision":
+			if systemConfig.Vision != "" {
+				return systemConfig.Vision
+			}
+		case "__yao.voice":
+			if systemConfig.Voice != "" {
+				return systemConfig.Voice
+			}
 		}
 
 		// Try system default
@@ -254,6 +266,40 @@ func resolveSystemConnector(agentID string) string {
 	return findCapableConnector()
 }
 
+// GetVisionConnector returns the connector for vision capabilities.
+// Priority: system.vision > system.default > defaultConnector > findCapableConnector
+func GetVisionConnector() string {
+	if systemConfig != nil {
+		if systemConfig.Vision != "" {
+			return systemConfig.Vision
+		}
+		if systemConfig.Default != "" {
+			return systemConfig.Default
+		}
+	}
+	if defaultConnector != "" {
+		return defaultConnector
+	}
+	return findCapableConnector()
+}
+
+// GetVoiceConnector returns the connector for voice/STT capabilities.
+// Priority: system.voice > system.default > defaultConnector > findCapableConnector
+func GetVoiceConnector() string {
+	if systemConfig != nil {
+		if systemConfig.Voice != "" {
+			return systemConfig.Voice
+		}
+		if systemConfig.Default != "" {
+			return systemConfig.Default
+		}
+	}
+	if defaultConnector != "" {
+		return defaultConnector
+	}
+	return findCapableConnector()
+}
+
 // findCapableConnector finds the first connector that supports tool calling
 func findCapableConnector() string {
 	for id, conn := range connector.Connectors {

agent/load.go

@@ -233,6 +233,8 @@ func initAssistant() error {
 			Prompt:     agentDSL.System.Prompt,
 			NeedSearch: agentDSL.System.NeedSearch,
 			Entity:     agentDSL.System.Entity,
+			Vision:     agentDSL.System.Vision,
+			Voice:      agentDSL.System.Voice,
 		})
 	}
 
@@ -472,6 +474,8 @@ func resolveEnvStrings(setting *types.DSL) {
 		setting.System.RobotPrompt = helper.EnvString(setting.System.RobotPrompt)
 		setting.System.NeedSearch = helper.EnvString(setting.System.NeedSearch)
 		setting.System.Entity = helper.EnvString(setting.System.Entity)
+		setting.System.Vision = helper.EnvString(setting.System.Vision)
+		setting.System.Voice = helper.EnvString(setting.System.Voice)
 	}
 
 	if setting.Uses != nil {

agent/load_test.go

@@ -228,6 +228,8 @@ func TestResolveEnvStrings(t *testing.T) {
 				RobotPrompt: "$ENV.TEST_CONNECTOR",
 				NeedSearch:  "$ENV.TEST_CONNECTOR",
 				Entity:      "$ENV.TEST_CONNECTOR",
+				Vision:      "$ENV.TEST_CONNECTOR",
+				Voice:       "$ENV.TEST_CONNECTOR",
 			},
 		}
 		resolveEnvStrings(setting)
@@ -240,6 +242,25 @@ func TestResolveEnvStrings(t *testing.T) {
 		assert.Equal(t, "openai.gpt-5", setting.System.RobotPrompt)
 		assert.Equal(t, "openai.gpt-5", setting.System.NeedSearch)
 		assert.Equal(t, "openai.gpt-5", setting.System.Entity)
+		assert.Equal(t, "openai.gpt-5", setting.System.Vision)
+		assert.Equal(t, "openai.gpt-5", setting.System.Voice)
+	})
+
+	t.Run("SystemVisionVoiceSeparateEnv", func(t *testing.T) {
+		t.Setenv("TEST_VISION_CONN", "openai.gpt-4o")
+		t.Setenv("TEST_VOICE_CONN", "whisper-1")
+		setting := &types.DSL{
+			System: &types.System{
+				Default: "$ENV.TEST_CONNECTOR",
+				Vision:  "$ENV.TEST_VISION_CONN",
+				Voice:   "$ENV.TEST_VOICE_CONN",
+			},
+		}
+		resolveEnvStrings(setting)
+
+		assert.Equal(t, "openai.gpt-5", setting.System.Default)
+		assert.Equal(t, "openai.gpt-4o", setting.System.Vision)
+		assert.Equal(t, "whisper-1", setting.System.Voice)
 	})
 
 	t.Run("UsesFields", func(t *testing.T) {

agent/types/types.go

@@ -101,6 +101,8 @@ type System struct {
 	RobotPrompt string `json:"robot_prompt,omitempty" yaml:"robot_prompt,omitempty"` // Connector for __yao.robot_prompt agent
 	NeedSearch  string `json:"needsearch,omitempty" yaml:"needsearch,omitempty"`     // Connector for __yao.needsearch agent
 	Entity      string `json:"entity,omitempty" yaml:"entity,omitempty"`             // Connector for __yao.entity agent
+	Vision      string `json:"vision,omitempty" yaml:"vision,omitempty"`             // Connector for vision capabilities
+	Voice       string `json:"voice,omitempty" yaml:"voice,omitempty"`               // Connector for voice/STT capabilities
 }
 
 // Mention Structure

yao/assistants/fetch/package.yao

@@ -0,0 +1,9 @@
+{
+  "name": "Fetch Helper",
+  "description": "Fetch and extract content from URLs",
+  "type": "worker",
+  "automated": true,
+  "public": true,
+  "uses": { "search": "disabled" },
+  "options": {}
+}

yao/assistants/fetch/prompts.yml

@@ -0,0 +1,8 @@
+- role: system
+  content: |
+    You are a content extraction assistant.
+    The user will provide a URL along with its fetched content.
+    Your job is to extract and summarize the key information from the fetched content.
+    Preserve important details, structure, and data.
+    If the content is HTML, focus on the main text and ignore navigation, ads, and boilerplate.
+    Match your response language to the user's language.

yao/assistants/fetch/src/index.ts

@@ -0,0 +1,113 @@
+/**
+ * Fetch Helper Agent - Create Hook
+ *
+ * Extracts URLs from the user's last message, fetches their content via http.Get,
+ * converts HTML to Markdown, and injects the content into the conversation.
+ */
+
+// @ts-nocheck
+
+const URL_REGEX = /https?:\/\/[^\s<>"')\]]+/gi;
+const MAX_CONTENT_LENGTH = 5000;
+
+/**
+ * Create hook - extracts URLs from the last user message, fetches and converts content
+ */
+function Create(
+  ctx: agent.Context,
+  messages: agent.Message[],
+  options?: Record<string, any>
+): agent.HookCreateResponse | null {
+  const lastMsg = findLastUserMessage(messages);
+  if (!lastMsg) return null;
+
+  const text = extractText(lastMsg);
+  if (!text) return null;
+
+  const urls = text.match(URL_REGEX);
+  if (!urls || urls.length === 0) return null;
+
+  const seen = new Set<string>();
+  const fetched: string[] = [];
+
+  for (const raw of urls) {
+    const url = raw.replace(/[.,;:!?)}\]]+$/, "");
+    if (seen.has(url)) continue;
+    seen.add(url);
+
+    try {
+      const resp = http.Get(url, {}, { "User-Agent": "YaoFetchHelper/1.0" });
+      if (resp.code >= 200 && resp.code < 300 && resp.data) {
+        let body =
+          typeof resp.data === "string"
+            ? resp.data
+            : JSON.stringify(resp.data);
+
+        const contentType: string = extractContentType(resp.headers);
+        if (contentType.includes("text/html") || looksLikeHTML(body)) {
+          try {
+            body = Process("text.HTMLToMarkdown", body);
+          } catch (_) {}
+        }
+
+        if (body.length > MAX_CONTENT_LENGTH) {
+          body = body.substring(0, MAX_CONTENT_LENGTH) + "\n... [truncated]";
+        }
+        fetched.push(`--- Content from ${url} ---\n${body}\n--- End ---`);
+      }
+    } catch (_) {
+      // Skip failed fetches silently
+    }
+  }
+
+  if (fetched.length === 0) return null;
+
+  return {
+    messages: [
+      {
+        role: "user",
+        content: `${text}\n\nFetched content:\n\n${fetched.join("\n\n")}`,
+      },
+    ],
+  };
+}
+
+function findLastUserMessage(
+  messages: agent.Message[]
+): agent.Message | null {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i].role === "user") return messages[i];
+  }
+  return null;
+}
+
+function extractText(msg: agent.Message): string {
+  if (typeof msg.content === "string") return msg.content;
+  if (Array.isArray(msg.content)) {
+    return msg.content
+      .filter((p: any) => p.type === "text" && p.text)
+      .map((p: any) => p.text)
+      .join("\n");
+  }
+  return "";
+}
+
+function extractContentType(headers: Record<string, any>): string {
+  if (!headers) return "";
+  for (const key of Object.keys(headers)) {
+    if (key.toLowerCase() === "content-type") {
+      const val = headers[key];
+      return (Array.isArray(val) ? val[0] : val || "").toLowerCase();
+    }
+  }
+  return "";
+}
+
+function looksLikeHTML(text: string): boolean {
+  const trimmed = text.trimStart();
+  return (
+    trimmed.startsWith("<!") ||
+    trimmed.startsWith("<html") ||
+    trimmed.startsWith("<HTML")
+  );
+}

yao/assistants/querydsl/package.yao

@@ -2,6 +2,8 @@
   "name": "Query Builder",
   "description": "Build database queries",
   "type": "worker",
+  "automated": true,
+  "public": true,
   "uses": { "search": "disabled" },
   "options": { "max_tokens": 8192 }
 }

yao/assistants/vision/package.yao

@@ -0,0 +1,7 @@
+{
+  "name": "Vision Helper",
+  "description": "Analyze images when the main model doesn't support vision",
+  "type": "worker",
+  "uses": { "search": "disabled" },
+  "options": {}
+}

yao/assistants/vision/prompts.yml

@@ -0,0 +1,7 @@
+- role: system
+  content: |
+    You are a vision analysis assistant.
+    Analyze images in detail and describe what you see.
+    Be precise, thorough, and objective in your descriptions.
+    Focus on text content, layout, objects, and any actionable information.
+    Match your response language to the user's language.