diff --git a/agent/api/agent.go b/agent/api/agent.go deleted file mode 100644 index d41e8d80..00000000 --- a/agent/api/agent.go +++ /dev/null @@ -1,38 +0,0 @@ -package api - -import ( - "github.com/gin-gonic/gin" - "github.com/yaoapp/yao/agent/assistant" - chatctx "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/types" -) - -// Agent the agent AI assistant -var Agent *API - -// API the agent API -type API struct { - *types.DSL -} - -// Answer reply the message -func (agent *API) Answer(ctx chatctx.Context, question string, c *gin.Context) error { - var err error - var ast assistant.API = Agent.Assistant - if ctx.AssistantID != "" { - ast, err = agent.Select(ctx.AssistantID) - if err != nil { - return err - } - } - _, err = ast.Execute(c, ctx, question, nil) - return err -} - -// Select select an assistant -func (agent *API) Select(id string) (assistant.API, error) { - if id == "" { - return Agent.Assistant, nil - } - return assistant.Get(id) -} diff --git a/agent/api/api.go b/agent/api/api.go deleted file mode 100644 index b9bfdae4..00000000 --- a/agent/api/api.go +++ /dev/null @@ -1,1061 +0,0 @@ -package api - -import ( - "fmt" - "net/url" - "strconv" - "strings" - "time" - - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/agent/assistant" - chatctx "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/message" - store "github.com/yaoapp/yao/agent/store/types" - "github.com/yaoapp/yao/agent/types" - "github.com/yaoapp/yao/helper" - "github.com/yaoapp/yao/openapi/oauth" -) - -// API registers the Agent API endpoints -func (agent *API) API(router *gin.Engine, path string) error { - - // Get the guards - middlewares, err := agent.getGuardHandlers() - if err != nil { - return err - } - - // Chat endpoint - // Chat endpoint - // Example: - // curl -X GET 'http://localhost:5099/api/__yao/agent?content=Hello&chat_id=chat_123&context=previous_context&token=xxx' - // curl -X POST 'http://localhost:5099/api/__yao/agent' \ - // -H 'Content-Type: application/json' \ - // -d '{"content": "Hello", "chat_id": "chat_123", "context": "previous_context", "token": "xxx"}' - router.GET(path, append(middlewares, agent.handleChat)...) - router.POST(path, append(middlewares, agent.handleChat)...) - - // Status check endpoint - // Example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/status?token=xxx' - router.GET(path+"/status", append(middlewares, agent.handleStatus)...) - - // Assistant API endpoints - // List assistants example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/assistants?page=1&pagesize=20&tags=tag1,tag2&token=xxx' - router.GET(path+"/assistants", append(middlewares, agent.HandleAssistantList)...) - // Get all assistant tags example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/assistants/tags?token=xxx' - router.GET(path+"/assistants/tags", append(middlewares, agent.HandleAssistantTags)...) - - // Get assistant details example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/assistants/assistant_123?token=xxx' - router.GET(path+"/assistants/:id", append(middlewares, agent.HandleAssistantDetail)...) - - // Execute assistant API example: - // curl -X POST 'http://localhost:5099/api/__yao/agent/assistants/assistant_123/api' \ - // -H 'Content-Type: application/json' \ - // -d '{"name": "Test", "payload": {"name": "yao", "age": 18}}' - router.POST(path+"/assistants/:id/call", append(middlewares, agent.HandleAssistantCall)...) - - // Create/Update assistant example: - // curl -X POST 'http://localhost:5099/api/__yao/agent/assistants' \ - // -H 'Content-Type: application/json' \ - // -d '{"name": "My Assistant", "type": "chat", "tags": ["tag1", "tag2"], "mentionable": true, "avatar": "path/to/avatar.png", "token": "xxx"}' - router.POST(path+"/assistants", append(middlewares, agent.HandleAssistantSave)...) - - // Delete assistant example: - // curl -X DELETE 'http://localhost:5099/api/__yao/agent/assistants/assistant_123?token=xxx' - router.DELETE(path+"/assistants/:id", append(middlewares, agent.HandleAssistantDelete)...) - - // Chat management endpoints - // List chats example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/chats?page=1&pagesize=20&keywords=search+term&order=desc&token=xxx' - router.GET(path+"/chats", append(middlewares, agent.handleChatList)...) - - // Get latest chat example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/chats/latest?assistant_id=assistant_123&token=xxx' - router.GET(path+"/chats/latest", append(middlewares, agent.handleChatLatest)...) - - // Get chat details example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/chats/chat_123?token=xxx' - router.GET(path+"/chats/:id", append(middlewares, agent.handleChatDetail)...) - - // Update chat example: - // curl -X POST 'http://localhost:5099/api/__yao/agent/chats/chat_123' \ - // -H 'Content-Type: application/json' \ - // -d '{"title": "New Title", "content": "Chat content for title generation", "token": "xxx"}' - router.POST(path+"/chats/:id", append(middlewares, agent.handleChatUpdate)...) - - // Delete chat example: - // curl -X DELETE 'http://localhost:5099/api/__yao/agent/chats/chat_123?token=xxx' - router.DELETE(path+"/chats/:id", append(middlewares, agent.handleChatDelete)...) - - // Chat history endpoint - // Example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/history?chat_id=chat_123&token=xxx' - router.GET(path+"/history", append(middlewares, agent.handleChatHistory)...) - - // File management endpoints - // Upload file example: - // curl -X POST 'http://localhost:5099/api/__yao/agent/upload?chat_id=chat_123&token=xxx' \ - // -F 'file=@/path/to/file.txt' - // router.POST(path+"/upload/:storage", append(middlewares, agent.handleUpload)...) - - // Download file example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/download?file_id=file_123&disposition=attachment&token=xxx' \ - // -o downloaded_file.txt - // router.GET(path+"/download", append(middlewares, agent.handleDownload)...) - - // Mentions endpoint - // Example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/mentions?keywords=assistant&token=xxx' - router.GET(path+"/mentions", append(middlewares, agent.handleMentions)...) - - // Generate title example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/generate/title?content=Chat+content&chat_id=chat_123&token=xxx' - // curl -X POST 'http://localhost:5099/api/__yao/agent/generate/title' \ - // -H 'Content-Type: application/json' \ - // -d '{"content": "Chat content", "chat_id": "chat_123", "token": "xxx"}' - router.GET(path+"/generate/title", append(middlewares, agent.handleGenerateTitle)...) - router.POST(path+"/generate/title", append(middlewares, agent.handleGenerateTitle)...) - - // Generate prompts example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/generate/prompts?content=Generate+prompts&chat_id=chat_123&token=xxx' - // curl -X POST 'http://localhost:5099/api/__yao/agent/generate/prompts' \ - // -H 'Content-Type: application/json' \ - // -d '{"content": "Generate prompts", "chat_id": "chat_123", "token": "xxx"}' - router.GET(path+"/generate/prompts", append(middlewares, agent.handleGeneratePrompts)...) - router.POST(path+"/generate/prompts", append(middlewares, agent.handleGeneratePrompts)...) - - // Utility endpoints - // List connectors example: - // curl -X GET 'http://localhost:5099/api/__yao/agent/utility/connectors?token=xxx' - router.GET(path+"/utility/connectors", append(middlewares, agent.handleConnectors)...) - - // Dangerous operations - // Dangerous operations - // Clear all chats example: - // curl -X DELETE 'http://localhost:5099/api/__yao/agent/dangerous/clear_chats?token=xxx' - router.DELETE(path+"/dangerous/clear_chats", append(middlewares, agent.handleChatsDeleteAll)...) - - return nil -} - -// handleStatus handles the status request -func (agent *API) handleStatus(c *gin.Context) { - c.Status(200) - c.Done() -} - -// handleChat handles the chat request -func (agent *API) handleChat(c *gin.Context) { - // Set headers for SSE - c.Header("Content-Type", "text/event-stream;charset=utf-8") - c.Header("Cache-Control", "no-cache") - c.Header("Connection", "keep-alive") - - sid := c.GetString("__sid") - if sid == "" { - sid = uuid.New().String() - } - - content := c.Query("content") - if content == "" { - msg := message.New().Error("content is required").Done() - msg.Write(c.Writer) - return - } - - chatID := c.Query("chat_id") - if chatID == "" { - // Only generate new chat_id if not provided - chatID = fmt.Sprintf("chat_%d", time.Now().UnixNano()) - } - - // Set the context with validated chat_id - ctx, cancel := chatctx.NewWithCancel(c.Request.Context(), nil, chatID, c.Query("context")) - defer cancel() - defer ctx.Release() // Release the context after the request is done - - // // Set the assistant ID - // assistantID := c.Query("assistant_id") - // if assistantID != "" { - // ctx = chatctx.WithAssistantID(ctx, assistantID) - // } - - // // Set the silent mode - // silent := c.Query("silent") - // if silent == "true" || silent == "1" { - // ctx = chatctx.WithSilent(ctx, true) - // } - - // // Set the history visible - // historyVisible := c.Query("history_visible") - // if historyVisible != "" { - // ctx = chatctx.WithHistoryVisible(ctx, historyVisible == "true" || historyVisible == "1") - // } - - // // Set the client type - // clientType := c.Query("client_type") - // if clientType != "" { - // ctx = chatctx.WithClientType(ctx, clientType) - // } - - err := agent.Answer(ctx, content, c) - - // Error handling - if err != nil { - message.New().Done().Error(err).Write(c.Writer) - c.Done() - return - } -} - -// handleChatList handles the chat list request -func (agent *API) handleChatList(c *gin.Context) { - sid := c.GetString("__sid") - sid = "temporary-mock-sid-123" - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - // Create filter from query parameters - filter := store.ChatFilter{ - Keywords: c.Query("keywords"), - Order: c.Query("order"), - } - - // Parse page and pagesize - if page := c.Query("page"); page != "" { - if n, err := strconv.Atoi(page); err == nil { - filter.Page = n - } - } - - if pageSize := c.Query("pagesize"); pageSize != "" { - if n, err := strconv.Atoi(pageSize); err == nil { - filter.PageSize = n - } - } - - locale := "en-us" - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - response, err := agent.Store.GetChats(sid, filter, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, map[string]interface{}{"data": response}) - c.Done() -} - -// handleChatHistory handles the chat history request -func (agent *API) handleChatHistory(c *gin.Context) { - sid := c.GetString("__sid") - sid = "temporary-mock-sid-123" - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - cid := c.Query("chat_id") - history, err := agent.Store.GetHistory(sid, cid) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, map[string]interface{}{"data": history}) - c.Done() -} - -// getOrigin returns the request origin -func (agent *API) getOrigin(c *gin.Context) string { - origin := c.Request.Header.Get("Origin") - if origin == "" { - origin = c.Request.Referer() - if origin != "" { - if u, err := url.Parse(origin); err == nil { - origin = fmt.Sprintf("%s://%s", u.Scheme, u.Host) - } - } - } - return origin -} - -// getGuardHandlers returns authentication middleware handlers -func (agent *API) getGuardHandlers() ([]gin.HandlerFunc, error) { - return []gin.HandlerFunc{}, nil -} - -// defaultGuard is the default authentication handler -func (agent *API) defaultGuard(c *gin.Context) { - - // Check if the request is for OpenAPI OAuth - if oauth.OAuth != nil { - agent.guardOpenapiOauth(c) - return - } - - token := strings.TrimSpace(strings.TrimPrefix(c.Query("token"), "Bearer ")) - if token == "" { - c.JSON(403, gin.H{"message": "token is required", "code": 403}) - c.Abort() - return - } - - user := helper.JwtValidate(token) - c.Set("__sid", user.SID) - c.Next() -} - -// Openapi Oauth -func (agent *API) guardOpenapiOauth(c *gin.Context) { - s := oauth.OAuth - token := agent.getAccessToken(c) - if token == "" { - c.JSON(403, gin.H{"code": 403, "message": "Not Authorized"}) - c.Abort() - return - } - - // Validate the token - _, err := s.VerifyToken(token) - if err != nil { - c.JSON(403, gin.H{"code": 403, "message": "Not Authorized"}) - c.Abort() - return - } - - // Get the session ID - sid := agent.getSessionID(c) - if sid == "" { - c.JSON(403, gin.H{"code": 403, "message": "Not Authorized"}) - c.Abort() - return - } - - c.Set("__sid", sid) -} - -func (agent *API) getAccessToken(c *gin.Context) string { - token := c.GetHeader("Authorization") - if token == "" || token == "Bearer undefined" { - cookie, err := c.Cookie("__Host-access_token") - if err != nil { - return "" - } - token = cookie - } - return strings.TrimPrefix(token, "Bearer ") -} - -func (agent *API) getSessionID(c *gin.Context) string { - sid, err := c.Cookie("__Host-session_id") - if err != nil { - return "" - } - return sid -} - -// handleChatLatest handles getting the latest chat -func (agent *API) handleChatLatest(c *gin.Context) { - sid := c.GetString("__sid") - sid = "temporary-mock-sid-123" - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - locale := "en-us" - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - // Get the chats - chats, err := agent.Store.GetChats(sid, store.ChatFilter{Page: 1}, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - // Create a new chat - if len(chats.Groups) == 0 || len(chats.Groups[0].Chats) == 0 { - - assistantID := agent.Uses.Default - queryAssistantID := c.Query("assistant_id") - if queryAssistantID != "" { - assistantID = queryAssistantID - } - - // Get the assistant info - ast, err := assistant.Get(assistantID) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, map[string]interface{}{"data": map[string]interface{}{ - "placeholder": ast.GetPlaceholder(locale), - "assistant_id": ast.ID, - "assistant_name": ast.GetName(locale), - "assistant_avatar": ast.Avatar, - "assistant_deleteable": agent.Uses.Default != ast.ID, - }}) - c.Done() - return - } - - // Get the chat_id - chatID, ok := chats.Groups[0].Chats[0]["chat_id"].(string) - if !ok { - c.JSON(404, gin.H{"message": "chat_id not found", "code": 404}) - c.Done() - return - } - - chat, err := agent.Store.GetChat(sid, chatID, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - // assistant_id is nil return the default assistant - if chat.Chat["assistant_id"] == nil { - chat.Chat["assistant_id"] = agent.Uses.Default - - // Get the assistant info - ast, err := assistant.Get(agent.Uses.Default) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - chat.Chat["assistant_name"] = ast.GetName(locale) - chat.Chat["assistant_avatar"] = ast.Avatar - } - - chat.Chat["assistant_deleteable"] = agent.Uses.Default != chat.Chat["assistant_id"] - c.JSON(200, map[string]interface{}{"data": chat}) - c.Done() -} - -// handleChatDetail handles getting a single chat's details -func (agent *API) handleChatDetail(c *gin.Context) { - sid := c.GetString("__sid") - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - chatID := c.Param("id") - if chatID == "" { - c.JSON(400, gin.H{"message": "chat id is required", "code": 400}) - c.Done() - return - } - - locale := "en-us" - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - // Get the chat details - chat, err := agent.Store.GetChat(sid, chatID, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - // assistant_id is nil return the default assistant - if chat.Chat["assistant_id"] == nil { - chat.Chat["assistant_id"] = agent.Uses.Default - - // Get the assistant info - ast, err := assistant.Get(agent.Uses.Default) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - chat.Chat["assistant_name"] = ast.GetName(locale) - chat.Chat["assistant_avatar"] = ast.Avatar - } - - chat.Chat["assistant_deleteable"] = agent.Uses.Default != chat.Chat["assistant_id"] - c.JSON(200, map[string]interface{}{"data": chat}) - c.Done() -} - -// handleMentions handles getting mentions for a chat -func (agent *API) handleMentions(c *gin.Context) { - sid := c.GetString("__sid") - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - locale := "en-us" - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - // Get keywords from query parameter - keywords := strings.ToLower(c.Query("keywords")) - mentionable := true - - // Query mentionable assistants - filter := store.AssistantFilter{ - Keywords: keywords, - Mentionable: &mentionable, - Page: 1, - PageSize: 20, - } - - response, err := agent.Store.GetAssistants(filter, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - // Convert assistants to mentions - mentions := []types.Mention{} - for _, assistant := range response.Data { - mention := types.Mention{ - ID: assistant.ID, - Name: assistant.Name, - Type: assistant.Type, - Avatar: assistant.Avatar, - } - mentions = append(mentions, mention) - } - - c.JSON(200, map[string]interface{}{"data": mentions}) - c.Done() -} - -// handleChatUpdate handles updating a chat's details -func (agent *API) handleChatUpdate(c *gin.Context) { - sid := c.GetString("__sid") - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - chatID := c.Param("id") - if chatID == "" { - c.JSON(400, gin.H{"message": "chat id is required", "code": 400}) - c.Done() - return - } - - // Get title from request body - var body struct { - Title string `json:"title"` - Content string `json:"content"` - } - if err := c.BindJSON(&body); err != nil { - c.JSON(400, gin.H{"message": "invalid request body", "code": 400}) - c.Done() - return - } - - if body.Title == "" { - c.JSON(400, gin.H{"message": "title is required", "code": 400}) - c.Done() - return - } - - err := agent.Store.UpdateChatTitle(sid, chatID, body.Title) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, gin.H{"message": "ok", "title": body.Title, "chat_id": chatID}) - c.Done() -} - -// handleChatDelete handles deleting a single chat -func (agent *API) handleChatDelete(c *gin.Context) { - sid := c.GetString("__sid") - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - chatID := c.Param("id") - if chatID == "" { - c.JSON(400, gin.H{"message": "chat id is required", "code": 400}) - c.Done() - return - } - - err := agent.Store.DeleteChat(sid, chatID) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, gin.H{"message": "ok"}) - c.Done() -} - -// handleChatsDeleteAll handles deleting all chats for a user -func (agent *API) handleChatsDeleteAll(c *gin.Context) { - sid := c.GetString("__sid") - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - err := agent.Store.DeleteAllChats(sid) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, gin.H{"message": "ok"}) - c.Done() -} - -// handleGenerateTitle handles generating a chat title -func (agent *API) handleGenerateTitle(c *gin.Context) { - // Set headers for SSE - c.Header("Content-Type", "text/event-stream;charset=utf-8") - c.Header("Cache-Control", "no-cache") - c.Header("Connection", "keep-alive") - - sid := c.GetString("__sid") - if sid == "" { - sid = uuid.New().String() - } - - content := c.Query("content") - if content == "" { - msg := message.New().Error("content is required").Done() - msg.Write(c.Writer) - return - } - - chatID := fmt.Sprintf("generate_title_%d", time.Now().UnixNano()) - - // Set the context with validated chat_id - ctx, cancel := chatctx.NewWithCancel(c.Request.Context(), nil, chatID, c.Query("context")) - defer cancel() - defer ctx.Release() // Release the context after the request is done - - // // Set the assistant ID - // ctx = chatctx.WithHistoryVisible(ctx, false) - // ctx = chatctx.WithAssistantID(ctx, agent.Uses.Title) - - err := agent.Answer(ctx, content, c) - - // Error handling - if err != nil { - message.New().Done().Error(err).Write(c.Writer) - c.Done() - return - } -} - -// handleGeneratePrompts handles generating prompts -func (agent *API) handleGeneratePrompts(c *gin.Context) { - // Set headers for SSE - c.Header("Content-Type", "text/event-stream;charset=utf-8") - c.Header("Cache-Control", "no-cache") - c.Header("Connection", "keep-alive") - - sid := c.GetString("__sid") - if sid == "" { - sid = uuid.New().String() - } - - content := c.Query("content") - if content == "" { - msg := message.New().Error("content is required").Done() - msg.Write(c.Writer) - return - } - - chatID := fmt.Sprintf("generate_prompts_%d", time.Now().UnixNano()) - - // Set the context with validated chat_id - ctx, cancel := chatctx.NewWithCancel(c.Request.Context(), nil, chatID, c.Query("context")) - defer cancel() - defer ctx.Release() // Release the context after the request is done - - // // Set the assistant ID - // ctx = chatctx.WithHistoryVisible(ctx, false) - // ctx = chatctx.WithAssistantID(ctx, agent.Uses.Prompt) - err := agent.Answer(ctx, content, c) - - // Error handling - if err != nil { - message.New().Done().Error(err).Write(c.Writer) - c.Done() - return - } -} - -// HandleAssistantList handles listing assistants (exported for use in openapi/agent) -func (agent *API) HandleAssistantList(c *gin.Context) { - // Parse filter parameters - filter := store.AssistantFilter{ - Type: "assistant", - Page: 1, - PageSize: 20, - } - - // Parse page and pagesize - if page := c.Query("page"); page != "" { - if n, err := strconv.Atoi(page); err == nil { - filter.Page = n - } - } - - if pageSize := c.Query("pagesize"); pageSize != "" { - if n, err := strconv.Atoi(pageSize); err == nil { - filter.PageSize = n - } - } - - // Parse tags - if tags := c.Query("tags"); tags != "" { - filter.Tags = strings.Split(tags, ",") - } - - // Parse keywords - if keywords := c.Query("keywords"); keywords != "" { - filter.Keywords = keywords - } - - // Parse connector - if connector := c.Query("connector"); connector != "" { - filter.Connector = connector - } - - // Parse select fields - if selectFields := c.Query("select"); selectFields != "" { - filter.Select = strings.Split(selectFields, ",") - } - - // Parse built_in (support various boolean formats) - if builtIn := c.Query("built_in"); builtIn != "" { - val := parseBoolValue(builtIn) - if val != nil { - filter.BuiltIn = val - } - } - - // Parse mentionable (support various boolean formats) - if mentionable := c.Query("mentionable"); mentionable != "" { - val := parseBoolValue(mentionable) - if val != nil { - filter.Mentionable = val - } - } - - // Parse automated (support various boolean formats) - if automated := c.Query("automated"); automated != "" { - val := parseBoolValue(automated) - if val != nil { - filter.Automated = val - } - } - - // Parse assistant_id - if assistantID := c.Query("assistant_id"); assistantID != "" { - filter.AssistantID = assistantID - } - - locale := "en-us" // Default locale - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - response, err := agent.Store.GetAssistants(filter, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, response) - c.Done() -} - -// parseBoolValue parses various string formats into a boolean pointer -// Supports: 1, 0, "1", "0", "true", "false", etc. -func parseBoolValue(value string) *bool { - value = strings.ToLower(strings.TrimSpace(value)) - switch value { - case "1", "true", "yes", "on": - v := true - return &v - case "0", "false", "no", "off": - v := false - return &v - default: - return nil - } -} - -// HandleAssistantCall handles the assistant API call (exported for use in openapi/agent) -func (agent *API) HandleAssistantCall(c *gin.Context) { - assistantID := c.Param("id") - if assistantID == "" { - c.JSON(400, gin.H{"message": "assistant id is required", "code": 400}) - c.Done() - return - } - ast, err := assistant.Get(assistantID) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - sid := c.GetString("__sid") - if sid == "" { - c.JSON(400, gin.H{"message": "sid is required", "code": 400}) - c.Done() - return - } - - payload := assistant.APIPayload{Sid: sid} - if err := c.BindJSON(&payload); err != nil { - c.JSON(400, gin.H{"message": "invalid request body", "code": 400}) - c.Done() - return - } - - result, err := ast.Call(c, payload) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, result) - c.Done() -} - -// HandleAssistantDetail handles getting a single assistant's details (exported for use in openapi/agent) -func (agent *API) HandleAssistantDetail(c *gin.Context) { - assistantID := c.Param("id") - if assistantID == "" { - c.JSON(400, gin.H{"message": "assistant id is required", "code": 400}) - c.Done() - return - } - - filter := store.AssistantFilter{ - AssistantID: assistantID, - Type: "assistant", - Page: 1, - PageSize: 1, - } - - locale := "en-us" // Default locale - // Translate the response - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - response, err := agent.Store.GetAssistants(filter, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - if len(response.Data) == 0 { - c.JSON(404, gin.H{"message": "assistant not found", "code": 404}) - c.Done() - return - } - - c.JSON(200, gin.H{"data": response.Data[0]}) - c.Done() -} - -// HandleAssistantSave handles creating or updating an assistant (exported for use in openapi/agent) -func (agent *API) HandleAssistantSave(c *gin.Context) { - var assistantData map[string]interface{} - if err := c.BindJSON(&assistantData); err != nil { - c.JSON(400, gin.H{"message": "invalid request body", "code": 400}) - c.Done() - return - } - - // Convert to AssistantModel - model, err := store.ToAssistantModel(assistantData) - if err != nil { - c.JSON(400, gin.H{"message": fmt.Sprintf("invalid assistant data: %s", err.Error()), "code": 400}) - c.Done() - return - } - - id, err := agent.Store.SaveAssistant(model) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - // Update the assistant map with the returned ID if it's not already set - if _, ok := assistantData["assistant_id"]; !ok { - assistantData["assistant_id"] = id - } - - // Remove the assistant from cache to ensure fresh data on next load - cache := assistant.GetCache() - if cache != nil { - cache.Remove(id) - } - - // Reload the assistant to ensure it's available in cache with updated data - _, err = assistant.Get(id) - if err != nil { - // Just log the error, don't fail the request - fmt.Printf("Error reloading assistant %s: %v\n", id, err) - } - - c.JSON(200, gin.H{"message": "ok", "data": assistantData}) - c.Done() -} - -// HandleAssistantDelete handles deleting an assistant (exported for use in openapi/agent) -func (agent *API) HandleAssistantDelete(c *gin.Context) { - assistantID := c.Param("id") - if assistantID == "" { - c.JSON(400, gin.H{"message": "assistant id is required", "code": 400}) - c.Done() - return - } - - err := agent.Store.DeleteAssistant(assistantID) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - // Remove the assistant from cache to ensure it's fully deleted - cache := assistant.GetCache() - if cache != nil { - cache.Remove(assistantID) - } - - c.JSON(200, gin.H{"message": "ok"}) - c.Done() -} - -// handleConnectors handles listing connectors -func (agent *API) handleConnectors(c *gin.Context) { - options := []map[string]interface{}{} - - // Filter and format connectors - for id, conn := range connector.Connectors { - if conn.Is(connector.OPENAI) || conn.Is(connector.MOAPI) { - setting := conn.Setting() - label := setting["label"] - if label == nil || label == "" { - label = setting["name"] - } - if label == nil || label == "" { - label = id - } - options = append(options, map[string]interface{}{ - "label": label, - "value": id, - }) - } - } - - c.JSON(200, gin.H{"data": options}) - c.Done() -} - -// HandleAssistantTags handles getting all assistant tags (exported for use in openapi/agent) -func (agent *API) HandleAssistantTags(c *gin.Context) { - locale := "en-us" // Default locale - if loc := c.Query("locale"); loc != "" { - locale = strings.ToLower(strings.TrimSpace(loc)) - } - - // Build filter for tags query - filter := store.AssistantFilter{} - - // Apply type filter (default to "assistant") - typeParam := strings.TrimSpace(c.Query("type")) - if typeParam == "" { - typeParam = "assistant" - } - filter.Type = typeParam - - // Apply other optional filters - if connector := strings.TrimSpace(c.Query("connector")); connector != "" { - filter.Connector = connector - } - - if builtInParam := c.Query("built_in"); builtInParam != "" { - if val, err := strconv.ParseBool(builtInParam); err == nil { - filter.BuiltIn = &val - } - } - - if mentionableParam := c.Query("mentionable"); mentionableParam != "" { - if val, err := strconv.ParseBool(mentionableParam); err == nil { - filter.Mentionable = &val - } - } - - if automatedParam := c.Query("automated"); automatedParam != "" { - if val, err := strconv.ParseBool(automatedParam); err == nil { - filter.Automated = &val - } - } - - if keywords := strings.TrimSpace(c.Query("keywords")); keywords != "" { - filter.Keywords = keywords - } - - tags, err := agent.Store.GetAssistantTags(filter, locale) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, gin.H{"data": tags}) - c.Done() -} diff --git a/agent/api/api_test.go b/agent/api/api_test.go deleted file mode 100644 index 20ede75f..00000000 --- a/agent/api/api_test.go +++ /dev/null @@ -1,233 +0,0 @@ -package api - -// import ( -// "context" -// "fmt" -// "net" -// "net/http" -// "net/http/httptest" -// "os" -// "strings" -// "testing" -// "time" - -// "github.com/gin-gonic/gin" -// "github.com/stretchr/testify/assert" -// httpTest "github.com/yaoapp/gou/http" -// "github.com/yaoapp/yao/config" -// "github.com/yaoapp/yao/helper" -// "github.com/yaoapp/yao/test" -// ) - -// func init() { -// // Set gin to release mode to reduce log output -// gin.SetMode(gin.ReleaseMode) -// } - -// func TestAPI(t *testing.T) { -// // Disable test logging -// test.Prepare(t, config.Conf) -// defer test.Clean() - -// // Redirect stdout to /dev/null -// oldStdout := os.Stdout -// null, _ := os.Open(os.DevNull) -// os.Stdout = null -// defer func() { -// os.Stdout = oldStdout -// null.Close() -// }() - -// // test router -// router := testRouter(t) -// err := Agent.API(router, "/agent/chat") -// if err != nil { -// t.Fatal(err) -// } - -// // test server -// host, shutdown := testServer(t, router) -// defer shutdown() - -// tests := []struct { -// name string -// url string -// method string -// headers http.Header -// expectCode int -// expectBody string -// }{ -// { -// name: "Basic Chat Request", -// url: fmt.Sprintf("/agent/chat?content=hello&token=%s", testToken()), -// method: "GET", -// headers: http.Header{"Content-Type": []string{"application/json"}}, -// expectBody: `{`, -// }, -// { -// name: "Chat with System Message", -// url: fmt.Sprintf("/agent/chat?content=hello&system=You are a helpful assistant&token=%s", testToken()), -// method: "GET", -// headers: http.Header{"Content-Type": []string{"application/json"}}, -// expectBody: `{`, -// }, -// { -// name: "Chat with Model Parameter", -// url: fmt.Sprintf("/agent/chat?content=hello&model=gpt-3.5-turbo&token=%s", testToken()), -// method: "GET", -// headers: http.Header{"Content-Type": []string{"application/json"}}, -// expectBody: `{`, -// }, -// } - -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// url := fmt.Sprintf("%s%s", host, tt.url) -// res := []byte{} -// req := httpTest.New(url).WithHeader(tt.headers) - -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// req.Stream(ctx, tt.method, nil, func(data []byte) int { -// res = append(res, data...) -// return 1 -// }) - -// assert.Contains(t, string(res), tt.expectBody) -// }) -// } -// } - -// func TestAPIAuth(t *testing.T) { -// test.Prepare(t, config.Conf) -// defer test.Clean() - -// // Redirect stdout and stderr to /dev/null -// oldStdout := os.Stdout -// oldStderr := os.Stderr -// null, _ := os.Open(os.DevNull) -// os.Stdout = null -// os.Stderr = null -// defer func() { -// os.Stdout = oldStdout -// os.Stderr = oldStderr -// null.Close() -// }() - -// router := testRouter(t) -// err := Agent.API(router, "/agent/chat") -// if err != nil { -// t.Fatal(err) -// } - -// // Separate tests for authentication errors and parameter validation errors -// authTests := []struct { -// name string -// url string -// method string -// expectCode int -// }{ -// { -// name: "Missing Token", -// url: "/agent/chat?content=hello", -// method: "GET", -// expectCode: http.StatusUnauthorized, -// }, -// { -// name: "Invalid Token", -// url: "/agent/chat?content=hello&token=invalid", -// method: "GET", -// expectCode: http.StatusUnauthorized, -// }, -// } - -// // Test authentication errors (will panic) -// for _, tt := range authTests { -// t.Run(tt.name, func(t *testing.T) { -// response := httptest.NewRecorder() -// req, _ := http.NewRequest(tt.method, tt.url, nil) -// assert.Panics(t, func() { -// router.ServeHTTP(response, req) -// }) -// }) -// } - -// // Test parameter validation errors (will return status code) -// validationTests := []struct { -// name string -// url string -// method string -// expectCode int -// }{ -// { -// name: "Missing Content", -// url: fmt.Sprintf("/agent/chat?token=%s", testToken()), -// method: "GET", -// expectCode: http.StatusBadRequest, -// }, -// } - -// // Test parameter validation errors (return status code) -// for _, tt := range validationTests { -// t.Run(tt.name, func(t *testing.T) { -// response := httptest.NewRecorder() -// req, _ := http.NewRequest(tt.method, tt.url, nil) -// router.ServeHTTP(response, req) -// assert.Equal(t, tt.expectCode, response.Code) -// }) -// } -// } - -// // Helper functions -// func testServer(t *testing.T, router *gin.Engine) (string, func()) { -// l, err := net.Listen("tcp4", ":0") -// if err != nil { -// t.Fatal(err) -// } - -// srv := &http.Server{Addr: ":0", Handler: router} - -// go func() { -// if err := srv.Serve(l); err != nil && err != http.ErrServerClosed { -// return -// } -// }() - -// addr := strings.Split(l.Addr().String(), ":") -// if len(addr) != 2 { -// t.Fatal("invalid address") -// } - -// host := fmt.Sprintf("http://127.0.0.1:%s", addr[1]) -// time.Sleep(50 * time.Millisecond) - -// shutdown := func() { -// srv.Close() -// l.Close() -// } -// return host, shutdown -// } - -// func testRouter(t *testing.T) *gin.Engine { -// err := Load(config.Conf) -// if err != nil { -// t.Fatal(err) -// } - -// router := gin.New() // Use gin.New() instead of gin.Default() to avoid default logging middleware -// return router -// } - -// func testToken() string { -// token := helper.JwtMake(1, -// map[string]interface{}{ -// "id": 1, -// "name": "Test", -// }, -// map[string]interface{}{ -// "exp": 3600, -// "sid": "123456", -// }) -// return token.Token -// } diff --git a/agent/assistant/api.go b/agent/assistant/api.go deleted file mode 100644 index 0534e3af..00000000 --- a/agent/assistant/api.go +++ /dev/null @@ -1,1286 +0,0 @@ -package assistant - -import ( - "context" - "encoding/base64" - "fmt" - "os" - "strings" - "time" - - "github.com/fatih/color" - "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/gou/fs" - "github.com/yaoapp/kun/exception" - "github.com/yaoapp/kun/log" - chatctx "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/i18n" - "github.com/yaoapp/yao/agent/message" - chatMessage "github.com/yaoapp/yao/agent/message" - store "github.com/yaoapp/yao/agent/store/types" -) - -// Get get the assistant by id -func Get(id string) (*Assistant, error) { - return LoadStore(id) -} - -// GetByConnector get the assistant by connector -func GetByConnector(connector string, name string) (*Assistant, error) { - id := "connector:" + connector - - assistant, exists := loaded.Get(id) - if exists { - return assistant, nil - } - - data := map[string]interface{}{ - "assistant_id": id, - "connector": connector, - "description": "Default assistant for " + connector, - "name": name, - "type": "assistant", - } - - assistant, err := loadMap(data) - if err != nil { - return nil, err - } - loaded.Put(assistant) - return assistant, nil -} - -// Execute implements the execute functionality -func (ast *Assistant) Execute(c *gin.Context, ctx chatctx.Context, input interface{}, options map[string]interface{}, callback ...interface{}) (interface{}, error) { - contents := chatMessage.NewContents() - messages, err := ast.withHistory(ctx, input) - if err != nil { - return nil, err - } - return ast.execute(c, ctx, messages, options, contents, callback...) -} - -// Execute implements the execute functionality -func (ast *Assistant) execute(c *gin.Context, ctx chatctx.Context, userInput interface{}, userOptions map[string]interface{}, contents *chatMessage.Contents, callback ...interface{}) (interface{}, error) { - - var input []chatMessage.Message - - switch v := userInput.(type) { - case string: - input = []chatMessage.Message{{Role: "user", Text: v}} - - case []interface{}: - raw, err := jsoniter.Marshal(v) - if err != nil { - return nil, fmt.Errorf("marshal input error: %s", err.Error()) - } - err = jsoniter.Unmarshal(raw, &input) - if err != nil { - return nil, fmt.Errorf("unmarshal input error: %s", err.Error()) - } - - case []chatMessage.Message: - input = v - } - - if contents == nil { - contents = chatMessage.NewContents() - } - options := ast.withOptions(userOptions) - - // Add RAG、Vision and Search support - // ctx.RAG = rag != nil - // ctx.Knowledge = false - // ctx.Vision = ast.vision - // ctx.Search = ast.search && search != nil - - // Run init hook - res, err := ast.HookCreate(c, ctx, input, options, contents) - if err != nil { - chatMessage.New(). - // Assistant(ast.ID, ast.Name, ast.Avatar). - Error(err). - Done(). - Write(c.Writer) - return nil, err - } - - // Update options if provided - if res != nil && res.Options != nil { - options = res.Options - } - - // messages - if res != nil && res.Input != nil { - input = res.Input - } - - // Has result return directly - if res != nil && res.Result != nil { - output := chatMessage.New(). - // Assistant(ast.ID, ast.Name, ast.Avatar). - SetResult(res.Result). - Done() - - // Has callback function - if len(callback) > 0 { - output.Callback(callback[0]).Write(c.Writer) - return res.Result, nil - } - output.Write(c.Writer) - return res.Result, nil - } - - // Switch to the new assistant if necessary - if res != nil && res.AssistantID != "" && res.AssistantID != ctx.AssistantID { - newAst, err := Get(res.AssistantID) - if err != nil { - chatMessage.New(). - // Assistant(ast.ID, ast.Name, ast.Avatar). - Error(err). - Done(). - Write(c.Writer) - return nil, err - } - - // Reset Message Contents - last := input[len(input)-1] - input, err = newAst.withHistory(ctx, last) - if err != nil { - return nil, err - } - - // Reset options - options = newAst.withOptions(userOptions) - - // Update options if provided - if res.Options != nil { - options = res.Options - } - - // Update assistant id - ctx.AssistantID = res.AssistantID - return newAst.handleChatStream(c, ctx, input, options, contents, callback...) - } - - // Only proceed with chat stream if no specific next action was handled - return ast.handleChatStream(c, ctx, input, options, contents, callback...) -} - -// Execute the next action -func (next *NextAction) Execute(c *gin.Context, ctx chatctx.Context, contents *chatMessage.Contents, callback ...interface{}) (interface{}, error) { - switch next.Action { - - case "assistant": - if next.Payload == nil { - return nil, fmt.Errorf("payload is required") - } - - // Get assistant id - id, ok := next.Payload["assistant_id"].(string) - if !ok { - return nil, fmt.Errorf("assistant id should be string") - } - - // Get assistant - assistant, err := Get(id) - if err != nil { - return nil, fmt.Errorf("get assistant error: %s", err.Error()) - } - - // Input - input := chatMessage.Message{} - _, has := next.Payload["input"] - if !has { - return nil, fmt.Errorf("input is required") - } - - // Retry mode - retry := false - _, has = next.Payload["retry"] - if has { - retry = next.Payload["retry"].(bool) - ctx.Retry = retry - } - - switch v := next.Payload["input"].(type) { - case string: - messages := chatMessage.Message{} - err := jsoniter.UnmarshalFromString(v, &messages) - if err != nil { - return nil, fmt.Errorf("unmarshal input error: %s", err.Error()) - } - input = messages - - case map[string]interface{}: - msg, err := chatMessage.NewMap(v) - if err != nil { - return nil, fmt.Errorf("unmarshal input error: %s", err.Error()) - } - input = *msg - - case *chatMessage.Message: - input = *v - - case chatMessage.Message: - input = v - - default: - return nil, fmt.Errorf("input should be string or []chatMessage.Message") - } - - // Options - options := map[string]interface{}{} - if v, ok := next.Payload["options"].(map[string]interface{}); ok { - options = v - } - - input.Hidden = true // not show in the history - if input.Name == "" && ctx.Sid != "" { // add user id to the input - input.Name = ctx.Sid - } - - messages, err := assistant.withHistory(ctx, input) - if err != nil { - return nil, fmt.Errorf("with history error: %s", err.Error()) - } - newContents := chatMessage.NewContents() - - // Update the context id - ctx.AssistantID = assistant.ID - return assistant.execute(c, ctx, messages, options, newContents, callback...) - - case "exit": - return nil, nil - - default: - return nil, fmt.Errorf("unknown action: %s", next.Action) - } -} - -// GetPlaceholder returns the placeholder of the assistant -func (ast *Assistant) GetPlaceholder(locale string) *store.Placeholder { - - prompts := []string{} - if ast.Placeholder.Prompts != nil { - prompts = i18n.Translate(ast.ID, locale, ast.Placeholder.Prompts).([]string) - } - title := i18n.Translate(ast.ID, locale, ast.Placeholder.Title).(string) - description := i18n.Translate(ast.ID, locale, ast.Placeholder.Description).(string) - return &store.Placeholder{ - Title: title, - Description: description, - Prompts: prompts, - } -} - -// GetName returns the name of the assistant -func (ast *Assistant) GetName(locale string) string { - return i18n.Translate(ast.ID, locale, ast.Name).(string) -} - -// GetDescription returns the description of the assistant -func (ast *Assistant) GetDescription(locale string) string { - return i18n.Translate(ast.ID, locale, ast.Description).(string) -} - -// Call implements the call functionality -func (ast *Assistant) Call(c *gin.Context, payload APIPayload) (interface{}, error) { - scriptCtx, err := ast.Script.NewContext(payload.Sid, nil) - if err != nil { - return nil, err - } - defer scriptCtx.Close() - ctx := c.Request.Context() - - method := fmt.Sprintf("%sAPI", payload.Name) - - // Check if the method exists - if !scriptCtx.Global().Has(method) { - color.Red("Assistant Call: %s Method %s not found", ast.ID, method) - return nil, fmt.Errorf(HookErrorMethodNotFound) - } - - if len(payload.Args) == 0 { - return scriptCtx.CallWith(ctx, method) - } - - return scriptCtx.CallWith(ctx, method, payload.Args...) -} - -// handleChatStream manages the streaming chat interaction with the AI -func (ast *Assistant) handleChatStream(c *gin.Context, ctx chatctx.Context, messages []chatMessage.Message, options map[string]interface{}, contents *chatMessage.Contents, callback ...interface{}) (interface{}, error) { - clientBreak := make(chan bool, 1) - done := make(chan bool, 1) - var result interface{} = nil - var err error = nil - - requestCtx := c.Request.Context() - go func() { - var res interface{} = nil - res, err = ast.streamChat(c, ctx, messages, options, clientBreak, contents, callback...) - result = res - done <- true - }() - - // Wait for completion or client disconnect - select { - case <-done: - if err != nil { - return nil, err - } - return result, nil - - case <-requestCtx.Done(): - clientBreak <- true - return nil, nil - } -} - -// streamChat handles the streaming chat interaction -func (ast *Assistant) streamChat( - c *gin.Context, - ctx chatctx.Context, - messages []chatMessage.Message, - options map[string]interface{}, - clientBreak chan bool, - contents *chatMessage.Contents, - callback ...interface{}, -) (interface{}, error) { - - var cb interface{} - if len(callback) > 0 { - cb = callback[0] - } - - errorRaw := "" - isFirst := true - isFirstThink := true - isThinking := false - - toolsCount := 0 - currentMessageID := "" - tokenID := "" - beganAt := int64(0) - var retry error = nil - var result interface{} = nil // To save the result - var content string = "" // To save the content - err := ast.Chat(c.Request.Context(), messages, options, func(data []byte) int { - - select { - case <-clientBreak: - return 0 // break - - default: - msg := chatMessage.NewOpenAI(data, isThinking) - if msg == nil { - return 1 // continue - } - - if msg.Pending { - errorRaw += msg.Text - return 1 // continue - } - - // Retry mode - msg.Retry = ctx.Retry // Retry mode - msg.Silent = ctx.Silent // Silent mode - - // Handle error - if msg.Type == "error" { - value := msg.String() - res, hookErr := ast.HookFail(c, ctx, messages, fmt.Errorf("%s", value), contents) - if hookErr == nil && res != nil && (res.Output != "" || res.Error != "") { - value = res.Output - if res.Error != "" { - value = res.Error - } - } - newMsg := chatMessage.New().Error(value).Done() - newMsg.Retry = ctx.Retry - newMsg.Silent = ctx.Silent - newMsg.Callback(cb).Write(c.Writer) - return 0 // break - } - - // for api reasoning_content response - if msg.Type == "think" { - if isFirstThink { - msg.Begin = time.Now().UnixNano() - msg.Text = "\n" + msg.Text // add the think begin tag - isFirstThink = false - isThinking = true - } - } - - // for api reasoning_content response - if isThinking && msg.Type != "think" { - // add the think close tag - end := chatMessage.New().Map(map[string]interface{}{"text": "\n\n", "type": "think", "delta": true}) - end.ID = currentMessageID - end.Retry = ctx.Retry - end.Silent = ctx.Silent - end.End = time.Now().UnixNano() - end.Begin = beganAt - end.ToolID = tokenID - - end.Callback(cb).Write(c.Writer) - end.AppendTo(contents) - contents.UpdateType("think", map[string]interface{}{"text": contents.Text()}, chatMessage.Extra{ID: currentMessageID, End: time.Now().UnixNano()}) - isThinking = false - - // Clear the token and make a new line - contents.NewText([]byte{}, chatMessage.Extra{ID: currentMessageID}) - - // Clear the token - contents.ClearToken(tokenID) - beganAt = 0 - tokenID = "" - } - - // for native tool_calls response, keep the first tool_calls_native message - if msg.Type == "tool_calls_native" { - - if toolsCount > 1 { - msg.Text = "" // clear the text - msg.Type = "text" - msg.IsNew = false - return 1 // continue - } - - if msg.IsBeginTool { - - if toolsCount == 1 { - msg.IsNew = false - msg.Text = "\n\n" // add the tool_calls close tag - } - - if toolsCount == 0 { - msg.Text = "\n\n" + msg.Text // add the tool_calls begin tag - } - - toolsCount++ - msg.Begin = time.Now().UnixNano() - } - - if msg.IsEndTool { - msg.Text = msg.Text + "\n\n" // add the tool_calls close tag - msg.End = time.Now().UnixNano() - } - } - - delta := msg.String() - - // Chunk the delta - if delta != "" { - - msg.AppendTo(contents) // Append content - - // Scan the tokens - contents.ScanTokens(currentMessageID, tokenID, beganAt, func(params message.ScanCallbackParams) { - currentMessageID = params.MessageID - msg.ID = params.MessageID - msg.Type = params.Token - msg.Text = "" // clear the text - msg.Props = map[string]interface{}{"text": params.Text, "id": params.TokenID} // Update props - msg.Begin = params.BeganAt - msg.End = params.EndAt - msg.ToolID = params.TokenID - - // End of the token clear the text - if params.Begin { - tokenID = params.TokenID - beganAt = params.BeganAt - return - } - - if params.End { - tokenID = "" - beganAt = 0 - return - } - - // New message with the tails - if params.Tails != "" { - newMsg, err := chatMessage.NewString(params.Tails, params.MessageID) - if err != nil { - return - } - messages = append(messages, *newMsg) - } - }) - - // Write the message to the stream - msgType := msg.Type - if msgType == "tool_calls_native" { - msgType = "tool" - } - - // Add the text content to the content - if msgType == "text" || msgType == "" { - content += msg.Text // Save the content - } - - output := chatMessage.New().Map(map[string]interface{}{ - "text": delta, - "type": msgType, - "done": msg.IsDone, - "delta": true, - }) - - output.Retry = ctx.Retry // Retry mode - output.Silent = ctx.Silent // Silent mode - if isFirst { - output.Assistant(ast.ID, ast.GetName(ctx.Locale), ast.Avatar) - isFirst = false - } - - if msg.Type == "think" || msg.Type == "tool" { - output.Begin = msg.Begin - output.End = msg.End - output.ToolID = msg.ToolID - } - - output.Callback(cb).Write(c.Writer) - } - - // Complete the stream - if msg.IsDone { - - // Send the last message to the client - if delta != "" { - chatMessage.New(). - Map(map[string]interface{}{ - "assistant_id": ast.ID, - "assistant_name": ast.GetName(ctx.Locale), - "assistant_avatar": ast.Avatar, - "text": delta, - "type": "text", - "delta": true, - "done": true, - "retry": ctx.Retry, - "silent": ctx.Silent, - }). - Callback(cb). - Write(c.Writer) - } - - // Remove the last empty data - contents.RemoveLastEmpty() - res, hookErr := ast.HookDone(c, ctx, messages, contents) - - // Some error occurred in the hook, return the error - if hookErr != nil { - retry = hookErr - return 0 // break - } - - // Save the chat history - ast.saveChatHistory(ctx, messages, contents) - - // If the hook is successful, execute the next action - if res != nil && res.Next != nil { - _, err := res.Next.Execute(c, ctx, contents, cb) - if err != nil { - chatMessage.New().Error(err.Error()).Done().Callback(cb).Write(c.Writer) - } - return 0 // break - } - - // if the result is not nil, save the result - if res != nil && res.Result != nil { - result = res.Result - } - - // The default output - output := chatMessage.New().Done() - if res != nil && res.Output != nil { - output = chatMessage.New().Map(map[string]interface{}{"text": res.Output, "done": true}) - output.Retry = ctx.Retry - output.Silent = ctx.Silent - } - - // has result - if res != nil && res.Result != nil { - output.SetResult(res.Result) - if cb != nil { - output.Callback(cb).Write(c.Writer) - return 0 // break - } - } - - // Send the result to the client - output.Write(c.Writer) - return 0 // break - } - - return 1 // continue - } - }) - - // retry - if retry != nil { - - // Update the retry times - ctx.RetryTimes = ctx.RetryTimes + 1 // Increment the retry times - ctx.Retry = true // Set the retry mode - - // The maximum retry times is 9 - if ctx.RetryTimes > 9 { - color.Red("Maximum retry times is 9, please check the error and fix it") - // chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) - return nil, retry - } - - // Hook retry - promptAny, retryErr := ast.HookRetry(c, ctx, messages, contents, exception.Trim(retry)) - if retryErr != nil { - color.Red("%s, try to fix the error %d times, but failed with %s", exception.Trim(retry), ctx.RetryTimes, exception.Trim(retryErr)) - // chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) - return nil, retry - } - - if promptAny == nil { - return nil, retry - } - - // Default prompt - var prompt string = fmt.Sprintf("Try to fix the error following the error message. error:\n %s", exception.Trim(retry)) - switch v := promptAny.(type) { - case bool: // Ignore the error, and return the specific result - if v == false { - return nil, retry - } - - case map[string]interface{}: // Ignore the error, and return the specific result - return v, nil - - case NextAction: // Execute the next action - result, err := v.Execute(c, ctx, contents, cb) - if err != nil { - // chatMessage.New().Error(err.Error()).Done().Callback(cb).Write(c.Writer) - return nil, retry - } - return result, nil - - case string: // Add the prompt to the messages - prompt = v - } - - // Add the prompt to the messages - retryMessages, retryErr := ast.retryMessages(messages, prompt) - if retryErr != nil { - color.Red("%s, try to fix the error %d times, but failed with %s", exception.Trim(retry), ctx.RetryTimes, exception.Trim(retryErr)) - // chatMessage.New().Error(retry.Error()).Done().Callback(cb).Write(c.Writer) - return nil, retry - } - - // Retry the chat - retryContents := chatMessage.NewContents() - return ast.execute(c, ctx, retryMessages, options, retryContents, cb) - } - - // Handle error - if err != nil { - return nil, err - } - - // raw error - if errorRaw != "" { - msg, err := chatMessage.NewStringError(errorRaw) - if err != nil { - return nil, fmt.Errorf("stream chat error %s", err.Error()) - } - msg.Retry = ctx.Retry - msg.Silent = ctx.Silent - msg.Done().Callback(cb).Write(c.Writer) - } - - // If the result is not nil, return the result - if result != nil { - return result, nil - } - - // Return the content - return strings.TrimSpace(content), nil -} - -func (ast *Assistant) retryMessages(messages []chatMessage.Message, prompt string) ([]chatMessage.Message, error) { - - // Get the last user message - var lastIndex int = -1 - for i := len(messages) - 1; i >= 0; i-- { - if messages[i].Role == "user" { - messages[i].Text = prompt - lastIndex = i - break - } - } - - if lastIndex == -1 { - return nil, fmt.Errorf("no user message found") - } - - // Remove the messages after the last user message - messages = messages[:lastIndex+1] - return messages, nil -} - -// saveChatHistory saves the chat history if storage is available -func (ast *Assistant) saveChatHistory(ctx chatctx.Context, messages []chatMessage.Message, contents *chatMessage.Contents) { - if len(contents.Data) > 0 && ctx.Sid != "" && len(messages) > 0 { - userMessage := messages[len(messages)-1] - data := []map[string]interface{}{ - { - "role": "user", - "content": userMessage.Content(), - "name": ctx.Sid, - }, - { - "role": "assistant", - "content": contents.JSON(), - "name": ast.ID, - "assistant_id": ast.ID, - "assistant_name": ast.GetName(ctx.Locale), - "assistant_avatar": ast.Avatar, - }, - } - - // if the user message is hidden, just save the assistant message - if userMessage.Hidden { - data = []map[string]interface{}{data[1]} - } - - storage.SaveHistory(ctx.Sid, data, ctx.ChatID, ctx.Map()) - } -} - -func (ast *Assistant) withOptions(options map[string]interface{}) map[string]interface{} { - if options == nil { - options = map[string]interface{}{} - } - - // Add Custom Options - if ast.Options != nil { - for key, value := range ast.Options { - options[key] = value - } - } - - // Add tool_calls - if ast.Tools != nil && ast.Tools.Tools != nil && len(ast.Tools.Tools) > 0 { - if capabilities, has := modelCapabilities[ast.Connector]; has && capabilities.Tools { - options["tools"] = ast.Tools.Tools - if options["tool_choice"] == nil { - options["tool_choice"] = "auto" - } - } - } - - return options -} - -func (ast *Assistant) withPrompts(messages []chatMessage.Message) []chatMessage.Message { - if ast.Prompts != nil { - for _, prompt := range ast.Prompts { - name := strings.ReplaceAll(ast.ID, ".", "_") // OpenAI only supports underscore in the name - if prompt.Name != "" { - name = prompt.Name - } - messages = append(messages, *chatMessage.New().Map(map[string]interface{}{"role": prompt.Role, "content": prompt.Content, "name": name})) - } - } - - // Add tool_calls - if ast.Tools != nil && ast.Tools.Tools != nil && len(ast.Tools.Tools) > 0 { - capabilities, has := modelCapabilities[ast.Connector] - if !has || !capabilities.Tools { - // Convert store tools to runtime tools if not already done - if ast.runtimeTools == nil { - runtimeTools, err := ToRuntimeTools(ast.Tools.Tools) - if err == nil { - ast.runtimeTools = runtimeTools - } - } - - raw, _ := jsoniter.MarshalToString(ast.runtimeTools) - - examples := []string{} - for _, tool := range ast.runtimeTools { - example := tool.Example() - examples = append(examples, example) - } - - examplesStr := "" - if len(examples) > 0 { - examplesStr = "Examples:\n" + strings.Join(examples, "\n\n") - } - - prompts := []map[string]interface{}{ - { - "role": "system", - "name": "TOOL_CALLS_SCHEMA", - "content": raw, - }, - { - "role": "system", - "name": "TOOL_CALLS_SCHEMA", - "content": "## Tool Calls Schema Definition\n" + - "Each tool call is defined with:\n" + - " - type: always 'function'\n" + - " - function:\n" + - " - name: function name\n" + - " - description: function description\n" + - " - parameters: function parameters with type and validation rules\n", - }, - { - "role": "system", - "name": "TOOL_CALLS", - "content": "## Tool Response Format\n" + - "1. Only use tool calls when a function matches your task exactly\n" + - "2. Each tool call must be wrapped in and tags\n" + - "3. Tool call must be a valid JSON with:\n" + - " {\"function\": \"function_name\", \"arguments\": {parameters}}\n" + - "4. Return the function's result as your response\n" + - "5. One tool call per response\n" + - "6. Arguments must match parameter types, rules and description\n\n" + - examplesStr, - }, - { - "role": "system", - "name": "TOOL_CALLS", - "content": "## Tool Usage Guidelines\n" + - "1. Use functions defined in TOOL_CALLS_SCHEMA only when they match your needs\n" + - "2. If no matching function exists, respond normally as a helpful assistant\n" + - "3. When using tools, arguments must match the schema definition exactly\n" + - "4. All parameter values must strictly adhere to the validation rules specified in properties\n" + - "5. Never skip or ignore any validation requirements defined in the schema", - }, - } - - // Add tool_calls developer prompts - if ast.Tools.Prompts != nil && len(ast.Tools.Prompts) > 0 { - for _, prompt := range ast.Tools.Prompts { - messages = append(messages, *chatMessage.New().Map(map[string]interface{}{ - "role": prompt.Role, - "content": prompt.Content, - "name": prompt.Name, - })) - } - } - - // Add the prompts - for _, prompt := range prompts { - messages = append(messages, *chatMessage.New().Map(prompt)) - } - - } - } - - return messages -} - -func (ast *Assistant) withHistory(ctx chatctx.Context, input interface{}) ([]chatMessage.Message, error) { - - var userMessage *chatMessage.Message - var inputMessages []*chatMessage.Message - switch v := input.(type) { - case string: - userMessage = chatMessage.New().Map(map[string]interface{}{"role": "user", "content": v}) - - case map[string]interface{}: - userMessage = chatMessage.New().Map(v) - - case []interface{}: - raw, err := jsoniter.Marshal(v) - if err != nil { - return nil, fmt.Errorf("marshal input error: %s", err.Error()) - } - err = jsoniter.Unmarshal(raw, &inputMessages) - if err != nil { - return nil, fmt.Errorf("unmarshal input error: %s", err.Error()) - } - - case chatMessage.Message: - userMessage = &v - case *chatMessage.Message: - userMessage = v - default: - return nil, fmt.Errorf("unknown input type: %T", input) - } - - messages := []chatMessage.Message{} - if storage != nil { - history, err := storage.GetHistory(ctx.Sid, ctx.ChatID) - if err != nil { - return nil, err - } - - // Add history messages - for _, h := range history { - msgs, err := chatMessage.NewHistory(h) - if err != nil { - return nil, err - } - messages = append(messages, msgs...) - } - } - - // Add system prompts - messages = ast.withPrompts(messages) - - // Add user message - if userMessage != nil { - messages = append(messages, *userMessage) - } - - // Add input messages - if len(inputMessages) > 0 { - for _, msg := range inputMessages { - if msg == nil || msg.Role == "" { - continue - } - messages = append(messages, *msg) - } - } - return messages, nil -} - -// Chat implements the chat functionality -func (ast *Assistant) Chat(ctx context.Context, messages []chatMessage.Message, option map[string]interface{}, cb func(data []byte) int) error { - if ast.openai == nil { - return fmt.Errorf("openai is not initialized") - } - - requestMessages, err := ast.requestMessages(ctx, messages) - if err != nil { - return fmt.Errorf("request messages error: %s", err.Error()) - } - - _, ext := ast.openai.ChatCompletionsWith(ctx, requestMessages, option, cb) - if ext != nil { - return fmt.Errorf("openai chat completions with error: %s", ext.Message) - } - - return nil -} - -// formatMessages processes messages to ensure they meet the required standards: -// 1. Filters out duplicate messages with identical content, role, and name -// 2. Moves system messages to the beginning while preserving the order of other messages -// 3. Ensures the first non-system message is a user message (removes leading assistant messages) -// 4. Ensures the last message is a user message (removes trailing assistant messages) -// 5. Merges consecutive assistant messages from the same assistant -func formatMessages(messages []map[string]interface{}) []map[string]interface{} { - // Filter out duplicate messages with identical content, role, and name - filteredMessages := []map[string]interface{}{ - { - "role": "system", - "name": "SYSTEM_TIME", - "content": "System Time: " + time.Now().Format(time.RFC3339) + "\n\n" + "It's the system time, please use it for reference.", - }, - } - seen := make(map[string]bool) - - for _, msg := range messages { - // Create a unique key for each message based on role, content, and name - role := msg["role"].(string) - content := fmt.Sprintf("%v", msg["content"]) // Convert to string regardless of type - - // Get name if it exists - name := "" - if nameVal, exists := msg["name"]; exists { - name = fmt.Sprintf("%v", nameVal) - } - - // Create a unique key for this message - key := fmt.Sprintf("%s:%s:%s", role, content, name) - - // If we haven't seen this message before, add it to filtered messages - if !seen[key] { - filteredMessages = append(filteredMessages, msg) - seen[key] = true - } - } - - // Separate system messages while preserving the order of other messages - systemMessages := []map[string]interface{}{} - otherMessages := []map[string]interface{}{} - - for _, msg := range filteredMessages { - if msg["role"].(string) == "system" { - systemMessages = append(systemMessages, msg) - } else { - otherMessages = append(otherMessages, msg) - } - } - - // Ensure the first non-system message is a user message - // If there are no user messages or the first message is not a user message, remove leading assistant messages - validOtherMessages := []map[string]interface{}{} - foundUserMessage := false - - for _, msg := range otherMessages { - if msg["role"].(string) == "user" { - foundUserMessage = true - validOtherMessages = append(validOtherMessages, msg) - } else if foundUserMessage { - // Only keep assistant messages that come after a user message - validOtherMessages = append(validOtherMessages, msg) - } - // Skip assistant messages that come before any user message - } - - // If no valid messages remain, return just the system messages - if len(validOtherMessages) == 0 { - return systemMessages - } - - // Ensure the last message is a user message - // Remove any trailing assistant messages - lastUserIndex := -1 - for i := len(validOtherMessages) - 1; i >= 0; i-- { - if validOtherMessages[i]["role"].(string) == "user" { - lastUserIndex = i - break - } - } - - // If we found a user message, trim any assistant messages after it - if lastUserIndex >= 0 && lastUserIndex < len(validOtherMessages)-1 { - validOtherMessages = validOtherMessages[:lastUserIndex+1] - } - - // If there are no user messages left after filtering, return just the system messages - if len(validOtherMessages) == 0 { - return systemMessages - } - - // Combine system messages first, followed by other valid messages in their original order - orderedMessages := append(systemMessages, validOtherMessages...) - - // Merge consecutive assistant messages - mergedMessages := []map[string]interface{}{} - var lastMessage map[string]interface{} - - for _, msg := range orderedMessages { - // If this is the first message, just add it - if lastMessage == nil { - mergedMessages = append(mergedMessages, msg) - lastMessage = msg - continue - } - - // If both current and last messages are from assistant, check if they can be merged - if msg["role"].(string) == "assistant" && lastMessage["role"].(string) == "assistant" { - // Get name information - nameVal, hasName := msg["name"] - - // Prepare name prefix for the content - namePrefix := "" - if hasName { - namePrefix = fmt.Sprintf("[%v]: ", nameVal) - } - - // Merge the content, including name information if available - lastContent := fmt.Sprintf("%v", lastMessage["content"]) - content := fmt.Sprintf("%v", msg["content"]) - - // Add the name prefix to the content - if namePrefix != "" { - content = namePrefix + content - } - - // Merge the messages - lastMessage["content"] = lastContent + "\n" + content - continue - } - - // If we can't merge, add as a new message - mergedMessages = append(mergedMessages, msg) - lastMessage = msg - } - - // Development log for DUI platform - return mergedMessages -} - -func (ast *Assistant) requestMessages(ctx context.Context, messages []chatMessage.Message) ([]map[string]interface{}, error) { - newMessages := []map[string]interface{}{} - length := len(messages) - - for index, message := range messages { - // Ignore the tool, think, error - if message.Type == "tool" || message.Type == "think" || message.Type == "error" { - continue - } - - role := message.Role - if role == "" { - if os.Getenv("YAO_AGENT_PRINT_REQUEST_MESSAGES") == "true" { - raw, _ := jsoniter.MarshalToString(message) - color.Red("Request Message Error, role is empty:") - fmt.Println(raw) - } - return nil, fmt.Errorf("role must be string") - } - - content := message.String() - if content == "" { - // fmt.Println("--------------------------------") - // fmt.Println("Request Message Error") - // utils.Dump(message) - // fmt.Println("--------------------------------") - // return nil, fmt.Errorf("content must be string") - continue - } - - newMessage := map[string]interface{}{ - "role": role, - "content": content, - } - - // Keep the name for user messages - if name := message.Name; name != "" { - if role != "system" { - newMessage["name"] = stringHash(name) - } else { - newMessage["name"] = name - } - } - - // Special handling for user messages with JSON content last message - if role == "user" && index == length-1 { - content = strings.TrimSpace(content) - msg, err := chatMessage.NewString(content) - if err != nil { - return nil, fmt.Errorf("new string error: %s", err.Error()) - } - - newMessage["content"] = msg.Text - if message.Attachments != nil { - contents, err := ast.withAttachments(ctx, &message) - if err != nil { - return nil, fmt.Errorf("with attachments error: %s", err.Error()) - } - - // if current assistant is vision capable, add the contents directly - if ast.vision { - newMessage["content"] = contents - continue - } - - // If current assistant is not vision capable, add the description of the image - if contents != nil { - for _, content := range contents { - newMessages = append(newMessages, content) - } - } - } - } - - newMessages = append(newMessages, newMessage) - } - - // Process messages to standardize format, filter duplicates, and merge consecutive assistant messages - processedMessages := formatMessages(newMessages) - - // For debug environment, print the request messages - if os.Getenv("YAO_AGENT_PRINT_REQUEST_MESSAGES") == "true" { - for _, message := range processedMessages { - raw, _ := jsoniter.MarshalToString(message) - log.Trace("[Request Message] %s", raw) - } - } - - return processedMessages, nil -} - -func (ast *Assistant) withAttachments(ctx context.Context, msg *chatMessage.Message) ([]map[string]interface{}, error) { - contents := []map[string]interface{}{{"type": "text", "text": msg.Text}} - if !ast.vision { - contents = []map[string]interface{}{{"role": "user", "content": msg.Text}} - } - - images := []string{} - for _, attachment := range msg.Attachments { - if strings.HasPrefix(attachment.ContentType, "image/") { - if ast.vision { - images = append(images, attachment.URL) - continue - } - - // If the current assistant is not vision capable, add the description of the image - raw, err := jsoniter.MarshalToString(attachment) - if err != nil { - return nil, fmt.Errorf("marshal attachment error: %s", err.Error()) - } - contents = append(contents, map[string]interface{}{ - "role": "system", - "content": raw, - }) - } - } - - if len(images) == 0 { - return contents, nil - } - - // If the current assistant is vision capable, add the image to the contents directly - if ast.vision { - for _, url := range images { - - // If the image is already a URL, add it directly - if strings.HasPrefix(url, "http") { - contents = append(contents, map[string]interface{}{ - "type": "image_url", - "image_url": map[string]string{ - "url": url, - }, - }) - continue - } - - // Read base64 - bytes64, err := ast.ReadBase64(ctx, url) - if err != nil { - return nil, fmt.Errorf("read base64 error: %s", err.Error()) - } - contents = append(contents, map[string]interface{}{ - "type": "image_url", - "image_url": map[string]string{ - "url": fmt.Sprintf("data:image/jpeg;base64,%s", bytes64), - }, - }) - } - return contents, nil - } - - // If the current assistant is not vision capable, add the description of the image - - return contents, nil -} - -// ReadBase64 implements base64 file reading functionality -func (ast *Assistant) ReadBase64(ctx context.Context, fileID string) (string, error) { - data, err := fs.Get("data") - if err != nil { - return "", fmt.Errorf("get filesystem error: %s", err.Error()) - } - - exists, err := data.Exists(fileID) - if err != nil { - return "", fmt.Errorf("check file error: %s", err.Error()) - } - if !exists { - return "", fmt.Errorf("file %s not found", fileID) - } - - content, err := data.ReadFile(fileID) - if err != nil { - return "", fmt.Errorf("read file error: %s", err.Error()) - } - - return base64.StdEncoding.EncodeToString(content), nil -} diff --git a/agent/assistant/assistant.go b/agent/assistant/assistant.go index b2a7b8ef..b80c221a 100644 --- a/agent/assistant/assistant.go +++ b/agent/assistant/assistant.go @@ -11,6 +11,62 @@ import ( sui "github.com/yaoapp/yao/sui/core" ) +// Get get the assistant by id +func Get(id string) (*Assistant, error) { + return LoadStore(id) +} + +// GetByConnector get the assistant by connector +func GetByConnector(connector string, name string) (*Assistant, error) { + id := "connector:" + connector + + assistant, exists := loaded.Get(id) + if exists { + return assistant, nil + } + + data := map[string]interface{}{ + "assistant_id": id, + "connector": connector, + "description": "Default assistant for " + connector, + "name": name, + "type": "assistant", + } + + assistant, err := loadMap(data) + if err != nil { + return nil, err + } + loaded.Put(assistant) + return assistant, nil +} + +// GetPlaceholder returns the placeholder of the assistant +func (ast *Assistant) GetPlaceholder(locale string) *store.Placeholder { + + prompts := []string{} + if ast.Placeholder.Prompts != nil { + prompts = i18n.Translate(ast.ID, locale, ast.Placeholder.Prompts).([]string) + } + title := i18n.Translate(ast.ID, locale, ast.Placeholder.Title).(string) + description := i18n.Translate(ast.ID, locale, ast.Placeholder.Description).(string) + return &store.Placeholder{ + Title: title, + Description: description, + Prompts: prompts, + } +} + +// GetName returns the name of the assistant +func (ast *Assistant) GetName(locale string) string { + return i18n.Translate(ast.ID, locale, ast.Name).(string) +} + +// GetDescription returns the description of the assistant +func (ast *Assistant) GetDescription(locale string) string { + return i18n.Translate(ast.ID, locale, ast.Description).(string) +} + // Save save the assistant func (ast *Assistant) Save() error { if storage == nil { diff --git a/agent/assistant/call.go b/agent/assistant/call.go deleted file mode 100644 index 1c737c5c..00000000 --- a/agent/assistant/call.go +++ /dev/null @@ -1,361 +0,0 @@ -package assistant - -import ( - "context" - "fmt" - - "github.com/fatih/color" - "github.com/google/uuid" - "github.com/yaoapp/gou/runtime/v8/bridge" - "github.com/yaoapp/kun/log" - chatctx "github.com/yaoapp/yao/agent/context" - chatMessage "github.com/yaoapp/yao/agent/message" - "rogchap.com/v8go" -) - -// objectCall is the object for the call function -type objectCall struct{} - -// OptionsCall is the options for the call function -type OptionsCall struct { - Retry OptionsCallRetry `json:"retry,omitempty"` // Retry options - Options map[string]interface{} `json:"options,omitempty"` // LLM API options - Silent bool `json:"silent,omitempty"` // Silent mode, default is true -} - -// OptionsCallRetry is the retry options for the call function -type OptionsCallRetry struct { - Times int `json:"times,omitempty"` // Retry times, default is 3 - Delay int `json:"delay,omitempty"` // Retry delay, default is 200 - DelayMax int `json:"delay_max,omitempty"` // Retry delay max, default is 5000 - Prompt string `json:"prompt,omitempty"` // Retry prompt, default is "Please fix the error. \n {{ error }}" -} - -// allowedEvents is the allowed events for the call function -var allowedEvents = map[string]bool{ - "done": true, - "retry": true, - "message": true, -} - -var callProps = []string{ - "assistant_id", - "input", - "options", - "retry_times", -} - -// jsNewPlan create a plan object and return it -func jsCall(info *v8go.FunctionCallbackInfo) *v8go.Value { - - args := info.Args() - if len(args) < 2 { - return bridge.JsException(info.Context(), "Run requires at least two arguments") - } - - options := v8go.Undefined(info.Context().Isolate()) - if len(args) > 2 { - options = args[2] - } - - // Export the object - obj := &objectCall{} - objectTmpl := obj.ExportObject(info) - this, err := objectTmpl.NewInstance(info.Context()) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - // Copy global properties - global := info.This() - for _, prop := range objectProperties { - if !global.Has(prop) { - continue - } - value, err := global.Get(prop) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get property %s: %s", prop, err.Error())) - } - this.Set(prop, value) - } - - this.Set("assistant_id", args[0]) - this.Set("input", args[1]) - this.Set("options", options) - this.Set("retry_times", int32(1)) - return this.Value -} - -// ExportObject Export as a FS Object -func (obj *objectCall) ExportObject(info *v8go.FunctionCallbackInfo) *v8go.ObjectTemplate { - tmpl := v8go.NewObjectTemplate(info.Context().Isolate()) - tmpl.Set("On", v8go.NewFunctionTemplate(info.Context().Isolate(), obj.on)) // On the call - tmpl.Set("Run", v8go.NewFunctionTemplate(info.Context().Isolate(), obj.run)) // Run the call - return tmpl -} - -// on bind the callback to the call object -func (obj *objectCall) on(info *v8go.FunctionCallbackInfo) *v8go.Value { - - args := info.Args() - if len(args) < 2 { - return bridge.JsException(info.Context(), "On requires at least one argument") - } - - if !args[0].IsString() { - return bridge.JsException(info.Context(), "The first argument should be a string") - } - - name := args[0].String() - if !allowedEvents[name] { - return bridge.JsException(info.Context(), fmt.Sprintf("Invalid event %s", name)) - } - - cb := args[1] - if !cb.IsFunction() { - return bridge.JsException(info.Context(), fmt.Sprintf("The second argument should be a function for event %s", name)) - } - - this := info.This() - this.Set(fmt.Sprintf("on_%s", name), cb) - return this.Value -} - -// run run the call -func (obj *objectCall) run(info *v8go.FunctionCallbackInfo) *v8go.Value { - - this := info.This() - args := info.Args() - - global, err := getGlobal(info.Context(), this) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - goArgs := []interface{}{} - jsArgs := []v8go.Valuer{} - if len(args) > 0 { - for _, arg := range args { - v, err := bridge.GoValue(arg, info.Context()) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - goArgs = append(goArgs, v) - jsArgs = append(jsArgs, arg) - } - } - - // Get the assistant id - jsAssistantID, err := this.Get("assistant_id") - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - assistantID := jsAssistantID.String() - - // Get the input - jsInput, err := this.Get("input") - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the input: %s", err.Error())) - } - - input, err := bridge.GoValue(jsInput, info.Context()) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to unmarshal the input: %s", err.Error())) - } - - // Get the retry input - if this.Has("retry_input") { - - jsRetryInput, err := this.Get("retry_input") - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the retry input: %s", err.Error())) - } - - input, err = bridge.GoValue(jsRetryInput, info.Context()) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to unmarshal the retry input: %s", err.Error())) - } - } - - // Options - options := OptionsCall{ - // Retry: OptionsCallRetry{ - // Times: 3, - // Delay: 200, - // DelayMax: 1000, - // Prompt: "{{ input }}\n**Answer is not correct, please try again.**\nError:\n{{ error }} \nAssistant's last answer:\n{{ output }}", - // }, - Silent: true, - Options: map[string]interface{}{}, // LLM API options - } - - // Get the options - if this.Has("options") { - jsOptions, err := this.Get("options") - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the options: %s", err.Error())) - } - - // Check if the options is undefined - if !jsOptions.IsUndefined() { - err = bridge.Unmarshal(jsOptions, &options) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to unmarshal the options: %s", err.Error())) - } - } - } - - // Get the assistant - newAst, err := Get(assistantID) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the assistant: %s", err.Error())) - } - - // Get the message event ( it will be used for the message event ) - eventMessage := "" - goCallProps := map[string]interface{}{} - if this.Has("on_message") { - jsEventMessage, err := this.Get("on_message") - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the message: %s", err.Error())) - } - eventMessage = jsEventMessage.String() - - for _, prop := range callProps { - if this.Has(prop) { - value, err := this.Get(prop) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the %s property: %s", prop, err.Error())) - } - goValue, err := bridge.GoValue(value, info.Context()) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the %s property: %s", prop, err.Error())) - } - goCallProps[prop] = goValue - } - } - } - - // Update the chat context - var chatCtx chatctx.Context = global.ChatContext - chatCtx.AssistantID = assistantID - chatCtx.ChatID = fmt.Sprintf("call_%s", uuid.New().String()) // New chat id - chatCtx.Silent = options.Silent - chatCtx.Referer = chatctx.RefererScript // Set the referer to hookscript - chatCtx.Args = goArgs // Arguments for call - - // Define the callback function - var cb func(msg *chatMessage.Message) = nil - var output = []chatMessage.Message{} - cb = func(msg *chatMessage.Message) { - output = append(output, *msg) - if eventMessage != "" { - err := obj.triggerAnonymous(chatCtx, global, goCallProps, eventMessage, goArgs, msg) - if err != nil { - color.Red("Failed to trigger the message event: %s", err.Error()) - log.Error("Failed to trigger the message event: %s", err.Error()) - return - } - } - } - - // Execute the assistant - result, err := newAst.Execute(global.GinContext, chatCtx, input, options.Options, cb) // Execute the assistant - if err != nil { - // result, err = obj.retry(jsArgs, err, input, output, info, options) - // if err != nil { - // return bridge.JsException(info.Context(), err.Error()) - // } - return bridge.JsException(info.Context(), err.Error()) - } - - // Copy props - for name, value := range goCallProps { - info.Context().Global().Set(name, value) - } - - // Trigger the done event - doneResult, err := obj.trigger(info, "done", jsArgs...) - if err != nil { - // result, err = obj.retry(jsArgs, err, input, output, info, options) - // if err != nil { - // return bridge.JsException(info.Context(), err.Error()) - // } - return bridge.JsException(info.Context(), err.Error()) - } - - // Return the done result - if doneResult != nil && !doneResult.IsUndefined() { - return doneResult - } - - // Return Value - switch v := result.(type) { - case *v8go.Value: - return v - case error: - return bridge.JsException(info.Context(), v.Error()) - } - - // Return Value - jsResult, err := bridge.JsValue(info.Context(), result) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("Failed to get the result: %s", err.Error())) - } - return jsResult -} - -func (obj *objectCall) triggerAnonymous(chatCtx chatctx.Context, global *GlobalVariables, goCallProps map[string]interface{}, source string, bindArgs []interface{}, fnArgs ...interface{}) error { - - ctx, err := global.Assistant.Script.NewContext(global.ChatContext.Sid, nil) - if err != nil { - return err - } - defer ctx.Close() - - // Update Context - global.Assistant.InitObject(ctx, global.GinContext, chatCtx, global.Contents) - - // Copy props - for k, v := range goCallProps { - ctx.WithGlobal(k, v) - } - - // Add the args - ctx.WithGlobal("args", bindArgs) - _, err = ctx.CallAnonymousWith(context.Background(), source, fnArgs...) - if err != nil { - return err - } - return nil - -} - -// trigger trigger the callback -func (obj *objectCall) trigger(info *v8go.FunctionCallbackInfo, name string, fnArgs ...v8go.Valuer) (*v8go.Value, error) { - // Try to get the callback - this := info.This() - if this.Has(fmt.Sprintf("on_%s", name)) { - event, err := this.Get(fmt.Sprintf("on_%s", name)) - if err != nil { - return nil, err - } - - if event.IsFunction() { - - cb, err := event.AsFunction() - if err != nil { - return nil, err - } - - result, err := cb.Call(this, fnArgs...) - if err != nil { - return nil, err - } - return result, nil - } - } - - return nil, nil -} diff --git a/agent/assistant/hook/script.go b/agent/assistant/hook/script.go index 6970e8b5..0f59a246 100644 --- a/agent/assistant/hook/script.go +++ b/agent/assistant/hook/script.go @@ -12,7 +12,12 @@ func (s *Script) Execute(ctx *context.Context, method string, args ...interface{ return nil, nil } - scriptCtx, err := s.NewContext(ctx.Sid, nil) + var sid = "" + if ctx.Authorized != nil { + sid = ctx.Authorized.SessionID + } + + scriptCtx, err := s.NewContext(sid, nil) if err != nil { return nil, err } diff --git a/agent/assistant/hooks.go b/agent/assistant/hooks.go deleted file mode 100644 index 57c68e51..00000000 --- a/agent/assistant/hooks.go +++ /dev/null @@ -1,316 +0,0 @@ -package assistant - -import ( - "context" - "fmt" - "os" - "strings" - "time" - - "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/kun/log" - chatctx "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/message" - chatMessage "github.com/yaoapp/yao/agent/message" -) - -// HookCreate create a new assistant -func (ast *Assistant) HookCreate(c *gin.Context, context chatctx.Context, input []chatMessage.Message, options map[string]interface{}, contents *chatMessage.Contents) (*ResHookInit, error) { - // Create timeout context - ctx := ast.createBackgroundContext() - v, err := ast.call(ctx, "Create", c, contents, context, input, options) - if err != nil { - if err.Error() == HookErrorMethodNotFound { - return nil, nil - } - return nil, err - } - - response := &ResHookInit{Result: nil} - switch v := v.(type) { - case map[string]interface{}: - if res, ok := v["assistant_id"].(string); ok { - response.AssistantID = res - } - if res, ok := v["chat_id"].(string); ok { - response.ChatID = res - } - - // input - if input, has := v["input"]; has { - raw, _ := jsoniter.MarshalToString(input) - vv := []message.Message{} - err := jsoniter.UnmarshalFromString(raw, &vv) - if err != nil { - return nil, err - } - response.Input = vv - } - - // result - if result, has := v["result"]; has { - response.Result = result - } - - if res, ok := v["next"].(map[string]interface{}); ok { - response.Next = &NextAction{} - if name, ok := res["action"].(string); ok { - response.Next.Action = name - } - if payload, ok := res["payload"].(map[string]interface{}); ok { - response.Next.Payload = payload - } - } - - case string: - response.AssistantID = v - response.ChatID = context.ChatID - - case nil: - response.AssistantID = ast.ID - response.ChatID = context.ChatID - } - - return response, nil -} - -// HookRetry Handle retry of assistant response -func (ast *Assistant) HookRetry(c *gin.Context, context chatctx.Context, input []message.Message, contents *chatMessage.Contents, errmsg string) (interface{}, error) { - ctx := ast.createBackgroundContext() - output := []message.Data{} - if len(input) < 1 { - return "", fmt.Errorf("no input") - } - - var lastInput message.Message = input[len(input)-1] - for _, data := range contents.Data { - if data.Type == "think" { - continue - } - output = append(output, data) - } - - v, err := ast.call(ctx, "Retry", c, contents, context, lastInput.String(), output, errmsg) - if err != nil { - if err.Error() == HookErrorMethodNotFound { - return nil, nil - } - return nil, err - } - - switch v := v.(type) { - case string, bool: - return v, nil - - case map[string]interface{}: - - // Has Action - if _, has := v["action"]; has { - var next NextAction - raw, _ := jsoniter.MarshalToString(v) - err := jsoniter.UnmarshalFromString(raw, &next) - if err != nil { - return nil, err - } - return &next, nil - } - - // Ignore the error, and return the specific result - return v, nil - - } - - return nil, nil -} - -// HookDone Handle completion of assistant response -func (ast *Assistant) HookDone(c *gin.Context, context chatctx.Context, input []message.Message, contents *chatMessage.Contents) (*ResHookDone, error) { - // Create timeout context - ctx := ast.createBackgroundContext() - - // format the output - // 1. Remove thinking message - // 2. Parse the tool call message content - output := []message.Data{} - if contents != nil && contents.Data != nil { - for _, data := range contents.Data { - if data.Type == "think" { - continue - } - - // parse the tool call message content - if data.Type == "tool" && data.Props != nil { - props := map[string]interface{}{} - if text, ok := data.Props["text"].(string); ok { - - // Extract the content between and tags more reliably - startTag := "" - endTag := "" - startIndex := strings.Index(text, startTag) - if startIndex != -1 { - // Find the content after - content := text[startIndex+len(startTag):] - endIndex := strings.LastIndex(content, endTag) - if endIndex != -1 { - // Extract the content between tags - text = content[:endIndex] - text = strings.TrimSpace(text) - if os.Getenv("YAO_AGENT_PRINT_TOOL_CALL") == "true" { - log.Trace("[TOOL CALL] %s", text) - } - } - } - - // Parse the text into props - err := ParseJSON(text, &props) - if err != nil { - props["error"] = fmt.Sprintf("Can not parse the tool call: %s\n--original--\n%s", err.Error(), text) - } - } - - output = append(output, message.Data{Type: "tool", Props: props}) - continue - } - output = append(output, data) - } - } - - v, err := ast.call(ctx, "Done", c, contents, context, input, output) - if err != nil { - if err.Error() == HookErrorMethodNotFound { - return nil, nil - } - return nil, err - } - - response := &ResHookDone{Input: input, Output: contents.Data} - - switch v := v.(type) { - case map[string]interface{}: - if res, ok := v["output"].(string); ok { - vv := []message.Data{} - err := jsoniter.UnmarshalFromString(res, &vv) - if err != nil { - return nil, err - } - response.Output = vv - } - - if res, ok := v["output"].([]interface{}); ok { - vv := []message.Data{} - raw, _ := jsoniter.MarshalToString(res) - err := jsoniter.UnmarshalFromString(raw, &vv) - if err != nil { - return nil, err - } - response.Output = vv - } - - // has result - if res, has := v["result"]; has { - response.Result = res - } - - if res, ok := v["next"].(map[string]interface{}); ok { - response.Next = &NextAction{} - if name, ok := res["action"].(string); ok { - response.Next.Action = name - } - if payload, ok := res["payload"].(map[string]interface{}); ok { - response.Next.Payload = payload - } - } - case string: - vv := []message.Data{} - err := jsoniter.UnmarshalFromString(v, &vv) - if err != nil { - return nil, err - } - response.Output = vv - } - - return response, nil -} - -// HookFail Handle failure of assistant response -func (ast *Assistant) HookFail(c *gin.Context, context chatctx.Context, input []message.Message, err error, contents *chatMessage.Contents) (*ResHookFail, error) { - // Create timeout context - ctx, cancel := ast.createTimeoutContext(5 * time.Second) - defer cancel() - - v, callErr := ast.call(ctx, "Fail", c, contents, context, input, err.Error()) - if callErr != nil { - if callErr.Error() == HookErrorMethodNotFound { - return nil, nil - } - return nil, callErr - } - - response := &ResHookFail{ - Input: input, - Output: contents.Text(), - Error: err.Error(), - } - - switch v := v.(type) { - case map[string]interface{}: - if res, ok := v["output"].(string); ok { - response.Output = res - } - if res, ok := v["error"].(string); ok { - response.Error = res - } - if res, ok := v["next"].(map[string]interface{}); ok { - response.Next = &NextAction{} - if name, ok := res["action"].(string); ok { - response.Next.Action = name - } - if payload, ok := res["payload"].(map[string]interface{}); ok { - response.Next.Payload = payload - } - } - case string: - response.Output = v - } - - return response, nil -} - -// createTimeoutContext creates a timeout context with 5 seconds timeout -func (ast *Assistant) createTimeoutContext(time time.Duration) (context.Context, context.CancelFunc) { - ctx, cancel := context.WithTimeout(context.Background(), time) - return ctx, cancel -} - -// createBackgroundContext creates a background context -func (ast *Assistant) createBackgroundContext() context.Context { - return context.Background() -} - -// Call the script method -func (ast *Assistant) call(ctx context.Context, method string, c *gin.Context, contents *chatMessage.Contents, context chatctx.Context, args ...any) (interface{}, error) { - if ast.Script == nil { - return nil, nil - } - - scriptCtx, err := ast.Script.NewContext(context.Sid, nil) - if err != nil { - return nil, err - } - defer scriptCtx.Close() - - // Initialize the object, add the global variables, methods to the script context - ast.InitObject(scriptCtx, c, context, contents) - - // Check if the method exists - if !scriptCtx.Global().Has(method) { - return nil, fmt.Errorf(HookErrorMethodNotFound) - } - - // Call the method directly in the current thread - if scriptCtx != nil { - return scriptCtx.CallWith(ctx, method, args...) - } - return nil, nil -} diff --git a/agent/assistant/load.go b/agent/assistant/load.go index 11105023..cf98ad55 100644 --- a/agent/assistant/load.go +++ b/agent/assistant/load.go @@ -17,7 +17,6 @@ import ( "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" store "github.com/yaoapp/yao/agent/store/types" - agentvision "github.com/yaoapp/yao/agent/vision" "github.com/yaoapp/yao/openai" "github.com/yaoapp/yao/share" "gopkg.in/yaml.v3" @@ -28,7 +27,6 @@ var loaded = NewCache(200) // 200 is the default capacity var storage store.Store = nil var search interface{} = nil var modelCapabilities map[string]ModelCapabilities = map[string]ModelCapabilities{} -var vision *agentvision.Vision = nil var defaultConnector string = "" // default connector var globalUses *context.Uses = nil // global uses configuration from agent.yml @@ -132,11 +130,6 @@ func SetStorage(s store.Store) { storage = s } -// SetVision set the vision -func SetVision(v *agentvision.Vision) { - vision = v -} - // SetModelCapabilities set the model capabilities configuration func SetModelCapabilities(capabilities map[string]ModelCapabilities) { modelCapabilities = capabilities @@ -710,7 +703,6 @@ func (ast *Assistant) initialize() error { return err } defer scriptCtx.Close() - ast.initHook = scriptCtx.Global().Has("init") } return nil diff --git a/agent/assistant/next.go b/agent/assistant/next.go index 6ad95648..14b5c294 100644 --- a/agent/assistant/next.go +++ b/agent/assistant/next.go @@ -54,7 +54,6 @@ func (ast *Assistant) handleDelegation( delegatedCtx := &agentContext.Context{ Context: ctx.Context, Locale: ctx.Locale, - Sid: ctx.Sid, Stack: ctx.Stack, // Maintain the call stack Authorized: ctx.Authorized, Metadata: ctx.Metadata, diff --git a/agent/assistant/object.go b/agent/assistant/object.go deleted file mode 100644 index 4ad34bbb..00000000 --- a/agent/assistant/object.go +++ /dev/null @@ -1,376 +0,0 @@ -package assistant - -import ( - "fmt" - "strings" - - "github.com/gin-gonic/gin" - v8 "github.com/yaoapp/gou/runtime/v8" - "github.com/yaoapp/gou/runtime/v8/bridge" - chatctx "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/message" - chatMessage "github.com/yaoapp/yao/agent/message" - sui "github.com/yaoapp/yao/sui/core" - "rogchap.com/v8go" -) - -// objectProperties is the properties of the assistant object -var objectProperties = []string{ - "__yao_agent_global", - "assistant", - "context", - "Plan", - "Send", - "Call", - "Assets", - "Set", - "Get", - "Del", - "Clear", -} - -// GlobalVariables is the global variables for the assistant -type GlobalVariables struct { - Assistant *Assistant - Contents *chatMessage.Contents - GinContext *gin.Context - ChatContext chatctx.Context -} - -// JsValue return the javascript value of the global variables -func (global *GlobalVariables) JsValue(ctx *v8go.Context) (*v8go.Value, error) { - return v8go.NewExternal(ctx.Isolate(), global) -} - -// InitObject add the global variables and methods to the script context -func (ast *Assistant) InitObject(v8ctx *v8.Context, c *gin.Context, context chatctx.Context, contents *chatMessage.Contents) { - - // Add global variables to the script context - global := &GlobalVariables{ - Assistant: ast, - Contents: contents, - GinContext: c, - ChatContext: context, - } - - // Add global variables to the script context - v8ctx.WithGlobal("__yao_agent_global", global) - - // Add assistant to the script context - v8ctx.WithGlobal("assistant", ast.Map()) - v8ctx.WithGlobal("context", context.Map()) - - // Add methods to the script contexts - v8ctx.WithFunction("Send", jsSend) - v8ctx.WithFunction("Assets", jsAssets) - v8ctx.WithFunction("MakeCall", jsCall) // Create a new call object - v8ctx.WithFunction("MakePlan", jsPlan) // Create a new plan object - - // Shared space methods - v8ctx.WithFunction("Set", jsSet) - v8ctx.WithFunction("Get", jsGet) - v8ctx.WithFunction("Del", jsDel) - v8ctx.WithFunction("Clear", jsClear) - - // Template methods - v8ctx.WithFunction("Replace", jsReplace) -} - -// jsSet function, set a value to the shared space -func jsSet(info *v8go.FunctionCallbackInfo) *v8go.Value { - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - if global.ChatContext.Space == nil { - return bridge.JsException(info.Context(), "Shared space is not set") - } - - args := info.Args() - if len(args) < 2 { - return bridge.JsException(info.Context(), "Set requires at least two arguments") - } - - if !args[0].IsString() { - return bridge.JsException(info.Context(), "Set requires a valid key") - } - - // Validate the key - key := args[0].String() - if key == "" { - return bridge.JsException(info.Context(), "Set requires a valid key") - } - - // Validate the value - value, err := bridge.GoValue(args[1], info.Context()) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - // Set the value - err = global.ChatContext.Space.Set(key, value) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - return nil -} - -// jsGet function, get a value from the shared space -func jsGet(info *v8go.FunctionCallbackInfo) *v8go.Value { - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - if global.ChatContext.Space == nil { - return bridge.JsException(info.Context(), "Shared space is not set") - } - - args := info.Args() - if len(args) < 1 { - return bridge.JsException(info.Context(), "Get requires at least one argument") - } - - if !args[0].IsString() { - return bridge.JsException(info.Context(), "Get requires a valid key") - } - - // Get the key - key := args[0].String() - if key == "" { - return bridge.JsException(info.Context(), "Get requires a valid key") - } - - // Get the value - value, err := global.ChatContext.Space.Get(key) - if err != nil { - // If the key is not found, return null - if strings.Contains(err.Error(), "not found") { - return v8go.Null(info.Context().Isolate()) - } - return bridge.JsException(info.Context(), err.Error()) - } - - jsValue, err := bridge.JsValue(info.Context(), value) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - return jsValue -} - -// jsDel function, delete a value from the shared space -func jsDel(info *v8go.FunctionCallbackInfo) *v8go.Value { - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - if global.ChatContext.Space == nil { - return bridge.JsException(info.Context(), "Shared space is not set") - } - - args := info.Args() - if len(args) < 1 { - return bridge.JsException(info.Context(), "Get requires at least one argument") - } - - if !args[0].IsString() { - return bridge.JsException(info.Context(), "Get requires a valid key") - } - - // Get the key - key := args[0].String() - if key == "" { - return bridge.JsException(info.Context(), "Get requires a valid key") - } - - err = global.ChatContext.Space.Delete(key) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - return nil -} - -func jsClear(info *v8go.FunctionCallbackInfo) *v8go.Value { - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - if global.ChatContext.Space == nil { - return bridge.JsException(info.Context(), "Shared space is not set") - } - - err = global.ChatContext.Space.Clear() - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - return nil -} - -// jsAssets function, get the assets content -func jsAssets(info *v8go.FunctionCallbackInfo) *v8go.Value { - - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - // Get the message - args := info.Args() - if len(args) < 1 { - return bridge.JsException(info.Context(), "Assets requires at least one argument") - } - - // Get the name - name := args[0].String() - - data := map[string]interface{}{} - if len(args) > 1 { - raw, err := bridge.GoValue(args[1], info.Context()) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - v, ok := raw.(map[string]interface{}) - if !ok { - return bridge.JsException(info.Context(), "Assets requires a map") - } - data = v - } - - content, err := global.Assistant.Assets(name, data) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - jsContent, err := bridge.JsValue(info.Context(), content) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - return jsContent -} - -// jsSend function, send a message to the http stream connection -func jsSend(info *v8go.FunctionCallbackInfo) *v8go.Value { - - // Get the message - args := info.Args() - if len(args) < 1 { - return bridge.JsException(info.Context(), "SendMessage requires at least one argument") - } - - input, err := bridge.GoValue(args[0], info.Context()) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - // Save history by default - saveHistory := true - if len(args) > 1 && args[1].IsBoolean() { - saveHistory = args[1].Boolean() - } - - switch v := input.(type) { - case string: - // Check if the message is json - msg, err := message.NewString(v) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - // Set the role to assistant - if msg.Role == "" { - msg.Role = "assistant" - } - - // Append the message to the contents - if saveHistory { - msg.AppendTo(global.Contents) - } - msg.Write(global.GinContext.Writer) - return nil - - case map[string]interface{}: - msg := message.New().Map(v) - if msg.Role == "" { - msg.Role = "assistant" - } - - // Append the message to the contents - if saveHistory { - msg.AppendTo(global.Contents) - } - msg.Write(global.GinContext.Writer) - return nil - - default: - return bridge.JsException(info.Context(), "Send requires a string or a map") - } -} - -func jsReplace(info *v8go.FunctionCallbackInfo) *v8go.Value { - args := info.Args() - if len(args) < 2 { - return bridge.JsException(info.Context(), "Replace requires at least two arguments") - } - - if !args[0].IsString() { - return bridge.JsException(info.Context(), "the first argument must be a string") - } - tmpl := args[0].String() - - raw, err := bridge.GoValue(args[1], info.Context()) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - data, ok := raw.(map[string]interface{}) - if !ok { - return bridge.JsException(info.Context(), "the second argument must be a map") - } - - replaced, _ := sui.Data(data).Replace(tmpl) - jsReplaced, err := bridge.JsValue(info.Context(), replaced) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - return jsReplaced -} - -// global get the global variables -func global(info *v8go.FunctionCallbackInfo) (global *GlobalVariables, err error) { - return getGlobal(info.Context(), info.This()) -} - -func getGlobal(ctx *v8go.Context, obj *v8go.Object) (global *GlobalVariables, err error) { - jsGlobal, err := obj.Get("__yao_agent_global") - if err != nil { - return nil, err - } - - // Convert to go interface - goGlobal, err := bridge.GoValue(jsGlobal, ctx) - if err != nil { - return nil, err - } - - global, ok := goGlobal.(*GlobalVariables) - if !ok { - return nil, fmt.Errorf("global is not a valid GlobalVariables. %#v", goGlobal) - } - - return global, nil -} diff --git a/agent/assistant/plan.go b/agent/assistant/plan.go deleted file mode 100644 index 8f5794f4..00000000 --- a/agent/assistant/plan.go +++ /dev/null @@ -1,140 +0,0 @@ -package assistant - -import ( - "context" - "fmt" - - "github.com/fatih/color" - "github.com/yaoapp/gou/runtime/v8/bridge" - v8plan "github.com/yaoapp/gou/runtime/v8/objects/plan" - "rogchap.com/v8go" -) - -// TaskFn is the task function -func TaskFn(plan_id string, task_id string, source bool, method string, args ...interface{}) (interface{}, error) { - - if !source { - return v8plan.DefaultTaskFn(plan_id, task_id, source, method, args...) - } - - // Data - plan, err := v8plan.GetPlan(plan_id) - if err != nil { - return nil, err - } - - global, ok := plan.Data().(*GlobalVariables) - if !ok { - return nil, fmt.Errorf("plan data is not a GlobalVariables") - } - - if global.Assistant == nil { - return nil, fmt.Errorf("assistant is not set") - } - - if global.Assistant.Script == nil { - return nil, fmt.Errorf("script is not set") - } - - scriptCtx, err := global.Assistant.Script.NewContext(global.ChatContext.Sid, nil) - if err != nil { - return nil, err - } - defer scriptCtx.Close() - - // Initialize the object - global.Assistant.InitObject(scriptCtx, global.GinContext, global.ChatContext, global.Contents) - - fnargs := []interface{}{plan_id, task_id} - fnargs = append(fnargs, args...) - - // Execute the anonymous function - return scriptCtx.CallAnonymousWith(context.Background(), method, fnargs...) - -} - -// SubscribeFn is the default subscribe function -func SubscribeFn(plan_id string, key string, value interface{}, source bool, method string, args ...interface{}) { - - if !source { - v8plan.DefaultSubscribeFn(plan_id, key, value, source, method, args...) - return - } - - // Data - plan, err := v8plan.GetPlan(plan_id) - if err != nil { - color.Red("Subscribe Failed to get the plan: %s", err.Error()) - return - } - - global, ok := plan.Data().(*GlobalVariables) - if !ok { - color.Red("Subscribe Failed: plan data is not a GlobalVariables") - return - } - - if global.Assistant == nil { - color.Red("Subscribe Failed: assistant is not set") - return - } - - if global.Assistant.Script == nil { - color.Red("Subscribe Failed: script is not set") - return - } - - scriptCtx, err := global.Assistant.Script.NewContext(global.ChatContext.Sid, nil) - if err != nil { - color.Red("Subscribe Failed: Failed to create the script context: %s", err.Error()) - return - } - defer scriptCtx.Close() - - fnargs := []interface{}{plan_id, key, value} - fnargs = append(fnargs, args...) - - // Initialize the object - global.Assistant.InitObject(scriptCtx, global.GinContext, global.ChatContext, global.Contents) - _, err = scriptCtx.CallAnonymousWith(context.Background(), method, fnargs...) - if err != nil { - return - } -} - -// jsNewPlan create a plan object and return it -func jsPlan(info *v8go.FunctionCallbackInfo) *v8go.Value { - - global, err := global(info) - if err != nil { - return bridge.JsException(info.Context(), err.Error()) - } - - obj := newPlanObject() - - args := info.Args() - if len(args) < 1 { - return bridge.JsException(info.Context(), "the first parameter should be a string") - } - - if !args[0].IsString() { - return bridge.JsException(info.Context(), "the first parameter should be a string") - } - - id := args[0].String() - objectTmpl := obj.ExportObject(info.Context().Isolate()) - plan, err := objectTmpl.NewInstance(info.Context()) - if err != nil { - return bridge.JsException(info.Context(), fmt.Sprintf("failed to create plan object %s", err.Error())) - } - - return obj.NewInstance(id, plan, global) -} - -func newPlanObject() *v8plan.Object { - obj := v8plan.New(v8plan.Options{ - TaskFn: TaskFn, - SubscribeFn: SubscribeFn, - }) - return obj -} diff --git a/agent/assistant/tool.go b/agent/assistant/tool.go deleted file mode 100644 index d6c82587..00000000 --- a/agent/assistant/tool.go +++ /dev/null @@ -1,156 +0,0 @@ -package assistant - -import ( - "fmt" - - jsoniter "github.com/json-iterator/go" - store "github.com/yaoapp/yao/agent/store/types" -) - -// Tool represents a tool -type Tool struct { - Type string `json:"type"` - Function struct { - Name string `json:"name"` - Description string `json:"description"` - Parameters Parameter `json:"parameters"` - Strict bool `json:"strict,omitempty"` - } `json:"function"` -} - -// SchemaProperty represents a JSON Schema property -type SchemaProperty struct { - Type string `json:"type,omitempty"` - Description string `json:"description,omitempty"` - Items *Parameter `json:"items,omitempty"` - OneOf []SchemaProperty `json:"oneOf,omitempty"` - Enum []interface{} `json:"enum,omitempty"` -} - -// Parameter represents the parameters field in function calling format -type Parameter struct { - Type string `json:"type,omitempty"` - Properties map[string]SchemaProperty `json:"properties,omitempty"` - Description string `json:"description,omitempty"` - Required []string `json:"required,omitempty"` - AdditionalProperties bool `json:"additionalProperties,omitempty"` - Strict bool `json:"strict,omitempty"` - OneOf []SchemaProperty `json:"oneOf,omitempty"` - Enum []interface{} `json:"enum,omitempty"` -} - -// Example returns a formatted example of how to use this tool -func (tool Tool) Example() string { - return fmt.Sprintf("\n{\"function\":\"%s\",\"arguments\":%s}\n", - tool.Function.Name, - jsoniter.Wrap(tool.ExampleArguments()).ToString()) -} - -// ExampleArguments generates example arguments for the tool based on parameter types -func (tool Tool) ExampleArguments() map[string]interface{} { - - args := map[string]interface{}{} - - // Handle the root parameter object - if tool.Function.Parameters.Type == "object" && tool.Function.Parameters.Properties != nil { - for name, prop := range tool.Function.Parameters.Properties { - args[name] = generateExampleValue(name, prop) - } - } - return args -} - -// generateExampleValue creates an example value for a parameter -func generateExampleValue(name string, prop SchemaProperty) interface{} { - if len(prop.OneOf) > 0 { - // Return the first non-null type example value from oneOf - for _, subProp := range prop.OneOf { - if subProp.Type != "null" { - return generateExampleValue(name, subProp) - } - } - return nil - } - - // If enum is defined, return the first enum value - if len(prop.Enum) > 0 { - return prop.Enum[0] - } - - switch prop.Type { - case "string": - return fmt.Sprintf("<%s:string>", name) - case "number": - return fmt.Sprintf("<%s:number>", name) - case "integer": - return fmt.Sprintf("<%s:integer>", name) - case "boolean": - return fmt.Sprintf("<%s:boolean>", name) - case "object": - return fmt.Sprintf("<%s:object>", name) - case "array": - return fmt.Sprintf("<%s:array>", name) - case "null": - return nil - default: - return fmt.Sprintf("<%s>", name) - } -} - -// ToRuntimeTool converts store.Tool to assistant.Tool (OpenAI format) -func ToRuntimeTool(storeTool store.Tool) (Tool, error) { - var tool Tool - - // Marshal and unmarshal to convert between formats - raw, err := jsoniter.Marshal(storeTool) - if err != nil { - return tool, fmt.Errorf("failed to marshal store tool: %w", err) - } - - // Try to unmarshal as OpenAI format first - err = jsoniter.Unmarshal(raw, &tool) - if err == nil && tool.Function.Name != "" { - return tool, nil - } - - // If it's a simple format, convert it - tool.Type = "function" - if storeTool.Type != "" { - tool.Type = storeTool.Type - } - tool.Function.Name = storeTool.Name - tool.Function.Description = storeTool.Description - - // Convert parameters - if storeTool.Parameters != nil { - raw, err := jsoniter.Marshal(storeTool.Parameters) - if err != nil { - return tool, fmt.Errorf("failed to marshal parameters: %w", err) - } - var params Parameter - err = jsoniter.Unmarshal(raw, ¶ms) - if err != nil { - return tool, fmt.Errorf("failed to unmarshal parameters: %w", err) - } - tool.Function.Parameters = params - } - - return tool, nil -} - -// ToRuntimeTools converts []store.Tool to []assistant.Tool -func ToRuntimeTools(storeTools []store.Tool) ([]Tool, error) { - if storeTools == nil { - return nil, nil - } - - tools := make([]Tool, 0, len(storeTools)) - for _, storeTool := range storeTools { - tool, err := ToRuntimeTool(storeTool) - if err != nil { - return nil, err - } - tools = append(tools, tool) - } - return tools, nil -} diff --git a/agent/assistant/types.go b/agent/assistant/types.go index cd85bb7c..542e76d6 100644 --- a/agent/assistant/types.go +++ b/agent/assistant/types.go @@ -1,14 +1,10 @@ package assistant import ( - "context" - "io" - - "github.com/gin-gonic/gin" jsoniter "github.com/json-iterator/go" "github.com/yaoapp/yao/agent/assistant/hook" chatctx "github.com/yaoapp/yao/agent/context" - "github.com/yaoapp/yao/agent/message" + outputMessage "github.com/yaoapp/yao/agent/output/message" store "github.com/yaoapp/yao/agent/store/types" api "github.com/yaoapp/yao/openai" @@ -21,56 +17,7 @@ const ( // API the assistant API interface type API interface { - Chat(ctx context.Context, messages []message.Message, option map[string]interface{}, cb func(data []byte) int) error GetPlaceholder(locale string) *store.Placeholder - Execute(c *gin.Context, ctx chatctx.Context, input interface{}, options map[string]interface{}, callback ...interface{}) (interface{}, error) - Call(c *gin.Context, payload APIPayload) (interface{}, error) -} - -// APIPayload the API payload -type APIPayload struct { - Sid string `json:"sid"` - Name string `json:"name"` - Args []interface{} `json:"args,omitempty"` -} - -// ResHookInit the response of the init hook -type ResHookInit struct { - AssistantID string `json:"assistant_id,omitempty"` - ChatID string `json:"chat_id,omitempty"` - Next *NextAction `json:"next,omitempty"` - Input []message.Message `json:"input,omitempty"` - Options map[string]interface{} `json:"options,omitempty"` - Result any `json:"result,omitempty"` -} - -// ResHookStream the response of the stream hook -type ResHookStream struct { - Silent bool `json:"silent,omitempty"` // Whether to suppress the output - Next *NextAction `json:"next,omitempty"` // The next action - Output []message.Data `json:"output,omitempty"` // The output -} - -// ResHookDone the response of the done hook -type ResHookDone struct { - Next *NextAction `json:"next,omitempty"` - Input []message.Message `json:"input,omitempty"` - Output []message.Data `json:"output,omitempty"` - Result any `json:"result,omitempty"` -} - -// ResHookFail the response of the fail hook -type ResHookFail struct { - Next *NextAction `json:"next,omitempty"` - Input []message.Message `json:"input,omitempty"` - Output string `json:"output,omitempty"` - Error string `json:"error,omitempty"` -} - -// NextAction the next action -type NextAction struct { - Action string `json:"action"` - Payload map[string]interface{} `json:"payload,omitempty"` } // SearchOption the search option @@ -79,21 +26,6 @@ type SearchOption struct { Knowledge *bool `json:"knowledge,omitempty" yaml:"knowledge,omitempty"` // Whether to search the knowledge } -// Prompt a prompt -type Prompt struct { - Role string `json:"role"` - Content string `json:"content"` - Name string `json:"name,omitempty"` -} - -// QueryParam the assistant query param -type QueryParam struct { - Limit uint `json:"limit"` - Order string `json:"order"` - After string `json:"after"` - Before string `json:"before"` -} - // Assistant the assistant type Assistant struct { store.AssistantModel @@ -106,8 +38,6 @@ type Assistant struct { search bool // Whether this assistant supports search vision bool // Whether this assistant supports vision // toolCalls bool // Whether this assistant supports tool_calls - initHook bool // Whether this assistant has an init hook - runtimeTools []Tool // Converted tools for business logic (OpenAI format) } // ModelCapabilities defines the capabilities of a language model @@ -148,25 +78,6 @@ var VisionCapableModels = map[string]bool{ "gpt-4o-mini": true, // Custom OpenAI compatible model - mini version } -// File the file -type File struct { - ID string `json:"file_id"` - Bytes int `json:"bytes"` - CreatedAt int `json:"created_at"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - Description string `json:"description,omitempty"` // Vision analysis result or other description - URL string `json:"url,omitempty"` // Vision URL for vision-capable models - DocIDs []string `json:"doc_ids,omitempty"` // RAG document IDs -} - -// FileResponse represents a file download response -type FileResponse struct { - Reader io.ReadCloser - ContentType string - Extension string -} - // MCPTool represents a simplified MCP tool for building LLM requests // This is an internal representation used when collecting tools from MCP servers // and preparing them for the LLM's tool calling interface diff --git a/agent/context/JSAPI.md b/agent/context/JSAPI.md index c295f31b..75a86970 100644 --- a/agent/context/JSAPI.md +++ b/agent/context/JSAPI.md @@ -123,20 +123,22 @@ const image_id = ctx.Send({ ```javascript // Scenario 1: Simple messages without block grouping (most common) -function Next(ctx, response) { +function Next(ctx, payload) { + const { completion } = payload; + // Each message is independent const loading_id = ctx.Send({ type: "loading", props: { message: "Thinking..." } }); - // Call LLM... - const result = Process("llms.chat", {...}); + // Process completion... + const result = completion.content; // Replace loading with result ctx.Replace(loading_id, { type: "text", - props: { content: result.content } + props: { content: result } }); } @@ -154,14 +156,14 @@ function Create(ctx, messages) { } // Scenario 3: LLM response + follow-up card in same block -function Next(ctx, response) { +function Next(ctx, payload) { + const { completion } = payload; const block_id = ctx.BlockID(); // LLM response - const result = Process("llms.chat", {...}); ctx.Send({ type: "text", - props: { content: result.content }, + props: { content: completion.content }, block_id: block_id }); @@ -948,9 +950,18 @@ Here's a comprehensive example using various Context API features: ```javascript /** * Next Hook - Process LLM response and enhance with tools + * @param {Context} ctx - Agent context + * @param {Object} payload - Hook payload + * @param {Array} payload.messages - Messages sent to the assistant + * @param {Object} payload.completion - Completion response from LLM + * @param {Array} payload.tools - Tool call results + * @param {string} payload.error - Error message if failed */ -function Next(ctx, messages, completion, tools) { +function Next(ctx, payload) { try { + // Destructure payload + const { messages, completion, tools, error } = payload; + // Create trace node for custom processing const process_node = ctx.Trace.Add( { completion, tools }, @@ -1001,14 +1012,11 @@ function Next(ctx, messages, completion, tools) { // Return enhanced response return { data: enhanced_response, - done: true, + metadata: { processed: true }, }; } catch (error) { ctx.Trace.Error("Processing failed", { error: error.message }); throw error; - } finally { - // Optional: Manual cleanup - ctx.Release(); } } ``` @@ -1043,13 +1051,17 @@ For TypeScript projects, the Context types are automatically inferred. You can a ```typescript import { Context, Message, TraceNodeOption } from "@yaoapps/types"; -function Next( - ctx: Context, - messages: Message[], - completion: any, - tools: any[] -): any { +interface NextPayload { + messages: Message[]; + completion: any; + tools: any[]; + error?: string; +} + +function Next(ctx: Context, payload: NextPayload): any { // Your code with full type checking + const { messages, completion, tools, error } = payload; + // ... } ``` diff --git a/agent/context/context.go b/agent/context/context.go index 8ec38aa0..2098ee3a 100644 --- a/agent/context/context.go +++ b/agent/context/context.go @@ -30,11 +30,12 @@ func New(parent context.Context, authorized *types.AuthorizedInfo, chatID, paylo // Validate the client type ctx := Context{ - Context: parent, - ID: generateContextID(), // Generate unique ID for the context - Space: plan.NewMemorySharedSpace(), - ChatID: chatID, - IDGenerator: message.NewIDGenerator(), // Initialize ID generator for this context + Context: parent, + ID: generateContextID(), // Generate unique ID for the context + Space: plan.NewMemorySharedSpace(), + ChatID: chatID, + IDGenerator: message.NewIDGenerator(), // Initialize ID generator for this context + messageMetadata: newMessageMetadataStore(), // Initialize message metadata store } if payload == "" { @@ -379,33 +380,22 @@ func (ctx *Context) TraceID() string { // recordMessageMetadata records metadata for a sent message // Used to inherit BlockID and ThreadID in subsequent delta operations func (ctx *Context) recordMessageMetadata(msg *message.Message) { - if msg.MessageID == "" { + if msg.MessageID == "" || ctx.messageMetadata == nil { return } - ctx.metadataMu.Lock() - defer ctx.metadataMu.Unlock() - - if ctx.messageMetadata == nil { - ctx.messageMetadata = make(map[string]*MessageMetadata) - } - - ctx.messageMetadata[msg.MessageID] = &MessageMetadata{ + ctx.messageMetadata.set(msg.MessageID, &MessageMetadata{ MessageID: msg.MessageID, BlockID: msg.BlockID, ThreadID: msg.ThreadID, - } + }) } // getMessageMetadata retrieves metadata for a message by ID // Returns nil if message metadata is not found func (ctx *Context) getMessageMetadata(messageID string) *MessageMetadata { - ctx.metadataMu.RLock() - defer ctx.metadataMu.RUnlock() - if ctx.messageMetadata == nil { return nil } - - return ctx.messageMetadata[messageID] + return ctx.messageMetadata.get(messageID) } diff --git a/agent/context/jsapi_test.go b/agent/context/jsapi_test.go index 06582f15..61708dcb 100644 --- a/agent/context/jsapi_test.go +++ b/agent/context/jsapi_test.go @@ -26,7 +26,6 @@ func TestJsValue(t *testing.T) { cxt := &context.Context{ ChatID: "ChatID-123456", AssistantID: "AssistantID-1234", - Sid: "Sid-1234", Context: stdContext.Background(), IDGenerator: message.NewIDGenerator(), } @@ -91,12 +90,10 @@ func TestJsValueConcurrent(t *testing.T) { for j := 0; j < iterationsPerGoroutine; j++ { chatID := fmt.Sprintf("ChatID-%d-%d", routineID, j) assistantID := fmt.Sprintf("AssistantID-%d-%d", routineID, j) - sid := fmt.Sprintf("Sid-%d-%d", routineID, j) cxt := &context.Context{ ChatID: chatID, AssistantID: assistantID, - Sid: sid, Context: stdContext.Background(), IDGenerator: message.NewIDGenerator(), } @@ -156,7 +153,6 @@ func TestJsValueRegistrationAndCleanup(t *testing.T) { cxt := &context.Context{ ChatID: fmt.Sprintf("ChatID-%d", i), AssistantID: fmt.Sprintf("AssistantID-%d", i), - Sid: fmt.Sprintf("Sid-%d", i), Context: stdContext.Background(), IDGenerator: message.NewIDGenerator(), } @@ -337,12 +333,6 @@ func TestJsValueAllFields(t *testing.T) { assert.Equal(t, "engineering", extra["department"], "constraints.extra.department mismatch") assert.Equal(t, "us-west", extra["region"], "constraints.extra.region mismatch") - // Verify deprecated fields are NOT exported - _, hasSid := result["sid"] - assert.False(t, hasSid, "sid (deprecated) should not be exported") - _, hasSilent := result["silent"] - assert.False(t, hasSilent, "silent (deprecated) should not be exported") - // Note: We can't directly check goMaps cleanup as it's in the bridge package } diff --git a/agent/context/types.go b/agent/context/types.go index 8e5b39a0..410cc422 100644 --- a/agent/context/types.go +++ b/agent/context/types.go @@ -202,23 +202,55 @@ type MessageMetadata struct { ThreadID string // Thread ID } +// messageMetadataStore provides thread-safe storage for message metadata +type messageMetadataStore struct { + data map[string]*MessageMetadata + mu sync.RWMutex +} + +// newMessageMetadataStore creates a new message metadata store +func newMessageMetadataStore() *messageMetadataStore { + return &messageMetadataStore{ + data: make(map[string]*MessageMetadata), + } +} + +// set stores metadata for a message (thread-safe) +func (s *messageMetadataStore) set(messageID string, metadata *MessageMetadata) { + s.mu.Lock() + defer s.mu.Unlock() + s.data[messageID] = metadata +} + +// get retrieves metadata for a message (thread-safe) +func (s *messageMetadataStore) get(messageID string) *MessageMetadata { + s.mu.RLock() + defer s.mu.RUnlock() + return s.data[messageID] +} + // Context the context type Context struct { // Context context.Context - ID string `json:"id"` // Context ID for external interrupt identification - Space plan.Space `json:"-"` // Shared data space, it will be used to share data between the request and the call - Cache store.Store `json:"-"` // Cache store, it will be used to store the message cache, default is "__yao.agent.cache" - Stack *Stack `json:"-"` // Stack, current active stack of the request - Stacks map[string]*Stack `json:"-"` // Stacks, all stacks in this request (for trace logging) - Writer Writer `json:"-"` // Writer, it will be used to write response data to the client - Skip *Skip `json:"skip,omitempty"` // Skip configuration (history, trace, etc.), nil means don't skip anything - trace traceTypes.Manager `json:"-"` // Trace manager, lazy initialized on first access - output *output.Output `json:"-"` // Output, it will be used to write response data to the client - IDGenerator *message.IDGenerator `json:"-"` // ID generator for this context (chunk, message, block, thread IDs) - messageMetadata map[string]*MessageMetadata `json:"-"` // Message metadata cache for delta operations (inheriting BlockID/ThreadID) - metadataMu sync.RWMutex `json:"-"` // Mutex for concurrent access to messageMetadata + + // External + ID string `json:"id"` // Context ID for external interrupt identification + Space plan.Space `json:"-"` // Shared data space, it will be used to share data between the request and the call + Cache store.Store `json:"-"` // Cache store, it will be used to store the message cache, default is "__yao.agent.cache" + Stack *Stack `json:"-"` // Stack, current active stack of the request + Stacks map[string]*Stack `json:"-"` // Stacks, all stacks in this request (for trace logging) + Writer Writer `json:"-"` // Writer, it will be used to write response data to the client + IDGenerator *message.IDGenerator `json:"-"` // ID generator for this context (chunk, message, block, thread IDs) + + // Internal + trace traceTypes.Manager `json:"-"` // Trace manager, lazy initialized on first access + output *output.Output `json:"-"` // Output, it will be used to write response data to the client + messageMetadata *messageMetadataStore `json:"-"` // Thread-safe message metadata store for delta operations + + // Skip configuration (history, trace, etc.), nil means don't skip anything + Skip *Skip `json:"skip,omitempty"` // Skip configuration (history, trace, etc.), nil means don't skip anything // Model capabilities (set by assistant, used by output adapters) Capabilities *ModelCapabilities `json:"-"` // Model capabilities for the current connector @@ -230,7 +262,6 @@ type Context struct { Authorized *types.AuthorizedInfo `json:"authorized,omitempty"` // Authorized information ChatID string `json:"chat_id,omitempty"` // Chat ID, use to select chat AssistantID string `json:"assistant_id,omitempty"` // Assistant ID, use to select assistant - Sid string `json:"sid" yaml:"-"` // Session ID (Deprecated, use Authorized instead) Connector string `json:"connector,omitempty"` // Connector, use to select the connector of the LLM Model, Default is Assistant.Connector Search *bool `json:"search,omitempty"` // Search mode, default is true @@ -251,8 +282,6 @@ type Context struct { // CUI Context information Route string `json:"route,omitempty"` // The route of the request, it will be used to identify the route of the request Metadata map[string]interface{} `json:"metadata,omitempty"` // The metadata of the request, it will be used to pass data to the page - - Silent bool `json:"silent,omitempty"` // Silent mode (Deprecated, use Referer instead) } // Stack represents the call stack node for tracing agent-to-agent calls diff --git a/agent/jsapi/jsapi.go b/agent/jsapi/jsapi.go deleted file mode 100644 index 6357f99e..00000000 --- a/agent/jsapi/jsapi.go +++ /dev/null @@ -1,4 +0,0 @@ -package jsapi - -// JSAPI Register the JavaScript API -// Agent API will be registered as a third party object diff --git a/agent/load.go b/agent/load.go index 9fa9bab4..b03819c5 100644 --- a/agent/load.go +++ b/agent/load.go @@ -6,19 +6,19 @@ import ( "github.com/yaoapp/gou/application" "github.com/yaoapp/gou/connector" - "github.com/yaoapp/kun/exception" - "github.com/yaoapp/yao/agent/api" "github.com/yaoapp/yao/agent/assistant" "github.com/yaoapp/yao/agent/context" "github.com/yaoapp/yao/agent/i18n" - mongoStore "github.com/yaoapp/yao/agent/store/mongo" - redisStore "github.com/yaoapp/yao/agent/store/redis" + storeMongo "github.com/yaoapp/yao/agent/store/mongo" + storeRedis "github.com/yaoapp/yao/agent/store/redis" store "github.com/yaoapp/yao/agent/store/types" - xunStore "github.com/yaoapp/yao/agent/store/xun" + "github.com/yaoapp/yao/agent/store/xun" "github.com/yaoapp/yao/agent/types" "github.com/yaoapp/yao/config" ) +var agentDSL *types.DSL + // Load load AIGC func Load(cfg config.Config) error { @@ -59,8 +59,7 @@ func Load(cfg config.Config) error { setting.Uses.Prompt = setting.Uses.Default } - // Initialize Agent API - api.Agent = &api.API{DSL: &setting} + agentDSL = &setting // Store Setting err = initStore() @@ -89,12 +88,9 @@ func Load(cfg config.Config) error { return nil } -// GetAgent returns the Agent instance -func GetAgent() *api.API { - if api.Agent == nil { - exception.New("Agent is not initialized", 500).Throw() - } - return api.Agent +// GetAgent returns the Agent settings +func GetAgent() *types.DSL { + return agentDSL } // initGlobalI18n initialize the global i18n @@ -126,7 +122,7 @@ func initModelCapabilities() error { return err } - api.Agent.DSL.Models = models + agentDSL.Models = models return nil } @@ -134,57 +130,52 @@ func initModelCapabilities() error { func initStore() error { var err error - if api.Agent.DSL.StoreSetting.Connector == "default" || api.Agent.DSL.StoreSetting.Connector == "" { - api.Agent.DSL.Store, err = xunStore.NewXun(api.Agent.DSL.StoreSetting) + if agentDSL.StoreSetting.Connector == "default" || agentDSL.StoreSetting.Connector == "" { + agentDSL.Store, err = xun.NewXun(agentDSL.StoreSetting) return err } // other connector - conn, err := connector.Select(api.Agent.DSL.StoreSetting.Connector) + conn, err := connector.Select(agentDSL.StoreSetting.Connector) if err != nil { return fmt.Errorf("load connectors error: %s", err.Error()) } if conn.Is(connector.DATABASE) { - api.Agent.DSL.Store, err = xunStore.NewXun(api.Agent.DSL.StoreSetting) + agentDSL.Store, err = xun.NewXun(agentDSL.StoreSetting) return err } else if conn.Is(connector.REDIS) { - api.Agent.DSL.Store = redisStore.NewRedis() + agentDSL.Store = storeRedis.NewRedis() return nil } else if conn.Is(connector.MONGO) { - api.Agent.DSL.Store = mongoStore.NewMongo() + agentDSL.Store = storeMongo.NewMongo() return nil } - return fmt.Errorf("Agent store connector %s not support", api.Agent.DSL.StoreSetting.Connector) + return fmt.Errorf("Agent store connector %s not support", agentDSL.StoreSetting.Connector) } // initAssistant initialize the assistant func initAssistant() error { // Set Storage - assistant.SetStorage(api.Agent.DSL.Store) - - // Assistant Vision - if api.Agent.DSL.Vision != nil { - assistant.SetVision(api.Agent.DSL.Vision) - } + assistant.SetStorage(agentDSL.Store) // Set global Uses configuration - if api.Agent.DSL.Uses != nil { + if agentDSL.Uses != nil { globalUses := &context.Uses{ - Vision: api.Agent.DSL.Uses.Vision, - Audio: api.Agent.DSL.Uses.Audio, - Search: api.Agent.DSL.Uses.Search, - Fetch: api.Agent.DSL.Uses.Fetch, + Vision: agentDSL.Uses.Vision, + Audio: agentDSL.Uses.Audio, + Search: agentDSL.Uses.Search, + Fetch: agentDSL.Uses.Fetch, } assistant.SetGlobalUses(globalUses) } - if api.Agent.DSL.Models != nil { - assistant.SetModelCapabilities(api.Agent.DSL.Models) + if agentDSL.Models != nil { + assistant.SetModelCapabilities(agentDSL.Models) } // Load Built-in Assistants @@ -199,14 +190,14 @@ func initAssistant() error { return err } - api.Agent.DSL.Assistant = defaultAssistant + agentDSL.Assistant = defaultAssistant return nil } // defaultAssistant get the default assistant func defaultAssistant() (*assistant.Assistant, error) { - if api.Agent.DSL.Uses == nil || api.Agent.DSL.Uses.Default == "" { + if agentDSL.Uses == nil || agentDSL.Uses.Default == "" { return nil, fmt.Errorf("default assistant not found") } - return assistant.Get(api.Agent.DSL.Uses.Default) + return assistant.Get(agentDSL.Uses.Default) } diff --git a/agent/message/contents.go b/agent/message/contents.go deleted file mode 100644 index 4d25f663..00000000 --- a/agent/message/contents.go +++ /dev/null @@ -1,411 +0,0 @@ -package message - -import ( - "fmt" - "math/rand" - "strings" - "time" - - "github.com/google/uuid" - jsoniter "github.com/json-iterator/go" -) - -const ( - // ContentStatusPending the content status pending - ContentStatusPending = iota - // ContentStatusDone the content status done - ContentStatusDone - // ContentStatusError the content status error - ContentStatusError -) - -var tokens = map[string][2]string{ - "think": {"", ""}, - "tool": {"", ""}, -} - -// Contents the contents -type Contents struct { - Current int `json:"current"` // the current content index - Data []Data `json:"data"` // the data - token string // the current token - id string // the id of the contents - stack [][]string // the token stack - mapping map[string]string // the mapping of the token stack -} - -// Data the data of the content -type Data struct { - Type string `json:"type"` // text, function, error, think, tool - ID string `json:"id"` // the id of the content - Bytes []byte `json:"bytes"` // the content bytes - Props map[string]interface{} `json:"props"` // the props - Begin int64 `json:"begin,omitempty"` // the begin time - End int64 `json:"end,omitempty"` // the end time -} - -// Extra the extra of the content -type Extra struct { - ID string `json:"id,omitempty"` // the id of the content - Begin int64 `json:"begin,omitempty"` // the begin time - End int64 `json:"end,omitempty"` // the end time -} - -// ScanCallbackParams the params of the scan callback -type ScanCallbackParams struct { - Token string - MessageID string - TokenID string - BeganAt int64 - EndAt int64 - Begin bool - End bool - Text string - Tails string -} - -// NewContents create a new contents -func NewContents() *Contents { - return &Contents{ - Current: -1, - Data: []Data{}, - } -} - -// ScanTokens scan the tokens -func (c *Contents) ScanTokens(messageID string, tokenID string, beganAt int64, cb func(params ScanCallbackParams)) { - - text := strings.TrimSpace(c.Text()) - - // check the end of the token - if c.token != "" { - - token := c.GetToken(c.token) - tokenType := c.GetTokenType(c.token) - // Check the end of the token - if index := strings.Index(text, token[1]); index >= 0 { - tails := "" - if index > 0 { - tails = text[index+len(token[1]):] - } - - extra := Extra{ - ID: c.id, - End: time.Now().UnixNano(), - } - - c.UpdateType(tokenType, map[string]interface{}{"text": text}, extra) - c.NewText([]byte(tails), extra) // Create new text with the tails - cb(ScanCallbackParams{Token: tokenType, MessageID: c.id, TokenID: tokenID, BeganAt: beganAt, Begin: false, End: true, Text: text, Tails: tails, EndAt: extra.End}) - c.ClearToken(c.token) // clear the token - return - } - - // call the callback for the scanning of the token - cb(ScanCallbackParams{Token: tokenType, MessageID: c.id, TokenID: tokenID, BeganAt: beganAt, Begin: false, End: false, Text: text, Tails: "", EndAt: 0}) - return - } - - // scan the begin of the token - begin := false - for name, token := range tokens { - if index := strings.Index(text, token[0]); index >= 0 { - - c.id = messageID - if c.id == "" { - c.id = GenerateNumericID("M") - } - - tokenType := name - if tokenID != "" { - tokenType = c.GetTokenType(tokenID) - } - - // First time scanning the token, generate the token ID and begin time - if tokenID == "" || tokenType != name { - tokenID = GenerateNumericID("T") - beganAt = time.Now().UnixNano() - begin = true - c.token = tokenID - c.AppendToken(tokenID, name) - c.UpdateType(name, map[string]interface{}{"text": text, "id": tokenID}, Extra{ID: c.id, Begin: beganAt, End: beganAt}) - } - - cb(ScanCallbackParams{Token: name, MessageID: c.id, TokenID: tokenID, BeganAt: beganAt, Begin: begin, End: false, Text: text, Tails: "", EndAt: 0}) // call the callback - } - } -} - -// ClearToken clear the token -func (c *Contents) ClearToken(id string) { - c.token = "" - next := 0 - - if c.stack == nil { - c.stack = [][]string{} - } - - if c.mapping == nil { - c.mapping = map[string]string{} - } - - for i, node := range c.stack { - if node[0] == id { - next = i + 1 - delete(c.mapping, id) - break - } - } - - // Remove the token from the stack, and set the next token - if next > 0 && next < len(c.stack) { - c.stack = c.stack[next:] - c.token = c.stack[len(c.stack)-1][0] - } -} - -// AppendToken append the token to the stack -func (c *Contents) AppendToken(id string, name string) { - if c.stack == nil { - c.stack = [][]string{} - } - if c.mapping == nil { - c.mapping = map[string]string{} - } - c.stack = append(c.stack, []string{id, name}) - c.mapping[id] = name - c.token = id -} - -// GetTokenType get the token type from the stack -func (c *Contents) GetTokenType(id string) string { - return c.mapping[id] -} - -// GetToken get the token from the stack -func (c *Contents) GetToken(name string) [2]string { - typ, ok := c.mapping[name] - if !ok { - return [2]string{} - } - return tokens[typ] -} - -// RemoveLastEmpty remove the last empty data -func (c *Contents) RemoveLastEmpty() { - if c.Current == -1 { - return - } - - // Remove the last empty data - if len(c.Data[c.Current].Bytes) == 0 && c.Data[c.Current].Type == "text" { - c.Data = c.Data[:c.Current] - c.Current-- - } -} - -// NewText create a new text data and append to the contents -func (c *Contents) NewText(bytes []byte, extra ...Extra) *Contents { - data := Data{Type: "text", Bytes: bytes} - - if len(extra) > 0 { - if extra[0].Begin != 0 { - data.Begin = extra[0].Begin - } - if extra[0].End != 0 { - data.End = extra[0].End - } - if extra[0].ID != "" { - data.ID = extra[0].ID - } - } - - c.Data = append(c.Data, data) - c.Current++ - return c -} - -// NewType create a new type data and append to the contents -func (c *Contents) NewType(typ string, props map[string]interface{}, extra ...Extra) *Contents { - - data := Data{ - Type: typ, - Props: props, - } - - if len(extra) > 0 { - if extra[0].Begin != 0 { - data.Begin = extra[0].Begin - } - if extra[0].End != 0 { - data.End = extra[0].End - } - if extra[0].ID != "" { - data.ID = extra[0].ID - } - } - - c.Data = append(c.Data, data) - c.Current++ - return c -} - -// UpdateType update the type of the current content -func (c *Contents) UpdateType(typ string, props map[string]interface{}, extra ...Extra) *Contents { - if c.Current == -1 { - c.NewType(typ, props, extra...) - return c - } - - if len(extra) > 0 { - if extra[0].Begin != 0 { - c.Data[c.Current].Begin = extra[0].Begin - } - if extra[0].End != 0 { - c.Data[c.Current].End = extra[0].End - } - if extra[0].ID != "" { - c.Data[c.Current].ID = extra[0].ID - } - } - c.Data[c.Current].Type = typ - if props != nil { - if c.Data[c.Current].Props == nil { - c.Data[c.Current].Props = map[string]interface{}{} - } - - for k, v := range props { - c.Data[c.Current].Props[k] = v - } - } - return c -} - -// NewError create a new error data and append to the contents -func (c *Contents) NewError(err []byte) *Contents { - c.Data = append(c.Data, Data{ - Type: "error", - Bytes: err, - }) - c.Current++ - return c -} - -// AppendText append the text to the current content -func (c *Contents) AppendText(bytes []byte, extra ...Extra) *Contents { - if c.Current == -1 { - c.NewText(bytes, extra...) - return c - } - - if len(extra) > 0 { - if extra[0].ID != "" { - c.Data[c.Current].ID = extra[0].ID - } - if extra[0].Begin != 0 { - c.Data[c.Current].Begin = extra[0].Begin - } - if extra[0].End != 0 { - c.Data[c.Current].End = extra[0].End - } - } - c.Data[c.Current].Bytes = append(c.Data[c.Current].Bytes, bytes...) - return c -} - -// AppendError append the error to the current content -func (c *Contents) AppendError(err []byte) *Contents { - if c.Current == -1 { - c.NewError(err) - return c - } - c.Data[c.Current].Bytes = append(c.Data[c.Current].Bytes, err...) - return c -} - -// JSON returns the json representation -func (c *Contents) JSON() string { - raw, _ := jsoniter.MarshalToString(c.Data) - return raw -} - -// Text returns the text of the current content -func (c *Contents) Text() string { - if c.Current == -1 { - return "" - } - return string(c.Data[c.Current].Bytes) -} - -// CurrentType returns the type of the current content -func (c *Contents) CurrentType() string { - if c.Current == -1 { - return "" - } - return c.Data[c.Current].Type -} - -// Map returns the map representation -func (data *Data) Map() (map[string]interface{}, error) { - v := map[string]interface{}{"type": data.Type} - - if data.ID != "" { - v["id"] = data.ID - } - - if data.Bytes != nil && data.Type == "text" { - v["text"] = string(data.Bytes) - } - - if data.Props != nil && data.Type != "text" { - v["props"] = data.Props - } - - return v, nil -} - -// MarshalJSON returns the json representation -func (data *Data) MarshalJSON() ([]byte, error) { - - v := map[string]interface{}{"type": data.Type} - - if data.ID != "" { - v["id"] = data.ID - } - - if data.Bytes != nil && data.Type == "text" { - v["text"] = string(data.Bytes) - } - - if data.Props != nil && data.Type != "text" { - v["props"] = data.Props - } - - // Add the begin and end time - if data.Begin != 0 { - v["begin"] = data.Begin - } - - if data.End != 0 { - v["end"] = data.End - } - - return jsoniter.Marshal(v) -} - -// GenerateNumericID generates a 10-digit number using UUID as seed -func GenerateNumericID(prefix string) string { - // Generate UUID and use it as seed - id := uuid.New() - seed := int64(id[0])<<56 | int64(id[1])<<48 | int64(id[2])<<40 | int64(id[3])<<32 | - int64(id[4])<<24 | int64(id[5])<<16 | int64(id[6])<<8 | int64(id[7]) - - // Create a new random source using the seed - source := rand.NewSource(seed) - r := rand.New(source) - - // Generate a number between 1000000000 and 9999999999 (10 digits) - num := r.Int63n(9000000000) + 1000000000 - - return fmt.Sprintf("%s%d", prefix, num) -} diff --git a/agent/message/message.go b/agent/message/message.go deleted file mode 100644 index 0ea72992..00000000 --- a/agent/message/message.go +++ /dev/null @@ -1,650 +0,0 @@ -package message - -import ( - "fmt" - "os" - "strings" - "sync" - - "github.com/fatih/color" - "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/gou/helper" - "github.com/yaoapp/kun/exception" - "github.com/yaoapp/kun/log" - "github.com/yaoapp/kun/maps" - "github.com/yaoapp/yao/attachment" - "github.com/yaoapp/yao/openai" -) - -var locker = sync.Mutex{} - -// New create a new message -func New() *Message { - return &Message{Actions: []Action{}, Props: map[string]interface{}{}} -} - -// NewHistory create a new message from history -func NewHistory(history map[string]interface{}) ([]Message, error) { - if history == nil { - return []Message{}, nil - } - - var copy map[string]interface{} = map[string]interface{}{} - for key, value := range history { - if key != "content" { - copy[key] = value - } - } - - globalMessage := New().Map(copy) - messages := []Message{} - if content, ok := history["content"].(string); ok { - if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { - var msg Message = *globalMessage - if err := jsoniter.UnmarshalFromString(content, &msg); err != nil { - return nil, err - } - messages = append(messages, msg) - } else if strings.HasPrefix(content, "[") && strings.HasSuffix(content, "]") { - var msgs []Message - if err := jsoniter.UnmarshalFromString(content, &msgs); err != nil { - return nil, err - } - for _, msg := range msgs { - msg.AssistantID = globalMessage.AssistantID - msg.AssistantName = globalMessage.AssistantName - msg.AssistantAvatar = globalMessage.AssistantAvatar - msg.Role = globalMessage.Role - msg.Name = globalMessage.Name - msg.Mentions = globalMessage.Mentions - messages = append(messages, msg) - } - } else { - messages = append(messages, Message{Text: content}) - } - } - - return messages, nil -} - -// NewContent create a new message from content -func NewContent(content string) ([]Message, error) { - messages := []Message{} - if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { - var msg Message - if err := jsoniter.UnmarshalFromString(content, &msg); err != nil { - return nil, err - } - messages = append(messages, msg) - } else if strings.HasPrefix(content, "[") && strings.HasSuffix(content, "]") { - var msgs []Message - if err := jsoniter.UnmarshalFromString(content, &msgs); err != nil { - return nil, err - } - for _, msg := range msgs { - messages = append(messages, msg) - } - } else { - messages = append(messages, Message{Text: content}) - } - return messages, nil -} - -// NewString create a new message from string -func NewString(content string, id ...string) (*Message, error) { - if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { - var msg Message - if err := jsoniter.UnmarshalFromString(content, &msg); err != nil { - return nil, err - } - return &msg, nil - } - if len(id) > 0 { - return &Message{ID: id[0], Text: content}, nil - } - return &Message{Text: content}, nil -} - -// NewStringError create a new message from string error -func NewStringError(content string) (*Message, error) { - if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { - var msg = New() - var errorMessage openai.ErrorMessage - if err := jsoniter.UnmarshalFromString(content, &errorMessage); err != nil { - msg.Text = err.Error() + "\n" + content - return msg, nil - } - msg.Type = "error" - msg.Text = errorMessage.Error.Message - return msg, nil - } - return &Message{Text: content}, nil -} - -// NewMap create a new message from map -func NewMap(content map[string]interface{}) (*Message, error) { - return New().Map(content), nil -} - -// NewAny create a new message from any content -func NewAny(content interface{}) (*Message, error) { - switch v := content.(type) { - case string: - return NewString(v) - case map[string]interface{}: - return NewMap(v) - } - return nil, fmt.Errorf("unknown content type: %T", content) -} - -// NewOpenAI create a new message from OpenAI response -func NewOpenAI(data []byte, isThinking bool) *Message { - - // For debug environment, print the response data - if os.Getenv("YAO_AGENT_PRINT_RESPONSE_DATA") == "true" { - log.Trace("[Response Data] %s", string(data)) - } - - if data == nil || len(data) == 0 { - return nil - } - - msg := New() - text := string(data) - data = []byte(strings.TrimPrefix(text, "data: ")) - - switch { - case strings.Contains(text, `"object":"chat.completion.chunk"`): // Delta content - var chunk openai.ChatCompletionChunk - err := jsoniter.Unmarshal(data, &chunk) - if err != nil { - color.Red("JSON parse error: %s", err.Error()) - color.White(string(data)) - msg.Text = "JSON parse error\n" + string(data) - msg.Type = "error" - msg.IsDone = true - } - - // Empty content, then it is a pending message - if len(chunk.Choices) == 0 { - msg.Pending = true - return msg - } - - // Tool calls - if len(chunk.Choices[0].Delta.ToolCalls) > 0 || chunk.Choices[0].FinishReason == "tool_calls" { - msg.Type = "tool_calls_native" - text := "" - if len(chunk.Choices[0].Delta.ToolCalls) > 0 { - id := chunk.Choices[0].Delta.ToolCalls[0].ID - function := chunk.Choices[0].Delta.ToolCalls[0].Function.Name - arguments := chunk.Choices[0].Delta.ToolCalls[0].Function.Arguments - text = arguments - if id != "" { - msg.IsBeginTool = true - msg.IsNew = true // mark as a new message - text = fmt.Sprintf(`{"id": "%s", "function": "%s", "arguments": %s`, id, function, arguments) - } - } - - if chunk.Choices[0].FinishReason == "tool_calls" { - msg.IsEndTool = true - } - - msg.Text = text - return msg - } - - // Text content - if chunk.Choices[0].Delta.Content != "" { - msg.Type = "text" - msg.Text = chunk.Choices[0].Delta.Content - msg.IsDone = chunk.Choices[0].FinishReason == "stop" // is done when the content is finished - return msg - } - - // Done messages - if chunk.Choices[0].FinishReason == "stop" || chunk.Choices[0].FinishReason == "tool_calls" { - msg.IsDone = true - return msg - } - - // Reasoning content - if chunk.Choices[0].Delta.ReasoningContent != "" { - msg.Type = "think" - msg.Text = chunk.Choices[0].Delta.ReasoningContent - return msg - } - // Content is empty and is thinking, then it is a thinking message pending - if isThinking { - msg.Type = "think" - msg.Text = "" - return msg - } - - msg.Text = "" - return msg - - case strings.Contains(text, `"usage":`): // usage content - msg.IsDone = true - break - - case strings.Contains(text, `[DONE]`): - msg.IsDone = true - return msg - - case len(data) > 2 && data[0] == '{' && data[len(data)-1] == '}': // JSON content (error) - - var error openai.Error - var errorMessage openai.ErrorMessage - if strings.Contains(string(data), `"error":`) { - if err := jsoniter.Unmarshal(data, &errorMessage); err != nil { - color.Red("JSON parse error: %s", err.Error()) - color.White(string(data)) - msg.Text = "JSON parse error\n" + string(data) - msg.Type = "error" - msg.IsDone = true - return msg - } - error = errorMessage.Error - } else { - err := jsoniter.Unmarshal(data, &error) - if err != nil { - color.Red("JSON parse error: %s", err.Error()) - color.White(string(data)) - msg.Text = "JSON parse error\n" + string(data) - msg.Type = "error" - msg.IsDone = true - return msg - } - } - - message := error.Message - if message == "" { - message = "Unknown error occurred\n" + string(data) - } - - msg.Type = "error" - msg.Text = message - msg.IsDone = true - return msg - - case !strings.Contains(text, `data: `): // unknown message or uncompleted message - msg.Pending = true - msg.Text = text - return msg - - default: // unknown message - str := strings.TrimPrefix(strings.Trim(string(data), "\""), "data: ") - msg.Type = "error" - msg.Text = str - return msg - } - - return msg -} - -// String returns the string representation -func (m *Message) String() string { - typ := m.Type - if typ == "" { - typ = "text" - } - - switch typ { - case "text", "think", "tool", "tool_calls_native": - return m.Text - - case "error": - return m.Text - - default: - raw, _ := jsoniter.MarshalToString(map[string]interface{}{"type": m.Type, "props": m.Props}) - return raw - } -} - -// SetText set the text -func (m *Message) SetText(text string) *Message { - m.Text = text - if m.Data != nil { - if replaced := helper.Bind(text, m.Data); replaced != nil { - if replacedText, ok := replaced.(string); ok { - m.Text = replacedText - } - } - } - return m -} - -// SetProps set the props -func (m *Message) SetProps(props map[string]interface{}) *Message { - m.Props = props - return m -} - -// Error set the error -func (m *Message) Error(message interface{}) *Message { - m.Type = "error" - switch v := message.(type) { - case error: - m.Text = v.Error() - case string: - m.Text = v - default: - m.Text = fmt.Sprintf("%v", message) - } - return m -} - -// SetResult set the result -func (m *Message) SetResult(result any) *Message { - m.Result = result - m.Type = "result" // set the type to result - return m -} - -// SetContent set the content -func (m *Message) SetContent(content string) *Message { - if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { - var msg Message - if err := jsoniter.UnmarshalFromString(content, &msg); err != nil { - m.Text = err.Error() + "\n" + content - return m - } - *m = msg - } else { - m.Text = content - m.Type = "text" - } - return m -} - -// AppendTo append the contents -func (m *Message) AppendTo(contents *Contents) *Message { - - // Set type - if m.Type == "" { - m.Type = "text" - } - - switch m.Type { - case "text", "think", "tool", "tool_calls_native": - if m.Text != "" { - if m.IsNew { - contents.NewText([]byte(m.Text), Extra{ID: m.ID, Begin: m.Begin, End: m.End}) - return m - } - contents.AppendText([]byte(m.Text), Extra{ID: m.ID, Begin: m.Begin, End: m.End}) - return m - } - return m - - case "loading", "error", "action", "progress", "plan", "result": // Ignore progress, loading, plan and error messages - return m - - default: - if m.IsNew { - contents.NewType(m.Type, m.Props) - return m - } - contents.UpdateType(m.Type, m.Props) - return m - } - -} - -// Content get the content -func (m *Message) Content() string { - content := map[string]interface{}{"text": m.Text} - if m.Attachments != nil { - content["attachments"] = m.Attachments - } - - if m.Type != "" { - content["type"] = m.Type - } - contentRaw, _ := jsoniter.MarshalToString(content) - return contentRaw -} - -// ToMap convert to map -func (m *Message) ToMap() map[string]interface{} { - return map[string]interface{}{ - "content": m.Content(), - "role": m.Role, - "name": m.Name, - } -} - -// Map set from map -func (m *Message) Map(msg map[string]interface{}) *Message { - if msg == nil { - return m - } - - // Content {"text": "xxxx", "attachments": ... } - if content, ok := msg["content"].(string); ok { - if strings.HasPrefix(content, "{") && strings.HasSuffix(content, "}") { - var msg Message - if err := jsoniter.UnmarshalFromString(content, &msg); err != nil { - m.Text = err.Error() + "\n" + content - return m - } - *m = msg - } else { - m.Text = content - m.Type = "text" - } - } - - // attachments - if attachments, has := msg["attachments"]; has { - raw, _ := jsoniter.Marshal(attachments) - m.Attachments = []attachment.Attachment{} - if err := jsoniter.Unmarshal(raw, &m.Attachments); err != nil { - color.Red("JSON parse error: %s", err.Error()) - color.White(string(raw)) - } - } - - if role, ok := msg["role"].(string); ok { - m.Role = role - } - - if name, ok := msg["name"].(string); ok { - m.Name = name - } - - if text, ok := msg["text"].(string); ok { - m.Text = text - } - if typ, ok := msg["type"].(string); ok { - m.Type = typ - } - if done, ok := msg["done"].(bool); ok { - m.IsDone = done - } - if props, ok := msg["props"].(map[string]interface{}); ok { - m.Props = props - } - - if isNew, ok := msg["new"].(bool); ok { - m.IsNew = isNew - } - - if isDelta, ok := msg["delta"].(bool); ok { - m.IsDelta = isDelta - } - - if assistantID, ok := msg["assistant_id"].(string); ok { - m.AssistantID = assistantID - - // Set name - if m.Role == "assistant" { - m.Name = m.AssistantID - } - } - - if assistantName, ok := msg["assistant_name"].(string); ok { - m.AssistantName = assistantName - } - - if assistantAvatar, ok := msg["assistant_avatar"].(string); ok { - m.AssistantAvatar = assistantAvatar - } - - if actions, ok := msg["actions"].([]interface{}); ok { - for _, action := range actions { - if v, ok := action.(map[string]interface{}); ok { - action := Action{} - if name, ok := v["name"].(string); ok { - action.Name = name - } - if t, ok := v["type"].(string); ok { - action.Type = t - } - if payload, ok := v["payload"].(map[string]interface{}); ok { - action.Payload = payload - } - m.Actions = append(m.Actions, action) - } - } - } - if data, ok := msg["data"].(map[string]interface{}); ok { - m.Data = data - } - return m -} - -// Done set the done flag -func (m *Message) Done() *Message { - m.IsDone = true - return m -} - -// Assistant set the assistant -func (m *Message) Assistant(id string, name string, avatar string) *Message { - m.AssistantID = id - m.AssistantName = name - m.AssistantAvatar = avatar - return m -} - -// Action add an action -func (m *Message) Action(name string, t string, payload interface{}, next string) *Message { - if m.Data != nil { - payload = helper.Bind(payload, m.Data) - } - m.Actions = append(m.Actions, Action{ - Name: name, - Type: t, - Payload: payload, - }) - return m -} - -// Bind replace with data -func (m *Message) Bind(data map[string]interface{}) *Message { - if data == nil { - return m - } - m.Data = maps.Of(data).Dot() - return m -} - -// Callback callback the message -func (m *Message) Callback(fn interface{}) *Message { - if fn != nil { - switch v := fn.(type) { - case func(msg *Message): - if v == nil { - break - } - v(m) - break - - case func(): - if v == nil { - break - } - v() - break - - default: - fmt.Println("no match callback") - break - } - } - return m -} - -// WriteError writes an error message to response writer -func (m *Message) WriteError(w gin.ResponseWriter, message string) { - errMsg := strings.Trim(exception.New(message, 500).Message, "\"") - data := []byte(fmt.Sprintf(`{"text":"%s","type":"error"`, errMsg)) - if m.IsDone { - data = []byte(fmt.Sprintf(`{"text":"%s","type":"error","done":true`, errMsg)) - } - data = append([]byte("data: "), data...) - data = append(data, []byte("}\n\n")...) - - if _, err := w.Write(data); err != nil { - color.Red("Write JSON Message Error: %s", message) - } - w.Flush() -} - -// MarshalJSON implements json.Marshaler interface -func (m *Message) MarshalJSON() ([]byte, error) { - type Alias Message - return jsoniter.Marshal(&struct { - *Alias - }{ - Alias: (*Alias)(m), - }) -} - -// UnmarshalJSON implements json.Unmarshaler interface -func (m *Message) UnmarshalJSON(data []byte) error { - type Alias Message - aux := &struct { - *Alias - }{ - Alias: (*Alias)(m), - } - if err := jsoniter.Unmarshal(data, &aux); err != nil { - return err - } - return nil -} - -// MarshalJSON implements json.Marshaler interface -func (a *Action) MarshalJSON() ([]byte, error) { - type Alias Action - return jsoniter.Marshal(&struct { - *Alias - }{ - Alias: (*Alias)(a), - }) -} - -// UnmarshalJSON implements json.Unmarshaler interface -func (a *Action) UnmarshalJSON(data []byte) error { - type Alias Action - aux := &struct { - *Alias - }{ - Alias: (*Alias)(a), - } - if err := jsoniter.Unmarshal(data, &aux); err != nil { - return err - } - return nil -} - -// Write writes the message to response writer using the message queue -func (m *Message) Write(w gin.ResponseWriter) bool { - return WriteMessageAsync(m, w) -} diff --git a/agent/message/queue.go b/agent/message/queue.go deleted file mode 100644 index 4a450efe..00000000 --- a/agent/message/queue.go +++ /dev/null @@ -1,153 +0,0 @@ -package message - -import ( - "sync" - "time" - - "github.com/fatih/color" - "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/kun/log" -) - -// AsyncMessageQueue represents a queue for handling message writes -type AsyncMessageQueue struct { - queue chan *AsyncTask - workers int - wg sync.WaitGroup - shutdown chan struct{} -} - -// AsyncTask represents a task to write a message -type AsyncTask struct { - message *Message - writer gin.ResponseWriter - done chan bool -} - -var ( - defaultQueue *AsyncMessageQueue - queueOnce sync.Once -) - -// GetQueue returns the default message queue instance -func GetQueue() *AsyncMessageQueue { - queueOnce.Do(func() { - defaultQueue = NewAsyncQueue(10) // Initialize with 10 workers - defaultQueue.Start() - }) - return defaultQueue -} - -// NewAsyncQueue creates a new message queue with the specified number of workers -func NewAsyncQueue(workers int) *AsyncMessageQueue { - return &AsyncMessageQueue{ - queue: make(chan *AsyncTask, 1000), // Buffer size of 1000 - workers: workers, - shutdown: make(chan struct{}), - } -} - -// Start starts the message queue workers -func (mq *AsyncMessageQueue) Start() { - for i := 0; i < mq.workers; i++ { - mq.wg.Add(1) - go mq.worker() - } -} - -// Stop stops the message queue workers -func (mq *AsyncMessageQueue) Stop() { - close(mq.shutdown) - mq.wg.Wait() -} - -// worker processes messages from the queue -func (mq *AsyncMessageQueue) worker() { - defer mq.wg.Done() - - for { - select { - case task := <-mq.queue: - if task == nil { - continue - } - success := writeMessageToResponse(task.message, task.writer) - if task.done != nil { - task.done <- success - } - case <-mq.shutdown: - return - } - } -} - -// WriteMessageAsync writes the message to response writer using the message queue -func WriteMessageAsync(m *Message, w gin.ResponseWriter) bool { - done := make(chan bool, 1) - task := &AsyncTask{ - message: m, - writer: w, - done: done, - } - - // Try to send the task to the queue with a timeout - select { - case GetQueue().queue <- task: - // Wait for the message to be processed with a longer timeout - select { - case success := <-done: - return success - case <-time.After(5 * time.Second): // Increased timeout to 5 seconds - log.Error("Message processing timeout") - return false - } - case <-time.After(1 * time.Second): // Increased queue timeout to 1 second - log.Error("Queue is full, message dropped") - return false - } -} - -// writeMessageToResponse writes the message directly to the response writer -func writeMessageToResponse(m *Message, w gin.ResponseWriter) bool { - // Sync write to response writer - locker.Lock() - defer locker.Unlock() - - defer func() { - if r := recover(); r != nil { - // Ignore if done is true - if m.IsDone { - return - } - - message := "Write Response Exception: (if client close the connection, it's normal) \n %s\n\n" - color.Red(message, r) - - // Print the message - raw, _ := jsoniter.MarshalToString(m) - color.White("Message:\n %s", raw) - } - }() - - // Ignore silent messages - if m.Silent { - return true - } - - data, err := jsoniter.Marshal(m) - if err != nil { - log.Error("%s", err.Error()) - return false - } - - data = append([]byte("data: "), data...) - data = append(data, []byte("\n\n")...) - - if _, err := w.Write(data); err != nil { - color.Red("Write JSON Message Error: %s", err.Error()) - return false - } - w.Flush() - return true -} diff --git a/agent/message/types.go b/agent/message/types.go deleted file mode 100644 index c0849e0f..00000000 --- a/agent/message/types.go +++ /dev/null @@ -1,48 +0,0 @@ -package message - -import "github.com/yaoapp/yao/attachment" - -// Message the message -type Message struct { - ID string `json:"id,omitempty"` // id for the message - ToolID string `json:"tool_id,omitempty"` // tool_id for the message - Text string `json:"text,omitempty"` // text content - Type string `json:"type,omitempty"` // error, text, plan, table, form, page, file, video, audio, image, markdown, json ... - Props map[string]interface{} `json:"props,omitempty"` // props for the types - IsDone bool `json:"done,omitempty"` // Mark as a done message from agent - IsNew bool `json:"new,omitempty"` // Mark as a new message from agent - IsDelta bool `json:"delta,omitempty"` // Mark as a delta message from agent - Actions []Action `json:"actions,omitempty"` // Conversation Actions for frontend - Attachments []attachment.Attachment `json:"attachments,omitempty"` // File attachments - Role string `json:"role,omitempty"` // user, assistant, system ... - Name string `json:"name,omitempty"` // name for the message - AssistantID string `json:"assistant_id,omitempty"` // assistant_id (for assistant role = assistant ) - AssistantName string `json:"assistant_name,omitempty"` // assistant_name (for assistant role = assistant ) - AssistantAvatar string `json:"assistant_avatar,omitempty"` // assistant_avatar (for assistant role = assistant ) - Mentions []Mention `json:"menions,omitempty"` // Mentions for the message ( for user role = user ) - Data map[string]interface{} `json:"-"` // data for the message - Pending bool `json:"-"` // pending for the message - Hidden bool `json:"hidden,omitempty"` // hidden for the message (not show in the UI and history) - Retry bool `json:"retry,omitempty"` // retry for the message - Silent bool `json:"silent,omitempty"` // silent for the message (not show in the UI and history) - IsTool bool `json:"-"` // is tool for the message for native tool_calls - IsBeginTool bool `json:"-"` // is new tool for the message for native tool_calls - IsEndTool bool `json:"-"` // is end tool for the message for native tool_calls - Result any `json:"result,omitempty"` // result for the message - Begin int64 `json:"begin,omitempty"` // begin at for the message // timestamp - End int64 `json:"end,omitempty"` // end at for the message // timestamp -} - -// Mention represents a mention -type Mention struct { - ID string `json:"assistant_id"` // assistant_id - Name string `json:"name"` // name - Avatar string `json:"avatar,omitempty"` // avatar -} - -// Action the action -type Action struct { - Name string `json:"name,omitempty"` - Type string `json:"type"` - Payload interface{} `json:"payload,omitempty"` -} diff --git a/agent/plan/plan.go b/agent/plan/plan.go deleted file mode 100644 index ec2d2c6b..00000000 --- a/agent/plan/plan.go +++ /dev/null @@ -1 +0,0 @@ -package plan diff --git a/agent/process.go b/agent/process.go deleted file mode 100644 index 7ce6d26c..00000000 --- a/agent/process.go +++ /dev/null @@ -1,332 +0,0 @@ -package agent - -import ( - "fmt" - "strconv" - "strings" - - "github.com/gin-gonic/gin" - "github.com/yaoapp/gou/process" - "github.com/yaoapp/kun/exception" - "github.com/yaoapp/yao/agent/message" - store "github.com/yaoapp/yao/agent/store/types" -) - -func init() { - process.RegisterGroup("agent", map[string]process.Handler{ - "write": ProcessWrite, - "assistant.create": processAssistantCreate, - "assistant.save": processAssistantSave, - "assistant.delete": processAssistantDelete, - "assistant.search": processAssistantSearch, - "assistant.find": processAssistantFind, - "assistant.match": processAssistantMatch, // Match assistant by content and params - }) - - // Neo is deprecated, use agent instead (for backward compatibility, It will be removed in the future) - process.RegisterGroup("neo", map[string]process.Handler{ - "write": ProcessWrite, - "assistant.create": processAssistantCreate, - "assistant.save": processAssistantSave, - "assistant.delete": processAssistantDelete, - "assistant.search": processAssistantSearch, - "assistant.find": processAssistantFind, - "assistant.match": processAssistantMatch, // Match assistant by content and params - }) -} - -// ProcessWrite process the write request -func ProcessWrite(process *process.Process) interface{} { - process.ValidateArgNums(2) - - w, ok := process.Args[0].(gin.ResponseWriter) - if !ok { - exception.New("The first argument must be a io.Writer", 400).Throw() - return nil - } - - data, ok := process.Args[1].([]interface{}) - if !ok { - exception.New("The second argument must be a Array", 400).Throw() - return nil - } - - for _, new := range data { - if v, ok := new.(map[string]interface{}); ok { - newMsg := message.New().Map(v) - newMsg.Write(w) - } - } - - return nil -} - -// processAssistantCreate process the assistant create request -func processAssistantCreate(process *process.Process) interface{} { - process.ValidateArgNums(1) - data := process.ArgsMap(0) - - agent := GetAgent() - if agent.Store == nil { - exception.New("Agent store is not initialized", 500).Throw() - } - - // Convert to AssistantModel - model, err := store.ToAssistantModel(data) - if err != nil { - exception.New("Invalid assistant data: %s", 400, err.Error()).Throw() - } - - id, err := agent.Store.SaveAssistant(model) - if err != nil { - exception.New("Failed to create assistant: %s", 500, err.Error()).Throw() - } - - return id -} - -// processAssistantSave process the assistant save request -func processAssistantSave(process *process.Process) interface{} { - process.ValidateArgNums(1) - data := process.ArgsMap(0) - - agent := GetAgent() - if agent.Store == nil { - exception.New("Agent store is not initialized", 500).Throw() - } - - // Convert to AssistantModel - model, err := store.ToAssistantModel(data) - if err != nil { - exception.New("Invalid assistant data: %s", 400, err.Error()).Throw() - } - - id, err := agent.Store.SaveAssistant(model) - if err != nil { - exception.New("Failed to save assistant: %s", 500, err.Error()).Throw() - } - - return id -} - -// processAssistantDelete process the assistant delete request -func processAssistantDelete(process *process.Process) interface{} { - process.ValidateArgNums(1) - assistantID := process.ArgsString(0) - - agent := GetAgent() - if agent.Store == nil { - exception.New("Agent store is not initialized", 500).Throw() - } - - err := agent.Store.DeleteAssistant(assistantID) - if err != nil { - exception.New("Failed to delete assistant: %s", 500, err.Error()).Throw() - } - - return gin.H{"message": "ok"} -} - -// processAssistantMatch process the assistant match request -func processAssistantMatch(process *process.Process) interface{} { - process.ValidateArgNums(1) - content := process.Args[0] - params := map[string]interface{}{} - if len(process.Args) > 1 { - params = process.ArgsMap(1) - } - - // Limit default to 20 - if _, has := params["limit"]; !has { - params["limit"] = 20 - } - - // Max limit to 100 - if limit, has := params["limit"]; has { - switch v := limit.(type) { - case int: - if v > 100 { - params["limit"] = 100 - } - case string: - limitInt, err := strconv.Atoi(v) - if err != nil { - exception.New("Invalid limit type: %T", 500, limit).Throw() - } - - params["limit"] = limitInt - if limitInt > 100 { - params["limit"] = 100 - } - - default: - exception.New("Invalid limit type: %T", 500, limit).Throw() - } - } - - // Match using Store - return assistantMatchStore(content, params) -} - -// parseAssistantFilter parse common filter parameters -func parseAssistantFilter(params map[string]interface{}) store.AssistantFilter { - filter := store.AssistantFilter{} - - // Parse page and pagesize - if page, ok := params["page"]; ok { - pageStr := fmt.Sprintf("%v", page) - if pageInt, err := strconv.Atoi(pageStr); err == nil { - filter.Page = pageInt - } - } - - if pagesize, ok := params["pagesize"]; ok { - pagesizeStr := fmt.Sprintf("%v", pagesize) - if pagesizeInt, err := strconv.Atoi(pagesizeStr); err == nil { - filter.PageSize = pagesizeInt - } - } - - // select - if sel, ok := params["select"]; ok { - switch v := sel.(type) { - case []interface{}: - filter.Select = []string{} - for _, field := range v { - switch v := field.(type) { - case string: - filter.Select = append(filter.Select, v) - case interface{}: - filter.Select = append(filter.Select, fmt.Sprintf("%v", v)) - } - } - - case []string: - filter.Select = v - - case string: - fields := strings.Split(v, ",") - filter.Select = fields - } - } - - // Parse tags - if tags, ok := params["tags"]; ok { - switch v := tags.(type) { - case []interface{}: - filter.Tags = make([]string, len(v)) - for i, tag := range v { - filter.Tags[i] = fmt.Sprintf("%v", tag) - } - case []string: - filter.Tags = v - } - } - - // Parse keywords - if keywords, ok := params["keywords"].(string); ok { - filter.Keywords = keywords - } - - // Parse connector - if connector, ok := params["connector"].(string); ok { - filter.Connector = connector - } - - // Parse mentionable - if mentionable, ok := params["mentionable"].(bool); ok { - filter.Mentionable = &mentionable - } - - // Parse automated - if automated, ok := params["automated"].(bool); ok { - filter.Automated = &automated - } - - return filter -} - -func assistantMatchStore(content interface{}, params map[string]interface{}) interface{} { - agent := GetAgent() - if agent.Store == nil { - exception.New("Agent store is not initialized", 500).Throw() - } - - // Convert limit to pagesize - if limit, has := params["limit"]; has { - params["pagesize"] = limit - } - params["page"] = 1 - - // Parse content to keywords if not empty - if content != nil { - contentStr := fmt.Sprintf("%v", content) - if contentStr != "" { - params["keywords"] = contentStr - } - } - - filter := parseAssistantFilter(params) - res, err := agent.Store.GetAssistants(filter) - if err != nil { - exception.New("get assistants error: %s", 500, err).Throw() - } - - return res.Data -} - -// processAssistantSearch process the assistant search request -func processAssistantSearch(process *process.Process) interface{} { - params := process.ArgsMap(0) - filter := parseAssistantFilter(params) - - // Get assistants - agent := GetAgent() - if agent.Store == nil { - exception.New("Agent store is not initialized", 500).Throw() - } - - locale := "en" - if len(process.Args) > 1 { - locale = process.ArgsString(1) - } - - res, err := agent.Store.GetAssistants(filter, locale) - if err != nil { - exception.New("get assistants error: %s", 500, err).Throw() - } - - return res -} - -// processAssistantFind process the assistant find request -func processAssistantFind(process *process.Process) interface{} { - process.ValidateArgNums(1) - assistantID := process.ArgsString(0) - - agent := GetAgent() - if agent.Store == nil { - exception.New("Agent store is not initialized", 500).Throw() - } - - filter := store.AssistantFilter{ - AssistantID: assistantID, - Page: 1, - PageSize: 1, - } - - locale := "en" - if len(process.Args) > 1 { - locale = process.ArgsString(1) - } - res, err := agent.Store.GetAssistants(filter, locale) - if err != nil { - exception.New("Failed to find assistant: %s", 500, err.Error()).Throw() - } - - if len(res.Data) == 0 { - exception.New("Assistant not found: %s", 404, assistantID).Throw() - } - - return res.Data[0] -} diff --git a/agent/process_test.go b/agent/process_test.go deleted file mode 100644 index fec24c80..00000000 --- a/agent/process_test.go +++ /dev/null @@ -1,513 +0,0 @@ -package agent - -// import ( -// "fmt" -// "testing" - -// "github.com/stretchr/testify/assert" -// "github.com/yaoapp/gou/process" -// "github.com/yaoapp/kun/any" -// "github.com/yaoapp/yao/config" -// "github.com/yaoapp/yao/test" -// ) - -// func prepare(t *testing.T) { -// test.Prepare(t, config.Conf) -// err := Load(config.Conf) -// if err != nil { -// t.Fatal(err) -// } - -// // Clean up the test data before each test -// p, err := process.Of("agent.assistant.search", map[string]interface{}{ -// "page": 1, -// "pagesize": 1000, // Use a large page size to get all records -// }) -// if err != nil { -// t.Fatal(err) -// } -// output, err := p.Exec() -// if err != nil { -// t.Fatal(err) -// } -// res := any.Of(output).Map() -// items := res.Get("data") -// if items != nil { -// for _, item := range items.([]map[string]interface{}) { -// assistantID := item["assistant_id"].(string) -// p, err = process.Of("agent.assistant.delete", assistantID) -// if err != nil { -// t.Fatal(err) -// } -// _, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } -// } -// } - -// // Verify cleanup -// p, err = process.Of("agent.assistant.search") -// if err != nil { -// t.Fatal(err) -// } -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } -// res = any.Of(output).Map() -// total := res.Get("total") -// if total != nil && any.Of(total).CInt() > 0 { -// t.Fatalf("Failed to clean up test data, %d records remaining", any.Of(total).CInt()) -// } - -// check(t) -// } - -// func TestProcessAssistantCRUD(t *testing.T) { -// prepare(t) -// defer test.Clean() - -// // Create an assistant with string JSON fields -// tagsJSON := `["tag1", "tag2", "tag3"]` -// optionsJSON := `{"model": "gpt-4"}` -// assistant := map[string]interface{}{ -// "name": "Test Assistant", -// "type": "assistant", -// "avatar": "https://example.com/avatar.png", -// "connector": "openai", -// "description": "Test Description", -// "tags": tagsJSON, -// "options": optionsJSON, -// "mentionable": true, -// "automated": true, -// } - -// // Test processAssistantCreate with string JSON -// p, err := process.Of("agent.assistant.create", assistant) -// if err != nil { -// t.Fatal(err) -// } - -// output, err := p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// assistantID := output -// assert.NotNil(t, assistantID) - -// // Test processAssistantFind -// p, err = process.Of("agent.assistant.find", assistantID) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// foundAssistant := output.(map[string]interface{}) -// assert.Equal(t, assistantID, foundAssistant["assistant_id"]) -// assert.Equal(t, "Test Assistant", foundAssistant["name"]) -// assert.Equal(t, []interface{}{"tag1", "tag2", "tag3"}, foundAssistant["tags"]) -// assert.Equal(t, map[string]interface{}{"model": "gpt-4"}, foundAssistant["options"]) - -// // Test processAssistantFind with non-existent ID -// p, err = process.Of("agent.assistant.find", "non-existent-id") -// if err != nil { -// t.Fatal(err) -// } - -// _, err = p.Exec() -// assert.NotNil(t, err) -// assert.Contains(t, err.Error(), "Assistant not found") - -// // Test with native type JSON fields -// assistant2 := map[string]interface{}{ -// "name": "Test Assistant 2", -// "type": "assistant", -// "avatar": "https://example.com/avatar2.png", -// "connector": "openai", -// "description": "Test Description 2", -// "tags": []string{"tag1", "tag2", "tag3"}, -// "options": map[string]interface{}{"model": "gpt-4"}, -// "prompts": []string{"prompt1", "prompt2"}, -// "flows": []string{"flow1", "flow2"}, -// "files": []string{"file1", "file2"}, -// "functions": []map[string]interface{}{{"name": "func1"}, {"name": "func2"}}, -// "permissions": map[string]interface{}{"read": true, "write": true}, -// "mentionable": true, -// "automated": true, -// } - -// // Test processAssistantCreate with native types -// p, err = process.Of("agent.assistant.create", assistant2) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// assistant2ID := output -// assert.NotNil(t, assistant2ID) - -// // Test with nil JSON fields -// assistant3 := map[string]interface{}{ -// "name": "Test Assistant 3", -// "type": "assistant", -// "connector": "openai", -// "description": "Test Description 3", -// "tags": nil, -// "options": nil, -// "prompts": nil, -// "flows": nil, -// "files": nil, -// "functions": nil, -// "permissions": nil, -// "mentionable": true, -// "automated": true, -// } - -// // Test processAssistantCreate with nil fields -// p, err = process.Of("agent.assistant.create", assistant3) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// assistant3ID := output -// assert.NotNil(t, assistant3ID) - -// // Test processAssistantSearch to verify all assistants -// p, err = process.Of("agent.assistant.search") -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// searchRes := any.Of(output).Map() -// total := searchRes.Get("total") -// if total == nil { -// total = int64(0) -// } -// assert.Equal(t, int64(3), total) - -// items := searchRes.Get("data") -// if items == nil { -// items = []map[string]interface{}{} -// } -// assert.Equal(t, 3, len(items.([]map[string]interface{}))) - -// // Verify each assistant's JSON fields -// for _, item := range items.([]map[string]interface{}) { -// switch item["assistant_id"].(string) { -// case assistantID: -// assert.Equal(t, []interface{}{"tag1", "tag2", "tag3"}, item["tags"]) -// assert.Equal(t, map[string]interface{}{"model": "gpt-4"}, item["options"]) -// case assistant2ID: -// assert.Equal(t, []interface{}{"tag1", "tag2", "tag3"}, item["tags"]) -// assert.Equal(t, map[string]interface{}{"model": "gpt-4"}, item["options"]) -// assert.Equal(t, []interface{}{"prompt1", "prompt2"}, item["prompts"]) -// assert.Equal(t, []interface{}{"flow1", "flow2"}, item["flows"]) -// assert.Equal(t, []interface{}{"file1", "file2"}, item["files"]) -// assert.Equal(t, -// []interface{}{ -// map[string]interface{}{"name": "func1"}, -// map[string]interface{}{"name": "func2"}, -// }, -// item["functions"]) -// assert.Equal(t, -// map[string]interface{}{ -// "read": true, -// "write": true, -// }, -// item["permissions"]) -// case assistant3ID: -// assert.Nil(t, item["tags"]) -// assert.Nil(t, item["options"]) -// assert.Nil(t, item["prompts"]) -// assert.Nil(t, item["flows"]) -// assert.Nil(t, item["files"]) -// assert.Nil(t, item["functions"]) -// assert.Nil(t, item["permissions"]) -// } -// } - -// // Test updating with mixed JSON formats -// assistant2["assistant_id"] = assistant2ID -// assistant2["tags"] = `["tag4", "tag5"]` -// assistant2["options"] = map[string]interface{}{"model": "gpt-3.5"} -// p, err = process.Of("agent.assistant.save", assistant2) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// savedID := output -// assert.NotNil(t, savedID) - -// // Double check with a new search -// p, err = process.Of("agent.assistant.search") -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// searchRes = any.Of(output).Map() -// items = searchRes.Get("data") -// found := false -// for _, item := range items.([]map[string]interface{}) { -// if item["assistant_id"].(string) == assistant2ID { -// found = true -// assert.Equal(t, []interface{}{"tag4", "tag5"}, item["tags"]) -// assert.Equal(t, map[string]interface{}{"model": "gpt-3.5"}, item["options"]) -// break -// } -// } -// assert.True(t, found) - -// // Test processAssistantDelete -// p, err = process.Of("agent.assistant.delete", assistantID) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// deleteRes := any.Of(output).Map() -// assert.Equal(t, "ok", deleteRes.Get("message")) - -// // Delete remaining assistants -// p, err = process.Of("agent.assistant.delete", assistant2ID) -// if err != nil { -// t.Fatal(err) -// } -// _, err = p.Exec() -// assert.Nil(t, err) - -// p, err = process.Of("agent.assistant.delete", assistant3ID) -// if err != nil { -// t.Fatal(err) -// } -// _, err = p.Exec() -// assert.Nil(t, err) - -// // Verify all assistants are deleted -// p, err = process.Of("agent.assistant.search") -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// searchRes = any.Of(output).Map() -// total = searchRes.Get("total") -// if total == nil { -// total = int64(0) -// } -// assert.Equal(t, int64(0), total) -// } - -// func TestProcessAssistantSearchPagination(t *testing.T) { -// prepare(t) -// defer test.Clean() - -// // Create multiple assistants for pagination testing -// for i := 0; i < 25; i++ { -// assistant := map[string]interface{}{ -// "name": fmt.Sprintf("Assistant %d", i), -// "type": "assistant", -// "connector": fmt.Sprintf("connector%d", i%3), -// "description": fmt.Sprintf("Description %d", i), -// "tags": []string{fmt.Sprintf("tag%d", i%5)}, -// "mentionable": i%2 == 0, -// "automated": i%3 == 0, -// } - -// p, err := process.Of("agent.assistant.create", assistant) -// if err != nil { -// t.Fatal(err) -// } - -// _, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } -// } - -// // Test first page -// p, err := process.Of("agent.assistant.search", map[string]interface{}{ -// "page": 1, -// "pagesize": 10, -// }) -// if err != nil { -// t.Fatal(err) -// } - -// output, err := p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// res := any.Of(output).Map() -// total := res.Get("total") -// if total == nil { -// total = int64(0) -// } -// assert.Equal(t, int64(25), total) - -// items := res.Get("data") -// if items == nil { -// items = []map[string]interface{}{} -// } -// assert.Equal(t, 10, len(items.([]map[string]interface{}))) - -// pageCnt := res.Get("pagecnt") -// if pageCnt == nil { -// pageCnt = 1 -// } -// assert.Equal(t, 3, pageCnt) - -// // Test second page -// p, err = process.Of("agent.assistant.search", map[string]interface{}{ -// "page": 2, -// "pagesize": 10, -// }) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// res = any.Of(output).Map() -// items = res.Get("data") -// if items == nil { -// items = []map[string]interface{}{} -// } -// assert.Equal(t, 10, len(items.([]map[string]interface{}))) - -// // Test last page -// p, err = process.Of("agent.assistant.search", map[string]interface{}{ -// "page": 3, -// "pagesize": 10, -// }) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// res = any.Of(output).Map() -// items = res.Get("data") -// if items == nil { -// items = []map[string]interface{}{} -// } -// assert.Equal(t, 5, len(items.([]map[string]interface{}))) - -// // Test filtering with tags -// p, err = process.Of("agent.assistant.search", map[string]interface{}{ -// "tags": []string{"tag0"}, -// "page": 1, -// "pagesize": 10, -// }) -// if err != nil { -// t.Fatal(err) -// } - -// output, err = p.Exec() -// if err != nil { -// t.Fatal(err) -// } - -// res = any.Of(output).Map() -// items = res.Get("data") -// if items == nil { -// items = []map[string]interface{}{} -// } -// assert.Equal(t, 5, len(items.([]map[string]interface{}))) -// } - -// func TestProcessAssistantValidation(t *testing.T) { -// prepare(t) -// defer test.Clean() - -// // Test missing required fields -// p, err := process.Of("agent.assistant.create", map[string]interface{}{}) -// if err != nil { -// t.Fatal(err) -// } - -// _, err = p.Exec() -// assert.NotNil(t, err) - -// // Test invalid assistant ID for delete -// p, err = process.Of("agent.assistant.delete", "non-existent-id") -// if err != nil { -// t.Fatal(err) -// } - -// _, err = p.Exec() -// assert.NotNil(t, err) - -// // Test invalid assistant ID for find -// p, err = process.Of("agent.assistant.find", "non-existent-id") -// if err != nil { -// t.Fatal(err) -// } - -// _, err = p.Exec() -// assert.NotNil(t, err) -// assert.Contains(t, err.Error(), "Assistant not found") - -// // Test invalid page number -// p, err = process.Of("agent.assistant.search", map[string]interface{}{ -// "page": -1, -// "pagesize": 10, -// }) -// if err != nil { -// t.Fatal(err) -// } - -// output, err := p.Exec() -// assert.Nil(t, err) - -// res := any.Of(output).Map() -// total := res.Get("total") -// if total == nil { -// total = int64(0) -// } -// assert.Equal(t, int64(0), total) -// } diff --git a/agent/types/types.go b/agent/types/types.go index 85e7ac42..ab53abea 100644 --- a/agent/types/types.go +++ b/agent/types/types.go @@ -4,7 +4,6 @@ import ( "github.com/gin-gonic/gin" "github.com/yaoapp/yao/agent/assistant" store "github.com/yaoapp/yao/agent/store/types" - "github.com/yaoapp/yao/agent/vision" ) // DSL AI assistant @@ -32,9 +31,9 @@ type DSL struct { // Internal // =============================== // ID string `json:"-" yaml:"-"` // The id of the instance - Assistant assistant.API `json:"-" yaml:"-"` // The default assistant - Store store.Store `json:"-" yaml:"-"` // The store of the assistant - Vision *vision.Vision `json:"-" yaml:"-"` + Assistant assistant.API `json:"-" yaml:"-"` // The default assistant + Store store.Store `json:"-" yaml:"-"` // The store of the assistant + // Vision *vision.Vision `json:"-" yaml:"-"` GuardHandlers []gin.HandlerFunc `json:"-" yaml:"-"` } diff --git a/agent/vision/driver/local/storage.go b/agent/vision/driver/local/storage.go deleted file mode 100644 index 81de81a3..00000000 --- a/agent/vision/driver/local/storage.go +++ /dev/null @@ -1,208 +0,0 @@ -package local - -import ( - "bytes" - "context" - "crypto/sha256" - "fmt" - "image" - "image/jpeg" - "image/png" - "io" - "path/filepath" - "strings" - "time" - - "github.com/yaoapp/gou/fs" -) - -// MaxImageSize maximum image size (1920x1080) -const MaxImageSize = 1920 - -// Storage the local storage driver -type Storage struct { - Path string `json:"path" yaml:"path"` - Compression bool `json:"compression" yaml:"compression"` - BaseURL string `json:"base_url" yaml:"base_url"` - PreviewURL func(fileID string) string `json:"-" yaml:"-"` -} - -// New create a new local storage -func New(options map[string]interface{}) (*Storage, error) { - storage := &Storage{ - Compression: true, - } - - if path, ok := options["path"].(string); ok { - storage.Path = path - } - - if compression, ok := options["compression"].(bool); ok { - storage.Compression = compression - } - - if baseURL, ok := options["base_url"].(string); ok { - storage.BaseURL = baseURL - } - - if previewURL, ok := options["preview_url"].(func(string) string); ok { - storage.PreviewURL = previewURL - } - - if storage.Path == "" { - return nil, fmt.Errorf("path is required") - } - - return storage, nil -} - -// Upload upload file to local storage -func (storage *Storage) Upload(ctx context.Context, filename string, reader io.Reader, contentType string) (string, error) { - data, err := fs.Get("data") - if err != nil { - return "", err - } - - ext := filepath.Ext(filename) - id := storage.makeID(filename, ext) - path := filepath.Join(storage.Path, id) - - // Create directory if not exists - dir := filepath.Dir(path) - if err := data.MkdirAll(dir, 0755); err != nil { - return "", err - } - - // Check if compression is enabled and if it's an image - if storage.Compression && isImage(contentType) { - // Read the entire image into memory - content, err := io.ReadAll(reader) - if err != nil { - return "", fmt.Errorf("failed to read image: %w", err) - } - - // Compress image - compressed, err := compressImage(content, contentType) - if err != nil { - return "", fmt.Errorf("failed to compress image: %w", err) - } - - // Write compressed image - _, err = data.Write(path, bytes.NewReader(compressed), 0644) - if err != nil { - return "", err - } - } else { - // Write file without compression - _, err = data.Write(path, reader, 0644) - if err != nil { - return "", err - } - } - - return id, nil -} - -// Download download file from local storage -func (storage *Storage) Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) { - data, err := fs.Get("data") - if err != nil { - return nil, "", err - } - - path := filepath.Join(storage.Path, fileID) - reader, err := data.ReadCloser(path) - if err != nil { - return nil, "", err - } - - contentType := "application/octet-stream" - if v, err := data.MimeType(path); err == nil { - contentType = v - } - - return reader, contentType, nil -} - -// URL get file url -func (storage *Storage) URL(ctx context.Context, fileID string) string { - if storage.PreviewURL != nil { - return storage.PreviewURL(fileID) - } - if storage.BaseURL != "" { - return fmt.Sprintf("%s/%s", strings.TrimRight(storage.BaseURL, "/"), fileID) - } - return fmt.Sprintf("%s/%s", storage.Path, fileID) -} - -func (storage *Storage) makeID(filename string, ext string) string { - date := time.Now().Format("20060102") - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(filename)))[:8] - name := strings.TrimSuffix(filepath.Base(filename), ext) - return fmt.Sprintf("%s/%s-%s%s", date, name, hash, ext) -} - -// isImage checks if the content type is an image -func isImage(contentType string) bool { - return strings.HasPrefix(contentType, "image/") -} - -// compressImage compresses the image while maintaining aspect ratio -func compressImage(data []byte, contentType string) ([]byte, error) { - // Decode image - img, _, err := image.Decode(bytes.NewReader(data)) - if err != nil { - return nil, fmt.Errorf("failed to decode image: %w", err) - } - - // Calculate new dimensions - bounds := img.Bounds() - width := bounds.Dx() - height := bounds.Dy() - var newWidth, newHeight int - - if width > height { - if width > MaxImageSize { - newWidth = MaxImageSize - newHeight = int(float64(height) * (float64(MaxImageSize) / float64(width))) - } else { - return data, nil // No need to resize - } - } else { - if height > MaxImageSize { - newHeight = MaxImageSize - newWidth = int(float64(width) * (float64(MaxImageSize) / float64(height))) - } else { - return data, nil // No need to resize - } - } - - // Create new image with new dimensions - newImg := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) - - // Scale the image using bilinear interpolation - for y := 0; y < newHeight; y++ { - for x := 0; x < newWidth; x++ { - srcX := float64(x) * float64(width) / float64(newWidth) - srcY := float64(y) * float64(height) / float64(newHeight) - newImg.Set(x, y, img.At(int(srcX), int(srcY))) - } - } - - // Encode image - var buf bytes.Buffer - switch contentType { - case "image/jpeg": - err = jpeg.Encode(&buf, newImg, &jpeg.Options{Quality: 85}) - case "image/png": - err = png.Encode(&buf, newImg) - default: - return data, nil // Unsupported format, return original - } - - if err != nil { - return nil, fmt.Errorf("failed to encode image: %w", err) - } - - return buf.Bytes(), nil -} diff --git a/agent/vision/driver/local/storage_test.go b/agent/vision/driver/local/storage_test.go deleted file mode 100644 index 36a53eee..00000000 --- a/agent/vision/driver/local/storage_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package local - -import ( - "bytes" - "context" - "image" - "image/png" - "io" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/test" -) - -func TestLocalStorage(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - t.Run("Create Storage", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "path": "/__vision_test", - "compression": true, - }) - assert.NoError(t, err) - assert.NotNil(t, storage) - assert.Equal(t, "/__vision_test", storage.Path) - assert.True(t, storage.Compression) - }) - - t.Run("Upload and Download", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "path": "/__vision_test", - "compression": true, - }) - assert.NoError(t, err) - - content := []byte("test content") - reader := bytes.NewReader(content) - fileID, err := storage.Upload(context.Background(), "test.txt", reader, "text/plain") - assert.NoError(t, err) - assert.NotEmpty(t, fileID) - - // Download - reader2, contentType, err := storage.Download(context.Background(), fileID) - assert.NoError(t, err) - assert.Contains(t, contentType, "text/plain") - - downloaded, err := io.ReadAll(reader2) - assert.NoError(t, err) - assert.Equal(t, content, downloaded) - }) - - t.Run("Upload and Download Image with Compression", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "path": "/__vision_test", - "compression": true, - }) - assert.NoError(t, err) - - // Create a test image (2000x2000 pixels) - img := image.NewRGBA(image.Rect(0, 0, 2000, 2000)) - var buf bytes.Buffer - err = png.Encode(&buf, img) - assert.NoError(t, err) - - // Upload - reader := bytes.NewReader(buf.Bytes()) - fileID, err := storage.Upload(context.Background(), "test.png", reader, "image/png") - assert.NoError(t, err) - assert.NotEmpty(t, fileID) - - // Download and verify size - reader2, contentType, err := storage.Download(context.Background(), fileID) - assert.NoError(t, err) - assert.Equal(t, "image/png", contentType) - - downloaded, err := io.ReadAll(reader2) - assert.NoError(t, err) - - // Decode the downloaded image - downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) - assert.NoError(t, err) - - // Verify dimensions - bounds := downloadedImg.Bounds() - assert.LessOrEqual(t, bounds.Dx(), MaxImageSize) - assert.LessOrEqual(t, bounds.Dy(), MaxImageSize) - }) - - t.Run("Upload Image without Compression", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "path": "/__vision_test", - "compression": false, - }) - assert.NoError(t, err) - - // Create a test image (2000x2000 pixels) - img := image.NewRGBA(image.Rect(0, 0, 2000, 2000)) - var buf bytes.Buffer - err = png.Encode(&buf, img) - assert.NoError(t, err) - - // Upload - reader := bytes.NewReader(buf.Bytes()) - fileID, err := storage.Upload(context.Background(), "test.png", reader, "image/png") - assert.NoError(t, err) - assert.NotEmpty(t, fileID) - - // Download and verify size - reader2, contentType, err := storage.Download(context.Background(), fileID) - assert.NoError(t, err) - assert.Equal(t, "image/png", contentType) - - downloaded, err := io.ReadAll(reader2) - assert.NoError(t, err) - - // Decode the downloaded image - downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) - assert.NoError(t, err) - - // Verify dimensions are unchanged - bounds := downloadedImg.Bounds() - assert.Equal(t, 2000, bounds.Dx()) - assert.Equal(t, 2000, bounds.Dy()) - }) - - t.Run("URL Generation", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "path": "/__vision_test", - "compression": true, - }) - assert.NoError(t, err) - - fileID := "20240101/test-12345678.txt" - url := storage.URL(context.Background(), fileID) - assert.Equal(t, "/__vision_test/20240101/test-12345678.txt", url) - }) - - t.Run("Download Non-existent File", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "path": "/__vision_test", - "compression": true, - }) - assert.NoError(t, err) - - _, _, err = storage.Download(context.Background(), "non-existent.txt") - assert.Error(t, err) - }) -} diff --git a/agent/vision/driver/openai/model.go b/agent/vision/driver/openai/model.go deleted file mode 100644 index 8a55abd1..00000000 --- a/agent/vision/driver/openai/model.go +++ /dev/null @@ -1,190 +0,0 @@ -package openai - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - - "github.com/yaoapp/gou/fs" -) - -// Model the OpenAI vision model -type Model struct { - APIKey string `json:"api_key" yaml:"api_key"` - Model string `json:"model" yaml:"model"` - Compression bool `json:"compression" yaml:"compression"` - Prompt string `json:"prompt" yaml:"prompt"` -} - -// New create a new OpenAI vision model -func New(options map[string]interface{}) (*Model, error) { - model := &Model{ - Model: "gpt-4-vision-preview", - Compression: true, - } - - if apiKey, ok := options["api_key"].(string); ok { - model.APIKey = apiKey - } - - if modelName, ok := options["model"].(string); ok { - model.Model = modelName - } - - if compression, ok := options["compression"].(bool); ok { - model.Compression = compression - } - - if prompt, ok := options["prompt"].(string); ok { - model.Prompt = prompt - } - - if model.APIKey == "" { - return nil, fmt.Errorf("api_key is required") - } - - return model, nil -} - -// Analyze analyze image using OpenAI vision model -func (model *Model) Analyze(ctx context.Context, fileID string, prompt ...string) (map[string]interface{}, error) { - if model.APIKey == "" { - return nil, fmt.Errorf("api_key is required") - } - - // Use default prompt if none provided - userPrompt := model.Prompt - if len(prompt) > 0 && prompt[0] != "" { - userPrompt = prompt[0] - } - - // Check if fileID is a URL or base64 data - var imageURL string - if strings.HasPrefix(fileID, "data:image/") { - // Already a base64 data URL - imageURL = fileID - } else if strings.HasPrefix(fileID, "http://") || strings.HasPrefix(fileID, "https://") { - // Already a URL - imageURL = fileID - } else { - // Try to read the file and convert to base64 - data, err := fs.Get("data") - if err != nil { - return nil, fmt.Errorf("failed to get data fs: %w", err) - } - - reader, err := data.ReadCloser(fileID) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } - defer reader.Close() - - content, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to read content: %w", err) - } - - // Get content type - contentType := "image/png" // default - if v, err := data.MimeType(fileID); err == nil { - contentType = v - } - - // Convert to base64 - base64Data := base64.StdEncoding.EncodeToString(content) - imageURL = fmt.Sprintf("data:%s;base64,%s", contentType, base64Data) - } - - // Prepare the request body - reqBody := map[string]interface{}{ - "model": model.Model, - "messages": []map[string]interface{}{ - { - "role": "user", - "content": []map[string]interface{}{ - { - "type": "text", - "text": userPrompt, - }, - { - "type": "image_url", - "image_url": map[string]interface{}{ - "url": imageURL, - }, - }, - }, - }, - }, - "max_tokens": 1000, - } - - jsonBody, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal request body: %w", err) - } - - // Create request - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.openai.com/v1/chat/completions", bytes.NewBuffer(jsonBody)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", model.APIKey)) - - // Send request - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - // Read response - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("OpenAI API error: %s", string(body)) - } - - // Parse response - var result map[string]interface{} - if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - - // Extract content - choices, ok := result["choices"].([]interface{}) - if !ok || len(choices) == 0 { - return nil, fmt.Errorf("invalid response format") - } - - message, ok := choices[0].(map[string]interface{})["message"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("invalid response format") - } - - content, ok := message["content"].(string) - if !ok { - return nil, fmt.Errorf("invalid response format") - } - - // Try to parse content as JSON - var description map[string]interface{} - if err := json.Unmarshal([]byte(content), &description); err != nil { - // If not JSON, use the content as description - description = map[string]interface{}{ - "description": content, - } - } - - return description, nil -} diff --git a/agent/vision/driver/openai/model_test.go b/agent/vision/driver/openai/model_test.go deleted file mode 100644 index dcf68af8..00000000 --- a/agent/vision/driver/openai/model_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package openai - -import ( - "bytes" - "context" - "encoding/base64" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/yaoapp/gou/fs" - "github.com/yaoapp/yao/agent/vision/driver/s3" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/test" -) - -var ( - // 1x1 transparent PNG - testImageBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" -) - -func TestOpenAIModel(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - t.Run("Create Model", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - }) - assert.NoError(t, err) - assert.NotNil(t, model) - if model != nil { - assert.Equal(t, os.Getenv("OPENAI_API_KEY"), model.APIKey) - assert.Equal(t, os.Getenv("VISION_MODEL"), model.Model) - assert.True(t, model.Compression) - } - }) - - t.Run("Create Model with Invalid API Key", func(t *testing.T) { - _, err := New(map[string]interface{}{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "api_key is required") - }) - - t.Run("Analyze with Base64 Image", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - }) - assert.NoError(t, err) - - // Use base64 image data - result, err := model.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "Describe this image in detail") - assert.NoError(t, err) - assert.NotNil(t, result) - assert.NotEmpty(t, result["description"]) - }) - - t.Run("Analyze with URL", func(t *testing.T) { - if os.Getenv("S3_API") == "" || os.Getenv("S3_ACCESS_KEY") == "" || - os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" { - t.Skip("S3 environment variables not set") - } - - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - }) - assert.NoError(t, err) - - // Create S3 client and upload test image - s3Client, err := s3.New(map[string]interface{}{ - "endpoint": os.Getenv("S3_API"), - "region": "auto", - "key": os.Getenv("S3_ACCESS_KEY"), - "secret": os.Getenv("S3_SECRET_KEY"), - "bucket": os.Getenv("S3_BUCKET"), - "prefix": "vision-test", - "expiration": "5m", - }) - assert.NoError(t, err) - - // Upload test image - imgData, err := base64.StdEncoding.DecodeString(testImageBase64) - assert.NoError(t, err) - reader := bytes.NewReader(imgData) - fileID, err := s3Client.Upload(context.Background(), "test.png", reader, "image/png") - assert.NoError(t, err) - - // Get URL from S3 - url := s3Client.URL(context.Background(), fileID) - assert.NotEmpty(t, url) - - // Use S3 URL for analysis - result, err := model.Analyze(context.Background(), url, "Describe this image in detail") - assert.NoError(t, err) - assert.NotNil(t, result) - assert.NotEmpty(t, result["description"]) - }) - - t.Run("Analyze with File ID", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - }) - assert.NoError(t, err) - - // Create test file - data, err := fs.Get("data") - assert.NoError(t, err) - - // Write test image data - imgData, err := base64.StdEncoding.DecodeString(testImageBase64) - assert.NoError(t, err) - _, err = data.WriteFile("/__vision_test/test.png", imgData, 0644) - assert.NoError(t, err) - - // Analyze using file ID - result, err := model.Analyze(context.Background(), "/__vision_test/test.png", "Describe this image in detail") - assert.NoError(t, err) - assert.NotNil(t, result) - assert.NotEmpty(t, result["description"]) - }) - - t.Run("Analyze with Invalid File ID", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - }) - assert.NoError(t, err) - - _, err = model.Analyze(context.Background(), "/non-existent.png", "Describe this image in detail") - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to read file") - }) - - t.Run("Analyze with Invalid API Key", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": "invalid-key", - "model": os.Getenv("VISION_MODEL"), - }) - assert.NoError(t, err) - - _, err = model.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "Describe this image in detail") - assert.Error(t, err) - assert.Contains(t, err.Error(), "OpenAI API error") - }) - - t.Run("Analyze with Default Prompt", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - "prompt": "Default test prompt", - }) - assert.NoError(t, err) - - // Use base64 image data without providing a prompt - result, err := model.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.NotEmpty(t, result["description"]) - }) - - t.Run("Analyze with Custom Prompt Overriding Default", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - "prompt": "Default test prompt", - }) - assert.NoError(t, err) - - // Use base64 image data with custom prompt - result, err := model.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "Custom test prompt") - assert.NoError(t, err) - assert.NotNil(t, result) - assert.NotEmpty(t, result["description"]) - }) - - t.Run("Analyze with Empty Custom Prompt", func(t *testing.T) { - model, err := New(map[string]interface{}{ - "api_key": os.Getenv("OPENAI_API_KEY"), - "model": os.Getenv("VISION_MODEL"), - "prompt": "Default test prompt", - }) - assert.NoError(t, err) - - // Use base64 image data with empty prompt (should use default) - result, err := model.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "") - assert.NoError(t, err) - assert.NotNil(t, result) - assert.NotEmpty(t, result["description"]) - }) -} diff --git a/agent/vision/driver/s3/storage.go b/agent/vision/driver/s3/storage.go deleted file mode 100644 index 3f55b7be..00000000 --- a/agent/vision/driver/s3/storage.go +++ /dev/null @@ -1,266 +0,0 @@ -package s3 - -import ( - "bytes" - "context" - "fmt" - "image" - "image/jpeg" - "image/png" - "io" - "path/filepath" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" -) - -// DefaultExpiration default expiration time for presigned URLs (5 minutes) -const DefaultExpiration = 5 * time.Minute - -// MaxImageSize maximum image size (1920x1080) -const MaxImageSize = 1920 - -// Storage the S3 storage driver -type Storage struct { - Endpoint string `json:"endpoint" yaml:"endpoint"` - Region string `json:"region" yaml:"region"` - Key string `json:"key" yaml:"key"` - Secret string `json:"secret" yaml:"secret"` - Bucket string `json:"bucket" yaml:"bucket"` - Expiration time.Duration `json:"expiration" yaml:"expiration"` - client *s3.Client - prefix string - compression bool -} - -// New create a new S3 storage -func New(options map[string]interface{}) (*Storage, error) { - storage := &Storage{ - Region: "auto", - Expiration: DefaultExpiration, - compression: true, - } - - if endpoint, ok := options["endpoint"].(string); ok { - storage.Endpoint = endpoint - } - - if region, ok := options["region"].(string); ok { - storage.Region = region - } - - if key, ok := options["key"].(string); ok { - storage.Key = key - } - - if secret, ok := options["secret"].(string); ok { - storage.Secret = secret - } - - if bucket, ok := options["bucket"].(string); ok { - storage.Bucket = bucket - } - - if prefix, ok := options["prefix"].(string); ok { - storage.prefix = prefix - } - - if exp, ok := options["expiration"].(time.Duration); ok { - storage.Expiration = exp - } - - if compression, ok := options["compression"].(bool); ok { - storage.compression = compression - } - - // Validate required fields - if storage.Key == "" || storage.Secret == "" { - return nil, fmt.Errorf("key and secret are required") - } - - if storage.Bucket == "" { - return nil, fmt.Errorf("bucket is required") - } - - // Create S3 client - opts := s3.Options{ - Region: storage.Region, - Credentials: credentials.NewStaticCredentialsProvider(storage.Key, storage.Secret, ""), - UsePathStyle: true, - } - - if storage.Endpoint != "" { - // Remove bucket name from endpoint if present - endpoint := storage.Endpoint - if strings.Contains(endpoint, "/"+storage.Bucket) { - endpoint = strings.TrimSuffix(endpoint, "/"+storage.Bucket) - } - opts.BaseEndpoint = aws.String(endpoint) - } - - storage.client = s3.New(opts) - return storage, nil -} - -// Upload upload file to S3 -func (storage *Storage) Upload(ctx context.Context, filename string, reader io.Reader, contentType string) (string, error) { - if storage.client == nil { - return "", fmt.Errorf("s3 client not initialized") - } - - // Generate file ID - fileID := storage.makeID(filename, filepath.Ext(filename)) - key := filepath.Join(storage.prefix, fileID) - - // Check if compression is enabled and if it's an image - var body io.Reader - if storage.compression && isImage(contentType) { - // Read the entire image into memory - content, err := io.ReadAll(reader) - if err != nil { - return "", fmt.Errorf("failed to read image: %w", err) - } - - // Compress image - compressed, err := compressImage(content, contentType) - if err != nil { - return "", fmt.Errorf("failed to compress image: %w", err) - } - - body = bytes.NewReader(compressed) - } else { - body = reader - } - - // Upload file - _, err := storage.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(storage.Bucket), - Key: aws.String(key), - Body: body, - ContentType: aws.String(contentType), - }) - if err != nil { - return "", fmt.Errorf("failed to upload file: %w", err) - } - - return fileID, nil -} - -// Download download file from S3 -func (storage *Storage) Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) { - if storage.client == nil { - return nil, "", fmt.Errorf("s3 client not initialized") - } - - key := filepath.Join(storage.prefix, fileID) - - // Get object - result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(storage.Bucket), - Key: aws.String(key), - }) - if err != nil { - return nil, "", fmt.Errorf("failed to download file: %w", err) - } - - contentType := "application/octet-stream" - if result.ContentType != nil { - contentType = *result.ContentType - } - - return result.Body, contentType, nil -} - -// URL get file url with expiration -func (storage *Storage) URL(ctx context.Context, fileID string) string { - if storage.client == nil { - return "" - } - - key := filepath.Join(storage.prefix, fileID) - presignClient := s3.NewPresignClient(storage.client) - request, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(storage.Bucket), - Key: aws.String(key), - }, s3.WithPresignExpires(storage.Expiration)) - - if err != nil { - return "" - } - - return request.URL -} - -func (storage *Storage) makeID(filename string, ext string) string { - date := time.Now().Format("20060102") - name := strings.TrimSuffix(filepath.Base(filename), ext) - return fmt.Sprintf("%s/%s-%d%s", date, name, time.Now().UnixNano(), ext) -} - -// isImage checks if the content type is an image -func isImage(contentType string) bool { - return strings.HasPrefix(contentType, "image/") -} - -// compressImage compresses the image while maintaining aspect ratio -func compressImage(data []byte, contentType string) ([]byte, error) { - // Decode image - img, _, err := image.Decode(bytes.NewReader(data)) - if err != nil { - return nil, fmt.Errorf("failed to decode image: %w", err) - } - - // Calculate new dimensions - bounds := img.Bounds() - width := bounds.Dx() - height := bounds.Dy() - var newWidth, newHeight int - - if width > height { - if width > MaxImageSize { - newWidth = MaxImageSize - newHeight = int(float64(height) * (float64(MaxImageSize) / float64(width))) - } else { - return data, nil // No need to resize - } - } else { - if height > MaxImageSize { - newHeight = MaxImageSize - newWidth = int(float64(width) * (float64(MaxImageSize) / float64(height))) - } else { - return data, nil // No need to resize - } - } - - // Create new image with new dimensions - newImg := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) - - // Scale the image using bilinear interpolation - for y := 0; y < newHeight; y++ { - for x := 0; x < newWidth; x++ { - srcX := float64(x) * float64(width) / float64(newWidth) - srcY := float64(y) * float64(height) / float64(newHeight) - newImg.Set(x, y, img.At(int(srcX), int(srcY))) - } - } - - // Encode image - var buf bytes.Buffer - switch contentType { - case "image/jpeg": - err = jpeg.Encode(&buf, newImg, &jpeg.Options{Quality: 85}) - case "image/png": - err = png.Encode(&buf, newImg) - default: - return data, nil // Unsupported format, return original - } - - if err != nil { - return nil, fmt.Errorf("failed to encode image: %w", err) - } - - return buf.Bytes(), nil -} diff --git a/agent/vision/driver/s3/storage_test.go b/agent/vision/driver/s3/storage_test.go deleted file mode 100644 index ca0737ce..00000000 --- a/agent/vision/driver/s3/storage_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package s3 - -import ( - "bytes" - "context" - "image" - "image/png" - "io" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/test" -) - -func TestS3Storage(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - t.Run("Create Storage", func(t *testing.T) { - options := map[string]interface{}{ - "endpoint": os.Getenv("S3_API"), - "region": "auto", - "key": os.Getenv("S3_ACCESS_KEY"), - "secret": os.Getenv("S3_SECRET_KEY"), - "bucket": os.Getenv("S3_BUCKET"), - "prefix": "vision-test", - "expiration": 10 * time.Minute, - "compression": true, - } - - storage, err := New(options) - if err != nil { - t.Logf("Error creating storage: %v", err) - } - assert.NoError(t, err) - assert.NotNil(t, storage) - if storage != nil { - assert.Equal(t, os.Getenv("S3_API"), storage.Endpoint) - assert.Equal(t, "auto", storage.Region) - assert.Equal(t, os.Getenv("S3_ACCESS_KEY"), storage.Key) - assert.Equal(t, os.Getenv("S3_SECRET_KEY"), storage.Secret) - assert.Equal(t, os.Getenv("S3_BUCKET"), storage.Bucket) - assert.Equal(t, "vision-test", storage.prefix) - assert.Equal(t, 10*time.Minute, storage.Expiration) - assert.True(t, storage.compression) - } - }) - - t.Run("Upload and Download Image with Compression", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "endpoint": os.Getenv("S3_API"), - "region": "auto", - "key": os.Getenv("S3_ACCESS_KEY"), - "secret": os.Getenv("S3_SECRET_KEY"), - "bucket": os.Getenv("S3_BUCKET"), - "prefix": "vision-test", - "expiration": 5 * time.Minute, - "compression": true, - }) - if err != nil { - t.Skip("S3 configuration not available") - } - - // Create a test image (2000x2000 pixels) - img := image.NewRGBA(image.Rect(0, 0, 2000, 2000)) - var buf bytes.Buffer - err = png.Encode(&buf, img) - assert.NoError(t, err) - - // Upload - reader := bytes.NewReader(buf.Bytes()) - fileID, err := storage.Upload(context.Background(), "test.png", reader, "image/png") - assert.NoError(t, err) - assert.NotEmpty(t, fileID) - - // Download and verify size - reader2, contentType, err := storage.Download(context.Background(), fileID) - assert.NoError(t, err) - assert.Equal(t, "image/png", contentType) - - downloaded, err := io.ReadAll(reader2) - assert.NoError(t, err) - - // Decode the downloaded image - downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) - assert.NoError(t, err) - - // Verify dimensions - bounds := downloadedImg.Bounds() - assert.LessOrEqual(t, bounds.Dx(), MaxImageSize) - assert.LessOrEqual(t, bounds.Dy(), MaxImageSize) - }) - - t.Run("Upload Image without Compression", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "endpoint": os.Getenv("S3_API"), - "region": "auto", - "key": os.Getenv("S3_ACCESS_KEY"), - "secret": os.Getenv("S3_SECRET_KEY"), - "bucket": os.Getenv("S3_BUCKET"), - "prefix": "vision-test", - "expiration": 5 * time.Minute, - "compression": false, - }) - if err != nil { - t.Skip("S3 configuration not available") - } - - // Create a test image (2000x2000 pixels) - img := image.NewRGBA(image.Rect(0, 0, 2000, 2000)) - var buf bytes.Buffer - err = png.Encode(&buf, img) - assert.NoError(t, err) - - // Upload - reader := bytes.NewReader(buf.Bytes()) - fileID, err := storage.Upload(context.Background(), "test.png", reader, "image/png") - assert.NoError(t, err) - assert.NotEmpty(t, fileID) - - // Download and verify size - reader2, contentType, err := storage.Download(context.Background(), fileID) - assert.NoError(t, err) - assert.Equal(t, "image/png", contentType) - - downloaded, err := io.ReadAll(reader2) - assert.NoError(t, err) - - // Decode the downloaded image - downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) - assert.NoError(t, err) - - // Verify dimensions are unchanged - bounds := downloadedImg.Bounds() - assert.Equal(t, 2000, bounds.Dx()) - assert.Equal(t, 2000, bounds.Dy()) - }) - - t.Run("Upload and Download Text File", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "endpoint": os.Getenv("S3_API"), - "region": "auto", - "key": os.Getenv("S3_ACCESS_KEY"), - "secret": os.Getenv("S3_SECRET_KEY"), - "bucket": os.Getenv("S3_BUCKET"), - "prefix": "vision-test", - "expiration": 5 * time.Minute, - "compression": true, - }) - if err != nil { - t.Skip("S3 configuration not available") - } - - content := []byte("test content") - reader := bytes.NewReader(content) - fileID, err := storage.Upload(context.Background(), "test.txt", reader, "text/plain") - assert.NoError(t, err) - assert.NotEmpty(t, fileID) - - // Get presigned URL - url := storage.URL(context.Background(), fileID) - assert.NotEmpty(t, url) - assert.Contains(t, url, "X-Amz-Signature") - assert.Contains(t, url, "X-Amz-Expires") - - // Download - reader2, contentType, err := storage.Download(context.Background(), fileID) - if err != nil { - t.Logf("Download error: %v", err) - t.FailNow() - } - assert.NoError(t, err) - assert.Contains(t, contentType, "text/plain") - - if reader2 != nil { - downloaded, err := io.ReadAll(reader2) - assert.NoError(t, err) - assert.Equal(t, content, downloaded) - reader2.Close() - } - }) - - t.Run("Download Non-existent File", func(t *testing.T) { - storage, err := New(map[string]interface{}{ - "endpoint": os.Getenv("S3_API"), - "region": "auto", - "key": os.Getenv("S3_ACCESS_KEY"), - "secret": os.Getenv("S3_SECRET_KEY"), - "bucket": os.Getenv("S3_BUCKET"), - "prefix": "vision-test", - "expiration": 5 * time.Minute, - "compression": true, - }) - if err != nil { - t.Skip("S3 configuration not available") - } - - _, _, err = storage.Download(context.Background(), "non-existent.txt") - assert.Error(t, err) - }) -} diff --git a/agent/vision/driver/types.go b/agent/vision/driver/types.go deleted file mode 100644 index 9cc2006e..00000000 --- a/agent/vision/driver/types.go +++ /dev/null @@ -1,45 +0,0 @@ -package driver - -import ( - "context" - "io" -) - -// Config the vision configuration -type Config struct { - Storage StorageConfig `json:"storage" yaml:"storage"` - Model ModelConfig `json:"model" yaml:"model"` -} - -// StorageConfig the storage configuration -type StorageConfig struct { - Driver string `json:"driver" yaml:"driver"` - Options map[string]interface{} `json:"options" yaml:"options"` -} - -// ModelConfig the model configuration -type ModelConfig struct { - Driver string `json:"driver" yaml:"driver"` - Options map[string]interface{} `json:"options" yaml:"options"` -} - -// Storage the storage interface -type Storage interface { - Upload(ctx context.Context, filename string, reader io.Reader, contentType string) (string, error) - Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) - URL(ctx context.Context, fileID string) string -} - -// Model the vision model interface -type Model interface { - // Analyze analyzes an image file - // If prompt is empty, it will use the default prompt from model.options.prompt - Analyze(ctx context.Context, fileID string, prompt ...string) (map[string]interface{}, error) -} - -// Response the vision response -type Response struct { - FileID string `json:"file_id" yaml:"file_id"` - URL string `json:"url" yaml:"url"` - Description map[string]interface{} `json:"description" yaml:"description"` -} diff --git a/agent/vision/vision.go b/agent/vision/vision.go deleted file mode 100644 index 81e3ecf5..00000000 --- a/agent/vision/vision.go +++ /dev/null @@ -1,139 +0,0 @@ -package vision - -import ( - "context" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/yaoapp/yao/agent/vision/driver" - "github.com/yaoapp/yao/agent/vision/driver/local" - "github.com/yaoapp/yao/agent/vision/driver/openai" - "github.com/yaoapp/yao/agent/vision/driver/s3" -) - -// parseEnvValue parse environment variable if the value starts with $ENV. -func parseEnvValue(value string) string { - if strings.HasPrefix(value, "$ENV.") { - envKey := strings.TrimPrefix(value, "$ENV.") - if envVal := os.Getenv(envKey); envVal != "" { - return envVal - } - } - return value -} - -// convertOptions convert interface{} options map to string map and parse environment variables -func convertOptions(options map[string]interface{}) map[string]interface{} { - converted := make(map[string]interface{}) - for k, v := range options { - if str, ok := v.(string); ok { - converted[k] = parseEnvValue(str) - } else { - converted[k] = v - } - } - return converted -} - -// Vision the vision service -type Vision struct { - storage driver.Storage - model driver.Model -} - -// New create a new vision service -func New(cfg *driver.Config) (*Vision, error) { - - // Parse environment variables in options - storageOptions := convertOptions(cfg.Storage.Options) - modelOptions := convertOptions(cfg.Model.Options) - - // Create storage driver - var storage driver.Storage - var err error - switch cfg.Storage.Driver { - case "local": - storage, err = local.New(storageOptions) - case "s3": - // Convert expiration string to duration if present - if exp, ok := storageOptions["expiration"].(string); ok { - if duration, err := time.ParseDuration(exp); err == nil { - storageOptions["expiration"] = duration - } - } - storage, err = s3.New(storageOptions) - default: - return nil, fmt.Errorf("storage driver %s not supported", cfg.Storage.Driver) - } - if err != nil { - return nil, fmt.Errorf("create storage driver error: %s", err.Error()) - } - - // Create model driver - var model driver.Model - switch cfg.Model.Driver { - case "openai": - model, err = openai.New(modelOptions) - default: - return nil, fmt.Errorf("model driver %s not supported", cfg.Model.Driver) - } - if err != nil { - return nil, fmt.Errorf("create model driver error: %s", err.Error()) - } - - return &Vision{ - storage: storage, - model: model, - }, nil -} - -// Upload upload file -func (v *Vision) Upload(ctx context.Context, filename string, reader io.Reader, contentType string) (*driver.Response, error) { - fileID, err := v.storage.Upload(ctx, filename, reader, contentType) - if err != nil { - return nil, err - } - - return &driver.Response{ - FileID: fileID, - URL: v.storage.URL(ctx, fileID), - }, nil -} - -// Analyze analyze image using vision model -func (v *Vision) Analyze(ctx context.Context, fileID string, prompt ...string) (*driver.Response, error) { - if v.model == nil { - return nil, fmt.Errorf("model is required") - } - - var url string - // If the input is already a base64 data URL or a HTTP(S) URL, use it directly - if strings.HasPrefix(fileID, "data:image/") || strings.HasPrefix(fileID, "http://") || strings.HasPrefix(fileID, "https://") { - url = fileID - } else { - // Otherwise, try to get the URL from storage - url = v.storage.URL(ctx, fileID) - if url == "" { - return nil, fmt.Errorf("failed to get URL for file %s", fileID) - } - } - - result, err := v.model.Analyze(ctx, url, prompt...) - if err != nil { - return nil, err - } - - return &driver.Response{ - FileID: fileID, - URL: url, - Description: result, - }, nil -} - -// Download download file -func (v *Vision) Download(ctx context.Context, fileID string) (io.ReadCloser, string, error) { - return v.storage.Download(ctx, fileID) -} diff --git a/agent/vision/vision_test.go b/agent/vision/vision_test.go deleted file mode 100644 index 99f2c531..00000000 --- a/agent/vision/vision_test.go +++ /dev/null @@ -1,502 +0,0 @@ -package vision - -// import ( -// "bytes" -// "context" -// "encoding/base64" -// "fmt" -// "image" -// "image/png" -// "io" -// "net/http" -// "net/http/httptest" -// "os" -// "testing" - -// "github.com/stretchr/testify/assert" -// "github.com/yaoapp/gou/fs" -// "github.com/yaoapp/yao/agent/vision/driver" -// "github.com/yaoapp/yao/agent/vision/driver/local" -// "github.com/yaoapp/yao/config" -// "github.com/yaoapp/yao/test" -// ) - -// var ( -// // 1x1 transparent PNG -// testImageBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" -// ) - -// // MaxImageSize maximum image size (1920x1080) -// const MaxImageSize = local.MaxImageSize - -// func TestVision(t *testing.T) { -// test.Prepare(t, config.Conf) -// defer test.Clean() - -// // Setup test server for image hosting -// imgServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { -// // Log request for debugging -// t.Logf("Received request for: %s", r.URL.Path) - -// // Always return the test image -// imgData, _ := base64.StdEncoding.DecodeString(testImageBase64) -// w.Header().Set("Content-Type", "image/png") -// w.Write(imgData) -// })) -// defer imgServer.Close() - -// t.Logf("Test server running at: %s", imgServer.URL) - -// t.Run("Create Vision Service", func(t *testing.T) { -// vision, err := createTestVision(imgServer.URL) -// assert.NoError(t, err) -// assert.NotNil(t, vision) -// }) - -// t.Run("Upload and Download with Local Storage", func(t *testing.T) { -// vision, err := createTestVision(imgServer.URL) -// assert.NoError(t, err) - -// // Test with text file -// content := []byte("test content") -// reader := bytes.NewReader(content) -// resp, err := vision.Upload(context.Background(), "test.txt", reader, "text/plain") -// assert.NoError(t, err) -// assert.NotEmpty(t, resp.FileID) -// assert.NotEmpty(t, resp.URL) - -// // Download -// reader2, contentType, err := vision.Download(context.Background(), resp.FileID) -// assert.NoError(t, err) -// assert.Contains(t, contentType, "text/plain") - -// if reader2 != nil { -// downloaded, err := io.ReadAll(reader2) -// assert.NoError(t, err) -// assert.Equal(t, content, downloaded) -// reader2.Close() -// } -// }) - -// t.Run("Upload and Download with S3 Storage", func(t *testing.T) { -// vision, err := createTestVisionWithS3() -// if err != nil { -// t.Skip("S3 configuration not available") -// } - -// // Test with text file -// content := []byte("test content") -// reader := bytes.NewReader(content) -// resp, err := vision.Upload(context.Background(), "test.txt", reader, "text/plain") -// assert.NoError(t, err) -// assert.NotEmpty(t, resp.FileID) -// assert.NotEmpty(t, resp.URL) - -// // Download -// reader2, contentType, err := vision.Download(context.Background(), resp.FileID) -// assert.NoError(t, err) -// assert.Contains(t, contentType, "text/plain") - -// if reader2 != nil { -// downloaded, err := io.ReadAll(reader2) -// assert.NoError(t, err) -// assert.Equal(t, content, downloaded) -// reader2.Close() -// } -// }) - -// t.Run("Analyze Image with Base64", func(t *testing.T) { -// // Create vision service -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// }, -// }, -// } - -// vision, err := New(cfg) -// assert.NoError(t, err) - -// // Use base64 data directly -// result, err := vision.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "Describe this image in detail") -// assert.NoError(t, err) -// assert.NotNil(t, result) -// assert.NotEmpty(t, result.Description) -// }) - -// t.Run("Analyze Image with File", func(t *testing.T) { -// // Create vision service -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// }, -// }, -// } - -// vision, err := New(cfg) -// assert.NoError(t, err) - -// // Create test file -// data, err := fs.Get("data") -// assert.NoError(t, err) - -// // Write test image data -// imgData, err := base64.StdEncoding.DecodeString(testImageBase64) -// assert.NoError(t, err) -// _, err = data.WriteFile("/test.png", imgData, 0644) -// assert.NoError(t, err) - -// // Analyze using file path -// result, err := vision.Analyze(context.Background(), "/test.png", "Describe this image in detail") -// assert.NoError(t, err) -// assert.NotNil(t, result) -// assert.NotEmpty(t, result.Description) -// }) - -// t.Run("Analyze Image with S3 URL", func(t *testing.T) { -// if os.Getenv("S3_API") == "" || os.Getenv("S3_ACCESS_KEY") == "" || -// os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" { -// t.Skip("S3 environment variables not set") -// } - -// // Create vision service -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "s3", -// Options: map[string]interface{}{ -// "endpoint": os.Getenv("S3_API"), -// "region": "auto", -// "key": os.Getenv("S3_ACCESS_KEY"), -// "secret": os.Getenv("S3_SECRET_KEY"), -// "bucket": os.Getenv("S3_BUCKET"), -// "prefix": "vision-test", -// "expiration": "5m", -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// }, -// }, -// } - -// vision, err := New(cfg) -// assert.NoError(t, err) - -// // Upload test image -// imgData, err := base64.StdEncoding.DecodeString(testImageBase64) -// assert.NoError(t, err) -// reader := bytes.NewReader(imgData) -// resp, err := vision.Upload(context.Background(), "test.png", reader, "image/png") -// assert.NoError(t, err) -// assert.NotEmpty(t, resp.FileID) -// assert.NotEmpty(t, resp.URL) - -// // Analyze using S3 URL -// result, err := vision.Analyze(context.Background(), resp.URL, "Describe this image in detail") -// assert.NoError(t, err) -// assert.NotNil(t, result) -// assert.NotEmpty(t, result.Description) -// }) - -// t.Run("Invalid Model", func(t *testing.T) { -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "invalid", -// Options: map[string]interface{}{}, -// }, -// } - -// _, err := New(cfg) -// assert.Error(t, err) -// assert.Contains(t, err.Error(), "model driver invalid not supported") -// }) - -// t.Run("Invalid Storage", func(t *testing.T) { -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "invalid", -// Options: map[string]interface{}{}, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": "test", -// }, -// }, -// } - -// _, err := New(cfg) -// assert.Error(t, err) -// assert.Contains(t, err.Error(), "storage driver invalid not supported") -// }) - -// t.Run("Upload and Download Image with Local Storage", func(t *testing.T) { -// vision, err := createTestVision(imgServer.URL) -// assert.NoError(t, err) - -// // Create test image (2000x2000 pixels) -// img := image.NewRGBA(image.Rect(0, 0, 2000, 2000)) -// var buf bytes.Buffer -// err = png.Encode(&buf, img) -// assert.NoError(t, err) - -// // Upload -// reader := bytes.NewReader(buf.Bytes()) -// resp, err := vision.Upload(context.Background(), "test.png", reader, "image/png") -// assert.NoError(t, err) -// assert.NotEmpty(t, resp.FileID) -// assert.NotEmpty(t, resp.URL) - -// // Download and verify size -// reader2, contentType, err := vision.Download(context.Background(), resp.FileID) -// assert.NoError(t, err) -// assert.Equal(t, "image/png", contentType) - -// downloaded, err := io.ReadAll(reader2) -// assert.NoError(t, err) - -// // Decode the downloaded image -// downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) -// assert.NoError(t, err) - -// // Verify dimensions -// bounds := downloadedImg.Bounds() -// assert.LessOrEqual(t, bounds.Dx(), MaxImageSize) -// assert.LessOrEqual(t, bounds.Dy(), MaxImageSize) -// }) - -// t.Run("Upload and Download Image with S3 Storage", func(t *testing.T) { -// vision, err := createTestVisionWithS3() -// if err != nil { -// t.Skip("S3 configuration not available") -// } - -// // Create test image (2000x2000 pixels) -// img := image.NewRGBA(image.Rect(0, 0, 2000, 2000)) -// var buf bytes.Buffer -// err = png.Encode(&buf, img) -// assert.NoError(t, err) - -// // Upload -// reader := bytes.NewReader(buf.Bytes()) -// resp, err := vision.Upload(context.Background(), "test.png", reader, "image/png") -// assert.NoError(t, err) -// assert.NotEmpty(t, resp.FileID) -// assert.NotEmpty(t, resp.URL) - -// // Download and verify size -// reader2, contentType, err := vision.Download(context.Background(), resp.FileID) -// assert.NoError(t, err) -// assert.Equal(t, "image/png", contentType) - -// downloaded, err := io.ReadAll(reader2) -// assert.NoError(t, err) - -// // Decode the downloaded image -// downloadedImg, _, err := image.Decode(bytes.NewReader(downloaded)) -// assert.NoError(t, err) - -// // Verify dimensions -// bounds := downloadedImg.Bounds() -// assert.LessOrEqual(t, bounds.Dx(), MaxImageSize) -// assert.LessOrEqual(t, bounds.Dy(), MaxImageSize) -// }) - -// t.Run("Analyze Image with Default Prompt", func(t *testing.T) { -// // Create vision service with default prompt -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// "prompt": "Default test prompt", -// }, -// }, -// } - -// vision, err := New(cfg) -// assert.NoError(t, err) - -// // Use base64 data without providing a prompt -// result, err := vision.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64) -// assert.NoError(t, err) -// assert.NotNil(t, result) -// assert.NotEmpty(t, result.Description) -// }) - -// t.Run("Analyze Image with Custom Prompt", func(t *testing.T) { -// // Create vision service with default prompt -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// "prompt": "Default test prompt", -// }, -// }, -// } - -// vision, err := New(cfg) -// assert.NoError(t, err) - -// // Use base64 data with custom prompt -// result, err := vision.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "Custom test prompt") -// assert.NoError(t, err) -// assert.NotNil(t, result) -// assert.NotEmpty(t, result.Description) -// }) - -// t.Run("Analyze Image with Empty Custom Prompt", func(t *testing.T) { -// // Create vision service with default prompt -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// "prompt": "Default test prompt", -// }, -// }, -// } - -// vision, err := New(cfg) -// assert.NoError(t, err) - -// // Use base64 data with empty prompt (should use default) -// result, err := vision.Analyze(context.Background(), "data:image/png;base64,"+testImageBase64, "") -// assert.NoError(t, err) -// assert.NotNil(t, result) -// assert.NotEmpty(t, result.Description) -// }) -// } - -// func createTestVision(baseURL string) (*Vision, error) { -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "local", -// Options: map[string]interface{}{ -// "path": "/__vision_test", -// "compression": true, -// "base_url": baseURL, -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// "prompt": `# Objective -// You are a vision assistant, you can help the user to understand the image and describe it. - -// ## Task Execution Steps -// 1. Understand the image/video and describe it. -// 2. Describe the image/video in detail. - -// ## Result Format -// { -// "description": "The description of the image/video", -// "content": "The content of the image/video" -// }`, -// }, -// }, -// } - -// return New(cfg) -// } - -// func createTestVisionWithS3() (*Vision, error) { -// // Check required S3 environment variables -// if os.Getenv("S3_API") == "" || os.Getenv("S3_ACCESS_KEY") == "" || -// os.Getenv("S3_SECRET_KEY") == "" || os.Getenv("S3_BUCKET") == "" { -// return nil, fmt.Errorf("S3 environment variables not set") -// } - -// cfg := &driver.Config{ -// Storage: driver.StorageConfig{ -// Driver: "s3", -// Options: map[string]interface{}{ -// "endpoint": os.Getenv("S3_API"), -// "region": "auto", -// "key": os.Getenv("S3_ACCESS_KEY"), -// "secret": os.Getenv("S3_SECRET_KEY"), -// "bucket": os.Getenv("S3_BUCKET"), -// "prefix": "vision-test", -// "expiration": "5m", -// }, -// }, -// Model: driver.ModelConfig{ -// Driver: "openai", -// Options: map[string]interface{}{ -// "api_key": os.Getenv("OPENAI_API_KEY"), -// "model": os.Getenv("VISION_MODEL"), -// "prompt": `# Objective -// You are a vision assistant, you can help the user to understand the image and describe it. - -// ## Task Execution Steps -// 1. Understand the image/video and describe it. -// 2. Describe the image/video in detail. - -// ## Result Format -// { -// "description": "The description of the image/video", -// "content": "The content of the image/video" -// }`, -// }, -// }, -// } - -// return New(cfg) -// } diff --git a/service/service.go b/service/service.go index 0888144a..715455f2 100644 --- a/service/service.go +++ b/service/service.go @@ -6,7 +6,6 @@ import ( "github.com/gin-gonic/gin" "github.com/yaoapp/gou/api" "github.com/yaoapp/gou/server/http" - agent "github.com/yaoapp/yao/agent/api" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/openapi" "github.com/yaoapp/yao/share" @@ -36,11 +35,6 @@ func Start(cfg config.Config) (*http.Server, error) { Timeout: 5 * time.Second, }) - // Agent API - if agent.Agent != nil { - agent.Agent.API(router, "/api/__yao/agent") - } - // OpenAPI Server if openapi.Server != nil { openapi.Server.Attach(router) diff --git a/widgets/app/app.go b/widgets/app/app.go index 2c5cb311..5f974c11 100644 --- a/widgets/app/app.go +++ b/widgets/app/app.go @@ -17,7 +17,7 @@ import ( "github.com/yaoapp/gou/session" "github.com/yaoapp/kun/exception" "github.com/yaoapp/kun/log" - agent "github.com/yaoapp/yao/agent/api" + "github.com/yaoapp/yao/agent" "github.com/yaoapp/yao/agent/assistant" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/data" @@ -550,15 +550,16 @@ func processXgen(process *process.Process) interface{} { // The default assistant agentConfig := map[string]interface{}{} - if agent.Agent != nil { + agent := agent.GetAgent() + if agent != nil { // Add Uses Settings - if agent.Agent.DSL != nil && agent.Agent.DSL.Uses != nil { - agentConfig["uses"] = agent.Agent.DSL.Uses + if agent.Uses != nil { + agentConfig["uses"] = agent.Uses } // Add Default Assistant Settings ( Will be removed later ) - if ast, ok := agent.Agent.Assistant.(*assistant.Assistant); ok { + if ast, ok := agent.Assistant.(*assistant.Assistant); ok { agentConfig["default"] = map[string]interface{}{ "assistant_id": ast.ID, "assistant_name": ast.Name,