From ef63718941657ef5c1ae2c79e63631bb6f7a390c Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 25 Jan 2026 17:32:50 +0800 Subject: [PATCH] Enhance Attachment Process with Plain Text and UTF-8 Support - Added tests for saving plain text and Chinese text directly, ensuring proper handling of content without data URI encoding. - Updated the `parseDataURI` function to support plain text storage, returning the correct content type and data bytes. - Improved the `getAssistants` method to support nested assistants, enhancing directory scanning for assistant pages. - Refactored the `Page` method to accommodate assistant IDs with nested structures, improving route handling for pages. --- attachment/process.go | 29 +++-- attachment/process_test.go | 89 +++++++++++++ sui/storages/agent/agent.go | 117 +++++++++++++---- sui/storages/agent/agent_test.go | 214 +++++++++++++++++++++++++++++++ sui/storages/agent/template.go | 9 +- 5 files changed, 423 insertions(+), 35 deletions(-) create mode 100644 sui/storages/agent/agent_test.go diff --git a/attachment/process.go b/attachment/process.go index 5422db36..3c19dfd1 100644 --- a/attachment/process.go +++ b/attachment/process.go @@ -408,11 +408,12 @@ func processGetText(p *process.Process) interface{} { // ============ Helper Functions ============ -// parseDataURI parses a data URI or plain base64 string -// Returns content type, decoded data, and error +// parseDataURI parses content as either: +// 1. Data URI format: data:image/png;base64,xxxxx (decoded from base64) +// 2. Plain text: stored as-is with text/plain content type +// +// Returns content type, data bytes, and error func parseDataURI(content string) (string, []byte, error) { - contentType := "application/octet-stream" - // Handle data URI format: data:image/png;base64,xxxxx if strings.HasPrefix(content, "data:") { // Split by comma to get the data part @@ -423,23 +424,27 @@ func parseDataURI(content string) (string, []byte, error) { // Parse the header: data:image/png;base64 header := parts[0] - content = parts[1] + base64Content := parts[1] // Extract content type from header + contentType := "application/octet-stream" header = strings.TrimPrefix(header, "data:") headerParts := strings.Split(header, ";") if len(headerParts) > 0 && headerParts[0] != "" { contentType = headerParts[0] } + + // Decode base64 + data, err := base64.StdEncoding.DecodeString(base64Content) + if err != nil { + return "", nil, fmt.Errorf("failed to decode base64: %v", err) + } + + return contentType, data, nil } - // Decode base64 - data, err := base64.StdEncoding.DecodeString(content) - if err != nil { - return "", nil, fmt.Errorf("failed to decode base64: %v", err) - } - - return contentType, data, nil + // Plain text content - store as-is + return "text/plain", []byte(content), nil } // generateFilename generates a filename based on content type diff --git a/attachment/process_test.go b/attachment/process_test.go index 8c103637..5eb24826 100644 --- a/attachment/process_test.go +++ b/attachment/process_test.go @@ -214,6 +214,95 @@ func TestProcessSave(t *testing.T) { t.Fatal("Expected error for invalid base64") } }) + + // Test 8: Save plain text directly (no data URI) + t.Run("SavePlainText", func(t *testing.T) { + content := "This is plain text content without data URI encoding." + + p := process.New("attachment.Save", "data.local", content, "plain-text.txt") + result := processSave(p) + + if err, ok := result.(error); ok { + t.Fatalf("Failed to save plain text: %v", err) + } + + file, ok := result.(*File) + if !ok { + t.Fatalf("Expected *File, got %T", result) + } + + if file.ID == "" { + t.Error("File ID should not be empty") + } + + // Content type should be text/plain for plain text + if !strings.HasPrefix(file.ContentType, "text/plain") { + t.Errorf("Expected content type 'text/plain', got '%s'", file.ContentType) + } + + // Read back and verify content + readP := process.New("attachment.Read", "data.local", file.ID) + readResult := processRead(readP) + + dataURI, ok := readResult.(string) + if !ok { + t.Fatalf("Expected string, got %T: %v", readResult, readResult) + } + + // Decode from data URI + parts := strings.SplitN(dataURI, ",", 2) + if len(parts) != 2 { + t.Fatalf("Invalid data URI format") + } + decoded, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatalf("Failed to decode: %v", err) + } + + if string(decoded) != content { + t.Errorf("Content mismatch: expected %q, got %q", content, string(decoded)) + } + }) + + // Test 9: Save Chinese text directly (UTF-8) + t.Run("SaveChineseText", func(t *testing.T) { + content := "这是一段中文内容，测试UTF-8编码。\n第二行内容。" + + p := process.New("attachment.Save", "data.local", content, "chinese.txt") + result := processSave(p) + + if err, ok := result.(error); ok { + t.Fatalf("Failed to save Chinese text: %v", err) + } + + file, ok := result.(*File) + if !ok { + t.Fatalf("Expected *File, got %T", result) + } + + // Read back and verify content + readP := process.New("attachment.Read", "data.local", file.ID) + readResult := processRead(readP) + + dataURI, ok := readResult.(string) + if !ok { + t.Fatalf("Expected string, got %T: %v", readResult, readResult) + } + + // Decode from data URI + parts := strings.SplitN(dataURI, ",", 2) + if len(parts) != 2 { + t.Fatalf("Invalid data URI format") + } + decoded, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatalf("Failed to decode: %v", err) + } + + if string(decoded) != content { + t.Errorf("Chinese content mismatch: expected %q, got %q", content, string(decoded)) + } + }) } func TestProcessRead(t *testing.T) { diff --git a/sui/storages/agent/agent.go b/sui/storages/agent/agent.go index 0c3aa8d9..edd4dbbf 100644 --- a/sui/storages/agent/agent.go +++ b/sui/storages/agent/agent.go @@ -150,37 +150,92 @@ func (agent *Agent) WithSid(sid string) { agent.DSL.Sid = sid } -// getAssistants get all assistant directories that have pages +// getAssistants get all assistant directories that have pages (supports nested assistants) +// Returns assistant IDs like: ["expense", "tasks", "tests.nested.demo"] +// Nested paths are joined with "." to form the assistant ID func (agent *Agent) getAssistants() ([]string, error) { if !agent.fs.IsDir(agent.assistantsRoot) { return []string{}, nil } - dirs, err := agent.fs.ReadDir(agent.assistantsRoot, false) + assistants := []string{} + err := agent.scanAssistantsRecursive(agent.assistantsRoot, "", &assistants) if err != nil { return nil, err } - assistants := []string{} - for _, dir := range dirs { - if !agent.fs.IsDir(dir) { - continue - } - - // Check if this assistant has a pages directory - pagesDir := filepath.Join(dir, "pages") - if agent.fs.IsDir(pagesDir) { - name := filepath.Base(dir) - assistants = append(assistants, name) - } - } - return assistants, nil } +// scanAssistantsRecursive recursively scans directories for assistants with pages +// prefix is the accumulated path prefix (e.g., "tests.nested") +func (agent *Agent) scanAssistantsRecursive(dir string, prefix string, assistants *[]string) error { + dirs, err := agent.fs.ReadDir(dir, false) + if err != nil { + return err + } + + for _, subdir := range dirs { + if !agent.fs.IsDir(subdir) { + continue + } + + name := filepath.Base(subdir) + // Skip hidden directories and special directories + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "__") { + continue + } + + // Build the assistant ID with prefix + assistantID := name + if prefix != "" { + assistantID = prefix + "." + name + } + + // Check if this directory has a pages subdirectory + pagesDir := filepath.Join(subdir, "pages") + if agent.fs.IsDir(pagesDir) { + *assistants = append(*assistants, assistantID) + } + + // Recursively scan subdirectories for nested assistants + // Only scan if there's no pages directory (to avoid scanning inside pages/) + // or if there are other subdirectories that might contain nested assistants + if !agent.fs.IsDir(pagesDir) { + err := agent.scanAssistantsRecursive(subdir, assistantID, assistants) + if err != nil { + log.Warn("[Agent] Error scanning subdirectory %s: %v", subdir, err) + continue + } + } else { + // Even if this has pages, check for nested assistants in other subdirectories + subdirs, _ := agent.fs.ReadDir(subdir, false) + for _, nested := range subdirs { + nestedName := filepath.Base(nested) + if agent.fs.IsDir(nested) && nestedName != "pages" && + !strings.HasPrefix(nestedName, ".") && + !strings.HasPrefix(nestedName, "__") { + err := agent.scanAssistantsRecursive(nested, assistantID+"."+nestedName, assistants) + if err != nil { + log.Warn("[Agent] Error scanning nested directory %s: %v", nested, err) + continue + } + } + } + } + } + + return nil +} + // getAssistantPagesRoot get the pages root for an assistant +// assistantID can be "expense" or "tests.nested.demo" +// Returns the actual filesystem path like "/assistants/tests/nested/demo/pages" func (agent *Agent) getAssistantPagesRoot(assistantID string) string { - return filepath.Join(agent.assistantsRoot, assistantID, "pages") + // Convert dot notation to path: "tests.nested.demo" -> "tests/nested/demo" + pathParts := strings.Split(assistantID, ".") + assistantPath := filepath.Join(pathParts...) + return filepath.Join(agent.assistantsRoot, assistantPath, "pages") } // Exists check if the agent storage is available @@ -192,7 +247,7 @@ func Exists() bool { return appFS.IsDir("/agent/template") } -// HasAssistantPages check if any assistant has pages +// HasAssistantPages check if any assistant has pages (supports nested assistants) func HasAssistantPages() bool { appFS, err := fs.Get("app") if err != nil { @@ -203,19 +258,37 @@ func HasAssistantPages() bool { return false } - dirs, err := appFS.ReadDir("/assistants", false) + return hasAssistantPagesRecursive(appFS, "/assistants") +} + +// hasAssistantPagesRecursive recursively checks for assistants with pages +func hasAssistantPagesRecursive(appFS fs.FileSystem, dir string) bool { + dirs, err := appFS.ReadDir(dir, false) if err != nil { return false } - for _, dir := range dirs { - if !appFS.IsDir(dir) { + for _, subdir := range dirs { + if !appFS.IsDir(subdir) { continue } - pagesDir := filepath.Join(dir, "pages") + + name := filepath.Base(subdir) + // Skip hidden directories and special directories + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "__") { + continue + } + + // Check if this directory has a pages subdirectory + pagesDir := filepath.Join(subdir, "pages") if appFS.IsDir(pagesDir) { return true } + + // Recursively check subdirectories + if hasAssistantPagesRecursive(appFS, subdir) { + return true + } } return false diff --git a/sui/storages/agent/agent_test.go b/sui/storages/agent/agent_test.go new file mode 100644 index 00000000..9b095adf --- /dev/null +++ b/sui/storages/agent/agent_test.go @@ -0,0 +1,214 @@ +package agent + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/sui/core" + "github.com/yaoapp/yao/test" +) + +func TestAgentExists(t *testing.T) { + prepare(t) + defer clean() + + exists := Exists() + assert.True(t, exists, "Agent template should exist") +} + +func TestHasAssistantPages(t *testing.T) { + prepare(t) + defer clean() + + hasPages := HasAssistantPages() + assert.True(t, hasPages, "Should have assistant pages") +} + +func TestGetAssistants(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + assistants, err := agent.getAssistants() + assert.Nil(t, err) + assert.NotEmpty(t, assistants) + + // Sort for consistent comparison + sort.Strings(assistants) + + // Should include both direct and nested assistants + // Direct: tests.sui-pages (has pages directly) + // Nested: tests.nested.demo (nested assistant with pages) + found := map[string]bool{} + for _, ast := range assistants { + found[ast] = true + } + + assert.True(t, found["tests.sui-pages"], "Should find tests.sui-pages assistant") + assert.True(t, found["tests.nested.demo"], "Should find tests.nested.demo assistant") +} + +func TestGetAssistantPagesRoot(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + + // Test direct assistant + root := agent.getAssistantPagesRoot("tests.sui-pages") + assert.Equal(t, "/assistants/tests/sui-pages/pages", root) + + // Test nested assistant + root = agent.getAssistantPagesRoot("tests.nested.demo") + assert.Equal(t, "/assistants/tests/nested/demo/pages", root) +} + +func TestGetTemplate(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + tmpl, err := agent.GetTemplate("agent") + assert.Nil(t, err) + assert.NotNil(t, tmpl) + assert.Equal(t, "agent", tmpl.(*Template).ID) +} + +func TestTemplatePages(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + tmpl, err := agent.GetTemplate("agent") + assert.Nil(t, err) + + pages, err := tmpl.Pages() + assert.Nil(t, err) + assert.NotEmpty(t, pages) + + // Check that we have pages from nested assistants + routes := map[string]bool{} + for _, page := range pages { + routes[page.Get().Route] = true + } + + // Should have pages from: + // 1. Agent global pages (/index) + // 2. Direct assistant (tests.sui-pages) -> /tests.sui-pages/dashboard + // 3. Nested assistant (tests.nested.demo) -> /tests.nested.demo/article + assert.True(t, routes["/index"], "Should have agent global page /index") + assert.True(t, routes["/tests.sui-pages/dashboard"], "Should have direct assistant page /tests.sui-pages/dashboard") + assert.True(t, routes["/tests.nested.demo/article"], "Should have nested assistant page /tests.nested.demo/article") +} + +func TestTemplatePage(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + tmpl, err := agent.GetTemplate("agent") + assert.Nil(t, err) + + // Test getting agent global page + page, err := tmpl.Page("/index") + assert.Nil(t, err) + assert.NotNil(t, page) + assert.Equal(t, "/index", page.Get().Route) + + // Test getting direct assistant page + page, err = tmpl.Page("/tests.sui-pages/dashboard") + assert.Nil(t, err) + assert.NotNil(t, page) + assert.Equal(t, "/tests.sui-pages/dashboard", page.Get().Route) + assert.Equal(t, "tests.sui-pages", page.(*Page).assistantID) + + // Test getting nested assistant page + page, err = tmpl.Page("/tests.nested.demo/article") + assert.Nil(t, err) + assert.NotNil(t, page) + assert.Equal(t, "/tests.nested.demo/article", page.Get().Route) + assert.Equal(t, "tests.nested.demo", page.(*Page).assistantID) + + // Test page not found + _, err = tmpl.Page("/non-existent/page") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func TestPageLoad(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + tmpl, err := agent.GetTemplate("agent") + assert.Nil(t, err) + + // Test loading nested assistant page + page, err := tmpl.Page("/tests.nested.demo/article") + assert.Nil(t, err) + + err = page.Load() + assert.Nil(t, err) + + // Check that content was loaded + p := page.Get() + assert.NotEmpty(t, p.Codes.HTML.Code, "HTML code should be loaded") + assert.NotEmpty(t, p.Codes.CSS.Code, "CSS code should be loaded") +} + +func TestPageBuild(t *testing.T) { + prepare(t) + defer clean() + + agent := createAgent(t) + + // Register the agent SUI so page build can find it + core.SUIs["agent"] = agent + + tmpl, err := agent.GetTemplate("agent") + assert.Nil(t, err) + + // Test building nested assistant page + page, err := tmpl.Page("/tests.nested.demo/article") + assert.Nil(t, err) + + err = page.Load() + assert.Nil(t, err) + + ctx := core.NewGlobalBuildContext(tmpl) + warnings, err := page.Build(ctx, &core.BuildOption{ + PublicRoot: "/agents", + AssetRoot: "/agents/assets", + }) + assert.Nil(t, err) + assert.Empty(t, warnings) +} + +func prepare(t *testing.T) { + test.Prepare(t, config.Conf, "YAO_TEST_APPLICATION") +} + +func clean() { + test.Clean() +} + +func createAgent(t *testing.T) *Agent { + dsl := &core.DSL{ + ID: "agent", + Name: "Agent", + Public: &core.Public{ + Root: "/agents", + Host: "/", + Index: "/index", + }, + } + + agent, err := New(dsl) + if err != nil { + t.Fatalf("Failed to create agent: %v", err) + } + + return agent +} diff --git a/sui/storages/agent/template.go b/sui/storages/agent/template.go index ac921480..481597f1 100644 --- a/sui/storages/agent/template.go +++ b/sui/storages/agent/template.go @@ -142,6 +142,11 @@ func (tmpl *Template) getPageBase(route string) string { } // Page get a specific page by route +// Route format: "/assistant-id/page-path" where assistant-id can contain dots for nested assistants +// Examples: +// - "/expense/test" -> assistant "expense", page "/test" +// - "/tests.nested.demo/article" -> assistant "tests.nested.demo", page "/article" +// - "/index" -> agent page (no assistant prefix) func (tmpl *Template) Page(route string) (core.IPage, error) { // Parse the route to determine if it's an assistant page or agent page parts := strings.Split(strings.Trim(route, "/"), "/") @@ -150,7 +155,7 @@ func (tmpl *Template) Page(route string) (core.IPage, error) { return nil, fmt.Errorf("Invalid route: %s", route) } - // Check if first part is an assistant ID + // Check if first part is an assistant ID (may contain dots for nested assistants) assistants, err := tmpl.agent.getAssistants() if err != nil { return nil, err @@ -160,6 +165,8 @@ func (tmpl *Template) Page(route string) (core.IPage, error) { pageRoute := route pagesRoot := filepath.Join(tmpl.agent.root, "pages") + // The first part of the route might be an assistant ID + // Assistant IDs can contain dots (e.g., "tests.nested.demo") for _, ast := range assistants { if parts[0] == ast { assistantID = ast