diff --git a/attachment/local/storage.go b/attachment/local/storage.go index 0ba0ce1c..65db9196 100644 --- a/attachment/local/storage.go +++ b/attachment/local/storage.go @@ -6,6 +6,8 @@ import ( "crypto/sha256" "fmt" "io" + "mime" + "net/http" "os" "path/filepath" "strings" @@ -259,3 +261,250 @@ func (storage *Storage) makeID(filename string, ext string) string { name := strings.TrimSuffix(filepath.Base(filename), ext) return fmt.Sprintf("%s/%s-%s%s", date, name, hash, ext) } + +// LocalPath returns the absolute path of the file and its content type +func (storage *Storage) LocalPath(ctx context.Context, path string) (string, string, error) { + fullPath := filepath.Join(storage.Path, path) + + // Check if file exists + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + return "", "", fmt.Errorf("file not found: %s", path) + } + + // For gzipped files, we need to detect the original content type, not the gzip wrapper + var contentType string + var err error + + if strings.HasSuffix(path, ".gz") { + // For gzipped files, detect content type of the decompressed content + originalPath := strings.TrimSuffix(path, ".gz") + ext := filepath.Ext(originalPath) + + // First try to detect by original file extension + contentType, err = detectContentTypeFromExtension(ext) + if err != nil || contentType == "application/octet-stream" { + // Fallback: decompress and detect from content + contentType, err = detectContentTypeFromGzippedFile(fullPath) + if err != nil { + return "", "", fmt.Errorf("failed to detect content type from gzipped file: %w", err) + } + } + } else { + // Regular file content type detection + contentType, err = detectContentType(fullPath) + if err != nil { + return "", "", fmt.Errorf("failed to detect content type: %w", err) + } + } + + // Return absolute path + absPath, err := filepath.Abs(fullPath) + if err != nil { + return "", "", fmt.Errorf("failed to get absolute path: %w", err) + } + + return absPath, contentType, nil +} + +// detectContentType detects content type based on file extension and content +func detectContentType(filePath string) (string, error) { + // First try to detect by file extension + ext := strings.ToLower(filepath.Ext(filePath)) + + // Common file extensions mapping + switch ext { + case ".txt": + return "text/plain", nil + case ".html", ".htm": + return "text/html", nil + case ".css": + return "text/css", nil + case ".js": + return "application/javascript", nil + case ".json": + return "application/json", nil + case ".xml": + return "application/xml", nil + case ".jpg", ".jpeg": + return "image/jpeg", nil + case ".png": + return "image/png", nil + case ".gif": + return "image/gif", nil + case ".webp": + return "image/webp", nil + case ".svg": + return "image/svg+xml", nil + case ".pdf": + return "application/pdf", nil + case ".doc": + return "application/msword", nil + case ".docx": + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil + case ".xls": + return "application/vnd.ms-excel", nil + case ".xlsx": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil + case ".ppt": + return "application/vnd.ms-powerpoint", nil + case ".pptx": + return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil + case ".zip": + return "application/zip", nil + case ".tar": + return "application/x-tar", nil + case ".gz": + return "application/gzip", nil + case ".mp3": + return "audio/mpeg", nil + case ".wav": + return "audio/wav", nil + case ".m4a": + return "audio/mp4", nil + case ".ogg": + return "audio/ogg", nil + case ".mp4": + return "video/mp4", nil + case ".avi": + return "video/x-msvideo", nil + case ".mov": + return "video/quicktime", nil + case ".webm": + return "video/webm", nil + case ".md", ".mdx": + return "text/markdown", nil + case ".yao": + return "application/yao", nil + case ".csv": + return "text/csv", nil + } + + // Try to detect by MIME package + if contentType := mime.TypeByExtension(ext); contentType != "" { + return contentType, nil + } + + // Fallback: detect by reading file content + file, err := os.Open(filePath) + if err != nil { + return "application/octet-stream", nil // Default fallback + } + defer file.Close() + + // Read first 512 bytes for content detection + buffer := make([]byte, 512) + n, err := file.Read(buffer) + if err != nil && err != io.EOF { + return "application/octet-stream", nil + } + + // Use http.DetectContentType to detect based on content + contentType := http.DetectContentType(buffer[:n]) + return contentType, nil +} + +// detectContentTypeFromExtension detects content type based only on file extension +func detectContentTypeFromExtension(ext string) (string, error) { + ext = strings.ToLower(ext) + + // Common file extensions mapping + switch ext { + case ".txt": + return "text/plain", nil + case ".html", ".htm": + return "text/html", nil + case ".css": + return "text/css", nil + case ".js": + return "application/javascript", nil + case ".json": + return "application/json", nil + case ".xml": + return "application/xml", nil + case ".jpg", ".jpeg": + return "image/jpeg", nil + case ".png": + return "image/png", nil + case ".gif": + return "image/gif", nil + case ".webp": + return "image/webp", nil + case ".svg": + return "image/svg+xml", nil + case ".pdf": + return "application/pdf", nil + case ".doc": + return "application/msword", nil + case ".docx": + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil + case ".xls": + return "application/vnd.ms-excel", nil + case ".xlsx": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil + case ".ppt": + return "application/vnd.ms-powerpoint", nil + case ".pptx": + return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil + case ".zip": + return "application/zip", nil + case ".tar": + return "application/x-tar", nil + case ".mp3": + return "audio/mpeg", nil + case ".wav": + return "audio/wav", nil + case ".m4a": + return "audio/mp4", nil + case ".ogg": + return "audio/ogg", nil + case ".mp4": + return "video/mp4", nil + case ".avi": + return "video/x-msvideo", nil + case ".mov": + return "video/quicktime", nil + case ".webm": + return "video/webm", nil + case ".md", ".mdx": + return "text/markdown", nil + case ".yao": + return "application/yao", nil + case ".csv": + return "text/csv", nil + } + + // Try to detect by MIME package + if contentType := mime.TypeByExtension(ext); contentType != "" { + return contentType, nil + } + + // Return default if not found + return "application/octet-stream", nil +} + +// detectContentTypeFromGzippedFile detects content type by decompressing and reading gzipped file +func detectContentTypeFromGzippedFile(gzippedFilePath string) (string, error) { + file, err := os.Open(gzippedFilePath) + if err != nil { + return "", err + } + defer file.Close() + + // Create gzip reader + gzipReader, err := gzip.NewReader(file) + if err != nil { + return "", err + } + defer gzipReader.Close() + + // Read first 512 bytes of decompressed content + buffer := make([]byte, 512) + n, err := gzipReader.Read(buffer) + if err != nil && err != io.EOF { + return "", err + } + + // Use http.DetectContentType to detect based on decompressed content + contentType := http.DetectContentType(buffer[:n]) + return contentType, nil +} diff --git a/attachment/local/storage_test.go b/attachment/local/storage_test.go index 793164ee..d1e41d82 100644 --- a/attachment/local/storage_test.go +++ b/attachment/local/storage_test.go @@ -223,4 +223,81 @@ func TestLocalStorage(t *testing.T) { exists = storage.Exists(context.Background(), fileID) assert.False(t, exists) }) + + t.Run("LocalPath", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + }) + assert.NoError(t, err) + + // Test different file types to verify content type detection + testFiles := []struct { + name string + content []byte + contentType string + expectedCT string + }{ + {"test.txt", []byte("Hello World"), "text/plain", "text/plain"}, + {"test.json", []byte(`{"key": "value"}`), "application/json", "application/json"}, + {"test.html", []byte("Test"), "text/html", "text/html"}, + {"test.csv", []byte("col1,col2\nval1,val2"), "text/csv", "text/csv"}, + {"test.md", []byte("# Markdown Content"), "text/markdown", "text/markdown"}, + {"test.yao", []byte("yao file content"), "application/yao", "application/yao"}, + } + + for _, tf := range testFiles { + // Upload file + _, err = storage.Upload(context.Background(), tf.name, bytes.NewReader(tf.content), tf.contentType) + assert.NoError(t, err, "Failed to upload %s", tf.name) + + // Get local path and content type + localPath, detectedCT, err := storage.LocalPath(context.Background(), tf.name) + assert.NoError(t, err, "Failed to get local path for %s", tf.name) + assert.NotEmpty(t, localPath, "Local path should not be empty for %s", tf.name) + assert.Equal(t, tf.expectedCT, detectedCT, "Content type mismatch for %s", tf.name) + + // Verify the path is absolute + assert.True(t, filepath.IsAbs(localPath), "Path should be absolute for %s", tf.name) + + // Verify the file exists at the returned path + _, err = os.Stat(localPath) + assert.NoError(t, err, "File should exist at local path for %s", tf.name) + + // Verify file content + fileContent, err := os.ReadFile(localPath) + assert.NoError(t, err, "Failed to read file at local path for %s", tf.name) + assert.Equal(t, tf.content, fileContent, "File content mismatch for %s", tf.name) + } + }) + + t.Run("LocalPath_NonExistentFile", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + }) + assert.NoError(t, err) + + // Test with non-existent file + _, _, err = storage.LocalPath(context.Background(), "non-existent.txt") + assert.Error(t, err) + assert.Contains(t, err.Error(), "file not found") + }) + + t.Run("LocalPath_ContentDetection", func(t *testing.T) { + storage, err := New(map[string]interface{}{ + "path": testPath, + }) + assert.NoError(t, err) + + // Upload a file without extension but with recognizable content + htmlContent := []byte("Test

Hello

") + _, err = storage.Upload(context.Background(), "noext", bytes.NewReader(htmlContent), "application/octet-stream") + assert.NoError(t, err) + + // Get local path - should detect HTML content type + localPath, contentType, err := storage.LocalPath(context.Background(), "noext") + assert.NoError(t, err) + assert.NotEmpty(t, localPath) + // Content detection should identify this as HTML + assert.Equal(t, "text/html; charset=utf-8", contentType) + }) } diff --git a/attachment/manager.go b/attachment/manager.go index fbe2585d..61303cd4 100644 --- a/attachment/manager.go +++ b/attachment/manager.go @@ -215,6 +215,18 @@ func New(option ManagerOption) (*Manager, error) { return manager, nil } +// LocalPath gets the local path of the file +func (manager Manager) LocalPath(ctx context.Context, fileID string) (string, string, error) { + // Get the real storage path from database + storagePath, err := manager.getStoragePathFromDatabase(ctx, fileID) + if err != nil { + return "", "", err + } + + // Call the storage implementation + return manager.storage.LocalPath(ctx, storagePath) +} + // Upload uploads a file, Content-Sync must be true for chunked upload func (manager Manager) Upload(ctx context.Context, fileheader *FileHeader, reader io.Reader, option UploadOption) (*File, error) { diff --git a/attachment/manager_test.go b/attachment/manager_test.go index 21ebc129..29e7e2d5 100644 --- a/attachment/manager_test.go +++ b/attachment/manager_test.go @@ -7,6 +7,7 @@ import ( "fmt" "mime/multipart" "os" + "path/filepath" "strings" "testing" "time" @@ -828,3 +829,282 @@ func TestList(t *testing.T) { t.Logf("Successfully tested list functionality with %d files", len(uploadedFiles)) } + +func TestManagerLocalPath(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Test with local storage + t.Run("LocalStorage", func(t *testing.T) { + // Create a local storage manager + manager, err := New(ManagerOption{ + Driver: "local", + MaxSize: "10M", + AllowedTypes: []string{"text/*", "image/*", "application/*", ".txt", ".json", ".html", ".csv", ".yao"}, + Options: map[string]interface{}{ + "path": "/tmp/test_localpath_attachments", + }, + }) + if err != nil { + t.Fatalf("Failed to create local manager: %v", err) + } + manager.Name = "localpath-test" + + // Test different file types + testFiles := []struct { + filename string + content string + contentType string + expectedCT string + }{ + {"test.txt", "Hello LocalPath", "text/plain", "text/plain"}, + {"test.json", `{"localpath": "test"}`, "application/json", "application/json"}, + {"test.html", "LocalPath Test", "text/html", "text/html"}, + {"test.csv", "col1,col2\nlocalpath,test", "text/csv", "text/csv"}, + {"test.yao", "localpath yao content", "application/yao", "application/yao"}, + } + + for _, tf := range testFiles { + // Upload file + reader := strings.NewReader(tf.content) + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: tf.filename, + Size: int64(len(tf.content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", tf.contentType) + + option := UploadOption{ + Groups: []string{"localpath", "test"}, + OriginalFilename: tf.filename, + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload file %s: %v", tf.filename, err) + } + + // Test LocalPath + localPath, detectedCT, err := manager.LocalPath(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to get local path for %s: %v", tf.filename, err) + } + + // Verify path is absolute + if !filepath.IsAbs(localPath) { + t.Errorf("Expected absolute path for %s, got: %s", tf.filename, localPath) + } + + // Verify content type + if detectedCT != tf.expectedCT { + t.Errorf("Expected content type %s for %s, got: %s", tf.expectedCT, tf.filename, detectedCT) + } + + // Verify file exists + if _, err := os.Stat(localPath); os.IsNotExist(err) { + t.Errorf("File should exist at local path %s for %s", localPath, tf.filename) + } + + // Verify file content + fileContent, err := os.ReadFile(localPath) + if err != nil { + t.Fatalf("Failed to read file at local path for %s: %v", tf.filename, err) + } + + if string(fileContent) != tf.content { + t.Errorf("File content mismatch for %s. Expected: %s, Got: %s", tf.filename, tf.content, string(fileContent)) + } + + t.Logf("File %s - ID: %s, LocalPath: %s, ContentType: %s", tf.filename, file.ID, localPath, detectedCT) + } + }) + + // Test with gzipped files in local storage + t.Run("LocalStorage_Gzipped", func(t *testing.T) { + manager, err := New(ManagerOption{ + Driver: "local", + MaxSize: "10M", + AllowedTypes: []string{"text/*"}, + Options: map[string]interface{}{ + "path": "/tmp/test_localpath_gzip_attachments", + }, + }) + if err != nil { + t.Fatalf("Failed to create local manager: %v", err) + } + manager.Name = "localpath-gzip-test" + + content := "This content will be gzipped" + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "gzipped.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{ + Groups: []string{"gzip", "test"}, + OriginalFilename: "gzipped.txt", + Gzip: true, // Enable gzip compression + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload gzipped file: %v", err) + } + + // Test LocalPath - should get decompressed content + localPath, contentType, err := manager.LocalPath(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to get local path for gzipped file: %v", err) + } + + // Verify content type + if contentType != "text/plain" { + t.Errorf("Expected content type text/plain, got: %s", contentType) + } + + // For gzipped files in local storage, the storage path ends with .gz + // but the content should be accessible normally through Read methods + fileContent, err := manager.Read(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to read gzipped file: %v", err) + } + + if string(fileContent) != content { + t.Errorf("Gzipped file content mismatch. Expected: %s, Got: %s", content, string(fileContent)) + } + + t.Logf("Gzipped file - ID: %s, LocalPath: %s, ContentType: %s", file.ID, localPath, contentType) + }) +} + +func TestManagerLocalPath_NonExistentFile(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager, err := New(ManagerOption{ + Driver: "local", + AllowedTypes: []string{"text/*"}, + Options: map[string]interface{}{ + "path": "/tmp/test_localpath_nonexistent", + }, + }) + if err != nil { + t.Fatalf("Failed to create manager: %v", err) + } + manager.Name = "nonexistent-test" + + // Test with non-existent file ID + _, _, err = manager.LocalPath(context.Background(), "non-existent-file-id") + if err == nil { + t.Error("Expected error for non-existent file ID") + } + + // Should contain "file not found" in the error chain + if !strings.Contains(err.Error(), "file not found") { + t.Errorf("Expected 'file not found' in error message, got: %s", err.Error()) + } +} + +func TestManagerLocalPath_ValidationFlow(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + manager, err := New(ManagerOption{ + Driver: "local", + AllowedTypes: []string{"text/*"}, + Options: map[string]interface{}{ + "path": "/tmp/test_localpath_validation", + }, + }) + if err != nil { + t.Fatalf("Failed to create manager: %v", err) + } + manager.Name = "validation-test" + + // Upload a file + content := "Validation flow test content" + reader := strings.NewReader(content) + + fileHeader := &FileHeader{ + FileHeader: &multipart.FileHeader{ + Filename: "validation.txt", + Size: int64(len(content)), + Header: make(map[string][]string), + }, + } + fileHeader.Header.Set("Content-Type", "text/plain") + + option := UploadOption{ + Groups: []string{"validation"}, + OriginalFilename: "original-validation.txt", + } + + file, err := manager.Upload(context.Background(), fileHeader, reader, option) + if err != nil { + t.Fatalf("Failed to upload file: %v", err) + } + + // Test complete flow: Upload -> LocalPath -> Verify -> Delete + t.Run("CompleteFlow", func(t *testing.T) { + // Get local path + localPath, contentType, err := manager.LocalPath(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to get local path: %v", err) + } + + // Verify all properties + if !filepath.IsAbs(localPath) { + t.Error("Path should be absolute") + } + + if contentType != "text/plain" { + t.Errorf("Expected content type text/plain, got: %s", contentType) + } + + // Verify file exists + stat, err := os.Stat(localPath) + if err != nil { + t.Fatalf("File should exist at local path: %v", err) + } + + if stat.Size() != int64(len(content)) { + t.Errorf("File size mismatch. Expected: %d, Got: %d", len(content), stat.Size()) + } + + // Verify file content matches + fileContent, err := os.ReadFile(localPath) + if err != nil { + t.Fatalf("Failed to read file: %v", err) + } + + if string(fileContent) != content { + t.Errorf("Content mismatch. Expected: %s, Got: %s", content, string(fileContent)) + } + + // Verify through manager's Read method as well + managerContent, err := manager.Read(context.Background(), file.ID) + if err != nil { + t.Fatalf("Failed to read through manager: %v", err) + } + + if string(managerContent) != content { + t.Errorf("Manager read content mismatch. Expected: %s, Got: %s", content, string(managerContent)) + } + + t.Logf("Validation complete - LocalPath: %s, Size: %d bytes, ContentType: %s", localPath, stat.Size(), contentType) + }) + + // Clean up + err = manager.Delete(context.Background(), file.ID) + if err != nil { + t.Logf("Warning: Failed to delete test file: %v", err) + } +} diff --git a/attachment/s3/storage.go b/attachment/s3/storage.go index 5209a481..7daa7c02 100644 --- a/attachment/s3/storage.go +++ b/attachment/s3/storage.go @@ -9,6 +9,9 @@ import ( "image/jpeg" "image/png" "io" + "mime" + "net/http" + "os" "path/filepath" "strings" "time" @@ -32,6 +35,7 @@ type Storage struct { Secret string `json:"secret" yaml:"secret"` Bucket string `json:"bucket" yaml:"bucket"` Expiration time.Duration `json:"expiration" yaml:"expiration"` + CacheDir string `json:"cache_dir" yaml:"cache_dir"` client *s3.Client prefix string compression bool @@ -69,6 +73,13 @@ func New(options map[string]interface{}) (*Storage, error) { storage.prefix = prefix } + if cacheDir, ok := options["cache_dir"].(string); ok { + storage.CacheDir = cacheDir + } else { + // Use system temp directory as default + storage.CacheDir = os.TempDir() + } + if exp, ok := options["expiration"].(time.Duration); ok { storage.Expiration = exp } @@ -103,6 +114,12 @@ func New(options map[string]interface{}) (*Storage, error) { } storage.client = s3.New(opts) + + // Ensure cache directory exists + if err := os.MkdirAll(storage.CacheDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create cache directory %s: %w", storage.CacheDir, err) + } + return storage, nil } @@ -445,3 +462,278 @@ func compressImage(data []byte, contentType string) ([]byte, error) { return buf.Bytes(), nil } + +// LocalPath downloads the file to cache directory and returns absolute path with content type +func (storage *Storage) LocalPath(ctx context.Context, path string) (string, string, error) { + if storage.client == nil { + return "", "", fmt.Errorf("s3 client not initialized") + } + + // Create cache file path using the same structure as storage path + cacheFilePath := filepath.Join(storage.CacheDir, "s3_cache", path) + + // Create directory for cache file + dir := filepath.Dir(cacheFilePath) + if err := os.MkdirAll(dir, 0755); err != nil { + return "", "", fmt.Errorf("failed to create cache directory: %w", err) + } + + // Check if file already exists in cache and is not outdated + if _, err := os.Stat(cacheFilePath); err == nil { + // File exists in cache, detect content type and return + contentType, err := detectContentType(cacheFilePath) + if err != nil { + return "", "", fmt.Errorf("failed to detect content type: %w", err) + } + return cacheFilePath, contentType, nil + } + + // Download file from S3 to cache + key := filepath.Join(storage.prefix, path) + result, err := storage.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(storage.Bucket), + Key: aws.String(key), + }) + if err != nil { + return "", "", fmt.Errorf("failed to download file %s: %w", path, err) + } + defer result.Body.Close() + + // Create cache file + cacheFile, err := os.Create(cacheFilePath) + if err != nil { + return "", "", fmt.Errorf("failed to create cache file: %w", err) + } + defer cacheFile.Close() + + // Handle gzipped files - decompress during download + var reader io.Reader = result.Body + if strings.HasSuffix(path, ".gz") { + gzipReader, err := gzip.NewReader(result.Body) + if err != nil { + return "", "", fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzipReader.Close() + reader = gzipReader + + // Remove .gz extension from cache file path since we're decompressing + newCacheFilePath := strings.TrimSuffix(cacheFilePath, ".gz") + cacheFile.Close() + os.Remove(cacheFilePath) + + cacheFile, err = os.Create(newCacheFilePath) + if err != nil { + return "", "", fmt.Errorf("failed to create decompressed cache file: %w", err) + } + defer cacheFile.Close() + cacheFilePath = newCacheFilePath + } + + // Copy file content to cache + _, err = io.Copy(cacheFile, reader) + if err != nil { + return "", "", fmt.Errorf("failed to copy file to cache: %w", err) + } + + // For files that were decompressed from .gz, we need to detect the original content type + var contentType string + if strings.HasSuffix(path, ".gz") { + // Original path was gzipped, detect content type of decompressed content + originalPath := strings.TrimSuffix(path, ".gz") + ext := filepath.Ext(originalPath) + + // First try to detect by original file extension + contentType, err = detectContentTypeFromExtension(ext) + if err != nil || contentType == "application/octet-stream" { + // Fallback: detect from decompressed content + contentType, err = detectContentType(cacheFilePath) + if err != nil { + return "", "", fmt.Errorf("failed to detect content type: %w", err) + } + } + } else { + // Regular file content type detection + contentType, err = detectContentType(cacheFilePath) + if err != nil { + return "", "", fmt.Errorf("failed to detect content type: %w", err) + } + } + + return cacheFilePath, contentType, nil +} + +// detectContentType detects content type based on file extension and content +func detectContentType(filePath string) (string, error) { + // First try to detect by file extension + ext := strings.ToLower(filepath.Ext(filePath)) + + // Common file extensions mapping + switch ext { + case ".txt": + return "text/plain", nil + case ".html", ".htm": + return "text/html", nil + case ".css": + return "text/css", nil + case ".js": + return "application/javascript", nil + case ".json": + return "application/json", nil + case ".xml": + return "application/xml", nil + case ".jpg", ".jpeg": + return "image/jpeg", nil + case ".png": + return "image/png", nil + case ".gif": + return "image/gif", nil + case ".webp": + return "image/webp", nil + case ".svg": + return "image/svg+xml", nil + case ".pdf": + return "application/pdf", nil + case ".doc": + return "application/msword", nil + case ".docx": + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil + case ".xls": + return "application/vnd.ms-excel", nil + case ".xlsx": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil + case ".ppt": + return "application/vnd.ms-powerpoint", nil + case ".pptx": + return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil + case ".zip": + return "application/zip", nil + case ".tar": + return "application/x-tar", nil + case ".gz": + return "application/gzip", nil + case ".mp3": + return "audio/mpeg", nil + case ".wav": + return "audio/wav", nil + case ".m4a": + return "audio/mp4", nil + case ".ogg": + return "audio/ogg", nil + case ".mp4": + return "video/mp4", nil + case ".avi": + return "video/x-msvideo", nil + case ".mov": + return "video/quicktime", nil + case ".webm": + return "video/webm", nil + case ".md", ".mdx": + return "text/markdown", nil + case ".yao": + return "application/yao", nil + case ".csv": + return "text/csv", nil + } + + // Try to detect by MIME package + if contentType := mime.TypeByExtension(ext); contentType != "" { + return contentType, nil + } + + // Fallback: detect by reading file content + file, err := os.Open(filePath) + if err != nil { + return "application/octet-stream", nil // Default fallback + } + defer file.Close() + + // Read first 512 bytes for content detection + buffer := make([]byte, 512) + n, err := file.Read(buffer) + if err != nil && err != io.EOF { + return "application/octet-stream", nil + } + + // Use http.DetectContentType to detect based on content + contentType := http.DetectContentType(buffer[:n]) + return contentType, nil +} + +// detectContentTypeFromExtension detects content type based only on file extension +func detectContentTypeFromExtension(ext string) (string, error) { + ext = strings.ToLower(ext) + + // Common file extensions mapping + switch ext { + case ".txt": + return "text/plain", nil + case ".html", ".htm": + return "text/html", nil + case ".css": + return "text/css", nil + case ".js": + return "application/javascript", nil + case ".json": + return "application/json", nil + case ".xml": + return "application/xml", nil + case ".jpg", ".jpeg": + return "image/jpeg", nil + case ".png": + return "image/png", nil + case ".gif": + return "image/gif", nil + case ".webp": + return "image/webp", nil + case ".svg": + return "image/svg+xml", nil + case ".pdf": + return "application/pdf", nil + case ".doc": + return "application/msword", nil + case ".docx": + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", nil + case ".xls": + return "application/vnd.ms-excel", nil + case ".xlsx": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", nil + case ".ppt": + return "application/vnd.ms-powerpoint", nil + case ".pptx": + return "application/vnd.openxmlformats-officedocument.presentationml.presentation", nil + case ".zip": + return "application/zip", nil + case ".tar": + return "application/x-tar", nil + case ".mp3": + return "audio/mpeg", nil + case ".wav": + return "audio/wav", nil + case ".m4a": + return "audio/mp4", nil + case ".ogg": + return "audio/ogg", nil + case ".mp4": + return "video/mp4", nil + case ".avi": + return "video/x-msvideo", nil + case ".mov": + return "video/quicktime", nil + case ".webm": + return "video/webm", nil + case ".md", ".mdx": + return "text/markdown", nil + case ".yao": + return "application/yao", nil + case ".csv": + return "text/csv", nil + } + + // Try to detect by MIME package + if contentType := mime.TypeByExtension(ext); contentType != "" { + return contentType, nil + } + + // Return default if not found + return "application/octet-stream", nil +} diff --git a/attachment/s3/storage_test.go b/attachment/s3/storage_test.go index b8750d59..06179fcc 100644 --- a/attachment/s3/storage_test.go +++ b/attachment/s3/storage_test.go @@ -5,9 +5,13 @@ import ( "context" "io" "os" + "path/filepath" + "strings" "testing" "time" + "compress/gzip" + "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -196,4 +200,164 @@ func TestS3Storage(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "bucket is required") }) + + t.Run("LocalPath", func(t *testing.T) { + skipIfNoS3Config(t) + + // Create storage with custom cache directory + tempCacheDir, err := os.MkdirTemp("", "s3_cache_test") + assert.NoError(t, err) + defer os.RemoveAll(tempCacheDir) + + config := getS3Config() + config["cache_dir"] = tempCacheDir + + storage, err := New(config) + assert.NoError(t, err) + + // Test different file types + testFiles := []struct { + name string + content []byte + contentType string + expectedCT string + }{ + {"test.txt", []byte("Hello S3 World"), "text/plain", "text/plain"}, + {"test.json", []byte(`{"s3": "test"}`), "application/json", "application/json"}, + {"test.html", []byte("S3 Test"), "text/html", "text/html"}, + {"test.csv", []byte("s3,test\nval1,val2"), "text/csv", "text/csv"}, + {"test.md", []byte("# S3 Markdown"), "text/markdown", "text/markdown"}, + {"test.yao", []byte("s3 yao content"), "application/yao", "application/yao"}, + } + + for _, tf := range testFiles { + // Upload file to S3 + fileID := "s3-localpath-" + uuid.New().String() + "-" + tf.name + _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(tf.content), tf.contentType) + assert.NoError(t, err, "Failed to upload %s", tf.name) + + // Get local path - first call should download to cache + localPath1, detectedCT1, err := storage.LocalPath(context.Background(), fileID) + assert.NoError(t, err, "Failed to get local path for %s", tf.name) + assert.NotEmpty(t, localPath1, "Local path should not be empty for %s", tf.name) + assert.Equal(t, tf.expectedCT, detectedCT1, "Content type mismatch for %s", tf.name) + + // Verify the path is absolute + assert.True(t, filepath.IsAbs(localPath1), "Path should be absolute for %s", tf.name) + + // Verify the file exists at the returned path + _, err = os.Stat(localPath1) + assert.NoError(t, err, "File should exist at local path for %s", tf.name) + + // Verify file content + fileContent, err := os.ReadFile(localPath1) + assert.NoError(t, err, "Failed to read file at local path for %s", tf.name) + assert.Equal(t, tf.content, fileContent, "File content mismatch for %s", tf.name) + + // Get local path again - should use cached version + localPath2, detectedCT2, err := storage.LocalPath(context.Background(), fileID) + assert.NoError(t, err, "Failed to get cached local path for %s", tf.name) + assert.Equal(t, localPath1, localPath2, "Cached path should be same as first call for %s", tf.name) + assert.Equal(t, detectedCT1, detectedCT2, "Cached content type should be same as first call for %s", tf.name) + + // Clean up from S3 + storage.Delete(context.Background(), fileID) + } + }) + + t.Run("LocalPath_GzippedFile", func(t *testing.T) { + skipIfNoS3Config(t) + + // Create storage with custom cache directory + tempCacheDir, err := os.MkdirTemp("", "s3_cache_gzip_test") + assert.NoError(t, err) + defer os.RemoveAll(tempCacheDir) + + config := getS3Config() + config["cache_dir"] = tempCacheDir + + storage, err := New(config) + assert.NoError(t, err) + + // Create gzipped content + originalContent := []byte("This content will be gzipped and stored in S3") + var gzipBuf bytes.Buffer + gzipWriter := gzip.NewWriter(&gzipBuf) + _, err = gzipWriter.Write(originalContent) + assert.NoError(t, err) + gzipWriter.Close() + + // Upload gzipped file + fileID := "gzipped-" + uuid.New().String() + ".txt.gz" + _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(gzipBuf.Bytes()), "text/plain") + assert.NoError(t, err) + + // Get local path - should decompress during download + localPath, contentType, err := storage.LocalPath(context.Background(), fileID) + assert.NoError(t, err) + assert.NotEmpty(t, localPath) + + // Verify the file is decompressed in cache (path should not end with .gz) + assert.False(t, strings.HasSuffix(localPath, ".gz"), "Cached file should be decompressed") + + // Verify content is decompressed + cachedContent, err := os.ReadFile(localPath) + assert.NoError(t, err) + assert.Equal(t, originalContent, cachedContent, "Cached file should contain decompressed content") + + // Verify content type + assert.Equal(t, "text/plain", contentType) + + // Clean up + storage.Delete(context.Background(), fileID) + }) + + t.Run("LocalPath_NonExistentFile", func(t *testing.T) { + skipIfNoS3Config(t) + + storage, err := New(getS3Config()) + assert.NoError(t, err) + + // Test with non-existent file + nonExistentFileID := "non-existent-" + uuid.New().String() + ".txt" + _, _, err = storage.LocalPath(context.Background(), nonExistentFileID) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to download file") + }) + + t.Run("LocalPath_CustomCacheDir", func(t *testing.T) { + skipIfNoS3Config(t) + + // Create custom cache directory + customCacheDir, err := os.MkdirTemp("", "custom_s3_cache") + assert.NoError(t, err) + defer os.RemoveAll(customCacheDir) + + config := getS3Config() + config["cache_dir"] = customCacheDir + + storage, err := New(config) + assert.NoError(t, err) + + // Verify cache directory is set correctly + assert.Equal(t, customCacheDir, storage.CacheDir) + + // Upload a test file + content := []byte("Custom cache directory test") + fileID := "custom-cache-" + uuid.New().String() + ".txt" + _, err = storage.Upload(context.Background(), fileID, bytes.NewReader(content), "text/plain") + assert.NoError(t, err) + + // Get local path + localPath, contentType, err := storage.LocalPath(context.Background(), fileID) + assert.NoError(t, err) + assert.NotEmpty(t, localPath) + assert.Equal(t, "text/plain", contentType) + + // Verify the file is cached in the custom directory + assert.True(t, strings.HasPrefix(localPath, customCacheDir), "File should be cached in custom directory") + + // Clean up + storage.Delete(context.Background(), fileID) + }) } diff --git a/attachment/types.go b/attachment/types.go index f5526729..99c9232c 100644 --- a/attachment/types.go +++ b/attachment/types.go @@ -43,6 +43,9 @@ type FileManager interface { // Delete deletes a file Delete(ctx context.Context, fileID string) error + + // LocalPath gets the local path of the file + LocalPath(ctx context.Context, fileID string) (string, string, error) } // File the file @@ -103,6 +106,7 @@ type Storage interface { URL(ctx context.Context, path string) string Exists(ctx context.Context, path string) bool Delete(ctx context.Context, path string) error + LocalPath(ctx context.Context, path string) (string, string, error) // Returns absolute path and content type } // ManagerOption the manager option diff --git a/openapi/kb/document.go b/openapi/kb/document.go index 533408fb..87ecd240 100644 --- a/openapi/kb/document.go +++ b/openapi/kb/document.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/yaoapp/yao/attachment" "github.com/yaoapp/yao/kb" "github.com/yaoapp/yao/openapi/response" ) @@ -12,8 +13,19 @@ import ( // AddFile adds a file to a collection func AddFile(c *gin.Context) { + var req AddFileRequest + // Check if kb.Instance is available + if kb.Instance == nil { + errorResp := &response.ErrorResponse{ + Code: response.ErrServerError.Code, + ErrorDescription: "Knowledge base not initialized", + } + response.RespondWithError(c, response.StatusInternalServerError, errorResp) + return + } + // Parse and bind JSON request if err := c.ShouldBindJSON(&req); err != nil { errorResp := &response.ErrorResponse{ @@ -34,24 +46,41 @@ func AddFile(c *gin.Context) { return } - // Check if kb.Instance is available - if kb.Instance == nil { + // Get file manager + m, ok := attachment.Managers[req.Uploader] + if !ok { + errorResp := &response.ErrorResponse{ + Code: response.ErrInvalidRequest.Code, + ErrorDescription: "Invalid uploader: " + req.Uploader + " not found", + } + response.RespondWithError(c, response.StatusNotFound, errorResp) + return + } + + // Check if the file exists + exists := m.Exists(c.Request.Context(), req.FileID) + if !exists { + errorResp := &response.ErrorResponse{ + Code: response.ErrInvalidRequest.Code, + ErrorDescription: "File not found: " + req.FileID, + } + response.RespondWithError(c, response.StatusNotFound, errorResp) + return + } + + // Get the options of the manager + path, contentType, err := m.LocalPath(c.Request.Context(), req.FileID) + if err != nil { errorResp := &response.ErrorResponse{ Code: response.ErrServerError.Code, - ErrorDescription: "Knowledge base not initialized", + ErrorDescription: "Failed to get local path: " + err.Error(), } response.RespondWithError(c, response.StatusInternalServerError, errorResp) return } - // TODO: Call external function to get file info - // filename, contentType, err := GetFileInfo(req.FileID) - // For now, use hardcoded values - filename := "document.pdf" - contentType := "application/pdf" - // Convert request to UpsertOptions - upsertOptions, err := req.BaseUpsertRequest.ToUpsertOptions(filename, contentType) + upsertOptions, err := req.BaseUpsertRequest.ToUpsertOptions(path, contentType) if err != nil { errorResp := &response.ErrorResponse{ Code: response.ErrInvalidRequest.Code, diff --git a/openapi/kb/utils.go b/openapi/kb/utils.go index 37fd1ef2..3321b421 100644 --- a/openapi/kb/utils.go +++ b/openapi/kb/utils.go @@ -91,7 +91,8 @@ type BaseUpsertRequest struct { // AddFileRequest represents the request for AddFile API type AddFileRequest struct { BaseUpsertRequest - FileID string `json:"file_id" binding:"required"` + FileID string `json:"file_id" binding:"required"` + Uploader string `json:"uploader,omitempty"` // The name of the uploader, e.g. "s3", "local", "webdav", etc. } // AddTextRequest represents the request for AddText API