diff --git a/fs/fs.go b/fs/fs.go index f14c8da7..f4c2a309 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -13,13 +13,16 @@ import ( func Load(cfg config.Config) error { scriptRoot := filepath.Join(cfg.AppSource, "scripts") + seedRoot := filepath.Join(cfg.AppSource, "seeds") dslDenyList := []string{scriptRoot, cfg.DataRoot} - fs.Register("system", system.New(cfg.DataRoot)) // alias Data - fs.RootRegister("dsl", dsl.New(cfg.AppSource).DenyAbs(dslDenyList...)) // DSL - fs.RootRegister("script", system.New(scriptRoot)) // Script + fs.Register("app", system.New(cfg.AppSource)) // App Soruce root path, it's an dangerous operation, be careful to use it. + fs.Register("data", system.New(cfg.DataRoot)) // Data root + fs.Register("seed", system.New(seedRoot).ReadOnly()) // Seed read only file system, for initial data seeding - fs.Register("app", system.New(cfg.AppSource)) // App Soruce root path, it's an dangerous operation, be careful to use it. - fs.Register("data", system.New(cfg.DataRoot)) // Data root + // Deprecated: DO NOT USE SYSTEM, DSL AND SCRIPT IN THE FUTURE, THEY WILL BE DEPRECATED IN THE FUTURE + fs.Register("system", system.New(cfg.DataRoot)) // alias Data + fs.RootRegister("dsl", dsl.New(cfg.AppSource).DenyAbs(dslDenyList...)) // DSL ( will be deprecated in the future) + fs.RootRegister("script", system.New(scriptRoot)) // Script ( will be deprecated in the future) return nil } diff --git a/main.go b/main.go index 420ff794..e5960a48 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( _ "github.com/yaoapp/yao/excel" _ "github.com/yaoapp/yao/helper" _ "github.com/yaoapp/yao/openai" + _ "github.com/yaoapp/yao/seed" _ "github.com/yaoapp/yao/wework" "github.com/yaoapp/yao/cmd" diff --git a/seed/process.go b/seed/process.go new file mode 100644 index 00000000..93878e66 --- /dev/null +++ b/seed/process.go @@ -0,0 +1,173 @@ +package seed + +import ( + "fmt" + + "github.com/yaoapp/gou/process" + "github.com/yaoapp/kun/exception" + "github.com/yaoapp/kun/maps" +) + +func init() { + process.RegisterGroup("seeds", map[string]process.Handler{ + "import": processSeedImport, + }) +} + +func processSeedImport(process *process.Process) interface{} { + process.ValidateArgNums(2) + filename := process.ArgsString(0) + modelName := process.ArgsString(1) + + // Default options + options := ImportOption{ + ChunkSize: ChunkSizeDefault, + Duplicate: DuplicateIgnore, + Mode: ImportModeBatch, + } + + // Parse options if provided + if process.NumOfArgs() > 2 { + opts, err := getOptions(process.Args[2]) + if err != nil { + exception.New(err.Error(), 500).Throw() + } + if opts.ChunkSize > 0 { + options.ChunkSize = opts.ChunkSize + } + if opts.Duplicate != "" { + options.Duplicate = opts.Duplicate + } + if opts.Mode != "" { + options.Mode = opts.Mode + } + } + + // Import seed data + result, err := Import(filename, modelName, options) + if err != nil { + exception.New(err.Error(), 500).Throw() + } + + return result +} + +// getOptions parses import options from interface +func getOptions(v interface{}) (ImportOption, error) { + opts := ImportOption{ + ChunkSize: ChunkSizeDefault, + Duplicate: DuplicateIgnore, + Mode: ImportModeBatch, + } + + switch val := v.(type) { + case map[string]interface{}: + if chunkSize, exists := val["chunk_size"]; exists { + if cs := toInt(chunkSize); cs > 0 { + opts.ChunkSize = cs + } + } + if duplicate, exists := val["duplicate"]; exists { + if dup := toString(duplicate); dup != "" { + opts.Duplicate = DuplicateMode(dup) + } + } + if mode, exists := val["mode"]; exists { + if m := toString(mode); m != "" { + opts.Mode = ImportMode(m) + } + } + + case maps.MapStr: + if chunkSize := val.Get("chunk_size"); chunkSize != nil { + if cs := toInt(chunkSize); cs > 0 { + opts.ChunkSize = cs + } + } + if duplicate := val.Get("duplicate"); duplicate != nil { + if dup := toString(duplicate); dup != "" { + opts.Duplicate = DuplicateMode(dup) + } + } + if mode := val.Get("mode"); mode != nil { + if m := toString(mode); m != "" { + opts.Mode = ImportMode(m) + } + } + + case ImportOption: + opts = val + + default: + return opts, nil + } + + // Validate options + if opts.ChunkSize <= 0 { + opts.ChunkSize = ChunkSizeDefault + } + + return opts, nil +} + +// toInt converts various types to int +func toInt(v interface{}) int { + if v == nil { + return 0 + } + + switch val := v.(type) { + case int: + return val + case int8: + return int(val) + case int16: + return int(val) + case int32: + return int(val) + case int64: + return int(val) + case uint: + return int(val) + case uint8: + return int(val) + case uint16: + return int(val) + case uint32: + return int(val) + case uint64: + return int(val) + case float32: + return int(val) + case float64: + return int(val) + case string: + // Try to parse string as number + if i, err := parseIntString(val); err == nil { + return i + } + } + return 0 +} + +// toString converts various types to string +func toString(v interface{}) string { + if v == nil { + return "" + } + + switch val := v.(type) { + case string: + return val + case []byte: + return string(val) + } + return "" +} + +// parseIntString parses a string to int +func parseIntString(s string) (int, error) { + var i int + _, err := fmt.Sscanf(s, "%d", &i) + return i, err +} diff --git a/seed/process_test.go b/seed/process_test.go new file mode 100644 index 00000000..3d17af84 --- /dev/null +++ b/seed/process_test.go @@ -0,0 +1,537 @@ +package seed + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +func TestProcessSeedImportCSV(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Test importing CSV file using process + p, err := process.Of("seeds.import", "roles.csv", "__yao.role") + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") +} + +func TestProcessSeedImportJSON(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Test importing JSON file using process + p, err := process.Of("seeds.import", "roles.json", "__yao.role") + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") + + // Check that JSON data was imported correctly + adminRoles, _ := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + if len(adminRoles) > 0 { + assert.Equal(t, "admin", adminRoles[0].Get("role_id")) + assert.NotNil(t, adminRoles[0].Get("permissions"), "Should have permissions") + } +} + +func TestProcessSeedImportXLSX(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Test importing XLSX file using process + p, err := process.Of("seeds.import", "roles.xlsx", "__yao.role") + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") + + // Check specific role + adminRoles, _ := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + if len(adminRoles) > 0 { + assert.Equal(t, "admin", adminRoles[0].Get("role_id")) + assert.Equal(t, "Administrator", adminRoles[0].Get("name")) + } +} + +func TestProcessSeedImportYao(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Test importing JSONC file using process + p, err := process.Of("seeds.import", "roles.yao", "__yao.role") + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") +} + +func TestProcessSeedImportWithBatchMode(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Test importing with batch mode and custom chunk size + options := map[string]interface{}{ + "chunk_size": 2, + "duplicate": "ignore", + "mode": "batch", + } + + p, err := process.Of("seeds.import", "roles.json", "__yao.role", options) + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") +} + +func TestProcessSeedImportWithEachMode(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Test importing with each mode + options := map[string]interface{}{ + "mode": "each", + "duplicate": "ignore", + } + + p, err := process.Of("seeds.import", "roles.json", "__yao.role", options) + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") +} + +func TestProcessSeedImportDuplicateStrategies(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Test ignore strategy + t.Run("DuplicateIgnore", func(t *testing.T) { + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // First import + p, err := process.Of("seeds.import", "roles.json", "__yao.role") + assert.NoError(t, err) + result1, err := p.Exec() + assert.NoError(t, err) + resultMap1 := result1.(*ImportResult) + firstSuccess := resultMap1.Success + + // Second import with ignore + p, err = process.Of("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "mode": "each", + "duplicate": "ignore", + }) + assert.NoError(t, err) + result2, err := p.Exec() + assert.NoError(t, err) + + resultMap2 := result2.(*ImportResult) + assert.Greater(t, resultMap2.Ignore, 0, "Should have ignored duplicates") + + // Verify count hasn't changed + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Equal(t, firstSuccess, len(roles), "Should have same number of roles") + }) + + // Test error strategy + t.Run("DuplicateError", func(t *testing.T) { + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // First import + p, err := process.Of("seeds.import", "roles.json", "__yao.role") + assert.NoError(t, err) + _, err = p.Exec() + assert.NoError(t, err) + + // Second import with error strategy + p, err = process.Of("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "mode": "each", + "duplicate": "error", + }) + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + + resultMap := result.(*ImportResult) + assert.Greater(t, resultMap.Failure, 0, "Should have failures for duplicates") + }) +} + +func TestProcessSeedImportInvalidArguments(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Test with missing arguments + t.Run("MissingArguments", func(t *testing.T) { + p, err := process.Of("seeds.import", "roles.csv") + assert.NoError(t, err) + _, err = p.Exec() + assert.Error(t, err, "Should fail with missing model argument") + }) + + // Test with invalid file + t.Run("InvalidFile", func(t *testing.T) { + p, err := process.Of("seeds.import", "nonexistent.csv", "__yao.role") + assert.NoError(t, err) + _, err = p.Exec() + assert.Error(t, err, "Should fail with non-existent file") + }) + + // Test with invalid model + t.Run("InvalidModel", func(t *testing.T) { + p, err := process.Of("seeds.import", "roles.csv", "nonexistent.model") + assert.NoError(t, err) + _, err = p.Exec() + assert.Error(t, err, "Should fail with non-existent model") + }) + + // Test with unsupported file format + t.Run("UnsupportedFormat", func(t *testing.T) { + p, err := process.Of("seeds.import", "roles.txt", "__yao.role") + assert.NoError(t, err) + _, err = p.Exec() + assert.Error(t, err, "Should fail with unsupported file format") + }) +} + +func TestProcessSeedImportOptions(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Test with various option types + t.Run("OptionsAsMap", func(t *testing.T) { + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + options := map[string]interface{}{ + "chunk_size": 100, + "duplicate": "ignore", + "mode": "batch", + } + + p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options) + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + assert.NotNil(t, result) + }) + + // Test with float64 chunk_size (JSON numbers) + t.Run("OptionsWithFloat64", func(t *testing.T) { + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + options := map[string]interface{}{ + "chunk_size": float64(200), + } + + p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options) + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + assert.NotNil(t, result) + }) + + // Test with partial options + t.Run("PartialOptions", func(t *testing.T) { + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + options := map[string]interface{}{ + "mode": "each", + } + + p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options) + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + assert.NotNil(t, result) + }) + + // Test with empty options + t.Run("EmptyOptions", func(t *testing.T) { + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + options := map[string]interface{}{} + + p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options) + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + assert.NotNil(t, result) + }) +} + +func TestProcessSeedImportResultStructure(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import data + p, err := process.Of("seeds.import", "roles.csv", "__yao.role") + if err != nil { + t.Fatal(err) + } + + result, err := p.Exec() + if err != nil { + t.Fatal(err) + } + + // Verify result structure + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult type") + + // Check all fields exist and have proper types + assert.GreaterOrEqual(t, resultMap.Total, 0, "Total should be non-negative") + assert.GreaterOrEqual(t, resultMap.Success, 0, "Success should be non-negative") + assert.GreaterOrEqual(t, resultMap.Failure, 0, "Failure should be non-negative") + assert.GreaterOrEqual(t, resultMap.Ignore, 0, "Ignore should be non-negative") + assert.NotNil(t, resultMap.Errors, "Errors should not be nil") + + // Verify total = success + failure + ignore + assert.Equal(t, resultMap.Total, resultMap.Success+resultMap.Failure+resultMap.Ignore, + "Total should equal sum of success, failure, and ignore") +} + +func TestProcessSeedImportMultipleFiles(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import CSV + t.Run("ImportCSV", func(t *testing.T) { + p, err := process.Of("seeds.import", "roles.csv", "__yao.role") + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + resultMap := result.(*ImportResult) + assert.Greater(t, resultMap.Success, 0) + }) + + // Clear and import JSON + t.Run("ImportJSON", func(t *testing.T) { + _, _ = mod.DestroyWhere(model.QueryParam{}) + p, err := process.Of("seeds.import", "roles.json", "__yao.role") + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + resultMap := result.(*ImportResult) + assert.Greater(t, resultMap.Success, 0) + }) + + // Clear and import XLSX + t.Run("ImportXLSX", func(t *testing.T) { + _, _ = mod.DestroyWhere(model.QueryParam{}) + p, err := process.Of("seeds.import", "roles.xlsx", "__yao.role") + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + resultMap := result.(*ImportResult) + assert.Greater(t, resultMap.Success, 0) + }) + + // Clear and import Yao + t.Run("ImportYao", func(t *testing.T) { + _, _ = mod.DestroyWhere(model.QueryParam{}) + p, err := process.Of("seeds.import", "roles.yao", "__yao.role") + assert.NoError(t, err) + result, err := p.Exec() + assert.NoError(t, err) + resultMap := result.(*ImportResult) + assert.Greater(t, resultMap.Success, 0) + }) +} diff --git a/seed/seed.go b/seed/seed.go new file mode 100644 index 00000000..2621ea2b --- /dev/null +++ b/seed/seed.go @@ -0,0 +1,503 @@ +package seed + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/xuri/excelize/v2" + "github.com/yaoapp/gou/application" + "github.com/yaoapp/gou/fs" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/kun/log" + "github.com/yaoapp/kun/maps" +) + +// Import imports seed data from file into model +func Import(filename string, modelName string, options ImportOption) (*ImportResult, error) { + // Get model + mod := model.Select(modelName) + + // Initialize result + result := &ImportResult{ + Total: 0, + Success: 0, + Failure: 0, + Ignore: 0, + Errors: []ImportError{}, + } + + // Determine file type and import + ext := strings.ToLower(filepath.Ext(filename)) + + switch ext { + case ".csv": + return result, importDataFromCSV(filename, mod, options, result) + case ".xlsx", ".xls": + return result, importDataFromXLSX(filename, mod, options, result) + case ".json": + return result, importDataFromJSON(filename, mod, options, result) + case ".yao", ".jsonc": + return result, importDataFromYao(filename, mod, options, result) + default: + return nil, fmt.Errorf("unsupported file format: %s", ext) + } +} + +// importDataFromCSV import data from CSV file +func importDataFromCSV(filename string, mod *model.Model, options ImportOption, result *ImportResult) error { + // Read file from seed filesystem + seedFS := fs.MustGet("seed") + data, err := seedFS.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read CSV file: %v", err) + } + + // Parse CSV + reader := csv.NewReader(strings.NewReader(string(data))) + reader.FieldsPerRecord = -1 // Allow variable number of fields + + // Read header + header, err := reader.Read() + if err != nil { + return fmt.Errorf("failed to read CSV header: %v", err) + } + + // Prepare handler + handler := createImportHandler(mod, header, options, result) + + // Read data in chunks + chunk := [][]interface{}{} + lineNum := 1 // Start from 1 (header is line 0) + + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + result.Errors = append(result.Errors, ImportError{ + Row: lineNum, + Message: err.Error(), + Code: 500, + }) + result.Failure++ + result.Total++ + lineNum++ + continue + } + + // Convert to interface slice + row := make([]interface{}, len(record)) + for i, v := range record { + row[i] = v + } + + chunk = append(chunk, row) + result.Total++ + + // Process chunk when size reached + if len(chunk) >= options.ChunkSize { + if err := handler(lineNum-len(chunk)+1, chunk); err != nil { + log.Error("Import chunk error: %v", err) + } + chunk = [][]interface{}{} + } + + lineNum++ + } + + // Process remaining chunk + if len(chunk) > 0 { + if err := handler(lineNum-len(chunk), chunk); err != nil { + log.Error("Import final chunk error: %v", err) + } + } + + return nil +} + +// importDataFromXLSX import data from XLSX file +func importDataFromXLSX(filename string, mod *model.Model, options ImportOption, result *ImportResult) error { + // Read file from seed filesystem + seedFS := fs.MustGet("seed") + data, err := seedFS.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read XLSX file: %v", err) + } + + // Open Excel file from bytes + file, err := excelize.OpenReader(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to open XLSX file: %v", err) + } + defer file.Close() + + // Get active sheet + sheetName := file.GetSheetName(file.GetActiveSheetIndex()) + rows, err := file.Rows(sheetName) + if err != nil { + return fmt.Errorf("failed to get rows: %v", err) + } + defer rows.Close() + + // Read header + if !rows.Next() { + return fmt.Errorf("empty XLSX file") + } + header, err := rows.Columns() + if err != nil { + return fmt.Errorf("failed to read header: %v", err) + } + + // Prepare handler + handler := createImportHandler(mod, header, options, result) + + // Read data in chunks + chunk := [][]interface{}{} + lineNum := 1 // Header is line 0, data starts from 1 + + for rows.Next() { + record, err := rows.Columns() + if err != nil { + result.Errors = append(result.Errors, ImportError{ + Row: lineNum, + Message: err.Error(), + Code: 500, + }) + result.Failure++ + result.Total++ + lineNum++ + continue + } + + // Check if row is empty + isEmpty := true + for _, v := range record { + if v != "" { + isEmpty = false + break + } + } + if isEmpty { + lineNum++ + continue + } + + // Convert to interface slice + row := make([]interface{}, len(record)) + for i, v := range record { + row[i] = v + } + + chunk = append(chunk, row) + result.Total++ + + // Process chunk when size reached + if len(chunk) >= options.ChunkSize { + if err := handler(lineNum-len(chunk)+1, chunk); err != nil { + log.Error("Import chunk error: %v", err) + } + chunk = [][]interface{}{} + } + + lineNum++ + } + + // Process remaining chunk + if len(chunk) > 0 { + if err := handler(lineNum-len(chunk), chunk); err != nil { + log.Error("Import final chunk error: %v", err) + } + } + + return nil +} + +// importDataFromJSON import data from JSON file +func importDataFromJSON(filename string, mod *model.Model, options ImportOption, result *ImportResult) error { + // Read file from seed filesystem + seedFS := fs.MustGet("seed") + data, err := seedFS.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read JSON file: %v", err) + } + + // Parse JSON - expect array of objects + var records []map[string]interface{} + if err := json.Unmarshal(data, &records); err != nil { + return fmt.Errorf("failed to parse JSON: %v", err) + } + + if len(records) == 0 { + return nil + } + + // Extract columns from first record + columns := []string{} + for key := range records[0] { + columns = append(columns, key) + } + + // Convert to rows format + handler := createJSONImportHandler(mod, columns, options, result) + + // Process records in chunks + chunk := []map[string]interface{}{} + for i, record := range records { + result.Total++ + chunk = append(chunk, record) + + if len(chunk) >= options.ChunkSize { + if err := handler(i-len(chunk)+1, chunk); err != nil { + log.Error("Import chunk error: %v", err) + } + chunk = []map[string]interface{}{} + } + } + + // Process remaining chunk + if len(chunk) > 0 { + if err := handler(len(records)-len(chunk), chunk); err != nil { + log.Error("Import final chunk error: %v", err) + } + } + + return nil +} + +// importDataFromYao import data from Yao file (JSONC) +func importDataFromYao(filename string, mod *model.Model, options ImportOption, result *ImportResult) error { + // Read file from seed filesystem + seedFS := fs.MustGet("seed") + data, err := seedFS.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read Yao file: %v", err) + } + + // Parse using application Parse (handles JSONC) + var records []map[string]interface{} + if err := application.Parse(filename, data, &records); err != nil { + return fmt.Errorf("failed to parse Yao file: %v", err) + } + + if len(records) == 0 { + return nil + } + + // Extract columns from first record + columns := []string{} + for key := range records[0] { + columns = append(columns, key) + } + + // Convert to rows format + handler := createJSONImportHandler(mod, columns, options, result) + + // Process records in chunks + chunk := []map[string]interface{}{} + for i, record := range records { + result.Total++ + chunk = append(chunk, record) + + if len(chunk) >= options.ChunkSize { + if err := handler(i-len(chunk)+1, chunk); err != nil { + log.Error("Import chunk error: %v", err) + } + chunk = []map[string]interface{}{} + } + } + + // Process remaining chunk + if len(chunk) > 0 { + if err := handler(len(records)-len(chunk), chunk); err != nil { + log.Error("Import final chunk error: %v", err) + } + } + + return nil +} + +// createImportHandler creates handler for CSV/XLSX format +func createImportHandler(mod *model.Model, columns []string, options ImportOption, result *ImportResult) ImportHandler { + return func(line int, data [][]interface{}) error { + if options.Mode == ImportModeEach { + // Single record mode - use Create + return importEach(mod, columns, data, line, options, result) + } + // Batch mode - use Insert + return importBatch(mod, columns, data, line, options, result) + } +} + +// createJSONImportHandler creates handler for JSON/Yao format +func createJSONImportHandler(mod *model.Model, columns []string, options ImportOption, result *ImportResult) func(line int, data []map[string]interface{}) error { + return func(line int, data []map[string]interface{}) error { + if options.Mode == ImportModeEach { + // Single record mode - use Create or Save + return importEachJSON(mod, data, line, options, result) + } + // Batch mode - convert to rows and use Insert + rows := make([][]interface{}, len(data)) + for i, record := range data { + row := make([]interface{}, len(columns)) + for j, col := range columns { + row[j] = record[col] + } + rows[i] = row + } + return importBatch(mod, columns, rows, line, options, result) + } +} + +// importBatch batch import using Model.Insert +func importBatch(mod *model.Model, columns []string, data [][]interface{}, startLine int, options ImportOption, result *ImportResult) error { + if len(data) == 0 { + return nil + } + + switch options.Duplicate { + case DuplicateIgnore: + // Try to insert, ignore errors + err := mod.Insert(columns, data) + if err != nil { + // Log error but don't fail + log.Warn("Batch insert with ignore strategy: %v", err) + result.Ignore += len(data) + } else { + result.Success += len(data) + } + + case DuplicateError: + // Insert and fail on error + err := mod.Insert(columns, data) + if err != nil { + for i := range data { + result.Errors = append(result.Errors, ImportError{ + Row: startLine + i, + Message: err.Error(), + Code: 500, + Data: data[i], + }) + } + result.Failure += len(data) + return err + } + result.Success += len(data) + + case DuplicateUpdate, DuplicateAbort: + // For update/abort, fall back to each mode + for i, row := range data { + rowMap := maps.MakeMapStrAny() + for j, col := range columns { + if j < len(row) { + rowMap[col] = row[j] + } + } + if err := handleDuplicate(mod, rowMap, startLine+i, options.Duplicate, result); err != nil { + if options.Duplicate == DuplicateAbort { + return err + } + } + } + } + + return nil +} + +// importEach single record import using Model.Create +func importEach(mod *model.Model, columns []string, data [][]interface{}, startLine int, options ImportOption, result *ImportResult) error { + for i, row := range data { + // Convert row to map + rowMap := maps.MakeMapStrAny() + for j, col := range columns { + if j < len(row) { + rowMap[col] = row[j] + } + } + + if err := handleDuplicate(mod, rowMap, startLine+i, options.Duplicate, result); err != nil { + if options.Duplicate == DuplicateAbort { + return err + } + } + } + return nil +} + +// importEachJSON single record import for JSON format +func importEachJSON(mod *model.Model, data []map[string]interface{}, startLine int, options ImportOption, result *ImportResult) error { + for i, record := range data { + rowMap := maps.MapStrAny(record) + if err := handleDuplicate(mod, rowMap, startLine+i, options.Duplicate, result); err != nil { + if options.Duplicate == DuplicateAbort { + return err + } + } + } + return nil +} + +// handleDuplicate handles duplicate strategy for single record +func handleDuplicate(mod *model.Model, row maps.MapStrAny, line int, duplicateMode DuplicateMode, result *ImportResult) error { + switch duplicateMode { + case DuplicateIgnore: + // Try to create, ignore if exists + _, err := mod.Create(row) + if err != nil { + result.Ignore++ + log.Debug("Row %d ignored: %v", line, err) + } else { + result.Success++ + } + + case DuplicateUpdate: + // Use Save (create or update) + _, err := mod.Save(row) + if err != nil { + result.Errors = append(result.Errors, ImportError{ + Row: line, + Message: err.Error(), + Code: 500, + }) + result.Failure++ + } else { + result.Success++ + } + + case DuplicateError: + // Create and fail on error + _, err := mod.Create(row) + if err != nil { + result.Errors = append(result.Errors, ImportError{ + Row: line, + Message: err.Error(), + Code: 500, + }) + result.Failure++ + return err + } + result.Success++ + + case DuplicateAbort: + // Create and abort on error + _, err := mod.Create(row) + if err != nil { + result.Errors = append(result.Errors, ImportError{ + Row: line, + Message: err.Error(), + Code: 500, + }) + result.Failure++ + return fmt.Errorf("import aborted at line %d: %v", line, err) + } + result.Success++ + } + + return nil +} + diff --git a/seed/seed_test.go b/seed/seed_test.go new file mode 100644 index 00000000..83bde7f6 --- /dev/null +++ b/seed/seed_test.go @@ -0,0 +1,324 @@ +package seed + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestSeedImportCSV tests importing roles from CSV file +func TestSeedImportCSV(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import CSV + p := process.New("seeds.import", "roles.csv", "__yao.role") + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + assert.Equal(t, resultMap.Total, resultMap.Success+resultMap.Failure+resultMap.Ignore, + "Total should equal sum of success, failure, and ignore") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") + + // Check specific role + adminRoles, _ := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + if len(adminRoles) > 0 { + assert.Equal(t, "admin", adminRoles[0].Get("role_id")) + assert.Equal(t, "Administrator", adminRoles[0].Get("name")) + } +} + +// TestSeedImportJSON tests importing roles from JSON file +func TestSeedImportJSON(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import JSON + p := process.New("seeds.import", "roles.json", "__yao.role") + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") + + // Check that permissions JSON was imported correctly + adminRoles, _ := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + if len(adminRoles) > 0 { + assert.Equal(t, "admin", adminRoles[0].Get("role_id")) + assert.NotNil(t, adminRoles[0].Get("permissions"), "Should have permissions") + } +} + +// TestSeedImportXLSX tests importing roles from XLSX file +func TestSeedImportXLSX(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import XLSX file + p := process.New("seeds.import", "roles.xlsx", "__yao.role") + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") + + // Check specific role + adminRoles, _ := mod.Get(model.QueryParam{ + Wheres: []model.QueryWhere{ + {Column: "role_id", Value: "admin"}, + }, + }) + if len(adminRoles) > 0 { + assert.Equal(t, "admin", adminRoles[0].Get("role_id")) + assert.Equal(t, "Administrator", adminRoles[0].Get("name")) + } +} + +// TestSeedImportYao tests importing roles from Yao file (JSONC) +func TestSeedImportYao(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import Yao file + p := process.New("seeds.import", "roles.yao", "__yao.role") + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok, "Result should be ImportResult") + assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record") + assert.Greater(t, resultMap.Success, 0, "Should have successful imports") + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0, "Should have roles in database") +} + +// TestSeedImportWithOptions tests importing with custom options +func TestSeedImportWithOptions(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import with batch mode + p := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "chunk_size": 2, + "duplicate": "ignore", + "mode": "batch", + }) + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok) + assert.Greater(t, resultMap.Success, 0) + + // Try importing again with ignore strategy + p2 := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "duplicate": "ignore", + }) + result2 := p2.Run() + + resultMap2, ok := result2.(*ImportResult) + assert.True(t, ok) + // With ignore strategy, duplicates should be ignored + assert.Greater(t, resultMap2.Ignore, 0, "Should have ignored duplicates") +} + +// TestSeedImportEachMode tests importing with each mode (single record) +func TestSeedImportEachMode(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import with each mode + p := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "mode": "each", + "duplicate": "ignore", + }) + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok) + assert.Greater(t, resultMap.Success, 0) + + // Verify data in database + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0) +} + +// TestSeedImportDuplicateIgnore tests importing with ignore duplicate strategy +func TestSeedImportDuplicateIgnore(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // First import + p1 := process.New("seeds.import", "roles.json", "__yao.role") + result1 := p1.Run() + resultMap1, ok := result1.(*ImportResult) + assert.True(t, ok) + firstSuccess := resultMap1.Success + + // Get the IDs of imported records + roles1, err := mod.Get(model.QueryParam{ + Select: []interface{}{"id", "role_id"}, + }) + assert.Nil(t, err) + assert.Greater(t, len(roles1), 0) + + // Second import with ignore mode + p2 := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "mode": "each", + "duplicate": "ignore", + }) + result2 := p2.Run() + + // Verify result + assert.NotNil(t, result2) + resultMap2, ok := result2.(*ImportResult) + assert.True(t, ok) + // With ignore strategy, duplicates should be ignored + assert.Greater(t, resultMap2.Ignore, 0, "Should have ignored duplicates") + + // Verify count hasn't changed (ignored duplicates, not created new) + roles2, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Equal(t, firstSuccess, len(roles2), "Should have same number of roles after re-import") +} + +// TestSeedImportChunkSize tests chunk processing +func TestSeedImportChunkSize(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + // Clear existing roles + mod := model.Select("__yao.role") + _, _ = mod.DestroyWhere(model.QueryParam{}) + + // Import with small chunk size + p := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{ + "chunk_size": 1, // Process one record at a time + "mode": "batch", + }) + result := p.Run() + + // Verify result + assert.NotNil(t, result) + resultMap, ok := result.(*ImportResult) + assert.True(t, ok) + assert.Greater(t, resultMap.Success, 0) + + // Verify all data imported correctly + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Greater(t, len(roles), 0) +} diff --git a/seed/types.go b/seed/types.go new file mode 100644 index 00000000..424cb0a5 --- /dev/null +++ b/seed/types.go @@ -0,0 +1,57 @@ +package seed + +// DuplicateMode the duplicate mode +type DuplicateMode string + +// ImportMode the import mode +type ImportMode string + +const ( + + // ImportModeBatch the batch import mode + ImportModeBatch ImportMode = "batch" + // ImportModeEach the each import mode + ImportModeEach ImportMode = "each" + + // DuplicateIgnore when the record is duplicate, ignore the record + DuplicateIgnore DuplicateMode = "ignore" + // DuplicateUpdate when the record is duplicate, update the record + DuplicateUpdate DuplicateMode = "update" + // DuplicateError when the record is duplicate, raise an error + DuplicateError DuplicateMode = "error" + // DuplicateAbort when the record is duplicate, abort the record + DuplicateAbort DuplicateMode = "abort" +) + +const ( + // ChunkSizeDefault the default chunk size + ChunkSizeDefault = 500 +) + +// ImportOption the seed import option +type ImportOption struct { + ChunkSize int `json:"chunk_size,omitempty"` + Duplicate DuplicateMode `json:"duplicate,omitempty"` + Mode ImportMode `json:"mode,omitempty"` +} + +// ImportHandler the seed import handler +type ImportHandler func(line int, data [][]interface{}) error + +// ImportResult the seed import result +type ImportResult struct { + Total int `json:"total,omitempty"` + Success int `json:"success,omitempty"` + Failure int `json:"failure,omitempty"` + Ignore int `json:"ignore,omitempty"` + Errors []ImportError `json:"errors,omitempty"` +} + +// ImportError the seed import error +type ImportError struct { + Row int `json:"row,omitempty"` + Message string `json:"message,omitempty"` + Code int `json:"code,omitempty"` + Data []interface{} `json:"data,omitempty"` +} +