Add seed file system support and deprecate legacy registrations
- Introduced a new read-only file system for seed data, enhancing initial data seeding capabilities. - Updated the file system registration to include the seed root and clarified comments regarding the use of app, data, system, DSL, and script registrations, marking them for future deprecation. - Improved code clarity and maintainability by restructuring file system registrations.
This commit is contained in:
Max
parent
0e260ffb6a
commit
8b9cd9f951
7 changed files with 1603 additions and 5 deletions
11
fs/fs.go
11
fs/fs.go
|
|
@ -13,13 +13,16 @@ import (
|
|||
func Load(cfg config.Config) error {
|
||||
|
||||
scriptRoot := filepath.Join(cfg.AppSource, "scripts")
|
||||
seedRoot := filepath.Join(cfg.AppSource, "seeds")
|
||||
dslDenyList := []string{scriptRoot, cfg.DataRoot}
|
||||
|
||||
fs.Register("system", system.New(cfg.DataRoot)) // alias Data
|
||||
fs.RootRegister("dsl", dsl.New(cfg.AppSource).DenyAbs(dslDenyList...)) // DSL
|
||||
fs.RootRegister("script", system.New(scriptRoot)) // Script
|
||||
|
||||
fs.Register("app", system.New(cfg.AppSource)) // App Soruce root path, it's an dangerous operation, be careful to use it.
|
||||
fs.Register("data", system.New(cfg.DataRoot)) // Data root
|
||||
fs.Register("seed", system.New(seedRoot).ReadOnly()) // Seed read only file system, for initial data seeding
|
||||
|
||||
// Deprecated: DO NOT USE SYSTEM, DSL AND SCRIPT IN THE FUTURE, THEY WILL BE DEPRECATED IN THE FUTURE
|
||||
fs.Register("system", system.New(cfg.DataRoot)) // alias Data
|
||||
fs.RootRegister("dsl", dsl.New(cfg.AppSource).DenyAbs(dslDenyList...)) // DSL ( will be deprecated in the future)
|
||||
fs.RootRegister("script", system.New(scriptRoot)) // Script ( will be deprecated in the future)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
1
main.go
1
main.go
|
|
@ -8,6 +8,7 @@ import (
|
|||
_ "github.com/yaoapp/yao/excel"
|
||||
_ "github.com/yaoapp/yao/helper"
|
||||
_ "github.com/yaoapp/yao/openai"
|
||||
_ "github.com/yaoapp/yao/seed"
|
||||
_ "github.com/yaoapp/yao/wework"
|
||||
|
||||
"github.com/yaoapp/yao/cmd"
|
||||
|
|
|
|||
173
seed/process.go
Normal file
173
seed/process.go
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
package seed
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/yaoapp/gou/process"
|
||||
"github.com/yaoapp/kun/exception"
|
||||
"github.com/yaoapp/kun/maps"
|
||||
)
|
||||
|
||||
func init() {
|
||||
process.RegisterGroup("seeds", map[string]process.Handler{
|
||||
"import": processSeedImport,
|
||||
})
|
||||
}
|
||||
|
||||
func processSeedImport(process *process.Process) interface{} {
|
||||
process.ValidateArgNums(2)
|
||||
filename := process.ArgsString(0)
|
||||
modelName := process.ArgsString(1)
|
||||
|
||||
// Default options
|
||||
options := ImportOption{
|
||||
ChunkSize: ChunkSizeDefault,
|
||||
Duplicate: DuplicateIgnore,
|
||||
Mode: ImportModeBatch,
|
||||
}
|
||||
|
||||
// Parse options if provided
|
||||
if process.NumOfArgs() > 2 {
|
||||
opts, err := getOptions(process.Args[2])
|
||||
if err != nil {
|
||||
exception.New(err.Error(), 500).Throw()
|
||||
}
|
||||
if opts.ChunkSize > 0 {
|
||||
options.ChunkSize = opts.ChunkSize
|
||||
}
|
||||
if opts.Duplicate != "" {
|
||||
options.Duplicate = opts.Duplicate
|
||||
}
|
||||
if opts.Mode != "" {
|
||||
options.Mode = opts.Mode
|
||||
}
|
||||
}
|
||||
|
||||
// Import seed data
|
||||
result, err := Import(filename, modelName, options)
|
||||
if err != nil {
|
||||
exception.New(err.Error(), 500).Throw()
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// getOptions parses import options from interface
|
||||
func getOptions(v interface{}) (ImportOption, error) {
|
||||
opts := ImportOption{
|
||||
ChunkSize: ChunkSizeDefault,
|
||||
Duplicate: DuplicateIgnore,
|
||||
Mode: ImportModeBatch,
|
||||
}
|
||||
|
||||
switch val := v.(type) {
|
||||
case map[string]interface{}:
|
||||
if chunkSize, exists := val["chunk_size"]; exists {
|
||||
if cs := toInt(chunkSize); cs > 0 {
|
||||
opts.ChunkSize = cs
|
||||
}
|
||||
}
|
||||
if duplicate, exists := val["duplicate"]; exists {
|
||||
if dup := toString(duplicate); dup != "" {
|
||||
opts.Duplicate = DuplicateMode(dup)
|
||||
}
|
||||
}
|
||||
if mode, exists := val["mode"]; exists {
|
||||
if m := toString(mode); m != "" {
|
||||
opts.Mode = ImportMode(m)
|
||||
}
|
||||
}
|
||||
|
||||
case maps.MapStr:
|
||||
if chunkSize := val.Get("chunk_size"); chunkSize != nil {
|
||||
if cs := toInt(chunkSize); cs > 0 {
|
||||
opts.ChunkSize = cs
|
||||
}
|
||||
}
|
||||
if duplicate := val.Get("duplicate"); duplicate != nil {
|
||||
if dup := toString(duplicate); dup != "" {
|
||||
opts.Duplicate = DuplicateMode(dup)
|
||||
}
|
||||
}
|
||||
if mode := val.Get("mode"); mode != nil {
|
||||
if m := toString(mode); m != "" {
|
||||
opts.Mode = ImportMode(m)
|
||||
}
|
||||
}
|
||||
|
||||
case ImportOption:
|
||||
opts = val
|
||||
|
||||
default:
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
// Validate options
|
||||
if opts.ChunkSize <= 0 {
|
||||
opts.ChunkSize = ChunkSizeDefault
|
||||
}
|
||||
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
// toInt converts various types to int
|
||||
func toInt(v interface{}) int {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
switch val := v.(type) {
|
||||
case int:
|
||||
return val
|
||||
case int8:
|
||||
return int(val)
|
||||
case int16:
|
||||
return int(val)
|
||||
case int32:
|
||||
return int(val)
|
||||
case int64:
|
||||
return int(val)
|
||||
case uint:
|
||||
return int(val)
|
||||
case uint8:
|
||||
return int(val)
|
||||
case uint16:
|
||||
return int(val)
|
||||
case uint32:
|
||||
return int(val)
|
||||
case uint64:
|
||||
return int(val)
|
||||
case float32:
|
||||
return int(val)
|
||||
case float64:
|
||||
return int(val)
|
||||
case string:
|
||||
// Try to parse string as number
|
||||
if i, err := parseIntString(val); err == nil {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// toString converts various types to string
|
||||
func toString(v interface{}) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
switch val := v.(type) {
|
||||
case string:
|
||||
return val
|
||||
case []byte:
|
||||
return string(val)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseIntString parses a string to int
|
||||
func parseIntString(s string) (int, error) {
|
||||
var i int
|
||||
_, err := fmt.Sscanf(s, "%d", &i)
|
||||
return i, err
|
||||
}
|
||||
537
seed/process_test.go
Normal file
537
seed/process_test.go
Normal file
|
|
@ -0,0 +1,537 @@
|
|||
package seed
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/gou/process"
|
||||
"github.com/yaoapp/yao/config"
|
||||
"github.com/yaoapp/yao/test"
|
||||
)
|
||||
|
||||
func TestProcessSeedImportCSV(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Test importing CSV file using process
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
}
|
||||
|
||||
func TestProcessSeedImportJSON(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Test importing JSON file using process
|
||||
p, err := process.Of("seeds.import", "roles.json", "__yao.role")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
|
||||
// Check that JSON data was imported correctly
|
||||
adminRoles, _ := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
if len(adminRoles) > 0 {
|
||||
assert.Equal(t, "admin", adminRoles[0].Get("role_id"))
|
||||
assert.NotNil(t, adminRoles[0].Get("permissions"), "Should have permissions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSeedImportXLSX(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Test importing XLSX file using process
|
||||
p, err := process.Of("seeds.import", "roles.xlsx", "__yao.role")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
|
||||
// Check specific role
|
||||
adminRoles, _ := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
if len(adminRoles) > 0 {
|
||||
assert.Equal(t, "admin", adminRoles[0].Get("role_id"))
|
||||
assert.Equal(t, "Administrator", adminRoles[0].Get("name"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSeedImportYao(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Test importing JSONC file using process
|
||||
p, err := process.Of("seeds.import", "roles.yao", "__yao.role")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
}
|
||||
|
||||
func TestProcessSeedImportWithBatchMode(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Test importing with batch mode and custom chunk size
|
||||
options := map[string]interface{}{
|
||||
"chunk_size": 2,
|
||||
"duplicate": "ignore",
|
||||
"mode": "batch",
|
||||
}
|
||||
|
||||
p, err := process.Of("seeds.import", "roles.json", "__yao.role", options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
}
|
||||
|
||||
func TestProcessSeedImportWithEachMode(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Test importing with each mode
|
||||
options := map[string]interface{}{
|
||||
"mode": "each",
|
||||
"duplicate": "ignore",
|
||||
}
|
||||
|
||||
p, err := process.Of("seeds.import", "roles.json", "__yao.role", options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
}
|
||||
|
||||
func TestProcessSeedImportDuplicateStrategies(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Test ignore strategy
|
||||
t.Run("DuplicateIgnore", func(t *testing.T) {
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// First import
|
||||
p, err := process.Of("seeds.import", "roles.json", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
result1, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
resultMap1 := result1.(*ImportResult)
|
||||
firstSuccess := resultMap1.Success
|
||||
|
||||
// Second import with ignore
|
||||
p, err = process.Of("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"mode": "each",
|
||||
"duplicate": "ignore",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
result2, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
|
||||
resultMap2 := result2.(*ImportResult)
|
||||
assert.Greater(t, resultMap2.Ignore, 0, "Should have ignored duplicates")
|
||||
|
||||
// Verify count hasn't changed
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, firstSuccess, len(roles), "Should have same number of roles")
|
||||
})
|
||||
|
||||
// Test error strategy
|
||||
t.Run("DuplicateError", func(t *testing.T) {
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// First import
|
||||
p, err := process.Of("seeds.import", "roles.json", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
_, err = p.Exec()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Second import with error strategy
|
||||
p, err = process.Of("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"mode": "each",
|
||||
"duplicate": "error",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
|
||||
resultMap := result.(*ImportResult)
|
||||
assert.Greater(t, resultMap.Failure, 0, "Should have failures for duplicates")
|
||||
})
|
||||
}
|
||||
|
||||
func TestProcessSeedImportInvalidArguments(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Test with missing arguments
|
||||
t.Run("MissingArguments", func(t *testing.T) {
|
||||
p, err := process.Of("seeds.import", "roles.csv")
|
||||
assert.NoError(t, err)
|
||||
_, err = p.Exec()
|
||||
assert.Error(t, err, "Should fail with missing model argument")
|
||||
})
|
||||
|
||||
// Test with invalid file
|
||||
t.Run("InvalidFile", func(t *testing.T) {
|
||||
p, err := process.Of("seeds.import", "nonexistent.csv", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
_, err = p.Exec()
|
||||
assert.Error(t, err, "Should fail with non-existent file")
|
||||
})
|
||||
|
||||
// Test with invalid model
|
||||
t.Run("InvalidModel", func(t *testing.T) {
|
||||
p, err := process.Of("seeds.import", "roles.csv", "nonexistent.model")
|
||||
assert.NoError(t, err)
|
||||
_, err = p.Exec()
|
||||
assert.Error(t, err, "Should fail with non-existent model")
|
||||
})
|
||||
|
||||
// Test with unsupported file format
|
||||
t.Run("UnsupportedFormat", func(t *testing.T) {
|
||||
p, err := process.Of("seeds.import", "roles.txt", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
_, err = p.Exec()
|
||||
assert.Error(t, err, "Should fail with unsupported file format")
|
||||
})
|
||||
}
|
||||
|
||||
func TestProcessSeedImportOptions(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Test with various option types
|
||||
t.Run("OptionsAsMap", func(t *testing.T) {
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
options := map[string]interface{}{
|
||||
"chunk_size": 100,
|
||||
"duplicate": "ignore",
|
||||
"mode": "batch",
|
||||
}
|
||||
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options)
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
})
|
||||
|
||||
// Test with float64 chunk_size (JSON numbers)
|
||||
t.Run("OptionsWithFloat64", func(t *testing.T) {
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
options := map[string]interface{}{
|
||||
"chunk_size": float64(200),
|
||||
}
|
||||
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options)
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
})
|
||||
|
||||
// Test with partial options
|
||||
t.Run("PartialOptions", func(t *testing.T) {
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
options := map[string]interface{}{
|
||||
"mode": "each",
|
||||
}
|
||||
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options)
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
})
|
||||
|
||||
// Test with empty options
|
||||
t.Run("EmptyOptions", func(t *testing.T) {
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
options := map[string]interface{}{}
|
||||
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role", options)
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestProcessSeedImportResultStructure(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import data
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := p.Exec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify result structure
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult type")
|
||||
|
||||
// Check all fields exist and have proper types
|
||||
assert.GreaterOrEqual(t, resultMap.Total, 0, "Total should be non-negative")
|
||||
assert.GreaterOrEqual(t, resultMap.Success, 0, "Success should be non-negative")
|
||||
assert.GreaterOrEqual(t, resultMap.Failure, 0, "Failure should be non-negative")
|
||||
assert.GreaterOrEqual(t, resultMap.Ignore, 0, "Ignore should be non-negative")
|
||||
assert.NotNil(t, resultMap.Errors, "Errors should not be nil")
|
||||
|
||||
// Verify total = success + failure + ignore
|
||||
assert.Equal(t, resultMap.Total, resultMap.Success+resultMap.Failure+resultMap.Ignore,
|
||||
"Total should equal sum of success, failure, and ignore")
|
||||
}
|
||||
|
||||
func TestProcessSeedImportMultipleFiles(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import CSV
|
||||
t.Run("ImportCSV", func(t *testing.T) {
|
||||
p, err := process.Of("seeds.import", "roles.csv", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
resultMap := result.(*ImportResult)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
})
|
||||
|
||||
// Clear and import JSON
|
||||
t.Run("ImportJSON", func(t *testing.T) {
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
p, err := process.Of("seeds.import", "roles.json", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
resultMap := result.(*ImportResult)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
})
|
||||
|
||||
// Clear and import XLSX
|
||||
t.Run("ImportXLSX", func(t *testing.T) {
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
p, err := process.Of("seeds.import", "roles.xlsx", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
resultMap := result.(*ImportResult)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
})
|
||||
|
||||
// Clear and import Yao
|
||||
t.Run("ImportYao", func(t *testing.T) {
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
p, err := process.Of("seeds.import", "roles.yao", "__yao.role")
|
||||
assert.NoError(t, err)
|
||||
result, err := p.Exec()
|
||||
assert.NoError(t, err)
|
||||
resultMap := result.(*ImportResult)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
})
|
||||
}
|
||||
503
seed/seed.go
Normal file
503
seed/seed.go
Normal file
|
|
@ -0,0 +1,503 @@
|
|||
package seed
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
"github.com/yaoapp/gou/application"
|
||||
"github.com/yaoapp/gou/fs"
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/kun/log"
|
||||
"github.com/yaoapp/kun/maps"
|
||||
)
|
||||
|
||||
// Import imports seed data from file into model
|
||||
func Import(filename string, modelName string, options ImportOption) (*ImportResult, error) {
|
||||
// Get model
|
||||
mod := model.Select(modelName)
|
||||
|
||||
// Initialize result
|
||||
result := &ImportResult{
|
||||
Total: 0,
|
||||
Success: 0,
|
||||
Failure: 0,
|
||||
Ignore: 0,
|
||||
Errors: []ImportError{},
|
||||
}
|
||||
|
||||
// Determine file type and import
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
|
||||
switch ext {
|
||||
case ".csv":
|
||||
return result, importDataFromCSV(filename, mod, options, result)
|
||||
case ".xlsx", ".xls":
|
||||
return result, importDataFromXLSX(filename, mod, options, result)
|
||||
case ".json":
|
||||
return result, importDataFromJSON(filename, mod, options, result)
|
||||
case ".yao", ".jsonc":
|
||||
return result, importDataFromYao(filename, mod, options, result)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported file format: %s", ext)
|
||||
}
|
||||
}
|
||||
|
||||
// importDataFromCSV import data from CSV file
|
||||
func importDataFromCSV(filename string, mod *model.Model, options ImportOption, result *ImportResult) error {
|
||||
// Read file from seed filesystem
|
||||
seedFS := fs.MustGet("seed")
|
||||
data, err := seedFS.ReadFile(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read CSV file: %v", err)
|
||||
}
|
||||
|
||||
// Parse CSV
|
||||
reader := csv.NewReader(strings.NewReader(string(data)))
|
||||
reader.FieldsPerRecord = -1 // Allow variable number of fields
|
||||
|
||||
// Read header
|
||||
header, err := reader.Read()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read CSV header: %v", err)
|
||||
}
|
||||
|
||||
// Prepare handler
|
||||
handler := createImportHandler(mod, header, options, result)
|
||||
|
||||
// Read data in chunks
|
||||
chunk := [][]interface{}{}
|
||||
lineNum := 1 // Start from 1 (header is line 0)
|
||||
|
||||
for {
|
||||
record, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Row: lineNum,
|
||||
Message: err.Error(),
|
||||
Code: 500,
|
||||
})
|
||||
result.Failure++
|
||||
result.Total++
|
||||
lineNum++
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert to interface slice
|
||||
row := make([]interface{}, len(record))
|
||||
for i, v := range record {
|
||||
row[i] = v
|
||||
}
|
||||
|
||||
chunk = append(chunk, row)
|
||||
result.Total++
|
||||
|
||||
// Process chunk when size reached
|
||||
if len(chunk) >= options.ChunkSize {
|
||||
if err := handler(lineNum-len(chunk)+1, chunk); err != nil {
|
||||
log.Error("Import chunk error: %v", err)
|
||||
}
|
||||
chunk = [][]interface{}{}
|
||||
}
|
||||
|
||||
lineNum++
|
||||
}
|
||||
|
||||
// Process remaining chunk
|
||||
if len(chunk) > 0 {
|
||||
if err := handler(lineNum-len(chunk), chunk); err != nil {
|
||||
log.Error("Import final chunk error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// importDataFromXLSX import data from XLSX file
|
||||
func importDataFromXLSX(filename string, mod *model.Model, options ImportOption, result *ImportResult) error {
|
||||
// Read file from seed filesystem
|
||||
seedFS := fs.MustGet("seed")
|
||||
data, err := seedFS.ReadFile(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read XLSX file: %v", err)
|
||||
}
|
||||
|
||||
// Open Excel file from bytes
|
||||
file, err := excelize.OpenReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open XLSX file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Get active sheet
|
||||
sheetName := file.GetSheetName(file.GetActiveSheetIndex())
|
||||
rows, err := file.Rows(sheetName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get rows: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Read header
|
||||
if !rows.Next() {
|
||||
return fmt.Errorf("empty XLSX file")
|
||||
}
|
||||
header, err := rows.Columns()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read header: %v", err)
|
||||
}
|
||||
|
||||
// Prepare handler
|
||||
handler := createImportHandler(mod, header, options, result)
|
||||
|
||||
// Read data in chunks
|
||||
chunk := [][]interface{}{}
|
||||
lineNum := 1 // Header is line 0, data starts from 1
|
||||
|
||||
for rows.Next() {
|
||||
record, err := rows.Columns()
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Row: lineNum,
|
||||
Message: err.Error(),
|
||||
Code: 500,
|
||||
})
|
||||
result.Failure++
|
||||
result.Total++
|
||||
lineNum++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if row is empty
|
||||
isEmpty := true
|
||||
for _, v := range record {
|
||||
if v != "" {
|
||||
isEmpty = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if isEmpty {
|
||||
lineNum++
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert to interface slice
|
||||
row := make([]interface{}, len(record))
|
||||
for i, v := range record {
|
||||
row[i] = v
|
||||
}
|
||||
|
||||
chunk = append(chunk, row)
|
||||
result.Total++
|
||||
|
||||
// Process chunk when size reached
|
||||
if len(chunk) >= options.ChunkSize {
|
||||
if err := handler(lineNum-len(chunk)+1, chunk); err != nil {
|
||||
log.Error("Import chunk error: %v", err)
|
||||
}
|
||||
chunk = [][]interface{}{}
|
||||
}
|
||||
|
||||
lineNum++
|
||||
}
|
||||
|
||||
// Process remaining chunk
|
||||
if len(chunk) > 0 {
|
||||
if err := handler(lineNum-len(chunk), chunk); err != nil {
|
||||
log.Error("Import final chunk error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// importDataFromJSON import data from JSON file
|
||||
func importDataFromJSON(filename string, mod *model.Model, options ImportOption, result *ImportResult) error {
|
||||
// Read file from seed filesystem
|
||||
seedFS := fs.MustGet("seed")
|
||||
data, err := seedFS.ReadFile(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read JSON file: %v", err)
|
||||
}
|
||||
|
||||
// Parse JSON - expect array of objects
|
||||
var records []map[string]interface{}
|
||||
if err := json.Unmarshal(data, &records); err != nil {
|
||||
return fmt.Errorf("failed to parse JSON: %v", err)
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Extract columns from first record
|
||||
columns := []string{}
|
||||
for key := range records[0] {
|
||||
columns = append(columns, key)
|
||||
}
|
||||
|
||||
// Convert to rows format
|
||||
handler := createJSONImportHandler(mod, columns, options, result)
|
||||
|
||||
// Process records in chunks
|
||||
chunk := []map[string]interface{}{}
|
||||
for i, record := range records {
|
||||
result.Total++
|
||||
chunk = append(chunk, record)
|
||||
|
||||
if len(chunk) >= options.ChunkSize {
|
||||
if err := handler(i-len(chunk)+1, chunk); err != nil {
|
||||
log.Error("Import chunk error: %v", err)
|
||||
}
|
||||
chunk = []map[string]interface{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Process remaining chunk
|
||||
if len(chunk) > 0 {
|
||||
if err := handler(len(records)-len(chunk), chunk); err != nil {
|
||||
log.Error("Import final chunk error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// importDataFromYao import data from Yao file (JSONC)
|
||||
func importDataFromYao(filename string, mod *model.Model, options ImportOption, result *ImportResult) error {
|
||||
// Read file from seed filesystem
|
||||
seedFS := fs.MustGet("seed")
|
||||
data, err := seedFS.ReadFile(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read Yao file: %v", err)
|
||||
}
|
||||
|
||||
// Parse using application Parse (handles JSONC)
|
||||
var records []map[string]interface{}
|
||||
if err := application.Parse(filename, data, &records); err != nil {
|
||||
return fmt.Errorf("failed to parse Yao file: %v", err)
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Extract columns from first record
|
||||
columns := []string{}
|
||||
for key := range records[0] {
|
||||
columns = append(columns, key)
|
||||
}
|
||||
|
||||
// Convert to rows format
|
||||
handler := createJSONImportHandler(mod, columns, options, result)
|
||||
|
||||
// Process records in chunks
|
||||
chunk := []map[string]interface{}{}
|
||||
for i, record := range records {
|
||||
result.Total++
|
||||
chunk = append(chunk, record)
|
||||
|
||||
if len(chunk) >= options.ChunkSize {
|
||||
if err := handler(i-len(chunk)+1, chunk); err != nil {
|
||||
log.Error("Import chunk error: %v", err)
|
||||
}
|
||||
chunk = []map[string]interface{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Process remaining chunk
|
||||
if len(chunk) > 0 {
|
||||
if err := handler(len(records)-len(chunk), chunk); err != nil {
|
||||
log.Error("Import final chunk error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createImportHandler creates handler for CSV/XLSX format
|
||||
func createImportHandler(mod *model.Model, columns []string, options ImportOption, result *ImportResult) ImportHandler {
|
||||
return func(line int, data [][]interface{}) error {
|
||||
if options.Mode == ImportModeEach {
|
||||
// Single record mode - use Create
|
||||
return importEach(mod, columns, data, line, options, result)
|
||||
}
|
||||
// Batch mode - use Insert
|
||||
return importBatch(mod, columns, data, line, options, result)
|
||||
}
|
||||
}
|
||||
|
||||
// createJSONImportHandler creates handler for JSON/Yao format
|
||||
func createJSONImportHandler(mod *model.Model, columns []string, options ImportOption, result *ImportResult) func(line int, data []map[string]interface{}) error {
|
||||
return func(line int, data []map[string]interface{}) error {
|
||||
if options.Mode == ImportModeEach {
|
||||
// Single record mode - use Create or Save
|
||||
return importEachJSON(mod, data, line, options, result)
|
||||
}
|
||||
// Batch mode - convert to rows and use Insert
|
||||
rows := make([][]interface{}, len(data))
|
||||
for i, record := range data {
|
||||
row := make([]interface{}, len(columns))
|
||||
for j, col := range columns {
|
||||
row[j] = record[col]
|
||||
}
|
||||
rows[i] = row
|
||||
}
|
||||
return importBatch(mod, columns, rows, line, options, result)
|
||||
}
|
||||
}
|
||||
|
||||
// importBatch batch import using Model.Insert
|
||||
func importBatch(mod *model.Model, columns []string, data [][]interface{}, startLine int, options ImportOption, result *ImportResult) error {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch options.Duplicate {
|
||||
case DuplicateIgnore:
|
||||
// Try to insert, ignore errors
|
||||
err := mod.Insert(columns, data)
|
||||
if err != nil {
|
||||
// Log error but don't fail
|
||||
log.Warn("Batch insert with ignore strategy: %v", err)
|
||||
result.Ignore += len(data)
|
||||
} else {
|
||||
result.Success += len(data)
|
||||
}
|
||||
|
||||
case DuplicateError:
|
||||
// Insert and fail on error
|
||||
err := mod.Insert(columns, data)
|
||||
if err != nil {
|
||||
for i := range data {
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Row: startLine + i,
|
||||
Message: err.Error(),
|
||||
Code: 500,
|
||||
Data: data[i],
|
||||
})
|
||||
}
|
||||
result.Failure += len(data)
|
||||
return err
|
||||
}
|
||||
result.Success += len(data)
|
||||
|
||||
case DuplicateUpdate, DuplicateAbort:
|
||||
// For update/abort, fall back to each mode
|
||||
for i, row := range data {
|
||||
rowMap := maps.MakeMapStrAny()
|
||||
for j, col := range columns {
|
||||
if j < len(row) {
|
||||
rowMap[col] = row[j]
|
||||
}
|
||||
}
|
||||
if err := handleDuplicate(mod, rowMap, startLine+i, options.Duplicate, result); err != nil {
|
||||
if options.Duplicate == DuplicateAbort {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// importEach single record import using Model.Create
|
||||
func importEach(mod *model.Model, columns []string, data [][]interface{}, startLine int, options ImportOption, result *ImportResult) error {
|
||||
for i, row := range data {
|
||||
// Convert row to map
|
||||
rowMap := maps.MakeMapStrAny()
|
||||
for j, col := range columns {
|
||||
if j < len(row) {
|
||||
rowMap[col] = row[j]
|
||||
}
|
||||
}
|
||||
|
||||
if err := handleDuplicate(mod, rowMap, startLine+i, options.Duplicate, result); err != nil {
|
||||
if options.Duplicate == DuplicateAbort {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importEachJSON single record import for JSON format
|
||||
func importEachJSON(mod *model.Model, data []map[string]interface{}, startLine int, options ImportOption, result *ImportResult) error {
|
||||
for i, record := range data {
|
||||
rowMap := maps.MapStrAny(record)
|
||||
if err := handleDuplicate(mod, rowMap, startLine+i, options.Duplicate, result); err != nil {
|
||||
if options.Duplicate == DuplicateAbort {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDuplicate handles duplicate strategy for single record
|
||||
func handleDuplicate(mod *model.Model, row maps.MapStrAny, line int, duplicateMode DuplicateMode, result *ImportResult) error {
|
||||
switch duplicateMode {
|
||||
case DuplicateIgnore:
|
||||
// Try to create, ignore if exists
|
||||
_, err := mod.Create(row)
|
||||
if err != nil {
|
||||
result.Ignore++
|
||||
log.Debug("Row %d ignored: %v", line, err)
|
||||
} else {
|
||||
result.Success++
|
||||
}
|
||||
|
||||
case DuplicateUpdate:
|
||||
// Use Save (create or update)
|
||||
_, err := mod.Save(row)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Row: line,
|
||||
Message: err.Error(),
|
||||
Code: 500,
|
||||
})
|
||||
result.Failure++
|
||||
} else {
|
||||
result.Success++
|
||||
}
|
||||
|
||||
case DuplicateError:
|
||||
// Create and fail on error
|
||||
_, err := mod.Create(row)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Row: line,
|
||||
Message: err.Error(),
|
||||
Code: 500,
|
||||
})
|
||||
result.Failure++
|
||||
return err
|
||||
}
|
||||
result.Success++
|
||||
|
||||
case DuplicateAbort:
|
||||
// Create and abort on error
|
||||
_, err := mod.Create(row)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Row: line,
|
||||
Message: err.Error(),
|
||||
Code: 500,
|
||||
})
|
||||
result.Failure++
|
||||
return fmt.Errorf("import aborted at line %d: %v", line, err)
|
||||
}
|
||||
result.Success++
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
324
seed/seed_test.go
Normal file
324
seed/seed_test.go
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
package seed
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/gou/process"
|
||||
"github.com/yaoapp/yao/config"
|
||||
"github.com/yaoapp/yao/test"
|
||||
)
|
||||
|
||||
// TestSeedImportCSV tests importing roles from CSV file
|
||||
func TestSeedImportCSV(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import CSV
|
||||
p := process.New("seeds.import", "roles.csv", "__yao.role")
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
assert.Equal(t, resultMap.Total, resultMap.Success+resultMap.Failure+resultMap.Ignore,
|
||||
"Total should equal sum of success, failure, and ignore")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
|
||||
// Check specific role
|
||||
adminRoles, _ := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
if len(adminRoles) > 0 {
|
||||
assert.Equal(t, "admin", adminRoles[0].Get("role_id"))
|
||||
assert.Equal(t, "Administrator", adminRoles[0].Get("name"))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeedImportJSON tests importing roles from JSON file
|
||||
func TestSeedImportJSON(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import JSON
|
||||
p := process.New("seeds.import", "roles.json", "__yao.role")
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
|
||||
// Check that permissions JSON was imported correctly
|
||||
adminRoles, _ := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
if len(adminRoles) > 0 {
|
||||
assert.Equal(t, "admin", adminRoles[0].Get("role_id"))
|
||||
assert.NotNil(t, adminRoles[0].Get("permissions"), "Should have permissions")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeedImportXLSX tests importing roles from XLSX file
|
||||
func TestSeedImportXLSX(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import XLSX file
|
||||
p := process.New("seeds.import", "roles.xlsx", "__yao.role")
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
|
||||
// Check specific role
|
||||
adminRoles, _ := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
if len(adminRoles) > 0 {
|
||||
assert.Equal(t, "admin", adminRoles[0].Get("role_id"))
|
||||
assert.Equal(t, "Administrator", adminRoles[0].Get("name"))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeedImportYao tests importing roles from Yao file (JSONC)
|
||||
func TestSeedImportYao(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import Yao file
|
||||
p := process.New("seeds.import", "roles.yao", "__yao.role")
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0, "Should have roles in database")
|
||||
}
|
||||
|
||||
// TestSeedImportWithOptions tests importing with custom options
|
||||
func TestSeedImportWithOptions(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import with batch mode
|
||||
p := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"chunk_size": 2,
|
||||
"duplicate": "ignore",
|
||||
"mode": "batch",
|
||||
})
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
|
||||
// Try importing again with ignore strategy
|
||||
p2 := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"duplicate": "ignore",
|
||||
})
|
||||
result2 := p2.Run()
|
||||
|
||||
resultMap2, ok := result2.(*ImportResult)
|
||||
assert.True(t, ok)
|
||||
// With ignore strategy, duplicates should be ignored
|
||||
assert.Greater(t, resultMap2.Ignore, 0, "Should have ignored duplicates")
|
||||
}
|
||||
|
||||
// TestSeedImportEachMode tests importing with each mode (single record)
|
||||
func TestSeedImportEachMode(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import with each mode
|
||||
p := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"mode": "each",
|
||||
"duplicate": "ignore",
|
||||
})
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
|
||||
// Verify data in database
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0)
|
||||
}
|
||||
|
||||
// TestSeedImportDuplicateIgnore tests importing with ignore duplicate strategy
|
||||
func TestSeedImportDuplicateIgnore(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// First import
|
||||
p1 := process.New("seeds.import", "roles.json", "__yao.role")
|
||||
result1 := p1.Run()
|
||||
resultMap1, ok := result1.(*ImportResult)
|
||||
assert.True(t, ok)
|
||||
firstSuccess := resultMap1.Success
|
||||
|
||||
// Get the IDs of imported records
|
||||
roles1, err := mod.Get(model.QueryParam{
|
||||
Select: []interface{}{"id", "role_id"},
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles1), 0)
|
||||
|
||||
// Second import with ignore mode
|
||||
p2 := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"mode": "each",
|
||||
"duplicate": "ignore",
|
||||
})
|
||||
result2 := p2.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result2)
|
||||
resultMap2, ok := result2.(*ImportResult)
|
||||
assert.True(t, ok)
|
||||
// With ignore strategy, duplicates should be ignored
|
||||
assert.Greater(t, resultMap2.Ignore, 0, "Should have ignored duplicates")
|
||||
|
||||
// Verify count hasn't changed (ignored duplicates, not created new)
|
||||
roles2, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, firstSuccess, len(roles2), "Should have same number of roles after re-import")
|
||||
}
|
||||
|
||||
// TestSeedImportChunkSize tests chunk processing
|
||||
func TestSeedImportChunkSize(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import with small chunk size
|
||||
p := process.New("seeds.import", "roles.json", "__yao.role", map[string]interface{}{
|
||||
"chunk_size": 1, // Process one record at a time
|
||||
"mode": "batch",
|
||||
})
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok)
|
||||
assert.Greater(t, resultMap.Success, 0)
|
||||
|
||||
// Verify all data imported correctly
|
||||
roles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0)
|
||||
}
|
||||
57
seed/types.go
Normal file
57
seed/types.go
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
package seed
|
||||
|
||||
// DuplicateMode the duplicate mode
|
||||
type DuplicateMode string
|
||||
|
||||
// ImportMode the import mode
|
||||
type ImportMode string
|
||||
|
||||
const (
|
||||
|
||||
// ImportModeBatch the batch import mode
|
||||
ImportModeBatch ImportMode = "batch"
|
||||
// ImportModeEach the each import mode
|
||||
ImportModeEach ImportMode = "each"
|
||||
|
||||
// DuplicateIgnore when the record is duplicate, ignore the record
|
||||
DuplicateIgnore DuplicateMode = "ignore"
|
||||
// DuplicateUpdate when the record is duplicate, update the record
|
||||
DuplicateUpdate DuplicateMode = "update"
|
||||
// DuplicateError when the record is duplicate, raise an error
|
||||
DuplicateError DuplicateMode = "error"
|
||||
// DuplicateAbort when the record is duplicate, abort the record
|
||||
DuplicateAbort DuplicateMode = "abort"
|
||||
)
|
||||
|
||||
const (
|
||||
// ChunkSizeDefault the default chunk size
|
||||
ChunkSizeDefault = 500
|
||||
)
|
||||
|
||||
// ImportOption the seed import option
|
||||
type ImportOption struct {
|
||||
ChunkSize int `json:"chunk_size,omitempty"`
|
||||
Duplicate DuplicateMode `json:"duplicate,omitempty"`
|
||||
Mode ImportMode `json:"mode,omitempty"`
|
||||
}
|
||||
|
||||
// ImportHandler the seed import handler
|
||||
type ImportHandler func(line int, data [][]interface{}) error
|
||||
|
||||
// ImportResult the seed import result
|
||||
type ImportResult struct {
|
||||
Total int `json:"total,omitempty"`
|
||||
Success int `json:"success,omitempty"`
|
||||
Failure int `json:"failure,omitempty"`
|
||||
Ignore int `json:"ignore,omitempty"`
|
||||
Errors []ImportError `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// ImportError the seed import error
|
||||
type ImportError struct {
|
||||
Row int `json:"row,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Code int `json:"code,omitempty"`
|
||||
Data []interface{} `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
Loading…
Add table
Reference in a new issue