Implement JSON field parsing for CSV and XLSX imports in seed module
- Added tests to verify correct parsing of JSON fields from both CSV and XLSX files during data import. - Enhanced import functions to build a column type map for detecting JSON fields and parse them appropriately. - Introduced helper functions for building column type maps and parsing JSON fields, improving data integrity during imports. - Updated existing tests to ensure successful imports and correct handling of JSON data structures.
This commit is contained in:
Max
parent
8b9cd9f951
commit
2e2abf1d5f
3 changed files with 195 additions and 5 deletions
58
seed/seed.go
58
seed/seed.go
|
|
@ -67,6 +67,9 @@ func importDataFromCSV(filename string, mod *model.Model, options ImportOption,
|
|||
return fmt.Errorf("failed to read CSV header: %v", err)
|
||||
}
|
||||
|
||||
// Build column type map for JSON field detection
|
||||
columnTypes := buildColumnTypeMap(mod, header)
|
||||
|
||||
// Prepare handler
|
||||
handler := createImportHandler(mod, header, options, result)
|
||||
|
||||
|
|
@ -91,10 +94,10 @@ func importDataFromCSV(filename string, mod *model.Model, options ImportOption,
|
|||
continue
|
||||
}
|
||||
|
||||
// Convert to interface slice
|
||||
// Convert to interface slice and parse JSON fields
|
||||
row := make([]interface{}, len(record))
|
||||
for i, v := range record {
|
||||
row[i] = v
|
||||
row[i] = parseJSONField(v, columnTypes[i])
|
||||
}
|
||||
|
||||
chunk = append(chunk, row)
|
||||
|
|
@ -154,6 +157,9 @@ func importDataFromXLSX(filename string, mod *model.Model, options ImportOption,
|
|||
return fmt.Errorf("failed to read header: %v", err)
|
||||
}
|
||||
|
||||
// Build column type map for JSON field detection
|
||||
columnTypes := buildColumnTypeMap(mod, header)
|
||||
|
||||
// Prepare handler
|
||||
handler := createImportHandler(mod, header, options, result)
|
||||
|
||||
|
|
@ -188,10 +194,10 @@ func importDataFromXLSX(filename string, mod *model.Model, options ImportOption,
|
|||
continue
|
||||
}
|
||||
|
||||
// Convert to interface slice
|
||||
// Convert to interface slice and parse JSON fields
|
||||
row := make([]interface{}, len(record))
|
||||
for i, v := range record {
|
||||
row[i] = v
|
||||
row[i] = parseJSONField(v, columnTypes[i])
|
||||
}
|
||||
|
||||
chunk = append(chunk, row)
|
||||
|
|
@ -501,3 +507,47 @@ func handleDuplicate(mod *model.Model, row maps.MapStrAny, line int, duplicateMo
|
|||
return nil
|
||||
}
|
||||
|
||||
// buildColumnTypeMap builds a map of column index to column type
|
||||
// Returns a slice where index matches the CSV/XLSX column position
|
||||
func buildColumnTypeMap(mod *model.Model, header []string) []string {
|
||||
columnTypes := make([]string, len(header))
|
||||
for i, colName := range header {
|
||||
if col, exists := mod.Columns[colName]; exists {
|
||||
columnTypes[i] = strings.ToLower(col.Type)
|
||||
} else {
|
||||
columnTypes[i] = ""
|
||||
}
|
||||
}
|
||||
return columnTypes
|
||||
}
|
||||
|
||||
// parseJSONField attempts to parse a value as JSON if the column type is json
|
||||
// Returns the parsed JSON object if successful, otherwise returns the original value
|
||||
func parseJSONField(value interface{}, columnType string) interface{} {
|
||||
// Check if column type is JSON
|
||||
if columnType != "json" && columnType != "jsonb" {
|
||||
return value
|
||||
}
|
||||
|
||||
// Try to parse string value as JSON
|
||||
strValue, ok := value.(string)
|
||||
if !ok || strValue == "" {
|
||||
return value
|
||||
}
|
||||
|
||||
// Trim whitespace
|
||||
strValue = strings.TrimSpace(strValue)
|
||||
if strValue == "" {
|
||||
return value
|
||||
}
|
||||
|
||||
// Try to parse as JSON
|
||||
var jsonValue interface{}
|
||||
if err := json.Unmarshal([]byte(strValue), &jsonValue); err != nil {
|
||||
// If parsing fails, return original value (might be empty or malformed)
|
||||
// Don't log error as this is expected for non-JSON strings
|
||||
return value
|
||||
}
|
||||
|
||||
return jsonValue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -322,3 +322,144 @@ func TestSeedImportChunkSize(t *testing.T) {
|
|||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(roles), 0)
|
||||
}
|
||||
|
||||
// TestSeedImportJSONFields tests that JSON fields are correctly parsed from CSV
|
||||
func TestSeedImportJSONFields(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import CSV file
|
||||
p := process.New("seeds.import", "roles.csv", "__yao.role")
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Get imported data
|
||||
roles, err := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(roles), "Should find admin role")
|
||||
|
||||
// Verify JSON fields are parsed as objects, not strings
|
||||
adminRole := roles[0]
|
||||
|
||||
// Check permissions field (should be a map, not a string)
|
||||
permissions := adminRole.Get("permissions")
|
||||
assert.NotNil(t, permissions, "Permissions should not be nil")
|
||||
permissionsMap, ok := permissions.(map[string]interface{})
|
||||
assert.True(t, ok, "Permissions should be parsed as map[string]interface{}, got %T", permissions)
|
||||
assert.NotNil(t, permissionsMap["users"], "Should have users permissions")
|
||||
|
||||
// Check metadata field (should be a map, not a string)
|
||||
metadata := adminRole.Get("metadata")
|
||||
assert.NotNil(t, metadata, "Metadata should not be nil")
|
||||
metadataMap, ok := metadata.(map[string]interface{})
|
||||
assert.True(t, ok, "Metadata should be parsed as map[string]interface{}, got %T", metadata)
|
||||
|
||||
// Verify nested values
|
||||
if usersPerms, ok := permissionsMap["users"].([]interface{}); ok {
|
||||
assert.Greater(t, len(usersPerms), 0, "Should have user permissions")
|
||||
assert.Contains(t, usersPerms, "create", "Should have create permission")
|
||||
}
|
||||
|
||||
if maxUsers, ok := metadataMap["max_users"].(float64); ok {
|
||||
assert.Equal(t, float64(5), maxUsers, "Max users should be 5")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeedImportXLSXJSONFields tests that JSON fields are correctly parsed from XLSX
|
||||
func TestSeedImportXLSXJSONFields(t *testing.T) {
|
||||
test.Prepare(t, config.Conf)
|
||||
defer test.Clean()
|
||||
|
||||
// Ensure __yao.role model exists
|
||||
if !model.Exists("__yao.role") {
|
||||
t.Skip("__yao.role model not loaded, skipping test")
|
||||
}
|
||||
|
||||
// Clear existing roles
|
||||
mod := model.Select("__yao.role")
|
||||
_, _ = mod.DestroyWhere(model.QueryParam{})
|
||||
|
||||
// Import XLSX file
|
||||
p := process.New("seeds.import", "roles.xlsx", "__yao.role")
|
||||
result := p.Run()
|
||||
|
||||
// Verify result
|
||||
assert.NotNil(t, result)
|
||||
resultMap, ok := result.(*ImportResult)
|
||||
assert.True(t, ok, "Result should be ImportResult")
|
||||
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")
|
||||
|
||||
// Get imported data
|
||||
roles, err := mod.Get(model.QueryParam{
|
||||
Wheres: []model.QueryWhere{
|
||||
{Column: "role_id", Value: "admin"},
|
||||
},
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(roles), "Should find admin role")
|
||||
|
||||
// Verify JSON fields are parsed as objects, not strings
|
||||
adminRole := roles[0]
|
||||
|
||||
// Check permissions field (should be a map, not a string)
|
||||
permissions := adminRole.Get("permissions")
|
||||
assert.NotNil(t, permissions, "Permissions should not be nil")
|
||||
permissionsMap, ok := permissions.(map[string]interface{})
|
||||
assert.True(t, ok, "Permissions should be parsed as map[string]interface{}, got %T", permissions)
|
||||
assert.NotNil(t, permissionsMap["users"], "Should have users permissions")
|
||||
|
||||
// Check metadata field (should be a map, not a string)
|
||||
metadata := adminRole.Get("metadata")
|
||||
assert.NotNil(t, metadata, "Metadata should not be nil")
|
||||
metadataMap, ok := metadata.(map[string]interface{})
|
||||
assert.True(t, ok, "Metadata should be parsed as map[string]interface{}, got %T", metadata)
|
||||
|
||||
// Verify nested values from XLSX
|
||||
if usersPerms, ok := permissionsMap["users"].([]interface{}); ok {
|
||||
assert.Greater(t, len(usersPerms), 0, "Should have user permissions")
|
||||
assert.Contains(t, usersPerms, "create", "Should have create permission")
|
||||
}
|
||||
|
||||
if maxUsers, ok := metadataMap["max_users"].(float64); ok {
|
||||
assert.Equal(t, float64(5), maxUsers, "Max users should be 5")
|
||||
}
|
||||
|
||||
// Also verify other roles to ensure all JSON fields are parsed
|
||||
allRoles, err := mod.Get(model.QueryParam{})
|
||||
assert.Nil(t, err)
|
||||
assert.Greater(t, len(allRoles), 1, "Should have multiple roles")
|
||||
|
||||
// Check that all roles have properly parsed JSON fields
|
||||
for _, role := range allRoles {
|
||||
roleID := role.Get("role_id")
|
||||
permissions := role.Get("permissions")
|
||||
if permissions != nil {
|
||||
_, ok := permissions.(map[string]interface{})
|
||||
assert.True(t, ok, "Role %s permissions should be parsed as map, got %T", roleID, permissions)
|
||||
}
|
||||
|
||||
metadata := role.Get("metadata")
|
||||
if metadata != nil {
|
||||
_, ok := metadata.(map[string]interface{})
|
||||
assert.True(t, ok, "Role %s metadata should be parsed as map, got %T", roleID, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,4 +54,3 @@ type ImportError struct {
|
|||
Code int `json:"code,omitempty"`
|
||||
Data []interface{} `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue