Refactor seed import logic to exclude auto-generated fields and improve error handling

- Updated import functions to filter out auto-generated fields (e.g., timestamps) when processing records.
- Enhanced error handling during batch inserts to provide clearer logging of failures.
- Removed detailed logging from TestSeedImportYao to streamline test output while maintaining essential assertions.
This commit is contained in:
Max 2025-10-20 11:33:19 +08:00
parent 9c82f12eca
commit 0f22d2d8c8
2 changed files with 38 additions and 14 deletions

View file

@ -244,10 +244,13 @@ func importDataFromJSON(filename string, mod *model.Model, options ImportOption,
}
// Extract columns from first record, but only include columns that exist in model
// Also exclude auto-generated fields (timestamps, etc.)
columns := []string{}
for key := range records[0] {
if _, exists := mod.Columns[key]; exists {
columns = append(columns, key)
if !isAutoGeneratedField(key, mod) {
columns = append(columns, key)
}
}
}
@ -301,10 +304,13 @@ func importDataFromYao(filename string, mod *model.Model, options ImportOption,
}
// Extract columns from first record, but only include columns that exist in model
// Also exclude auto-generated fields (timestamps, etc.)
columns := []string{}
for key := range records[0] {
if _, exists := mod.Columns[key]; exists {
columns = append(columns, key)
if !isAutoGeneratedField(key, mod) {
columns = append(columns, key)
}
}
}
@ -362,7 +368,16 @@ func createJSONImportHandler(mod *model.Model, columns []string, options ImportO
for i, record := range data {
row := make([]interface{}, len(columns))
for j, col := range columns {
row[j] = record[col]
value, exists := record[col]
if !exists {
// Field missing in record, use default value from model
if column, ok := mod.Columns[col]; ok && column.Default != nil {
value = column.Default
}
// If no default and field is missing, value remains nil
// which should work for nullable fields
}
row[j] = value
}
rows[i] = row
}
@ -382,7 +397,7 @@ func importBatch(mod *model.Model, columns []string, data [][]interface{}, start
err := mod.Insert(columns, data)
if err != nil {
// Log error but don't fail
log.Warn("Batch insert with ignore strategy: %v", err)
log.Warn("Batch insert with ignore strategy failed: %v", err)
result.Ignore += len(data)
} else {
result.Success += len(data)
@ -574,3 +589,22 @@ func sortColumns(columns []string) {
}
}
}
// isAutoGeneratedField checks if a field is auto-generated (timestamps, etc.)
func isAutoGeneratedField(fieldName string, mod *model.Model) bool {
// Skip timestamp fields that will be auto-added by Insert
if mod.MetaData.Option.Timestamps {
if fieldName == "created_at" || fieldName == "updated_at" || fieldName == "deleted_at" {
return true
}
}
// Skip tracking fields
if mod.MetaData.Option.Trackings {
if fieldName == "created_by" || fieldName == "updated_by" || fieldName == "deleted_by" {
return true
}
}
return false
}

View file

@ -160,16 +160,6 @@ func TestSeedImportYao(t *testing.T) {
assert.NotNil(t, result)
resultMap, ok := result.(*ImportResult)
assert.True(t, ok, "Result should be ImportResult")
// Print detailed result for debugging
if resultMap.Total == 0 || resultMap.Success == 0 {
t.Logf("Import result: Total=%d, Success=%d, Failure=%d, Ignore=%d",
resultMap.Total, resultMap.Success, resultMap.Failure, resultMap.Ignore)
if len(resultMap.Errors) > 0 {
t.Logf("Errors: %+v", resultMap.Errors)
}
}
assert.Greater(t, resultMap.Total, 0, "Should import at least 1 record")
assert.Greater(t, resultMap.Success, 0, "Should have successful imports")