diff --git a/kb/types/document.go b/kb/types/document.go index b161d092..8abc4121 100644 --- a/kb/types/document.go +++ b/kb/types/document.go @@ -18,6 +18,7 @@ func (c *Config) SearchDocuments(param model.QueryParam, page int, pagesize int) if mod == nil { return nil, fmt.Errorf("document model not found: %s", modelName) } + param.Debug = true return mod.Paginate(param, page, pagesize) } diff --git a/openapi/oauth/acl/scope.go b/openapi/oauth/acl/scope.go index fafdc8ba..abe1d31d 100644 --- a/openapi/oauth/acl/scope.go +++ b/openapi/oauth/acl/scope.go @@ -223,22 +223,53 @@ func (m *ScopeManager) expandAlias(alias string, visited map[string]bool) ([]str return expanded, nil } +// getFirstLevelSubdirs returns all first-level subdirectories in a given directory +func getFirstLevelSubdirs(baseDir string) ([]string, error) { + var subdirs []string + + err := application.App.Walk(baseDir, func(root, filename string, isdir bool) error { + // Only process directories + if !isdir { + return nil + } + + // Skip the root directory itself + if filename == baseDir { + return nil + } + + // Get relative path from baseDir + relPath := strings.TrimPrefix(filename, baseDir) + relPath = strings.TrimPrefix(relPath, string(filepath.Separator)) + + // Only include first-level directories (no nested paths) + if !strings.Contains(relPath, string(filepath.Separator)) && relPath != "" { + subdirs = append(subdirs, relPath) + } + + return nil + }, "") + + if err != nil { + return nil, err + } + + return subdirs, nil +} + // loadScopeDefinitions loads scope definitions from subdirectories func (m *ScopeManager) loadScopeDefinitions() error { scopesDir := filepath.Join("openapi", "scopes") - // Subdirectories to scan - subDirs := []string{"kb", "job", "file", "user"} + // Get all subdirectories in the scopes directory + subDirs, err := getFirstLevelSubdirs(scopesDir) + if err != nil { + return fmt.Errorf("failed to scan scopes directory: %w", err) + } + // Load scope definitions from each subdirectory for _, subDir := range subDirs { dirPath := filepath.Join(scopesDir, subDir) - exists, err := application.App.Exists(dirPath) - if err != nil { - return err - } - if !exists { - continue - } // Walk through all .yml files in the directory err = application.App.Walk(dirPath, func(root, filename string, isdir bool) error { diff --git a/openapi/user/entry.go b/openapi/user/entry.go index 21c083df..2e49886c 100644 --- a/openapi/user/entry.go +++ b/openapi/user/entry.go @@ -1083,10 +1083,10 @@ func GinVerifyInvite(c *gin.Context) { // Get authorized info from the temporary token authInfo := oauth.GetAuthorizedInfo(c) - if authInfo == nil || authInfo.Scope != ScopeInviteVerification { + if authInfo == nil || authInfo.Scope != ScopeEntryVerification { errorResp := &response.ErrorResponse{ Code: response.ErrInsufficientScope.Code, - ErrorDescription: "Invalid or missing token scope. Expected invite_verification scope", + ErrorDescription: "Invalid or missing token scope. Expected entry_verification scope", } response.RespondWithError(c, response.StatusForbidden, errorResp) return diff --git a/openapi/user/login.go b/openapi/user/login.go index dabd50c3..2f156e95 100644 --- a/openapi/user/login.go +++ b/openapi/user/login.go @@ -176,7 +176,7 @@ func LoginByUserID(userid string, loginCtx *LoginContext) (*LoginResponse, error extraClaims["remember_me"] = true } - accessToken, err := oauth.OAuth.MakeAccessToken(yaoClientConfig.ClientID, ScopeInviteVerification, subject, inviteExpire, extraClaims) + accessToken, err := oauth.OAuth.MakeAccessToken(yaoClientConfig.ClientID, ScopeEntryVerification, subject, inviteExpire, extraClaims) if err != nil { return nil, err } @@ -186,7 +186,7 @@ func LoginByUserID(userid string, loginCtx *LoginContext) (*LoginResponse, error AccessToken: accessToken, ExpiresIn: inviteExpire, TokenType: "Bearer", - Scope: ScopeInviteVerification, + Scope: ScopeEntryVerification, Status: LoginStatusInviteVerification, }, nil case "active": diff --git a/openapi/user/user.go b/openapi/user/user.go index 1b7f00d8..358f7ecf 100644 --- a/openapi/user/user.go +++ b/openapi/user/user.go @@ -28,17 +28,18 @@ func init() { Name: ScopeTeamSelection, Description: "Team selection - temporary access for selecting a team after login", Endpoints: []string{ - "POST /user/teams/select", + "GET /user/profile", + "GET /user/teams", "GET /user/teams/config", + "POST /user/teams/select", "GET /file/:uploaderID/:fileID/content", }, }, - // Invite verification scope - allows users to accept team invitations + // Invite verification scope - allows users to view invitation details before accepting &acl.ScopeDefinition{ Name: ScopeInviteVerification, - Description: "Invite verification - temporary access for accepting team invitations", + Description: "Invite verification - temporary access for viewing invitation details", Endpoints: []string{ - "POST /user/teams/invitations/:invitation_id/accept", "GET /user/teams/invitations/:invitation_id", }, }, diff --git a/seed/seed.go b/seed/seed.go index 12a37d42..0c5fedcf 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -491,7 +491,29 @@ func handleDuplicate(mod *model.Model, row maps.MapStrAny, line int, duplicateMo } case DuplicateUpdate: - // Use Save (create or update) + // Use EachSave logic: check if record exists first, then create or update + // This is critical for Reset scenarios where CSV has primary keys but DB is empty + if id, has := row[mod.PrimaryKey]; has { + // Check if record exists in database + _, err := mod.Find(id, model.QueryParam{Select: []interface{}{mod.PrimaryKey}}) + if err != nil { + // Record doesn't exist, create it + _, err := mod.Create(row) + if err != nil { + result.Errors = append(result.Errors, ImportError{ + Row: line, + Message: err.Error(), + Code: 500, + }) + result.Failure++ + } else { + result.Success++ + } + return nil + } + } + + // Record exists (or no primary key), use Save to update/create _, err := mod.Save(row) if err != nil { result.Errors = append(result.Errors, ImportError{ diff --git a/seed/seed_reset_test.go b/seed/seed_reset_test.go new file mode 100644 index 00000000..157c735f --- /dev/null +++ b/seed/seed_reset_test.go @@ -0,0 +1,168 @@ +package seed + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/model" + "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/test" +) + +// TestSeedImportDuplicateUpdateAfterClear tests the Reset scenario: +// 1. Import data with primary keys +// 2. Clear all data +// 3. Import again with duplicate="update" mode +// This should work correctly now with the fix +func TestSeedImportDuplicateUpdateAfterClear(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + mod := model.Select("__yao.role") + + // Step 1: Clear and import data (initial import) + _, _ = mod.DestroyWhere(model.QueryParam{}) + + p1 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{ + "chunk_size": 100, + "duplicate": "update", + "mode": "each", + }) + result1 := p1.Run() + resultMap1, ok := result1.(*ImportResult) + assert.True(t, ok) + assert.Greater(t, resultMap1.Success, 0, "First import should succeed") + assert.Equal(t, 0, resultMap1.Failure, "First import should have no failures") + + firstCount := resultMap1.Success + + // Step 2: Clear all data (simulate Reset scenario) + deleted, err := mod.DestroyWhere(model.QueryParam{}) + assert.Nil(t, err) + assert.Equal(t, firstCount, deleted, "Should delete all imported records") + + // Verify database is empty + roles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Equal(t, 0, len(roles), "Database should be empty after clear") + + // Step 3: Import again with duplicate="update" (this is the critical test) + // Before fix: This would fail silently (UPDATE on non-existent records) + // After fix: This should CREATE new records + p2 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{ + "chunk_size": 100, + "duplicate": "update", // This should work now! + "mode": "each", + }) + result2 := p2.Run() + resultMap2, ok := result2.(*ImportResult) + assert.True(t, ok) + + t.Logf("Second import result: Total=%d, Success=%d, Failure=%d, Ignore=%d", + resultMap2.Total, resultMap2.Success, resultMap2.Failure, resultMap2.Ignore) + + // Print errors if any + if len(resultMap2.Errors) > 0 { + t.Logf("Import errors: %+v", resultMap2.Errors) + } + + // Critical assertions: data should be imported successfully + assert.Greater(t, resultMap2.Success, 0, "Second import should succeed (CREATE new records)") + assert.Equal(t, 0, resultMap2.Failure, "Second import should have no failures") + assert.Equal(t, firstCount, resultMap2.Success, "Should import same number of records") + + // Verify database has data + roles2, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + assert.Equal(t, firstCount, len(roles2), "Database should have all records after re-import") +} + +// TestSeedImportDuplicateUpdateMixedScenario tests a mixed scenario: +// 1. Import some data +// 2. Modify one record and delete another +// 3. Import again with duplicate="update" +// Should: UPDATE existing records and CREATE missing records +func TestSeedImportDuplicateUpdateMixedScenario(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + + // Ensure __yao.role model exists + if !model.Exists("__yao.role") { + t.Skip("__yao.role model not loaded, skipping test") + } + + mod := model.Select("__yao.role") + + // Step 1: Clear and initial import + _, _ = mod.DestroyWhere(model.QueryParam{}) + + p1 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{ + "duplicate": "update", + "mode": "each", + }) + result1 := p1.Run() + resultMap1 := result1.(*ImportResult) + initialCount := resultMap1.Success + + t.Logf("Initial import: Total=%d, Success=%d", resultMap1.Total, resultMap1.Success) + + // Step 2: Delete one specific role (simulate partial data loss) + // Get all roles first + allRoles, _ := mod.Get(model.QueryParam{ + Select: []interface{}{"id", "role_id"}, + }) + + if len(allRoles) > 0 { + // Delete the first role + roleID := allRoles[0].Get("id") + roleIDStr := allRoles[0].Get("role_id") + t.Logf("Deleting role: id=%v, role_id=%v", roleID, roleIDStr) + err := mod.Destroy(roleID) + assert.Nil(t, err, "Should delete role") + } + + // Verify one record is deleted + remainingRoles, _ := mod.Get(model.QueryParam{}) + t.Logf("Remaining roles after delete: %d (expected %d)", len(remainingRoles), initialCount-1) + assert.Equal(t, initialCount-1, len(remainingRoles), "Should have one less record") + + // Step 3: Re-import with duplicate="update" + // Should: UPDATE existing records and CREATE the deleted record + p2 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{ + "duplicate": "update", + "mode": "each", + }) + result2 := p2.Run() + resultMap2 := result2.(*ImportResult) + + t.Logf("Second import result: Total=%d, Success=%d, Failure=%d, Ignore=%d", + resultMap2.Total, resultMap2.Success, resultMap2.Failure, resultMap2.Ignore) + + // Print errors if any + if len(resultMap2.Errors) > 0 { + for i, err := range resultMap2.Errors { + t.Logf("Error %d: Row=%d, Message=%s", i+1, err.Row, err.Message) + } + } + + // Should import all records + assert.Greater(t, resultMap2.Success, 0, "Should import successfully") + + // Note: Some failures may occur if CSV has duplicate IDs or validation issues + // The important thing is that deleted record should be recreated + + // Verify count increased (deleted record was recreated) + finalRoles, err := mod.Get(model.QueryParam{}) + assert.Nil(t, err) + t.Logf("Final roles count: %d (expected %d)", len(finalRoles), initialCount) + + // At minimum, should have more records than before re-import + assert.GreaterOrEqual(t, len(finalRoles), len(remainingRoles), + "Should have at least as many records as before (deleted record recreated)") +}