Merge pull request #1263 from trheyi/main

Improve document search, seed duplicate handling, update token scopes, add user endpoints, and load scopes dynamically

kb/types/document.go

@@ -18,6 +18,7 @@ func (c *Config) SearchDocuments(param model.QueryParam, page int, pagesize int)
 	if mod == nil {
 		return nil, fmt.Errorf("document model not found: %s", modelName)
 	}
+	param.Debug = true
 	return mod.Paginate(param, page, pagesize)
 }
 

openapi/oauth/acl/scope.go

@@ -223,22 +223,53 @@ func (m *ScopeManager) expandAlias(alias string, visited map[string]bool) ([]str
 	return expanded, nil
 }
 
+// getFirstLevelSubdirs returns all first-level subdirectories in a given directory
+func getFirstLevelSubdirs(baseDir string) ([]string, error) {
+	var subdirs []string
+
+	err := application.App.Walk(baseDir, func(root, filename string, isdir bool) error {
+		// Only process directories
+		if !isdir {
+			return nil
+		}
+
+		// Skip the root directory itself
+		if filename == baseDir {
+			return nil
+		}
+
+		// Get relative path from baseDir
+		relPath := strings.TrimPrefix(filename, baseDir)
+		relPath = strings.TrimPrefix(relPath, string(filepath.Separator))
+
+		// Only include first-level directories (no nested paths)
+		if !strings.Contains(relPath, string(filepath.Separator)) && relPath != "" {
+			subdirs = append(subdirs, relPath)
+		}
+
+		return nil
+	}, "")
+
+	if err != nil {
+		return nil, err
+	}
+
+	return subdirs, nil
+}
+
 // loadScopeDefinitions loads scope definitions from subdirectories
 func (m *ScopeManager) loadScopeDefinitions() error {
 	scopesDir := filepath.Join("openapi", "scopes")
 
-	// Subdirectories to scan
-	subDirs := []string{"kb", "job", "file", "user"}
+	// Get all subdirectories in the scopes directory
+	subDirs, err := getFirstLevelSubdirs(scopesDir)
+	if err != nil {
+		return fmt.Errorf("failed to scan scopes directory: %w", err)
+	}
 
+	// Load scope definitions from each subdirectory
 	for _, subDir := range subDirs {
 		dirPath := filepath.Join(scopesDir, subDir)
-		exists, err := application.App.Exists(dirPath)
-		if err != nil {
-			return err
-		}
-		if !exists {
-			continue
-		}
 
 		// Walk through all .yml files in the directory
 		err = application.App.Walk(dirPath, func(root, filename string, isdir bool) error {

openapi/user/entry.go

@@ -1083,10 +1083,10 @@ func GinVerifyInvite(c *gin.Context) {
 
 	// Get authorized info from the temporary token
 	authInfo := oauth.GetAuthorizedInfo(c)
-	if authInfo == nil || authInfo.Scope != ScopeInviteVerification {
+	if authInfo == nil || authInfo.Scope != ScopeEntryVerification {
 		errorResp := &response.ErrorResponse{
 			Code:             response.ErrInsufficientScope.Code,
-			ErrorDescription: "Invalid or missing token scope. Expected invite_verification scope",
+			ErrorDescription: "Invalid or missing token scope. Expected entry_verification scope",
 		}
 		response.RespondWithError(c, response.StatusForbidden, errorResp)
 		return

openapi/user/login.go

@@ -176,7 +176,7 @@ func LoginByUserID(userid string, loginCtx *LoginContext) (*LoginResponse, error
 			extraClaims["remember_me"] = true
 		}
 
-		accessToken, err := oauth.OAuth.MakeAccessToken(yaoClientConfig.ClientID, ScopeInviteVerification, subject, inviteExpire, extraClaims)
+		accessToken, err := oauth.OAuth.MakeAccessToken(yaoClientConfig.ClientID, ScopeEntryVerification, subject, inviteExpire, extraClaims)
 		if err != nil {
 			return nil, err
 		}
@@ -186,7 +186,7 @@ func LoginByUserID(userid string, loginCtx *LoginContext) (*LoginResponse, error
 			AccessToken: accessToken,
 			ExpiresIn:   inviteExpire,
 			TokenType:   "Bearer",
-			Scope:       ScopeInviteVerification,
+			Scope:       ScopeEntryVerification,
 			Status:      LoginStatusInviteVerification,
 		}, nil
 	case "active":

openapi/user/user.go

@@ -28,17 +28,18 @@ func init() {
 			Name:        ScopeTeamSelection,
 			Description: "Team selection - temporary access for selecting a team after login",
 			Endpoints: []string{
-				"POST /user/teams/select",
+				"GET /user/profile",
+				"GET /user/teams",
 				"GET /user/teams/config",
+				"POST /user/teams/select",
 				"GET /file/:uploaderID/:fileID/content",
 			},
 		},
-		// Invite verification scope - allows users to accept team invitations
+		// Invite verification scope - allows users to view invitation details before accepting
 		&acl.ScopeDefinition{
 			Name:        ScopeInviteVerification,
-			Description: "Invite verification - temporary access for accepting team invitations",
+			Description: "Invite verification - temporary access for viewing invitation details",
 			Endpoints: []string{
-				"POST /user/teams/invitations/:invitation_id/accept",
 				"GET /user/teams/invitations/:invitation_id",
 			},
 		},

seed/seed.go

@@ -491,7 +491,29 @@ func handleDuplicate(mod *model.Model, row maps.MapStrAny, line int, duplicateMo
 		}
 
 	case DuplicateUpdate:
-		// Use Save (create or update)
+		// Use EachSave logic: check if record exists first, then create or update
+		// This is critical for Reset scenarios where CSV has primary keys but DB is empty
+		if id, has := row[mod.PrimaryKey]; has {
+			// Check if record exists in database
+			_, err := mod.Find(id, model.QueryParam{Select: []interface{}{mod.PrimaryKey}})
+			if err != nil {
+				// Record doesn't exist, create it
+				_, err := mod.Create(row)
+				if err != nil {
+					result.Errors = append(result.Errors, ImportError{
+						Row:     line,
+						Message: err.Error(),
+						Code:    500,
+					})
+					result.Failure++
+				} else {
+					result.Success++
+				}
+				return nil
+			}
+		}
+
+		// Record exists (or no primary key), use Save to update/create
 		_, err := mod.Save(row)
 		if err != nil {
 			result.Errors = append(result.Errors, ImportError{

seed/seed_reset_test.go

@@ -0,0 +1,168 @@
+package seed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/yaoapp/gou/model"
+	"github.com/yaoapp/gou/process"
+	"github.com/yaoapp/yao/config"
+	"github.com/yaoapp/yao/test"
+)
+
+// TestSeedImportDuplicateUpdateAfterClear tests the Reset scenario:
+// 1. Import data with primary keys
+// 2. Clear all data
+// 3. Import again with duplicate="update" mode
+// This should work correctly now with the fix
+func TestSeedImportDuplicateUpdateAfterClear(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	// Ensure __yao.role model exists
+	if !model.Exists("__yao.role") {
+		t.Skip("__yao.role model not loaded, skipping test")
+	}
+
+	mod := model.Select("__yao.role")
+
+	// Step 1: Clear and import data (initial import)
+	_, _ = mod.DestroyWhere(model.QueryParam{})
+
+	p1 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{
+		"chunk_size": 100,
+		"duplicate":  "update",
+		"mode":       "each",
+	})
+	result1 := p1.Run()
+	resultMap1, ok := result1.(*ImportResult)
+	assert.True(t, ok)
+	assert.Greater(t, resultMap1.Success, 0, "First import should succeed")
+	assert.Equal(t, 0, resultMap1.Failure, "First import should have no failures")
+
+	firstCount := resultMap1.Success
+
+	// Step 2: Clear all data (simulate Reset scenario)
+	deleted, err := mod.DestroyWhere(model.QueryParam{})
+	assert.Nil(t, err)
+	assert.Equal(t, firstCount, deleted, "Should delete all imported records")
+
+	// Verify database is empty
+	roles, err := mod.Get(model.QueryParam{})
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(roles), "Database should be empty after clear")
+
+	// Step 3: Import again with duplicate="update" (this is the critical test)
+	// Before fix: This would fail silently (UPDATE on non-existent records)
+	// After fix: This should CREATE new records
+	p2 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{
+		"chunk_size": 100,
+		"duplicate":  "update", // This should work now!
+		"mode":       "each",
+	})
+	result2 := p2.Run()
+	resultMap2, ok := result2.(*ImportResult)
+	assert.True(t, ok)
+
+	t.Logf("Second import result: Total=%d, Success=%d, Failure=%d, Ignore=%d",
+		resultMap2.Total, resultMap2.Success, resultMap2.Failure, resultMap2.Ignore)
+
+	// Print errors if any
+	if len(resultMap2.Errors) > 0 {
+		t.Logf("Import errors: %+v", resultMap2.Errors)
+	}
+
+	// Critical assertions: data should be imported successfully
+	assert.Greater(t, resultMap2.Success, 0, "Second import should succeed (CREATE new records)")
+	assert.Equal(t, 0, resultMap2.Failure, "Second import should have no failures")
+	assert.Equal(t, firstCount, resultMap2.Success, "Should import same number of records")
+
+	// Verify database has data
+	roles2, err := mod.Get(model.QueryParam{})
+	assert.Nil(t, err)
+	assert.Equal(t, firstCount, len(roles2), "Database should have all records after re-import")
+}
+
+// TestSeedImportDuplicateUpdateMixedScenario tests a mixed scenario:
+// 1. Import some data
+// 2. Modify one record and delete another
+// 3. Import again with duplicate="update"
+// Should: UPDATE existing records and CREATE missing records
+func TestSeedImportDuplicateUpdateMixedScenario(t *testing.T) {
+	test.Prepare(t, config.Conf)
+	defer test.Clean()
+
+	// Ensure __yao.role model exists
+	if !model.Exists("__yao.role") {
+		t.Skip("__yao.role model not loaded, skipping test")
+	}
+
+	mod := model.Select("__yao.role")
+
+	// Step 1: Clear and initial import
+	_, _ = mod.DestroyWhere(model.QueryParam{})
+
+	p1 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{
+		"duplicate": "update",
+		"mode":      "each",
+	})
+	result1 := p1.Run()
+	resultMap1 := result1.(*ImportResult)
+	initialCount := resultMap1.Success
+
+	t.Logf("Initial import: Total=%d, Success=%d", resultMap1.Total, resultMap1.Success)
+
+	// Step 2: Delete one specific role (simulate partial data loss)
+	// Get all roles first
+	allRoles, _ := mod.Get(model.QueryParam{
+		Select: []interface{}{"id", "role_id"},
+	})
+
+	if len(allRoles) > 0 {
+		// Delete the first role
+		roleID := allRoles[0].Get("id")
+		roleIDStr := allRoles[0].Get("role_id")
+		t.Logf("Deleting role: id=%v, role_id=%v", roleID, roleIDStr)
+		err := mod.Destroy(roleID)
+		assert.Nil(t, err, "Should delete role")
+	}
+
+	// Verify one record is deleted
+	remainingRoles, _ := mod.Get(model.QueryParam{})
+	t.Logf("Remaining roles after delete: %d (expected %d)", len(remainingRoles), initialCount-1)
+	assert.Equal(t, initialCount-1, len(remainingRoles), "Should have one less record")
+
+	// Step 3: Re-import with duplicate="update"
+	// Should: UPDATE existing records and CREATE the deleted record
+	p2 := process.New("seeds.import", "roles.csv", "__yao.role", map[string]interface{}{
+		"duplicate": "update",
+		"mode":      "each",
+	})
+	result2 := p2.Run()
+	resultMap2 := result2.(*ImportResult)
+
+	t.Logf("Second import result: Total=%d, Success=%d, Failure=%d, Ignore=%d",
+		resultMap2.Total, resultMap2.Success, resultMap2.Failure, resultMap2.Ignore)
+
+	// Print errors if any
+	if len(resultMap2.Errors) > 0 {
+		for i, err := range resultMap2.Errors {
+			t.Logf("Error %d: Row=%d, Message=%s", i+1, err.Row, err.Message)
+		}
+	}
+
+	// Should import all records
+	assert.Greater(t, resultMap2.Success, 0, "Should import successfully")
+
+	// Note: Some failures may occur if CSV has duplicate IDs or validation issues
+	// The important thing is that deleted record should be recreated
+
+	// Verify count increased (deleted record was recreated)
+	finalRoles, err := mod.Get(model.QueryParam{})
+	assert.Nil(t, err)
+	t.Logf("Final roles count: %d (expected %d)", len(finalRoles), initialCount)
+
+	// At minimum, should have more records than before re-import
+	assert.GreaterOrEqual(t, len(finalRoles), len(remainingRoles),
+		"Should have at least as many records as before (deleted record recreated)")
+}