Implement pagination and filtering for document listing in ListDocuments function
- Enhanced ListDocuments function to support pagination with customizable page size and sorting options. - Added filtering capabilities for keywords, tags, collection IDs, and status, allowing for more refined document retrieval. - Introduced validation for requested fields and sorting parameters to ensure only valid options are processed. - Improved error handling for document search operations, returning appropriate error responses when necessary.
This commit is contained in:
Max
parent
3bef5a6222
commit
d51874abb3
1 changed files with 252 additions and 8 deletions
|
|
@ -2,15 +2,54 @@ package kb
|
|||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/yaoapp/gou/graphrag/types"
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/yao/attachment"
|
||||
"github.com/yaoapp/yao/kb"
|
||||
"github.com/yaoapp/yao/openapi/response"
|
||||
)
|
||||
|
||||
// Document field definitions
|
||||
var (
|
||||
// availableDocumentFields defines all available fields for security filtering
|
||||
availableDocumentFields = map[string]bool{
|
||||
"id": true, "document_id": true, "collection_id": true, "name": true,
|
||||
"description": true, "status": true, "type": true, "size": true,
|
||||
"segment_count": true, "job_id": true, "uploader_id": true, "tags": true,
|
||||
"locale": true, "system": true, "sort": true, "cover": true,
|
||||
"file_name": true, "file_path": true, "file_mime_type": true,
|
||||
"url": true, "url_title": true, "text_content": true,
|
||||
"converter_provider_id": true, "converter_option_id": true, "converter_properties": true,
|
||||
"fetcher_provider_id": true, "fetcher_option_id": true, "fetcher_properties": true,
|
||||
"chunking_provider_id": true, "chunking_option_id": true, "chunking_properties": true,
|
||||
"extraction_provider_id": true, "extraction_option_id": true, "extraction_properties": true,
|
||||
"processed_at": true, "error_message": true, "created_at": true, "updated_at": true,
|
||||
}
|
||||
|
||||
// defaultDocumentFields defines the default compact field list
|
||||
defaultDocumentFields = []interface{}{
|
||||
"id", "document_id", "collection_id", "name", "description",
|
||||
"cover", "tags", "type", "size", "segment_count", "status", "locale",
|
||||
"error_message", "created_at", "updated_at",
|
||||
}
|
||||
|
||||
// validSortFields defines valid fields for sorting
|
||||
validSortFields = map[string]bool{
|
||||
"created_at": true,
|
||||
"updated_at": true,
|
||||
"name": true,
|
||||
"size": true,
|
||||
"segment_count": true,
|
||||
"sort": true,
|
||||
"processed_at": true,
|
||||
}
|
||||
)
|
||||
|
||||
// SimpleJob represents a simple job for async operations
|
||||
// TODO: replace with proper job system later
|
||||
type SimpleJob struct {
|
||||
|
|
@ -35,14 +74,219 @@ func (j *SimpleJob) Run(fn func()) string {
|
|||
|
||||
// ListDocuments lists documents with pagination
|
||||
func ListDocuments(c *gin.Context) {
|
||||
// TODO: Implement list documents logic
|
||||
// Query parameters for pagination: page, limit, filter, etc.
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"documents": []interface{}{},
|
||||
"total": 0,
|
||||
"page": 1,
|
||||
"limit": 20,
|
||||
})
|
||||
// Check if kb.Instance is available
|
||||
if !checkKBInstance(c) {
|
||||
return
|
||||
}
|
||||
|
||||
// Parse pagination parameters
|
||||
page := 1
|
||||
if pageStr := c.Query("page"); pageStr != "" {
|
||||
if p, err := strconv.Atoi(pageStr); err == nil && p > 0 {
|
||||
page = p
|
||||
}
|
||||
}
|
||||
|
||||
pagesize := 20
|
||||
if pagesizeStr := c.Query("pagesize"); pagesizeStr != "" {
|
||||
if ps, err := strconv.Atoi(pagesizeStr); err == nil && ps > 0 && ps <= 100 {
|
||||
pagesize = ps
|
||||
}
|
||||
}
|
||||
|
||||
// Get KB instance and config
|
||||
kbInstance := kb.Instance.(*kb.KnowledgeBase)
|
||||
config := kbInstance.Config
|
||||
|
||||
// Parse select parameter
|
||||
var selectFields []interface{}
|
||||
if selectParam := strings.TrimSpace(c.Query("select")); selectParam != "" {
|
||||
requestedFields := strings.Split(selectParam, ",")
|
||||
for _, field := range requestedFields {
|
||||
field = strings.TrimSpace(field)
|
||||
if field != "" && availableDocumentFields[field] {
|
||||
selectFields = append(selectFields, field)
|
||||
}
|
||||
}
|
||||
// If no valid fields found, use default
|
||||
if len(selectFields) == 0 {
|
||||
selectFields = defaultDocumentFields
|
||||
}
|
||||
} else {
|
||||
selectFields = defaultDocumentFields
|
||||
}
|
||||
|
||||
// Build query parameters
|
||||
param := model.QueryParam{
|
||||
Select: selectFields,
|
||||
}
|
||||
|
||||
// Add filters
|
||||
var wheres []model.QueryWhere
|
||||
|
||||
// Filter by keywords (search in name and description)
|
||||
if keywords := strings.TrimSpace(c.Query("keywords")); keywords != "" {
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "name",
|
||||
Value: "%" + keywords + "%",
|
||||
OP: "like",
|
||||
})
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "description",
|
||||
Value: "%" + keywords + "%",
|
||||
OP: "like",
|
||||
Wheres: []model.QueryWhere{},
|
||||
Method: "orwhere",
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by tag
|
||||
if tag := strings.TrimSpace(c.Query("tag")); tag != "" {
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "tags",
|
||||
Value: "%" + tag + "%",
|
||||
OP: "like",
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by collection_id
|
||||
if collectionID := strings.TrimSpace(c.Query("collection_id")); collectionID != "" {
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "collection_id",
|
||||
Value: collectionID,
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by status (support multiple values separated by comma)
|
||||
if statusParam := strings.TrimSpace(c.Query("status")); statusParam != "" {
|
||||
statusList := strings.Split(statusParam, ",")
|
||||
var statusValues []interface{}
|
||||
for _, status := range statusList {
|
||||
status = strings.TrimSpace(status)
|
||||
if status != "" {
|
||||
statusValues = append(statusValues, status)
|
||||
}
|
||||
}
|
||||
|
||||
if len(statusValues) > 0 {
|
||||
if len(statusValues) == 1 {
|
||||
// Single status
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "status",
|
||||
Value: statusValues[0],
|
||||
})
|
||||
} else {
|
||||
// Multiple status - use IN clause
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "status",
|
||||
Value: statusValues,
|
||||
OP: "in",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by status_not (exclude specific statuses)
|
||||
if statusNotParam := strings.TrimSpace(c.Query("status_not")); statusNotParam != "" {
|
||||
statusNotList := strings.Split(statusNotParam, ",")
|
||||
var statusNotValues []interface{}
|
||||
for _, status := range statusNotList {
|
||||
status = strings.TrimSpace(status)
|
||||
if status != "" {
|
||||
statusNotValues = append(statusNotValues, status)
|
||||
}
|
||||
}
|
||||
|
||||
if len(statusNotValues) > 0 {
|
||||
if len(statusNotValues) == 1 {
|
||||
// Single status exclusion
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "status",
|
||||
Value: statusNotValues[0],
|
||||
OP: "!=",
|
||||
})
|
||||
} else {
|
||||
// Multiple status exclusion - use NOT IN clause
|
||||
// Since gou/model doesn't support "notin" OP directly,
|
||||
// we need to use a different approach or multiple != conditions
|
||||
for _, status := range statusNotValues {
|
||||
wheres = append(wheres, model.QueryWhere{
|
||||
Column: "status",
|
||||
Value: status,
|
||||
OP: "!=",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
param.Wheres = wheres
|
||||
|
||||
// Add ordering
|
||||
sortParam := strings.TrimSpace(c.Query("sort"))
|
||||
if sortParam == "" {
|
||||
sortParam = "created_at desc" // Default sort
|
||||
}
|
||||
|
||||
// Parse sort parameter (format: "field1 direction1,field2 direction2")
|
||||
var orders []model.QueryOrder
|
||||
sortItems := strings.Split(sortParam, ",")
|
||||
|
||||
for _, sortItem := range sortItems {
|
||||
sortItem = strings.TrimSpace(sortItem)
|
||||
if sortItem == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Parse each sort item (format: "field direction")
|
||||
sortParts := strings.Fields(sortItem)
|
||||
sortField := "created_at" // Default field
|
||||
sortOrder := "desc" // Default order
|
||||
|
||||
if len(sortParts) >= 1 {
|
||||
sortField = sortParts[0]
|
||||
}
|
||||
if len(sortParts) >= 2 {
|
||||
sortOrder = strings.ToLower(sortParts[1])
|
||||
}
|
||||
|
||||
// Validate sort field
|
||||
if !validSortFields[sortField] {
|
||||
continue // Skip invalid fields
|
||||
}
|
||||
|
||||
// Validate sort order
|
||||
if sortOrder != "asc" && sortOrder != "desc" {
|
||||
sortOrder = "desc" // Default order
|
||||
}
|
||||
|
||||
orders = append(orders, model.QueryOrder{
|
||||
Column: sortField,
|
||||
Option: sortOrder,
|
||||
})
|
||||
}
|
||||
|
||||
// If no valid orders found, use default
|
||||
if len(orders) == 0 {
|
||||
orders = []model.QueryOrder{
|
||||
{Column: "created_at", Option: "desc"},
|
||||
}
|
||||
}
|
||||
|
||||
param.Orders = orders
|
||||
|
||||
// Query documents using KB config
|
||||
result, err := config.SearchDocuments(param, page, pagesize)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to search documents: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// ScrollDocuments scrolls through documents with iterator-style pagination
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue