Enhance collection management and segment handling in the API
- Added GetCollection endpoint to retrieve collections by ID, including error handling for missing IDs and uninitialized knowledge base. - Updated CreateCollectionConfig to use more descriptive field names for embedding provider and option IDs. - Refactored UpdateSegments function to improve validation and error handling, ensuring document IDs and segment texts are properly checked. - Introduced new request structures for updating votes, scores, and weights for segments, enhancing the API's capabilities. - Removed the outdated vote handling file to streamline the codebase.
This commit is contained in:
parent
c4c598db35
commit
91d67d3b54
11 changed files with 634 additions and 252 deletions
272
data/bindata.go
272
data/bindata.go
File diff suppressed because one or more lines are too long
|
|
@ -201,6 +201,52 @@ func CollectionExists(c *gin.Context) {
|
|||
response.RespondWithSuccess(c, response.StatusOK, successData)
|
||||
}
|
||||
|
||||
// GetCollection retrieves a collection by ID
|
||||
func GetCollection(c *gin.Context) {
|
||||
collectionID := c.Param("collectionID")
|
||||
if collectionID == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Collection ID is required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if kb.Instance is available
|
||||
if kb.Instance == nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Knowledge base not initialized",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Use the dedicated GetCollection method
|
||||
collection, err := kb.Instance.GetCollection(c.Request.Context(), collectionID)
|
||||
if err != nil {
|
||||
// Check if it's a "not found" error
|
||||
if err.Error() == fmt.Sprintf("collection with ID '%s' not found", collectionID) {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Collection not found",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusNotFound, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to get collection: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
response.RespondWithSuccess(c, response.StatusOK, collection)
|
||||
}
|
||||
|
||||
// GetCollections retrieves collections with optional filtering
|
||||
func GetCollections(c *gin.Context) {
|
||||
// Check if kb.Instance is available
|
||||
|
|
@ -332,9 +378,9 @@ type CreateCollectionRequest struct {
|
|||
|
||||
// CreateCollectionConfig represents the request structure for creating a collection
|
||||
type CreateCollectionConfig struct {
|
||||
EmbeddingProvider string `json:"embedding_provider" binding:"required"` // embedding provider id
|
||||
EmbeddingOption string `json:"embedding_option" binding:"required"` // embedding option value
|
||||
Locale string `json:"locale,omitempty"` // locale for provider reading
|
||||
EmbeddingProviderID string `json:"embedding_provider_id" binding:"required"` // embedding provider id
|
||||
EmbeddingOptionID string `json:"embedding_option_id" binding:"required"` // embedding option id
|
||||
Locale string `json:"locale,omitempty"` // locale for provider reading
|
||||
*types.CreateCollectionOptions
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ func Attach(group *gin.RouterGroup, oauth types.OAuth) {
|
|||
// Collection Management
|
||||
group.POST("/collections", CreateCollection)
|
||||
group.DELETE("/collections/:collectionID", RemoveCollection)
|
||||
group.GET("/collections/:collectionID", GetCollection)
|
||||
group.GET("/collections/:collectionID/exists", CollectionExists)
|
||||
group.GET("/collections", GetCollections)
|
||||
group.PUT("/collections/:collectionID/metadata", UpdateCollectionMetadata)
|
||||
|
|
@ -39,12 +40,14 @@ func Attach(group *gin.RouterGroup, oauth types.OAuth) {
|
|||
|
||||
// Segment Management
|
||||
group.POST("/documents/:docID/segments", AddSegments)
|
||||
group.PUT("/segments", UpdateSegments)
|
||||
group.DELETE("/segments", RemoveSegments)
|
||||
group.PUT("/documents/:docID/segments", UpdateSegments)
|
||||
group.DELETE("/documents/:docID/segments", RemoveSegmentsByDocID)
|
||||
group.GET("/documents/:docID/segments", ScrollSegments)
|
||||
|
||||
// Global segment operations (not tied to specific document)
|
||||
group.DELETE("/segments", RemoveSegments)
|
||||
group.GET("/segments", GetSegments)
|
||||
group.GET("/segments/:segmentID", GetSegment)
|
||||
group.GET("/documents/:docID/segments", ScrollSegments)
|
||||
|
||||
// Segment Voting, Scoring, Weighting
|
||||
group.PUT("/segments/vote", UpdateVote)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,17 @@
|
|||
package kb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/yaoapp/gou/graphrag/types"
|
||||
"github.com/yaoapp/gou/graphrag/utils"
|
||||
"github.com/yaoapp/gou/model"
|
||||
"github.com/yaoapp/yao/kb"
|
||||
"github.com/yaoapp/yao/kb/providers/factory"
|
||||
"github.com/yaoapp/yao/openapi/response"
|
||||
)
|
||||
|
||||
|
|
@ -83,26 +87,21 @@ func AddSegments(c *gin.Context) {
|
|||
|
||||
// UpdateSegments updates segments manually
|
||||
func UpdateSegments(c *gin.Context) {
|
||||
var req UpdateSegmentsRequest
|
||||
|
||||
// Parse and bind JSON request
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
// Extract docID from URL path
|
||||
docID := c.Param("docID")
|
||||
if docID == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Invalid request format: " + err.Error(),
|
||||
ErrorDescription: "Document ID is required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request
|
||||
if err := req.Validate(); err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
// Parse CollectionID from docID to find the right collection
|
||||
collectionID, _ := utils.ExtractCollectionIDFromDocID(docID)
|
||||
if collectionID == "" {
|
||||
collectionID = "default"
|
||||
}
|
||||
|
||||
// Check if kb.Instance is available
|
||||
|
|
@ -115,17 +114,147 @@ func UpdateSegments(c *gin.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
// Convert request to UpsertOptions
|
||||
upsertOptions, err := req.BaseUpsertRequest.ToUpsertOptions()
|
||||
// Get Embedding Provider ID from collection
|
||||
knowledgeBase := kb.Instance.(*kb.KnowledgeBase)
|
||||
|
||||
// Get Extraction Provider ID from document
|
||||
document, err := knowledgeBase.Config.FindDocument(docID, model.QueryParam{Select: []interface{}{
|
||||
"collection_id",
|
||||
"embedding_provider_id", "embedding_option_id", "embedding_properties",
|
||||
"extraction_provider_id", "extraction_option_id", "extraction_properties",
|
||||
}})
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Failed to convert request to upsert options: " + err.Error(),
|
||||
ErrorDescription: "Failed to find document: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("--------------------------------")
|
||||
fmt.Println(document)
|
||||
|
||||
var req UpdateSegmentsRequest
|
||||
// Parse and bind JSON request
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Invalid request format: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate segment texts
|
||||
if len(req.SegmentTexts) == 0 {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "segment_texts is required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
for i, segmentText := range req.SegmentTexts {
|
||||
if strings.TrimSpace(segmentText.Text) == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: fmt.Sprintf("segment_texts[%d].text cannot be empty", i),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(segmentText.ID) == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: fmt.Sprintf("segment_texts[%d].id cannot be empty", i),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Construct UpsertOptions from database document configuration
|
||||
upsertOptions := &types.UpsertOptions{
|
||||
CollectionID: document["collection_id"].(string),
|
||||
DocID: docID,
|
||||
Metadata: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Build Embedding provider configuration from document using Factory
|
||||
if embeddingProviderID, ok := document["embedding_provider_id"].(string); ok && embeddingProviderID != "" {
|
||||
embeddingConfig := &ProviderConfig{
|
||||
ProviderID: embeddingProviderID,
|
||||
}
|
||||
|
||||
if embeddingOptionID, ok := document["embedding_option_id"].(string); ok && embeddingOptionID != "" {
|
||||
embeddingConfig.OptionID = embeddingOptionID
|
||||
}
|
||||
|
||||
// Use Factory to resolve and create embedding provider
|
||||
embeddingOption, err := embeddingConfig.ProviderOption("embedding", "en")
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Failed to resolve embedding provider: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
embeddingProvider, err := factory.MakeEmbedding(embeddingProviderID, embeddingOption)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Failed to create embedding provider: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
upsertOptions.Embedding = embeddingProvider
|
||||
}
|
||||
|
||||
// Build Extraction provider configuration from document (if available)
|
||||
if extractionProviderID, ok := document["extraction_provider_id"].(string); ok && extractionProviderID != "" {
|
||||
extractionConfig := &ProviderConfig{
|
||||
ProviderID: extractionProviderID,
|
||||
}
|
||||
|
||||
if extractionOptionID, ok := document["extraction_option_id"].(string); ok && extractionOptionID != "" {
|
||||
extractionConfig.OptionID = extractionOptionID
|
||||
}
|
||||
|
||||
// Use Factory to resolve and create extraction provider
|
||||
extractionOption, err := extractionConfig.ProviderOption("extraction", "en")
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Failed to resolve extraction provider: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
extractionProvider, err := factory.MakeExtraction(extractionProviderID, extractionOption)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Failed to create extraction provider: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
upsertOptions.Extraction = extractionProvider
|
||||
}
|
||||
|
||||
fmt.Println("--- UpdateSegments ---")
|
||||
fmt.Println(req.SegmentTexts)
|
||||
fmt.Println(upsertOptions.DocID)
|
||||
fmt.Println(upsertOptions.CollectionID)
|
||||
|
||||
// Perform update segments operation
|
||||
updatedCount, err := kb.Instance.UpdateSegments(c.Request.Context(), req.SegmentTexts, upsertOptions)
|
||||
if err != nil {
|
||||
|
|
@ -140,7 +269,7 @@ func UpdateSegments(c *gin.Context) {
|
|||
// Return success response
|
||||
result := gin.H{
|
||||
"message": "Segments updated successfully",
|
||||
"collection_id": req.CollectionID,
|
||||
"collection_id": upsertOptions.CollectionID,
|
||||
"updated_count": updatedCount,
|
||||
"segments_count": len(req.SegmentTexts),
|
||||
}
|
||||
|
|
@ -150,14 +279,109 @@ func UpdateSegments(c *gin.Context) {
|
|||
|
||||
// RemoveSegments removes segments by IDs
|
||||
func RemoveSegments(c *gin.Context) {
|
||||
// TODO: Implement remove segments logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Segments removed"})
|
||||
// Parse segment_ids from query parameter (comma-separated string)
|
||||
segmentIDsParam := strings.TrimSpace(c.Query("segment_ids"))
|
||||
if segmentIDsParam == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "segment_ids query parameter is required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Split comma-separated segment IDs
|
||||
segmentIDs := strings.Split(segmentIDsParam, ",")
|
||||
var validSegmentIDs []string
|
||||
for _, id := range segmentIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id != "" {
|
||||
validSegmentIDs = append(validSegmentIDs, id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(validSegmentIDs) == 0 {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "No valid segment IDs provided",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if kb.Instance is available
|
||||
if kb.Instance == nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Knowledge base not initialized",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Perform remove segments operation
|
||||
removedCount, err := kb.Instance.RemoveSegments(c.Request.Context(), validSegmentIDs)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to remove segments: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Return success response
|
||||
result := gin.H{
|
||||
"message": "Segments removed successfully",
|
||||
"segment_ids": validSegmentIDs,
|
||||
"removed_count": removedCount,
|
||||
}
|
||||
|
||||
response.RespondWithSuccess(c, response.StatusOK, result)
|
||||
}
|
||||
|
||||
// RemoveSegmentsByDocID removes all segments of a document
|
||||
func RemoveSegmentsByDocID(c *gin.Context) {
|
||||
// TODO: Implement remove segments by document ID logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Segments removed by document ID"})
|
||||
// Parse docID from URL path parameter
|
||||
docID := c.Param("docID")
|
||||
if docID == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "docID is required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if kb.Instance is available
|
||||
if kb.Instance == nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Knowledge base not initialized",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Perform remove segments by document ID operation
|
||||
removedCount, err := kb.Instance.RemoveSegmentsByDocID(c.Request.Context(), docID)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to remove segments by document ID: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Return success response
|
||||
result := gin.H{
|
||||
"message": "Segments removed successfully",
|
||||
"doc_id": docID,
|
||||
"removed_count": removedCount,
|
||||
}
|
||||
|
||||
response.RespondWithSuccess(c, response.StatusOK, result)
|
||||
}
|
||||
|
||||
// GetSegments gets segments by IDs
|
||||
|
|
|
|||
78
openapi/kb/store.go
Normal file
78
openapi/kb/store.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package kb
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/yaoapp/yao/kb"
|
||||
"github.com/yaoapp/yao/openapi/response"
|
||||
)
|
||||
|
||||
// Segment Voting, Scoring, Weighting Handlers
|
||||
|
||||
// UpdateVote updates votes for segments
|
||||
func UpdateVote(c *gin.Context) {
|
||||
// TODO: Implement update vote logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Vote updated"})
|
||||
}
|
||||
|
||||
// UpdateScore updates scores for segments
|
||||
func UpdateScore(c *gin.Context) {
|
||||
// TODO: Implement update score logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Score updated"})
|
||||
}
|
||||
|
||||
// UpdateWeight updates weights for segments
|
||||
func UpdateWeight(c *gin.Context) {
|
||||
var req UpdateWeightRequest
|
||||
|
||||
// Parse and bind JSON request
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Invalid request format: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request
|
||||
if err := req.Validate(); err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if kb.Instance is available
|
||||
if kb.Instance == nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Knowledge base not initialized",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Perform update weight operation
|
||||
updatedCount, err := kb.Instance.UpdateWeight(c.Request.Context(), req.Segments)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to update segment weights: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Return success response
|
||||
result := gin.H{
|
||||
"message": "Segment weights updated successfully",
|
||||
"segments": req.Segments,
|
||||
"updated_count": updatedCount,
|
||||
}
|
||||
|
||||
response.RespondWithSuccess(c, response.StatusOK, result)
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package kb
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/yaoapp/gou/graphrag/types"
|
||||
"github.com/yaoapp/yao/kb"
|
||||
|
|
@ -124,10 +125,25 @@ type AddSegmentsRequest struct {
|
|||
|
||||
// UpdateSegmentsRequest represents the request for UpdateSegments API
|
||||
type UpdateSegmentsRequest struct {
|
||||
BaseUpsertRequest
|
||||
// Segment texts to update
|
||||
SegmentTexts []types.SegmentText `json:"segment_texts" binding:"required"`
|
||||
}
|
||||
|
||||
// UpdateVoteRequest represents the request for UpdateVote API
|
||||
type UpdateVoteRequest struct {
|
||||
Segments []types.SegmentVote `json:"segments" binding:"required"`
|
||||
}
|
||||
|
||||
// UpdateScoreRequest represents the request for UpdateScore API
|
||||
type UpdateScoreRequest struct {
|
||||
Segments []types.SegmentScore `json:"segments" binding:"required"`
|
||||
}
|
||||
|
||||
// UpdateWeightRequest represents the request for UpdateWeight API
|
||||
type UpdateWeightRequest struct {
|
||||
Segments []types.SegmentWeight `json:"segments" binding:"required"`
|
||||
}
|
||||
|
||||
// ProviderOption resolves a ProviderConfig to a *kbtypes.ProviderOption
|
||||
// If OptionID is provided, it looks up the option from the provider
|
||||
// If Option is provided directly, it uses the Option field
|
||||
|
|
@ -386,13 +402,18 @@ func (r *AddSegmentsRequest) Validate() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Validate validates the UpdateSegmentsRequest fields
|
||||
func (r *UpdateSegmentsRequest) Validate() error {
|
||||
if err := r.BaseUpsertRequest.Validate(); err != nil {
|
||||
return err
|
||||
// Validate validates the UpdateWeightRequest fields
|
||||
func (r *UpdateWeightRequest) Validate() error {
|
||||
if len(r.Segments) == 0 {
|
||||
return fmt.Errorf("segments is required")
|
||||
}
|
||||
if len(r.SegmentTexts) == 0 {
|
||||
return fmt.Errorf("segment_texts is required")
|
||||
for i, segment := range r.Segments {
|
||||
if strings.TrimSpace(segment.ID) == "" {
|
||||
return fmt.Errorf("segments[%d].id cannot be empty", i)
|
||||
}
|
||||
if segment.Weight < 0 {
|
||||
return fmt.Errorf("segments[%d].weight cannot be negative", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -452,6 +473,15 @@ func (r *BaseUpsertRequest) AddBaseFields(data map[string]interface{}) {
|
|||
data["chunking_properties"] = r.Chunking.Option.Properties
|
||||
}
|
||||
}
|
||||
if r.Embedding != nil {
|
||||
data["embedding_provider_id"] = r.Embedding.ProviderID
|
||||
if r.Embedding.OptionID != "" {
|
||||
data["embedding_option_id"] = r.Embedding.OptionID
|
||||
}
|
||||
if r.Embedding.Option != nil {
|
||||
data["embedding_properties"] = r.Embedding.Option.Properties
|
||||
}
|
||||
}
|
||||
if r.Extraction != nil {
|
||||
data["extraction_provider_id"] = r.Extraction.ProviderID
|
||||
if r.Extraction.OptionID != "" {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ func PrepareCreateCollection(c *gin.Context) (*CreateCollectionRequest, map[stri
|
|||
}
|
||||
|
||||
// Get provider settings first to resolve dimension
|
||||
providerSettings, err := getProviderSettings(req.Config.EmbeddingProvider, req.Config.EmbeddingOption, req.Config.Locale)
|
||||
providerSettings, err := getProviderSettings(req.Config.EmbeddingProviderID, req.Config.EmbeddingOptionID, req.Config.Locale)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to resolve provider settings: %w", err)
|
||||
}
|
||||
|
|
@ -26,12 +26,23 @@ func PrepareCreateCollection(c *gin.Context) (*CreateCollectionRequest, map[stri
|
|||
// Set dimension from provider settings
|
||||
req.Config.Dimension = providerSettings.Dimension
|
||||
|
||||
// Store embedding properties if available
|
||||
var embeddingProperties map[string]interface{} = nil
|
||||
if providerSettings.Properties != nil {
|
||||
embeddingProperties = providerSettings.Properties
|
||||
}
|
||||
|
||||
// Add metadata with provider information
|
||||
if req.Metadata == nil {
|
||||
req.Metadata = make(map[string]interface{})
|
||||
}
|
||||
req.Metadata["__embedding_provider"] = req.Config.EmbeddingProvider
|
||||
req.Metadata["__embedding_option"] = req.Config.EmbeddingOption
|
||||
req.Metadata["__embedding_provider"] = req.Config.EmbeddingProviderID
|
||||
req.Metadata["__embedding_option"] = req.Config.EmbeddingOptionID
|
||||
|
||||
if embeddingProperties != nil {
|
||||
req.Metadata["__embedding_properties"] = embeddingProperties
|
||||
}
|
||||
|
||||
if req.Config.Locale != "" {
|
||||
req.Metadata["__locale"] = req.Config.Locale
|
||||
}
|
||||
|
|
@ -43,15 +54,16 @@ func PrepareCreateCollection(c *gin.Context) (*CreateCollectionRequest, map[stri
|
|||
|
||||
// Prepare collection data for database
|
||||
data := map[string]interface{}{
|
||||
"collection_id": req.ID,
|
||||
"name": req.Metadata["name"],
|
||||
"description": req.Metadata["description"],
|
||||
"status": "creating",
|
||||
"embedding_provider": req.Config.EmbeddingProvider,
|
||||
"embedding_option": req.Config.EmbeddingOption,
|
||||
"locale": req.Config.Locale,
|
||||
"distance": req.Config.Distance,
|
||||
"index_type": req.Config.IndexType,
|
||||
"collection_id": req.ID,
|
||||
"name": req.Metadata["name"],
|
||||
"description": req.Metadata["description"],
|
||||
"status": "creating",
|
||||
"embedding_provider_id": req.Config.EmbeddingProviderID,
|
||||
"embedding_option_id": req.Config.EmbeddingOptionID,
|
||||
"embedding_properties": embeddingProperties,
|
||||
"locale": req.Config.Locale,
|
||||
"distance": req.Config.Distance,
|
||||
"index_type": req.Config.IndexType,
|
||||
}
|
||||
|
||||
// Add optional HNSW parameters
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
package kb
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Segment Voting, Scoring, Weighting Handlers
|
||||
|
||||
// UpdateVote updates votes for segments
|
||||
func UpdateVote(c *gin.Context) {
|
||||
// TODO: Implement update vote logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Vote updated"})
|
||||
}
|
||||
|
||||
// UpdateScore updates scores for segments
|
||||
func UpdateScore(c *gin.Context) {
|
||||
// TODO: Implement update score logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Score updated"})
|
||||
}
|
||||
|
||||
// UpdateWeight updates weights for segments
|
||||
func UpdateWeight(c *gin.Context) {
|
||||
// TODO: Implement update weight logic
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Weight updated"})
|
||||
}
|
||||
|
|
@ -401,63 +401,50 @@ func TestAddSegmentsRequest_Validate(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestUpdateSegmentsRequest_Validate(t *testing.T) {
|
||||
func TestUpdateSegmentsRequest_Structure(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
request *kb.UpdateSegmentsRequest
|
||||
expectError bool
|
||||
errorMsg string
|
||||
expectValid bool
|
||||
}{
|
||||
{
|
||||
name: "valid update segments request",
|
||||
request: &kb.UpdateSegmentsRequest{
|
||||
BaseUpsertRequest: kb.BaseUpsertRequest{
|
||||
CollectionID: "test_collection",
|
||||
Chunking: &kb.ProviderConfig{
|
||||
ProviderID: "chunking_provider",
|
||||
},
|
||||
Embedding: &kb.ProviderConfig{
|
||||
ProviderID: "embedding_provider",
|
||||
},
|
||||
},
|
||||
SegmentTexts: []types.SegmentText{
|
||||
{Text: "Updated segment"},
|
||||
{ID: "segment_1", Text: "Updated segment"},
|
||||
},
|
||||
},
|
||||
expectError: false,
|
||||
expectValid: true,
|
||||
},
|
||||
{
|
||||
name: "missing segment_texts",
|
||||
name: "empty segment_texts",
|
||||
request: &kb.UpdateSegmentsRequest{
|
||||
BaseUpsertRequest: kb.BaseUpsertRequest{
|
||||
CollectionID: "test_collection",
|
||||
Chunking: &kb.ProviderConfig{
|
||||
ProviderID: "chunking_provider",
|
||||
},
|
||||
Embedding: &kb.ProviderConfig{
|
||||
ProviderID: "embedding_provider",
|
||||
},
|
||||
},
|
||||
SegmentTexts: []types.SegmentText{},
|
||||
},
|
||||
expectError: true,
|
||||
errorMsg: "segment_texts is required",
|
||||
expectValid: false,
|
||||
},
|
||||
{
|
||||
name: "multiple segments",
|
||||
request: &kb.UpdateSegmentsRequest{
|
||||
SegmentTexts: []types.SegmentText{
|
||||
{ID: "segment_1", Text: "First updated segment"},
|
||||
{ID: "segment_2", Text: "Second updated segment"},
|
||||
},
|
||||
},
|
||||
expectValid: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.request.Validate()
|
||||
|
||||
if tt.expectError {
|
||||
if err == nil {
|
||||
t.Errorf("Expected error but got none")
|
||||
} else if err.Error() != tt.errorMsg {
|
||||
t.Errorf("Expected error message '%s', got '%s'", tt.errorMsg, err.Error())
|
||||
// Test basic structure validation
|
||||
if tt.expectValid {
|
||||
if len(tt.request.SegmentTexts) == 0 {
|
||||
t.Errorf("Expected valid request to have segment_texts")
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("Expected no error but got: %v", err)
|
||||
if len(tt.request.SegmentTexts) > 0 {
|
||||
t.Errorf("Expected invalid request to have empty segment_texts")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -94,22 +94,28 @@
|
|||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "embedding_provider",
|
||||
"name": "embedding_provider_id",
|
||||
"type": "string",
|
||||
"label": "Embedding Provider",
|
||||
"comment": "Embedding provider ID",
|
||||
"label": "Embedding Provider ID",
|
||||
"comment": "Knowledge embedding provider ID (optional)",
|
||||
"length": 128,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "embedding_option",
|
||||
"name": "embedding_option_id",
|
||||
"type": "string",
|
||||
"label": "Embedding Option",
|
||||
"comment": "Embedding model option value",
|
||||
"label": "Embedding Option ID",
|
||||
"comment": "Knowledge embedding provider option ID (optional)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "embedding_properties",
|
||||
"type": "json",
|
||||
"label": "Embedding Properties",
|
||||
"comment": "Embedding provider configuration properties",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "locale",
|
||||
"type": "string",
|
||||
|
|
|
|||
|
|
@ -281,6 +281,29 @@
|
|||
"comment": "Chunking provider configuration properties (includes split_mode, chunk_size, chunk_overlap, etc.)",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "embedding_provider_id",
|
||||
"type": "string",
|
||||
"label": "Embedding Provider ID",
|
||||
"comment": "Knowledge embedding provider ID (optional)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "embedding_option_id",
|
||||
"type": "string",
|
||||
"label": "Embedding Option ID",
|
||||
"comment": "Knowledge embedding provider option ID (optional)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "embedding_properties",
|
||||
"type": "json",
|
||||
"label": "Embedding Properties",
|
||||
"comment": "Embedding provider configuration properties",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "extraction_provider_id",
|
||||
"type": "string",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue