Merge pull request #1107 from trheyi/main

Update asset modification timestamps and enhance chunking options configuration

kb/providers/chunking.go

@@ -35,7 +35,9 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
 			Overlap:        20,
 			MaxDepth:       3,
 			SizeMultiplier: 3,
-			MaxConcurrent:  10,
+			MaxConcurrent:  1,
+			Separator:      "",
+			EnableDebug:    false,
 		}, nil
 	}
 
@@ -45,7 +47,9 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
 		Overlap:        20,
 		MaxDepth:       3,
 		SizeMultiplier: 3,
-		MaxConcurrent:  10,
+		MaxConcurrent:  1,
+		Separator:      "",
+		EnableDebug:    false,
 	}
 
 	// Extract values from Properties map
@@ -89,6 +93,18 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
 				options.MaxConcurrent = int(maxConcurrentFloat)
 			}
 		}
+
+		if separator, ok := option.Properties["separator"]; ok {
+			if separatorStr, ok := separator.(string); ok {
+				options.Separator = separatorStr
+			}
+		}
+
+		if enableDebug, ok := option.Properties["enable_debug"]; ok {
+			if enableDebugBool, ok := enableDebug.(bool); ok {
+				options.EnableDebug = enableDebugBool
+			}
+		}
 	}
 
 	return options, nil
@@ -115,12 +131,13 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
 			Overlap:        50,
 			MaxDepth:       3,
 			SizeMultiplier: 3,
-			MaxConcurrent:  10,
+			MaxConcurrent:  1,
 			SemanticOptions: &types.SemanticOptions{
+				Connector:     "openai.gpt-4o-mini",
 				ContextSize:   1800, // Default L1 Size (ChunkSize * 6)
 				MaxRetry:      3,
-				MaxConcurrent: 10,
-				Toolcall:      false,
+				MaxConcurrent: 1,
+				Toolcall:      true,
 			},
 		}, nil
 	}
@@ -131,12 +148,13 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
 		Overlap:        50,
 		MaxDepth:       3,
 		SizeMultiplier: 3,
-		MaxConcurrent:  10,
+		MaxConcurrent:  1,
 		SemanticOptions: &types.SemanticOptions{
+			Connector:     "openai.gpt-4o-mini",
 			ContextSize:   1800, // Default L1 Size (ChunkSize * 6)
 			MaxRetry:      3,
-			MaxConcurrent: 10,
-			Toolcall:      false,
+			MaxConcurrent: 1,
+			Toolcall:      true,
 		},
 	}
 
@@ -186,52 +204,56 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
 			}
 		}
 
-		// Semantic-specific options
-		if connector, ok := option.Properties["connector"]; ok {
-			if connectorStr, ok := connector.(string); ok {
-				options.SemanticOptions.Connector = connectorStr
-			}
-		}
+		// Handle nested semantic options
+		if semanticProps, ok := option.Properties["semantic"]; ok {
+			if semanticMap, ok := semanticProps.(map[string]interface{}); ok {
+				if connector, ok := semanticMap["connector"]; ok {
+					if connectorStr, ok := connector.(string); ok {
+						options.SemanticOptions.Connector = connectorStr
+					}
+				}
 
-		if toolcall, ok := option.Properties["toolcall"]; ok {
-			if toolcallBool, ok := toolcall.(bool); ok {
-				options.SemanticOptions.Toolcall = toolcallBool
-			}
-		}
+				if toolcall, ok := semanticMap["toolcall"]; ok {
+					if toolcallBool, ok := toolcall.(bool); ok {
+						options.SemanticOptions.Toolcall = toolcallBool
+					}
+				}
 
-		if contextSize, ok := option.Properties["context_size"]; ok {
-			if contextSizeInt, ok := contextSize.(int); ok {
-				options.SemanticOptions.ContextSize = contextSizeInt
-			} else if contextSizeFloat, ok := contextSize.(float64); ok {
-				options.SemanticOptions.ContextSize = int(contextSizeFloat)
-			}
-		}
+				if contextSize, ok := semanticMap["context_size"]; ok {
+					if contextSizeInt, ok := contextSize.(int); ok {
+						options.SemanticOptions.ContextSize = contextSizeInt
+					} else if contextSizeFloat, ok := contextSize.(float64); ok {
+						options.SemanticOptions.ContextSize = int(contextSizeFloat)
+					}
+				}
 
-		if optionsStr, ok := option.Properties["options"]; ok {
-			if optionsString, ok := optionsStr.(string); ok {
-				options.SemanticOptions.Options = optionsString
-			}
-		}
+				if maxRetry, ok := semanticMap["max_retry"]; ok {
+					if maxRetryInt, ok := maxRetry.(int); ok {
+						options.SemanticOptions.MaxRetry = maxRetryInt
+					} else if maxRetryFloat, ok := maxRetry.(float64); ok {
+						options.SemanticOptions.MaxRetry = int(maxRetryFloat)
+					}
+				}
 
-		if prompt, ok := option.Properties["prompt"]; ok {
-			if promptStr, ok := prompt.(string); ok {
-				options.SemanticOptions.Prompt = promptStr
-			}
-		}
+				if semanticMaxConcurrent, ok := semanticMap["semantic_max_concurrent"]; ok {
+					if semanticMaxConcurrentInt, ok := semanticMaxConcurrent.(int); ok {
+						options.SemanticOptions.MaxConcurrent = semanticMaxConcurrentInt
+					} else if semanticMaxConcurrentFloat, ok := semanticMaxConcurrent.(float64); ok {
+						options.SemanticOptions.MaxConcurrent = int(semanticMaxConcurrentFloat)
+					}
+				}
 
-		if maxRetry, ok := option.Properties["max_retry"]; ok {
-			if maxRetryInt, ok := maxRetry.(int); ok {
-				options.SemanticOptions.MaxRetry = maxRetryInt
-			} else if maxRetryFloat, ok := maxRetry.(float64); ok {
-				options.SemanticOptions.MaxRetry = int(maxRetryFloat)
-			}
-		}
+				if prompt, ok := semanticMap["prompt"]; ok {
+					if promptStr, ok := prompt.(string); ok {
+						options.SemanticOptions.Prompt = promptStr
+					}
+				}
 
-		if semanticMaxConcurrent, ok := option.Properties["semantic_max_concurrent"]; ok {
-			if semanticMaxConcurrentInt, ok := semanticMaxConcurrent.(int); ok {
-				options.SemanticOptions.MaxConcurrent = semanticMaxConcurrentInt
-			} else if semanticMaxConcurrentFloat, ok := semanticMaxConcurrent.(float64); ok {
-				options.SemanticOptions.MaxConcurrent = int(semanticMaxConcurrentFloat)
+				if optionsStr, ok := semanticMap["options"]; ok {
+					if optionsJSON, ok := optionsStr.(string); ok {
+						options.SemanticOptions.Options = optionsJSON
+					}
+				}
 			}
 		}
 	}

openapi/kb/types.go

@@ -405,31 +405,58 @@ func (r *BaseUpsertRequest) AddBaseFields(data map[string]interface{}) {
 	if r.DocID != "" {
 		data["document_id"] = r.DocID
 	}
+
+	// Extract fields from metadata
 	if r.Metadata != nil {
-		data["tags"] = r.Metadata
+		// Extract specific fields from metadata
+		if description, ok := r.Metadata["description"]; ok && description != nil {
+			data["description"] = description
+		}
+		if cover, ok := r.Metadata["cover"]; ok && cover != nil {
+			data["cover"] = cover
+		}
+		if tags, ok := r.Metadata["tags"]; ok && tags != nil {
+			data["tags"] = tags
+		}
+		if name, ok := r.Metadata["name"]; ok && name != nil {
+			data["name"] = name
+		}
+		// Note: Other metadata fields like size, type, etc. are handled by specific request types
 	}
 
 	// Add provider configurations
 	if r.Converter != nil {
 		data["converter_provider_id"] = r.Converter.ProviderID
+		if r.Converter.OptionID != "" {
+			data["converter_option_id"] = r.Converter.OptionID
+		}
 		if r.Converter.Option != nil {
 			data["converter_properties"] = r.Converter.Option.Properties
 		}
 	}
 	if r.Fetcher != nil {
 		data["fetcher_provider_id"] = r.Fetcher.ProviderID
+		if r.Fetcher.OptionID != "" {
+			data["fetcher_option_id"] = r.Fetcher.OptionID
+		}
 		if r.Fetcher.Option != nil {
 			data["fetcher_properties"] = r.Fetcher.Option.Properties
 		}
 	}
 	if r.Chunking != nil {
 		data["chunking_provider_id"] = r.Chunking.ProviderID
+		if r.Chunking.OptionID != "" {
+			data["chunking_option_id"] = r.Chunking.OptionID
+		}
 		if r.Chunking.Option != nil {
 			data["chunking_properties"] = r.Chunking.Option.Properties
 		}
 	}
 	if r.Extraction != nil {
 		data["extraction_provider_id"] = r.Extraction.ProviderID
+		if r.Extraction.OptionID != "" {
+			data["extraction_option_id"] = r.Extraction.OptionID
+		}
 		if r.Extraction.Option != nil {
 			data["extraction_properties"] = r.Extraction.Option.Properties
 		}

yao/models/kb/document.mod.yao

@@ -147,6 +147,14 @@
       "default": 0,
       "nullable": false
     },
+    {
+      "name": "cover",
+      "type": "string",
+      "label": "Cover Image",
+      "comment": "Document cover image URL",
+      "length": 512,
+      "nullable": true
+    },
     {
       "name": "file_name",
       "type": "string",
@@ -202,6 +210,14 @@
       "length": 128,
       "nullable": true
     },
+    {
+      "name": "converter_option_id",
+      "type": "string",
+      "label": "Converter Option ID",
+      "comment": "Document converter provider option ID (for file type)",
+      "length": 128,
+      "nullable": true
+    },
     {
       "name": "converter_properties",
       "type": "json",
@@ -217,6 +233,14 @@
       "length": 128,
       "nullable": true
     },
+    {
+      "name": "fetcher_option_id",
+      "type": "string",
+      "label": "Fetcher Option ID",
+      "comment": "URL fetcher provider option ID (for url type)",
+      "length": 128,
+      "nullable": true
+    },
     {
       "name": "fetcher_properties",
       "type": "json",
@@ -233,6 +257,14 @@
       "nullable": false,
       "index": true
     },
+    {
+      "name": "chunking_option_id",
+      "type": "string",
+      "label": "Chunking Option ID",
+      "comment": "Text chunking provider option ID",
+      "length": 128,
+      "nullable": true
+    },
     {
       "name": "chunking_properties",
       "type": "json",
@@ -248,6 +280,14 @@
       "length": 128,
       "nullable": true
     },
+    {
+      "name": "extraction_option_id",
+      "type": "string",
+      "label": "Extraction Option ID",
+      "comment": "Knowledge extraction provider option ID (optional)",
+      "length": 128,
+      "nullable": true
+    },
     {
       "name": "extraction_properties",
       "type": "json",