Merge pull request #1092 from trheyi/main
Add provider management functionality and enhance schema retrieval
This commit is contained in:
Max
GitHub
commit
e6c5631b57
18 changed files with 1242 additions and 107 deletions
504
data/bindata.go
504
data/bindata.go
File diff suppressed because one or more lines are too long
137
kb/kb.go
137
kb/kb.go
|
|
@ -1,7 +1,9 @@
|
|||
package kb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
"github.com/yaoapp/gou/application"
|
||||
"github.com/yaoapp/gou/graphrag"
|
||||
|
|
@ -73,3 +75,138 @@ func Load(appConfig config.Config) (*KnowledgeBase, error) {
|
|||
Instance = instance
|
||||
return instance, nil
|
||||
}
|
||||
|
||||
// GetProviders returns all providers
|
||||
func GetProviders(typ string, ids []string, locale string) ([]kbtypes.Provider, error) {
|
||||
if Instance == nil {
|
||||
return nil, fmt.Errorf("knowledge base not initialized")
|
||||
}
|
||||
|
||||
// Get the providers from the instance
|
||||
knowledgeBase, ok := Instance.(*KnowledgeBase)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("knowledge base not initialized")
|
||||
}
|
||||
|
||||
// Get the configuration
|
||||
conf := knowledgeBase.Config
|
||||
if conf == nil {
|
||||
return nil, fmt.Errorf("configuration not found")
|
||||
}
|
||||
|
||||
providers := []*kbtypes.Provider{}
|
||||
switch typ {
|
||||
case "chunking":
|
||||
providers = conf.Chunkings
|
||||
|
||||
case "converter":
|
||||
providers = conf.Converters
|
||||
|
||||
case "embedding":
|
||||
providers = conf.Embeddings
|
||||
|
||||
case "extractor":
|
||||
providers = conf.Extractors
|
||||
|
||||
case "fetcher":
|
||||
providers = conf.Fetchers
|
||||
|
||||
case "searcher":
|
||||
providers = conf.Searchers
|
||||
|
||||
case "reranker":
|
||||
providers = conf.Rerankers
|
||||
|
||||
case "vote":
|
||||
providers = conf.Votes
|
||||
|
||||
case "weight":
|
||||
providers = conf.Weights
|
||||
|
||||
case "score":
|
||||
providers = conf.Scores
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid provider type: %s", typ)
|
||||
|
||||
}
|
||||
|
||||
// Filter empty ids
|
||||
filteredIds := []string{}
|
||||
for _, id := range ids {
|
||||
if id != "" {
|
||||
filteredIds = append(filteredIds, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Filter the providers by ids
|
||||
filteredProviders := []kbtypes.Provider{}
|
||||
for _, provider := range providers {
|
||||
if len(filteredIds) == 0 || slices.Contains(ids, provider.ID) {
|
||||
filteredProviders = append(filteredProviders, *provider)
|
||||
}
|
||||
}
|
||||
return filteredProviders, nil
|
||||
}
|
||||
|
||||
// GetProvider returns a provider by id
|
||||
func GetProvider(typ string, id string) (*kbtypes.Provider, error) {
|
||||
if Instance == nil {
|
||||
return nil, fmt.Errorf("knowledge base not initialized")
|
||||
}
|
||||
|
||||
knowledgeBase, ok := Instance.(*KnowledgeBase)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("knowledge base not initialized")
|
||||
}
|
||||
|
||||
conf := knowledgeBase.Config
|
||||
if conf == nil {
|
||||
return nil, fmt.Errorf("configuration not found")
|
||||
}
|
||||
|
||||
providers := []*kbtypes.Provider{}
|
||||
switch typ {
|
||||
case "chunking":
|
||||
providers = conf.Chunkings
|
||||
|
||||
case "converter":
|
||||
providers = conf.Converters
|
||||
|
||||
case "embedding":
|
||||
providers = conf.Embeddings
|
||||
|
||||
case "extractor":
|
||||
providers = conf.Extractors
|
||||
|
||||
case "fetcher":
|
||||
providers = conf.Fetchers
|
||||
|
||||
case "searcher":
|
||||
providers = conf.Searchers
|
||||
|
||||
case "reranker":
|
||||
providers = conf.Rerankers
|
||||
|
||||
case "vote":
|
||||
providers = conf.Votes
|
||||
|
||||
case "weight":
|
||||
providers = conf.Weights
|
||||
|
||||
case "score":
|
||||
providers = conf.Scores
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid provider type: %s", typ)
|
||||
}
|
||||
|
||||
// Find the provider by id
|
||||
for _, provider := range providers {
|
||||
if provider.ID == id {
|
||||
return provider, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("provider %s not found", id)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ func (s *Structured) Options(option *kbtypes.ProviderOption) (*types.ChunkingOpt
|
|||
|
||||
// Schema returns the schema for the structured chunking provider
|
||||
func (s *Structured) Schema(provider *kbtypes.Provider, locale string) (*kbtypes.ProviderSchema, error) {
|
||||
return nil, nil
|
||||
return factory.GetSchemaFromBindata(factory.ProviderTypeChunking, "structured", locale)
|
||||
}
|
||||
|
||||
// === Semantic Chunking ===
|
||||
|
|
@ -241,5 +241,5 @@ func (s *Semantic) Options(option *kbtypes.ProviderOption) (*types.ChunkingOptio
|
|||
|
||||
// Schema returns the schema for the semantic chunking provider
|
||||
func (s *Semantic) Schema(provider *kbtypes.Provider, locale string) (*kbtypes.ProviderSchema, error) {
|
||||
return nil, nil
|
||||
return factory.GetSchemaFromBindata(factory.ProviderTypeChunking, "semantic", locale)
|
||||
}
|
||||
|
|
|
|||
91
kb/providers/factory/utils.go
Normal file
91
kb/providers/factory/utils.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package factory
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/yaoapp/yao/data"
|
||||
kbtypes "github.com/yaoapp/yao/kb/types"
|
||||
)
|
||||
|
||||
// GetSchemaFromBindata reads the schema from bindata
|
||||
func GetSchemaFromBindata(typ ProviderType, name string, locale string) (*kbtypes.ProviderSchema, error) {
|
||||
local := strings.ToLower(locale)
|
||||
if local == "" {
|
||||
local = "en"
|
||||
}
|
||||
|
||||
// Read the schema from bindata
|
||||
raw, err := data.Asset("yao/data/kb/providers/" + string(typ) + "/" + name + "/" + local + ".json")
|
||||
if err != nil {
|
||||
// fallback to en
|
||||
raw, err = data.Asset("yao/data/kb/providers/" + string(typ) + "/" + name + "/en.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Replace the {{ $limit... }} with the actual limit values
|
||||
raw, err = replaceVars(raw, map[string]interface{}{
|
||||
"limit.max_concurrent": 10,
|
||||
"limit.task.max_concurrent": 10,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
schema := &kbtypes.ProviderSchema{}
|
||||
if err := jsoniter.Unmarshal(raw, schema); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return schema, nil
|
||||
}
|
||||
|
||||
// ReplaceVars replaces the variables in the raw data {{ $... }}
|
||||
func replaceVars(raw []byte, vars map[string]interface{}) ([]byte, error) {
|
||||
result := string(raw)
|
||||
|
||||
// Regular expressions to match quoted and unquoted variables
|
||||
// Match "{{ $variable }}" (quoted) or {{ $variable }} (unquoted)
|
||||
var regQuoted = regexp.MustCompile(`"\{\{\s*\$([a-zA-Z_][a-zA-Z0-9_.]*)\s*\}\}"`)
|
||||
var regUnquoted = regexp.MustCompile(`\{\{\s*\$([a-zA-Z_][a-zA-Z0-9_.]*)\s*\}\}`)
|
||||
|
||||
// First, process quoted variables
|
||||
quotedMatches := regQuoted.FindAllStringSubmatch(result, -1)
|
||||
for _, match := range quotedMatches {
|
||||
fullMatch := match[0] // Full match, e.g. "{{ $limit.max_concurrent }}"
|
||||
varName := match[1] // Variable name, e.g. "limit.max_concurrent"
|
||||
|
||||
if value, exists := vars[varName]; exists {
|
||||
// For quoted variables, replace the entire quoted part with JSON-encoded value
|
||||
valueBytes, err := jsoniter.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal variable %s: %v", varName, err)
|
||||
}
|
||||
replacement := string(valueBytes)
|
||||
result = strings.ReplaceAll(result, fullMatch, replacement)
|
||||
}
|
||||
}
|
||||
|
||||
// Then, process unquoted variables
|
||||
unquotedMatches := regUnquoted.FindAllStringSubmatch(result, -1)
|
||||
for _, match := range unquotedMatches {
|
||||
fullMatch := match[0] // Full match, e.g. {{ $limit.max_concurrent }}
|
||||
varName := match[1] // Variable name, e.g. "limit.max_concurrent"
|
||||
|
||||
if value, exists := vars[varName]; exists {
|
||||
// For unquoted variables, also use JSON encoding
|
||||
valueBytes, err := jsoniter.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal variable %s: %v", varName, err)
|
||||
}
|
||||
replacement := string(valueBytes)
|
||||
result = strings.ReplaceAll(result, fullMatch, replacement)
|
||||
}
|
||||
}
|
||||
|
||||
return []byte(result), nil
|
||||
}
|
||||
|
|
@ -57,4 +57,8 @@ func Attach(group *gin.RouterGroup, oauth types.OAuth) {
|
|||
// Collection Backup and Restore
|
||||
group.POST("/collections/:collectionID/backup", Backup)
|
||||
group.POST("/collections/:collectionID/restore", Restore)
|
||||
|
||||
// Provider Management (Chunking, Converter, Embedding, Extractor, Fetcher ...)
|
||||
group.GET("/providers/:providerType", GetProviders)
|
||||
group.GET("/providers/:providerType/:providerID/schema", GetProviderSchema)
|
||||
}
|
||||
|
|
|
|||
86
openapi/kb/provider.go
Normal file
86
openapi/kb/provider.go
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
package kb
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/yaoapp/yao/kb"
|
||||
"github.com/yaoapp/yao/kb/providers/factory"
|
||||
"github.com/yaoapp/yao/openapi/response"
|
||||
)
|
||||
|
||||
// GetProviders get all providers
|
||||
func GetProviders(c *gin.Context) {
|
||||
providerType := c.Param("providerType")
|
||||
locale := c.Query("locale")
|
||||
if locale == "" {
|
||||
locale = "en"
|
||||
}
|
||||
|
||||
if providerType == "" {
|
||||
// Create a custom error with the same structure but specific message
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Invalid request format: providerType is required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// Filter providers by ids
|
||||
ids := strings.Split(c.Query("ids"), ",")
|
||||
providers, err := kb.GetProviders(providerType, ids, locale)
|
||||
if err != nil {
|
||||
// Create a custom error with the same structure but specific message
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to get providers: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
// response with success
|
||||
response.RespondWithSuccess(c, response.StatusOK, providers)
|
||||
}
|
||||
|
||||
// GetProviderSchema get provider schema
|
||||
func GetProviderSchema(c *gin.Context) {
|
||||
providerType := c.Param("providerType")
|
||||
providerID := c.Param("providerID")
|
||||
locale := c.Query("locale")
|
||||
if locale == "" {
|
||||
locale = "en"
|
||||
}
|
||||
|
||||
if providerType == "" || providerID == "" {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrInvalidRequest.Code,
|
||||
ErrorDescription: "Invalid request format: providerType and providerID are required",
|
||||
}
|
||||
response.RespondWithError(c, response.StatusBadRequest, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
provider, err := kb.GetProvider(providerType, providerID)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to get provider: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
schema, err := factory.GetSchema(factory.ProviderType(providerType), provider, locale)
|
||||
if err != nil {
|
||||
errorResp := &response.ErrorResponse{
|
||||
Code: response.ErrServerError.Code,
|
||||
ErrorDescription: "Failed to get provider schema: " + err.Error(),
|
||||
}
|
||||
response.RespondWithError(c, response.StatusInternalServerError, errorResp)
|
||||
return
|
||||
}
|
||||
|
||||
response.RespondWithSuccess(c, response.StatusOK, schema)
|
||||
}
|
||||
176
yao/data/kb/providers/chunking/semantic/en.json
Normal file
176
yao/data/kb/providers/chunking/semantic/en.json
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
{
|
||||
"id": "__yao.semantic",
|
||||
"title": "Smart Text Splitting",
|
||||
"description": "AI-powered intelligent document splitting that understands content meaning and context. Uses large language models to split text at natural topic boundaries, preserving subject coherence and logical flow for better search retrieval quality.",
|
||||
"required": ["size", "overlap", "max_depth"],
|
||||
"properties": {
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"title": "Segment Size",
|
||||
"description": "Target characters per text segment.",
|
||||
"default": 300,
|
||||
"minimum": 50,
|
||||
"maximum": 4000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 1
|
||||
},
|
||||
"overlap": {
|
||||
"type": "integer",
|
||||
"title": "Overlap",
|
||||
"description": "Overlapping characters between adjacent segments.",
|
||||
"default": 50,
|
||||
"minimum": 0,
|
||||
"maximum": 1000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 2
|
||||
},
|
||||
"max_depth": {
|
||||
"type": "integer",
|
||||
"title": "Max Depth",
|
||||
"description": "Maximum hierarchy depth to traverse for splitting.",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 3
|
||||
},
|
||||
"size_multiplier": {
|
||||
"type": "integer",
|
||||
"title": "Size Multiplier",
|
||||
"description": "Multiplier to adjust effective size at deeper levels.",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 4
|
||||
},
|
||||
"max_concurrent": {
|
||||
"type": "integer",
|
||||
"title": "Max Concurrent",
|
||||
"description": "Parallelism when splitting documents.",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": "{{ $limit.max_concurrent }}",
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 5
|
||||
},
|
||||
"semantic": {
|
||||
"type": "object",
|
||||
"title": "Semantic Options",
|
||||
"description": "Advanced options for semantic model calls.",
|
||||
"component": "Nested",
|
||||
"order": 6,
|
||||
"required": false,
|
||||
"requiredFields": ["connector", "context_size"],
|
||||
"properties": {
|
||||
"connector": {
|
||||
"type": "string",
|
||||
"title": "Connector",
|
||||
"description": "Connector ID for the AI model (e.g. openai.gpt-4o-mini, deepseek.v3).",
|
||||
"default": "",
|
||||
"component": "Select",
|
||||
"enum": [
|
||||
{
|
||||
"groupLabel": "OpenAI Models",
|
||||
"options": [
|
||||
{
|
||||
"label": "GPT-4o Mini",
|
||||
"value": "openai.gpt-4o-mini",
|
||||
"description": "Fast and cost-effective, best for most use cases",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"label": "GPT-4o",
|
||||
"value": "openai.gpt-4o",
|
||||
"description": "Highest quality AI splitting for complex documents"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"groupLabel": "Alternative Models",
|
||||
"options": [
|
||||
{
|
||||
"label": "Deepseek V3",
|
||||
"value": "deepseek.v3",
|
||||
"description": "Cost-effective alternative with good performance"
|
||||
},
|
||||
{
|
||||
"label": "Claude 3.5 Sonnet",
|
||||
"value": "anthropic.claude-3-5-sonnet",
|
||||
"description": "Excellent reasoning and context understanding"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"width": "half",
|
||||
"order": 1
|
||||
},
|
||||
"toolcall": {
|
||||
"type": "boolean",
|
||||
"title": "Enable Tool Call",
|
||||
"description": "Allow tool calls during semantic analysis.",
|
||||
"default": false,
|
||||
"component": "Switch",
|
||||
"width": "half",
|
||||
"order": 2
|
||||
},
|
||||
"context_size": {
|
||||
"type": "integer",
|
||||
"title": "Context Size",
|
||||
"description": "Approximate characters provided to the model for context (defaults to size * 6).",
|
||||
"default": 1800,
|
||||
"minimum": 200,
|
||||
"maximum": 32000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 3
|
||||
},
|
||||
"options": {
|
||||
"type": "string",
|
||||
"title": "Model Options (JSON)",
|
||||
"description": "Optional model-specific options in JSON string.",
|
||||
"default": "",
|
||||
"component": "CodeEditor",
|
||||
"width": "full",
|
||||
"order": 4
|
||||
},
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"title": "Custom Prompt",
|
||||
"description": "Override default prompting for semantic splitting.",
|
||||
"default": "",
|
||||
"component": "TextArea",
|
||||
"width": "full",
|
||||
"order": 5
|
||||
},
|
||||
"max_retry": {
|
||||
"type": "integer",
|
||||
"title": "Max Retry",
|
||||
"description": "Maximum retries for model calls.",
|
||||
"default": 3,
|
||||
"minimum": 0,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 6
|
||||
},
|
||||
"semantic_max_concurrent": {
|
||||
"type": "integer",
|
||||
"title": "Semantic Max Concurrent",
|
||||
"description": "Parallelism for semantic model calls.",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": "{{ $limit.task.max_concurrent }}",
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 7
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
176
yao/data/kb/providers/chunking/semantic/zh-cn.json
Normal file
176
yao/data/kb/providers/chunking/semantic/zh-cn.json
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
{
|
||||
"id": "__yao.semantic",
|
||||
"title": "智能文本分割",
|
||||
"description": "AI驱动的智能文档分割，能够理解内容含义和上下文。使用大语言模型在自然主题边界处分割文本，保持主题连贯性和逻辑流程，提供更好的搜索检索质量。",
|
||||
"required": ["size", "overlap", "max_depth"],
|
||||
"properties": {
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"title": "片段大小",
|
||||
"description": "每个片段的目标字符数。",
|
||||
"default": 300,
|
||||
"minimum": 50,
|
||||
"maximum": 4000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 1
|
||||
},
|
||||
"overlap": {
|
||||
"type": "integer",
|
||||
"title": "重叠字符",
|
||||
"description": "相邻片段间的重叠字符数。",
|
||||
"default": 50,
|
||||
"minimum": 0,
|
||||
"maximum": 1000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 2
|
||||
},
|
||||
"max_depth": {
|
||||
"type": "integer",
|
||||
"title": "最大深度",
|
||||
"description": "分割时遍历的最大层级深度。",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 3
|
||||
},
|
||||
"size_multiplier": {
|
||||
"type": "integer",
|
||||
"title": "大小倍数",
|
||||
"description": "调整更深层级有效大小的倍数。",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 4
|
||||
},
|
||||
"max_concurrent": {
|
||||
"type": "integer",
|
||||
"title": "最大并发数",
|
||||
"description": "分割文档时的并行度。",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": "{{ $limit.max_concurrent }}",
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 5
|
||||
},
|
||||
"semantic": {
|
||||
"type": "object",
|
||||
"title": "语义选项",
|
||||
"description": "语义模型调用的高级选项。",
|
||||
"component": "Nested",
|
||||
"order": 6,
|
||||
"required": false,
|
||||
"requiredFields": ["connector", "context_size"],
|
||||
"properties": {
|
||||
"connector": {
|
||||
"type": "string",
|
||||
"title": "连接器",
|
||||
"description": "AI模型的连接器ID（例如：openai.gpt-4o-mini, deepseek.v3）。",
|
||||
"default": "",
|
||||
"component": "Select",
|
||||
"enum": [
|
||||
{
|
||||
"groupLabel": "OpenAI 模型",
|
||||
"options": [
|
||||
{
|
||||
"label": "GPT-4o Mini",
|
||||
"value": "openai.gpt-4o-mini",
|
||||
"description": "快速且经济，适合大多数使用场景",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"label": "GPT-4o",
|
||||
"value": "openai.gpt-4o",
|
||||
"description": "最高质量的AI分割，适用于复杂文档"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"groupLabel": "替代模型",
|
||||
"options": [
|
||||
{
|
||||
"label": "Deepseek V3",
|
||||
"value": "deepseek.v3",
|
||||
"description": "经济实惠的替代方案，性能良好"
|
||||
},
|
||||
{
|
||||
"label": "Claude 3.5 Sonnet",
|
||||
"value": "anthropic.claude-3-5-sonnet",
|
||||
"description": "出色的推理和上下文理解能力"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"width": "half",
|
||||
"order": 1
|
||||
},
|
||||
"toolcall": {
|
||||
"type": "boolean",
|
||||
"title": "启用工具调用",
|
||||
"description": "在语义分析过程中允许工具调用。",
|
||||
"default": false,
|
||||
"component": "Switch",
|
||||
"width": "half",
|
||||
"order": 2
|
||||
},
|
||||
"context_size": {
|
||||
"type": "integer",
|
||||
"title": "上下文大小",
|
||||
"description": "提供给模型的近似字符数上下文（默认为大小的6倍）。",
|
||||
"default": 1800,
|
||||
"minimum": 200,
|
||||
"maximum": 32000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 3
|
||||
},
|
||||
"options": {
|
||||
"type": "string",
|
||||
"title": "模型选项（JSON）",
|
||||
"description": "可选的模型特定选项，JSON字符串格式。",
|
||||
"default": "",
|
||||
"component": "CodeEditor",
|
||||
"width": "full",
|
||||
"order": 4
|
||||
},
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"title": "自定义提示词",
|
||||
"description": "覆盖语义分割的默认提示词。",
|
||||
"default": "",
|
||||
"component": "TextArea",
|
||||
"width": "full",
|
||||
"order": 5
|
||||
},
|
||||
"max_retry": {
|
||||
"type": "integer",
|
||||
"title": "最大重试次数",
|
||||
"description": "模型调用的最大重试次数。",
|
||||
"default": 3,
|
||||
"minimum": 0,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 6
|
||||
},
|
||||
"semantic_max_concurrent": {
|
||||
"type": "integer",
|
||||
"title": "语义最大并发数",
|
||||
"description": "语义模型调用的并行度。",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": "{{ $limit.task.max_concurrent }}",
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 7
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
82
yao/data/kb/providers/chunking/structured/en.json
Normal file
82
yao/data/kb/providers/chunking/structured/en.json
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
{
|
||||
"id": "__yao.structured",
|
||||
"title": "Hierarchical Text Splitting",
|
||||
"description": "Splits documents into multiple layers of text segments with configurable sizes and overlaps. Creates parent-child relationships between segments at different depths, allowing for both detailed and broader context retrieval in search applications.",
|
||||
"required": ["size", "overlap", "max_depth"],
|
||||
"properties": {
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"title": "Segment Size",
|
||||
"description": "Target characters per text segment.",
|
||||
"default": 300,
|
||||
"minimum": 50,
|
||||
"maximum": 4000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 1
|
||||
},
|
||||
"overlap": {
|
||||
"type": "integer",
|
||||
"title": "Overlap",
|
||||
"description": "Overlapping characters between adjacent segments.",
|
||||
"default": 20,
|
||||
"minimum": 0,
|
||||
"maximum": 1000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 2
|
||||
},
|
||||
"max_depth": {
|
||||
"type": "integer",
|
||||
"title": "Max Depth",
|
||||
"description": "Maximum hierarchy depth to traverse for splitting.",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 3
|
||||
},
|
||||
"separator": {
|
||||
"type": "string",
|
||||
"title": "Custom Separator",
|
||||
"description": "Custom separator pattern (regex supported).",
|
||||
"default": "",
|
||||
"placeholder": "e.g. \\n\\n, ---",
|
||||
"component": "Input",
|
||||
"width": "half",
|
||||
"order": 4
|
||||
},
|
||||
"enable_debug": {
|
||||
"type": "boolean",
|
||||
"title": "Enable Debug Mode",
|
||||
"description": "Output detailed splitting information for debugging.",
|
||||
"default": false,
|
||||
"component": "Switch",
|
||||
"width": "half",
|
||||
"order": 5
|
||||
},
|
||||
"size_multiplier": {
|
||||
"type": "integer",
|
||||
"title": "Size Multiplier",
|
||||
"description": "Multiplier to adjust effective size at deeper levels.",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 6
|
||||
},
|
||||
"max_concurrent": {
|
||||
"type": "integer",
|
||||
"title": "Max Concurrent",
|
||||
"description": "Parallelism when splitting documents.",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": "{{ $limit.max_concurrent }}",
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 7
|
||||
}
|
||||
}
|
||||
}
|
||||
82
yao/data/kb/providers/chunking/structured/zh-cn.json
Normal file
82
yao/data/kb/providers/chunking/structured/zh-cn.json
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
{
|
||||
"id": "__yao.structured",
|
||||
"title": "分层文本分割",
|
||||
"description": "将文档分割为多层文本片段，可配置大小和重叠度。在不同深度的片段间创建父子关系，支持在搜索应用中进行详细和宽泛的上下文检索。",
|
||||
"required": ["size", "overlap", "max_depth"],
|
||||
"properties": {
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"title": "片段大小",
|
||||
"description": "每个片段的目标字符数。",
|
||||
"default": 300,
|
||||
"minimum": 50,
|
||||
"maximum": 4000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 1
|
||||
},
|
||||
"overlap": {
|
||||
"type": "integer",
|
||||
"title": "重叠字符",
|
||||
"description": "相邻片段间的重叠字符数。",
|
||||
"default": 20,
|
||||
"minimum": 0,
|
||||
"maximum": 1000,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 2
|
||||
},
|
||||
"max_depth": {
|
||||
"type": "integer",
|
||||
"title": "最大深度",
|
||||
"description": "分割时遍历的最大层级深度。",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 3
|
||||
},
|
||||
"separator": {
|
||||
"type": "string",
|
||||
"title": "自定义分隔符",
|
||||
"description": "自定义分隔符模式（支持正则表达式）。",
|
||||
"default": "",
|
||||
"placeholder": "例如：\\n\\n, ---",
|
||||
"component": "Input",
|
||||
"width": "half",
|
||||
"order": 4
|
||||
},
|
||||
"enable_debug": {
|
||||
"type": "boolean",
|
||||
"title": "启用调试模式",
|
||||
"description": "输出详细的分割信息用于调试。",
|
||||
"default": false,
|
||||
"component": "Switch",
|
||||
"width": "half",
|
||||
"order": 5
|
||||
},
|
||||
"size_multiplier": {
|
||||
"type": "integer",
|
||||
"title": "大小倍数",
|
||||
"description": "调整更深层级有效大小的倍数。",
|
||||
"default": 3,
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 6
|
||||
},
|
||||
"max_concurrent": {
|
||||
"type": "integer",
|
||||
"title": "最大并发数",
|
||||
"description": "分割文档时的并行度。",
|
||||
"default": 1,
|
||||
"minimum": 1,
|
||||
"maximum": "{{ $limit.max_concurrent }}",
|
||||
"component": "InputNumber",
|
||||
"width": "half",
|
||||
"order": 7
|
||||
}
|
||||
}
|
||||
}
|
||||
1
yao/data/kb/providers/converter/en.json
Normal file
1
yao/data/kb/providers/converter/en.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
1
yao/data/kb/providers/converter/zh-cn.json
Normal file
1
yao/data/kb/providers/converter/zh-cn.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
1
yao/data/kb/providers/embedding/en.json
Normal file
1
yao/data/kb/providers/embedding/en.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
1
yao/data/kb/providers/embedding/zh-cn.json
Normal file
1
yao/data/kb/providers/embedding/zh-cn.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
1
yao/data/kb/providers/extractor/en.json
Normal file
1
yao/data/kb/providers/extractor/en.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
1
yao/data/kb/providers/extractor/zh-cn.json
Normal file
1
yao/data/kb/providers/extractor/zh-cn.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
0
yao/data/kb/providers/fetcher/en.json
Normal file
0
yao/data/kb/providers/fetcher/en.json
Normal file
1
yao/data/kb/providers/fetcher/zh-cn.json
Normal file
1
yao/data/kb/providers/fetcher/zh-cn.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
Loading…
Add table
Reference in a new issue