Refactor knowledge base model structure to support new collection and document formats
- Replaced the existing kb.mod.yao file with two new files: collection.mod.yao and document.mod.yao, enhancing the organization of knowledge base models. - Updated references in system and test models to reflect the new structure, ensuring compatibility with the updated model paths. - Adjusted asset loading functions to accommodate the new model files, improving the overall asset management system.
This commit is contained in:
Max
parent
862733d1f4
commit
bab6eec3c3
6 changed files with 641 additions and 281 deletions
323
data/bindata.go
323
data/bindata.go
File diff suppressed because one or more lines are too long
|
|
@ -27,7 +27,8 @@ var systemModels = map[string]string{
|
|||
"__yao.config": "yao/models/config.mod.yao",
|
||||
"__yao.dsl": "yao/models/dsl.mod.yao",
|
||||
"__yao.history": "yao/models/history.mod.yao",
|
||||
"__yao.kb": "yao/models/kb.mod.yao",
|
||||
"__yao.kb.collection": "yao/models/kb/collection.mod.yao",
|
||||
"__yao.kb.document": "yao/models/kb/document.mod.yao",
|
||||
"__yao.user": "yao/models/user.mod.yao",
|
||||
"__yao.user_role": "yao/models/user_role.mod.yao",
|
||||
"__yao.user_type": "yao/models/user_type.mod.yao",
|
||||
|
|
|
|||
|
|
@ -43,7 +43,8 @@ var testSystemModels = map[string]string{
|
|||
"__yao.config": "yao/models/config.mod.yao",
|
||||
"__yao.dsl": "yao/models/dsl.mod.yao",
|
||||
"__yao.history": "yao/models/history.mod.yao",
|
||||
"__yao.kb": "yao/models/kb.mod.yao",
|
||||
"__yao.kb.collection": "yao/models/kb/collection.mod.yao",
|
||||
"__yao.kb.document": "yao/models/kb/document.mod.yao",
|
||||
"__yao.user": "yao/models/user.mod.yao",
|
||||
"__yao.user_role": "yao/models/user_role.mod.yao",
|
||||
"__yao.user_type": "yao/models/user_type.mod.yao",
|
||||
|
|
|
|||
|
|
@ -1,130 +0,0 @@
|
|||
{
|
||||
"name": "knowledge",
|
||||
"label": "Knowledge",
|
||||
"description": "Knowledge table for storing knowledge collections for AI assistants",
|
||||
"tags": ["system"],
|
||||
"builtin": true,
|
||||
"readonly": true,
|
||||
"sort": 9999,
|
||||
"table": {
|
||||
"name": "knowledge",
|
||||
"comment": "Knowledge table"
|
||||
},
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"type": "ID",
|
||||
"label": "Knowledge ID",
|
||||
"comment": "Unique knowledge identifier"
|
||||
},
|
||||
{
|
||||
"name": "collection_id",
|
||||
"type": "string",
|
||||
"label": "Collection ID",
|
||||
"comment": "Knowledge collection identifier",
|
||||
"length": 200,
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "string",
|
||||
"label": "Name",
|
||||
"comment": "Knowledge collection name",
|
||||
"length": 200,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"type": "string",
|
||||
"label": "Description",
|
||||
"comment": "Knowledge collection description",
|
||||
"length": 600,
|
||||
"nullable": true,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "tags",
|
||||
"type": "json",
|
||||
"label": "Tags",
|
||||
"comment": "Knowledge tags",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "builtin",
|
||||
"type": "boolean",
|
||||
"label": "Builtin",
|
||||
"comment": "Whether this is a builtin knowledge",
|
||||
"default": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "readonly",
|
||||
"type": "boolean",
|
||||
"label": "Readonly",
|
||||
"comment": "Whether knowledge is readonly",
|
||||
"default": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "sort",
|
||||
"type": "integer",
|
||||
"label": "Sort",
|
||||
"comment": "Knowledge sort order",
|
||||
"default": 9999,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "uid",
|
||||
"type": "string",
|
||||
"label": "User ID",
|
||||
"comment": "User identifier",
|
||||
"length": 255,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "public",
|
||||
"type": "boolean",
|
||||
"label": "Public",
|
||||
"comment": "Whether knowledge is public",
|
||||
"default": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "scope",
|
||||
"type": "json",
|
||||
"label": "Scope",
|
||||
"comment": "Knowledge access scope",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "option",
|
||||
"type": "json",
|
||||
"label": "Option",
|
||||
"comment": "Knowledge options",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "system",
|
||||
"type": "boolean",
|
||||
"label": "System",
|
||||
"comment": "Whether this is a system knowledge",
|
||||
"default": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "cover",
|
||||
"type": "string",
|
||||
"label": "Cover",
|
||||
"comment": "Knowledge cover image URL",
|
||||
"length": 500,
|
||||
"nullable": true
|
||||
}
|
||||
],
|
||||
"relations": {},
|
||||
"indexes": [],
|
||||
"option": { "timestamps": true, "soft_deletes": false }
|
||||
}
|
||||
185
yao/models/kb/collection.mod.yao
Normal file
185
yao/models/kb/collection.mod.yao
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
{
|
||||
"name": "collection",
|
||||
"label": "Collection",
|
||||
"description": "Collection table for storing Knowledge Base collections",
|
||||
"tags": ["system"],
|
||||
"builtin": true,
|
||||
"readonly": false,
|
||||
"sort": 9999,
|
||||
"table": {
|
||||
"name": "kb_collection",
|
||||
"comment": "Knowledge Base Collection table"
|
||||
},
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"type": "ID",
|
||||
"label": "ID",
|
||||
"comment": "Auto-increment primary key"
|
||||
},
|
||||
{
|
||||
"name": "collection_id",
|
||||
"type": "string",
|
||||
"label": "Collection ID",
|
||||
"comment": "Unique string identifier for the collection (from API)",
|
||||
"length": 64,
|
||||
"nullable": false,
|
||||
"unique": true
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "string",
|
||||
"label": "Collection Name",
|
||||
"comment": "Display name of the collection",
|
||||
"length": 255,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"label": "Description",
|
||||
"comment": "Collection description",
|
||||
"nullable": true
|
||||
},
|
||||
|
||||
{
|
||||
"name": "status",
|
||||
"type": "enum",
|
||||
"label": "Status",
|
||||
"comment": "Collection status",
|
||||
"option": [
|
||||
"creating", // Collection being initialized
|
||||
"active", // Normal operation, ready for read/write
|
||||
"maintenance", // Under maintenance, read-only mode
|
||||
"restoring", // Backup restore in progress, no access
|
||||
"error", // Collection has errors, needs attention
|
||||
"disabled" // Manually disabled, no access
|
||||
],
|
||||
"default": "creating",
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
|
||||
{
|
||||
"name": "system",
|
||||
"type": "boolean",
|
||||
"label": "System Collection",
|
||||
"comment": "Whether this is a system collection",
|
||||
"default": false,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "sort",
|
||||
"type": "integer",
|
||||
"label": "Sort Order",
|
||||
"comment": "Sort order for display",
|
||||
"default": 0,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "cover",
|
||||
"type": "string",
|
||||
"label": "Cover Image",
|
||||
"comment": "Collection cover image URL",
|
||||
"length": 512,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "document_count",
|
||||
"type": "integer",
|
||||
"label": "Document Count",
|
||||
"comment": "Number of documents in the collection",
|
||||
"default": 0,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "embedding_provider",
|
||||
"type": "string",
|
||||
"label": "Embedding Provider",
|
||||
"comment": "Embedding provider ID",
|
||||
"length": 128,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "embedding_option",
|
||||
"type": "string",
|
||||
"label": "Embedding Option",
|
||||
"comment": "Embedding model option value",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "locale",
|
||||
"type": "string",
|
||||
"label": "Locale",
|
||||
"comment": "Locale setting for the collection",
|
||||
"length": 10,
|
||||
"nullable": true,
|
||||
"default": "en"
|
||||
},
|
||||
{
|
||||
"name": "distance",
|
||||
"type": "enum",
|
||||
"label": "Distance Metric",
|
||||
"comment": "Vector distance calculation method",
|
||||
"option": ["cosine", "euclidean", "dot"],
|
||||
"default": "cosine",
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "index_type",
|
||||
"type": "enum",
|
||||
"label": "Index Type",
|
||||
"comment": "Vector index algorithm type",
|
||||
"option": ["hnsw", "ivf", "flat"],
|
||||
"default": "hnsw",
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "m",
|
||||
"type": "integer",
|
||||
"label": "HNSW M Parameter",
|
||||
"comment": "HNSW algorithm M parameter (bidirectional links per node)",
|
||||
"nullable": true,
|
||||
"default": 16
|
||||
},
|
||||
{
|
||||
"name": "ef_construction",
|
||||
"type": "integer",
|
||||
"label": "HNSW EF Construction",
|
||||
"comment": "HNSW algorithm EF construction parameter",
|
||||
"nullable": true,
|
||||
"default": 200
|
||||
},
|
||||
{
|
||||
"name": "ef_search",
|
||||
"type": "integer",
|
||||
"label": "HNSW EF Search",
|
||||
"comment": "HNSW algorithm EF search parameter",
|
||||
"nullable": true,
|
||||
"default": 64
|
||||
},
|
||||
{
|
||||
"name": "num_lists",
|
||||
"type": "integer",
|
||||
"label": "IVF Number of Lists",
|
||||
"comment": "IVF algorithm number of clusters",
|
||||
"nullable": true,
|
||||
"default": 100
|
||||
},
|
||||
{
|
||||
"name": "num_probes",
|
||||
"type": "integer",
|
||||
"label": "IVF Number of Probes",
|
||||
"comment": "IVF algorithm number of clusters to search",
|
||||
"nullable": true,
|
||||
"default": 10
|
||||
}
|
||||
],
|
||||
"option": {
|
||||
"permission": true,
|
||||
"timestamps": true
|
||||
}
|
||||
}
|
||||
278
yao/models/kb/document.mod.yao
Normal file
278
yao/models/kb/document.mod.yao
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
{
|
||||
"name": "document",
|
||||
"label": "Document",
|
||||
"description": "Document table for storing Knowledge Base documents",
|
||||
"tags": ["system"],
|
||||
"builtin": true,
|
||||
"readonly": false,
|
||||
"sort": 9999,
|
||||
"table": {
|
||||
"name": "kb_document",
|
||||
"comment": "Knowledge Base Document table"
|
||||
},
|
||||
"columns": [
|
||||
{
|
||||
"name": "id",
|
||||
"type": "ID",
|
||||
"label": "ID",
|
||||
"comment": "Auto-increment primary key"
|
||||
},
|
||||
{
|
||||
"name": "document_id",
|
||||
"type": "string",
|
||||
"label": "Document ID",
|
||||
"comment": "Unique string identifier for the document (from API)",
|
||||
"length": 64,
|
||||
"nullable": false,
|
||||
"unique": true
|
||||
},
|
||||
{
|
||||
"name": "collection_id",
|
||||
"type": "string",
|
||||
"label": "Collection ID",
|
||||
"comment": "Reference to the collection this document belongs to",
|
||||
"length": 64,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "string",
|
||||
"label": "Document Name",
|
||||
"comment": "Display name of the document",
|
||||
"length": 255,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"label": "Description",
|
||||
"comment": "Document description",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"type": "enum",
|
||||
"label": "Status",
|
||||
"comment": "Document processing status",
|
||||
"option": [
|
||||
"pending", // Document created, waiting to start processing
|
||||
"converting", // Converting file to text using converter provider
|
||||
"chunking", // Splitting text into chunks using chunking provider
|
||||
"extracting", // Extracting entities and relationships (optional)
|
||||
"embedding", // Creating embeddings for chunks and entities
|
||||
"storing", // Storing documents to vector and graph databases
|
||||
"completed", // All processing steps finished successfully
|
||||
"maintenance", // Under maintenance, read-only mode
|
||||
"restoring", // Backup restore in progress, no access
|
||||
"error" // Processing failed, check error_message field
|
||||
],
|
||||
"default": "pending",
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "type",
|
||||
"type": "enum",
|
||||
"label": "Document Type",
|
||||
"comment": "Type of document content",
|
||||
"option": ["file", "text", "url"],
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "size",
|
||||
"type": "integer",
|
||||
"label": "Document Size",
|
||||
"comment": "Size of document in bytes",
|
||||
"default": 0,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "segment_count",
|
||||
"type": "integer",
|
||||
"label": "Segment Count",
|
||||
"comment": "Number of segments generated from this document",
|
||||
"default": 0,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "job_id",
|
||||
"type": "string",
|
||||
"label": "Job ID",
|
||||
"comment": "Processing job identifier for tracking document processing tasks",
|
||||
"length": 128,
|
||||
"nullable": true,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "uploader_id",
|
||||
"type": "string",
|
||||
"label": "Uploader ID",
|
||||
"comment": "Uploader identifier used for file management and attachment system",
|
||||
"length": 128,
|
||||
"nullable": true,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "tags",
|
||||
"type": "json",
|
||||
"label": "Tags",
|
||||
"comment": "Document tags for categorization and filtering",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "locale",
|
||||
"type": "string",
|
||||
"label": "Locale",
|
||||
"comment": "Locale setting for provider reading and processing",
|
||||
"length": 10,
|
||||
"nullable": true,
|
||||
"default": "en"
|
||||
},
|
||||
{
|
||||
"name": "system",
|
||||
"type": "boolean",
|
||||
"label": "System Document",
|
||||
"comment": "Whether this is a system document",
|
||||
"default": false,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "sort",
|
||||
"type": "integer",
|
||||
"label": "Sort Order",
|
||||
"comment": "Sort order for display",
|
||||
"default": 0,
|
||||
"nullable": false
|
||||
},
|
||||
{
|
||||
"name": "file_name",
|
||||
"type": "string",
|
||||
"label": "File Name",
|
||||
"comment": "Original file name (for file type)",
|
||||
"length": 255,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "file_path",
|
||||
"type": "string",
|
||||
"label": "File Path",
|
||||
"comment": "Storage path of the file (for file type)",
|
||||
"length": 512,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "file_mime_type",
|
||||
"type": "string",
|
||||
"label": "File MIME Type",
|
||||
"comment": "MIME type of the file (for file type)",
|
||||
"length": 100,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "url",
|
||||
"type": "string",
|
||||
"label": "URL",
|
||||
"comment": "Source URL (for url type)",
|
||||
"length": 2048,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "url_title",
|
||||
"type": "string",
|
||||
"label": "URL Title",
|
||||
"comment": "Title extracted from URL (for url type)",
|
||||
"length": 255,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "text_content",
|
||||
"type": "text",
|
||||
"label": "Text Content",
|
||||
"comment": "Raw text content (for text type or processed content)",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "converter_provider_id",
|
||||
"type": "string",
|
||||
"label": "Converter Provider ID",
|
||||
"comment": "Document converter provider ID (for file type)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "converter_properties",
|
||||
"type": "json",
|
||||
"label": "Converter Properties",
|
||||
"comment": "Converter provider configuration properties",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "fetcher_provider_id",
|
||||
"type": "string",
|
||||
"label": "Fetcher Provider ID",
|
||||
"comment": "URL fetcher provider ID (for url type)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "fetcher_properties",
|
||||
"type": "json",
|
||||
"label": "Fetcher Properties",
|
||||
"comment": "Fetcher provider configuration properties",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "chunking_provider_id",
|
||||
"type": "string",
|
||||
"label": "Chunking Provider ID",
|
||||
"comment": "Text chunking provider ID",
|
||||
"length": 128,
|
||||
"nullable": false,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "chunking_properties",
|
||||
"type": "json",
|
||||
"label": "Chunking Properties",
|
||||
"comment": "Chunking provider configuration properties (includes split_mode, chunk_size, chunk_overlap, etc.)",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "extractor_provider_id",
|
||||
"type": "string",
|
||||
"label": "Extractor Provider ID",
|
||||
"comment": "Knowledge extractor provider ID (optional)",
|
||||
"length": 128,
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "extractor_properties",
|
||||
"type": "json",
|
||||
"label": "Extractor Properties",
|
||||
"comment": "Extractor provider configuration properties",
|
||||
"nullable": true
|
||||
},
|
||||
{
|
||||
"name": "processed_at",
|
||||
"type": "timestamp",
|
||||
"label": "Processed At",
|
||||
"comment": "Timestamp when document processing completed",
|
||||
"nullable": true,
|
||||
"index": true
|
||||
},
|
||||
{
|
||||
"name": "error_message",
|
||||
"type": "text",
|
||||
"label": "Error Message",
|
||||
"comment": "Error message if processing failed",
|
||||
"nullable": true
|
||||
}
|
||||
],
|
||||
"option": {
|
||||
"permission": true,
|
||||
"timestamps": true
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue