From 6288e25a8a7a56b9f199896ee5c61c121e893e8d Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 2 Jan 2025 16:22:10 +0800 Subject: [PATCH 1/4] Refactor conversation table handling and update settings structure in Neo API - Changed the 'Table' field to 'Prefix' in the Setting struct to better represent its purpose as a table name prefix. - Updated all relevant instances in the codebase to use the new 'Prefix' field, ensuring consistency across the application. - Adjusted logging and table retrieval methods to reflect the new naming convention, enhancing clarity and maintainability. --- neo/load.go | 2 +- neo/store/types.go | 2 +- neo/store/xun.go | 8 ++++---- neo/store/xun_test.go | 20 ++++++++++---------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/neo/load.go b/neo/load.go index ef45f86a..34cf3704 100644 --- a/neo/load.go +++ b/neo/load.go @@ -38,7 +38,7 @@ func Load(cfg config.Config) error { Option: map[string]interface{}{}, Allows: []string{}, StoreSetting: store.Setting{ - Table: "yao_neo_conversation", + Prefix: "yao_neo_", Connector: "default", }, } diff --git a/neo/store/types.go b/neo/store/types.go index 925c5ecd..1d6a889b 100644 --- a/neo/store/types.go +++ b/neo/store/types.go @@ -5,7 +5,7 @@ package store type Setting struct { Connector string `json:"connector,omitempty"` // Name of the connector used to specify data storage method UserField string `json:"user_field,omitempty"` // User ID field name, defaults to "user_id" - Table string `json:"table,omitempty"` // Database table name + Prefix string `json:"prefix,omitempty"` // Database table name prefix MaxSize int `json:"max_size,omitempty" yaml:"max_size,omitempty"` // Maximum storage size limit TTL int `json:"ttl,omitempty" yaml:"ttl,omitempty"` // Time To Live in seconds } diff --git a/neo/store/xun.go b/neo/store/xun.go index 05c9c068..aadb4583 100644 --- a/neo/store/xun.go +++ b/neo/store/xun.go @@ -99,7 +99,7 @@ func (conv *Xun) clean() { } if nums > 0 { - log.Trace("Clean the conversation table: %s %d", conv.setting.Table, nums) + log.Trace("Clean the conversation table: %s %d", conv.setting.Prefix, nums) } } @@ -283,15 +283,15 @@ func (conv *Xun) getUserID(sid string) (string, error) { } func (conv *Xun) getHistoryTable() string { - return conv.setting.Table + "_history" + return conv.setting.Prefix + "history" } func (conv *Xun) getChatTable() string { - return conv.setting.Table + "_chat" + return conv.setting.Prefix + "chat" } func (conv *Xun) getAssistantTable() string { - return conv.setting.Table + "_assistant" + return conv.setting.Prefix + "assistant" } // UpdateChatTitle update the chat title diff --git a/neo/store/xun_test.go b/neo/store/xun_test.go index dc42de04..4dbd936a 100644 --- a/neo/store/xun_test.go +++ b/neo/store/xun_test.go @@ -40,7 +40,7 @@ func TestNewXunDefault(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { @@ -139,7 +139,7 @@ func TestNewXunConnector(t *testing.T) { store, err := NewXun(Setting{ Connector: "mysql", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { @@ -201,7 +201,7 @@ func TestXunSaveAndGetHistory(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", TTL: 3600, }) @@ -239,7 +239,7 @@ func TestXunSaveAndGetHistoryWithCID(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", TTL: 3600, }) @@ -308,7 +308,7 @@ func TestXunGetChats(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { t.Fatal(err) @@ -364,7 +364,7 @@ func TestXunDeleteChat(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { t.Fatal(err) @@ -404,7 +404,7 @@ func TestXunDeleteAllChats(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { t.Fatal(err) @@ -455,7 +455,7 @@ func TestXunAssistantCRUD(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { t.Fatal(err) @@ -926,7 +926,7 @@ func TestXunAssistantPagination(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { t.Fatal(err) @@ -1250,7 +1250,7 @@ func TestGetAssistantTags(t *testing.T) { store, err := NewXun(Setting{ Connector: "default", - Table: "__unit_test_conversation", + Prefix: "__unit_test_conversation_", }) if err != nil { t.Fatal(err) From 2903602c6bee9546aa1b2b109f1fb632578039ed Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 2 Jan 2025 16:55:21 +0800 Subject: [PATCH 2/4] Refactor Neo API initialization and enhance assistant management - Introduced new methods for initializing RAG and store, improving modularity and clarity in the Load function. - Refactored the assistant initialization process to streamline the loading of built-in assistants and set RAG configurations. - Moved the defaultAssistant method from neo.go to load.go for better organization and accessibility. - Enhanced error handling in store initialization to support multiple connector types, including Redis and Mongo. - Updated the assistant struct to include RAG settings, improving the overall assistant management capabilities. --- neo/assistant/assistant.go | 14 +++++ neo/assistant/types.go | 8 +++ neo/load.go | 104 ++++++++++++++++++++++++++++++------- neo/neo.go | 14 ----- 4 files changed, 108 insertions(+), 32 deletions(-) diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go index ee57d890..2460a44d 100644 --- a/neo/assistant/assistant.go +++ b/neo/assistant/assistant.go @@ -9,6 +9,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/yaoapp/gou/fs" + "github.com/yaoapp/gou/rag/driver" v8 "github.com/yaoapp/gou/runtime/v8" "github.com/yaoapp/yao/neo/store" "github.com/yaoapp/yao/share" @@ -18,6 +19,7 @@ import ( // loaded the loaded assistant var loaded = NewCache(200) // 200 is the default capacity var storage store.Store = nil +var rag *RAG = nil // LoadBuiltIn load the built-in assistants func LoadBuiltIn() error { @@ -85,6 +87,18 @@ func SetStorage(s store.Store) { storage = s } +// SetRAG set the RAG engine +// e: the RAG engine +// u: the RAG file uploader +// v: the RAG vectorizer +func SetRAG(e driver.Engine, u driver.FileUpload, v driver.Vectorizer) { + rag = &RAG{ + Engine: e, + Uploader: u, + Vectorizer: v, + } +} + // SetCache set the cache func SetCache(capacity int) { ClearCache() diff --git a/neo/assistant/types.go b/neo/assistant/types.go index 977cf5e9..17f17bf0 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -5,6 +5,7 @@ import ( "io" "mime/multipart" + "github.com/yaoapp/gou/rag/driver" v8 "github.com/yaoapp/gou/runtime/v8" ) @@ -16,6 +17,13 @@ type API interface { ReadBase64(ctx context.Context, fileID string) (string, error) } +// RAG the RAG interface +type RAG struct { + Engine driver.Engine + Uploader driver.FileUpload + Vectorizer driver.Vectorizer +} + // Prompt a prompt type Prompt struct { Role string `json:"role"` diff --git a/neo/load.go b/neo/load.go index 34cf3704..cfafc65c 100644 --- a/neo/load.go +++ b/neo/load.go @@ -1,10 +1,12 @@ package neo import ( + "fmt" "path/filepath" "github.com/fatih/color" "github.com/yaoapp/gou/application" + "github.com/yaoapp/gou/connector" "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/neo/assistant" @@ -15,20 +17,6 @@ import ( // Neo the neo AI assistant var Neo *DSL -// initRAG initialize the RAG instance -func (neo *DSL) initRAG() { - if neo.RAGSetting.Engine.Driver == "" { - return - } - instance, err := rag.New(neo.RAGSetting) - if err != nil { - color.Red("[Neo] Failed to initialize RAG: %v", err) - log.Error("[Neo] Failed to initialize RAG: %v", err) - return - } - neo.RAG = instance -} - // Load load AIGC func Load(cfg config.Config) error { @@ -60,7 +48,7 @@ func Load(cfg config.Config) error { Neo = &setting // Store Setting - err = Neo.createStore() + err = Neo.initStore() if err != nil { return err } @@ -68,13 +56,79 @@ func Load(cfg config.Config) error { // Initialize RAG Neo.initRAG() - // Load Built-in Assistants - assistant.SetStorage(Neo.Store) - err = assistant.LoadBuiltIn() + // Initialize Assistant + err = Neo.initAssistant() if err != nil { return err } + return nil +} + +// initRAG initialize the RAG instance +func (neo *DSL) initRAG() { + if neo.RAGSetting.Engine.Driver == "" { + return + } + instance, err := rag.New(neo.RAGSetting) + if err != nil { + color.Red("[Neo] Failed to initialize RAG: %v", err) + log.Error("[Neo] Failed to initialize RAG: %v", err) + return + } + + neo.RAG = instance +} + +// initStore initialize the store +func (neo *DSL) initStore() error { + + var err error + if neo.StoreSetting.Connector == "default" || neo.StoreSetting.Connector == "" { + neo.Store, err = store.NewXun(neo.StoreSetting) + return err + } + + // other connector + conn, err := connector.Select(neo.StoreSetting.Connector) + if err != nil { + return err + } + + if conn.Is(connector.DATABASE) { + neo.Store, err = store.NewXun(neo.StoreSetting) + return err + + } else if conn.Is(connector.REDIS) { + neo.Store = store.NewRedis() + return nil + + } else if conn.Is(connector.MONGO) { + neo.Store = store.NewMongo() + return nil + } + + return fmt.Errorf("%s store connector %s not support", neo.ID, neo.StoreSetting.Connector) +} + +// initAssistant initialize the assistant +func (neo *DSL) initAssistant() error { + + // Set Storage + assistant.SetStorage(Neo.Store) + + // Assistant RAG + if Neo.RAG != nil { + assistant.SetRAG(Neo.RAG.Engine(), Neo.RAG.FileUpload(), Neo.RAG.Vectorizer()) + } + + // Load Built-in Assistants + err := assistant.LoadBuiltIn() + if err != nil { + return err + } + + // Default Assistant defaultAssistant, err := Neo.defaultAssistant() if err != nil { return err @@ -83,3 +137,17 @@ func Load(cfg config.Config) error { Neo.Assistant = defaultAssistant.API return nil } + +// defaultAssistant get the default assistant +func (neo *DSL) defaultAssistant() (*assistant.Assistant, error) { + if neo.Use != "" { + return assistant.Get(neo.Use) + } + + name := neo.Name + if name == "" { + name = "Neo" + } + + return assistant.GetByConnector(neo.Connector, name) +} diff --git a/neo/neo.go b/neo/neo.go index e5cf0399..f072dc1c 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -346,20 +346,6 @@ func (neo *DSL) chat(ast assistant.API, ctx Context, messages []map[string]inter } } -// defaultAssistant get the default assistant -func (neo *DSL) defaultAssistant() (*assistant.Assistant, error) { - if neo.Use != "" { - return assistant.Get(neo.Use) - } - - name := neo.Name - if name == "" { - name = "Neo" - } - - return assistant.GetByConnector(neo.Connector, name) -} - // updateAssistantList update the assistant list func (neo *DSL) updateAssistantList(list []assistant.Assistant) { lock.Lock() From 973fb5dd3d9d85503ce0aff20b2541e768c2f7f4 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 2 Jan 2025 17:00:50 +0800 Subject: [PATCH 3/4] Remove assistant management files to streamline codebase - Deleted the assistant.go and assistant_test.go files, which contained the implementation and tests for assistant management. - This cleanup reduces complexity and focuses on core functionalities, paving the way for future enhancements in assistant handling. --- neo/assistant/{assistant.go => load.go} | 0 neo/assistant/{assistant_test.go => load_test.go} | 12 ++++++------ 2 files changed, 6 insertions(+), 6 deletions(-) rename neo/assistant/{assistant.go => load.go} (100%) rename neo/assistant/{assistant_test.go => load_test.go} (97%) diff --git a/neo/assistant/assistant.go b/neo/assistant/load.go similarity index 100% rename from neo/assistant/assistant.go rename to neo/assistant/load.go diff --git a/neo/assistant/assistant_test.go b/neo/assistant/load_test.go similarity index 97% rename from neo/assistant/assistant_test.go rename to neo/assistant/load_test.go index fb4f20e1..5c6d21ba 100644 --- a/neo/assistant/assistant_test.go +++ b/neo/assistant/load_test.go @@ -14,7 +14,7 @@ func prepare(t *testing.T) { test.Prepare(t, config.Conf) } -func TestAssistant_LoadPath(t *testing.T) { +func TestLoad_LoadPath(t *testing.T) { prepare(t) defer test.Clean() @@ -37,7 +37,7 @@ func TestAssistant_LoadPath(t *testing.T) { assert.Error(t, err) } -func TestAssistant_LoadStore(t *testing.T) { +func TestLoad_LoadStore(t *testing.T) { prepare(t) defer test.Clean() @@ -79,7 +79,7 @@ func TestAssistant_LoadStore(t *testing.T) { assert.Error(t, err) } -func TestAssistant_Cache(t *testing.T) { +func TestLoad_Cache(t *testing.T) { prepare(t) defer test.Clean() @@ -127,7 +127,7 @@ func TestAssistant_Cache(t *testing.T) { assert.NotNil(t, loaded) } -func TestAssistant_Validate(t *testing.T) { +func TestLoad_Validate(t *testing.T) { tests := []struct { name string ast *Assistant @@ -178,7 +178,7 @@ func TestAssistant_Validate(t *testing.T) { } } -func TestAssistant_Clone(t *testing.T) { +func TestLoad_Clone(t *testing.T) { // Create a test assistant with all fields populated original := &Assistant{ ID: "test-id", @@ -233,7 +233,7 @@ func TestAssistant_Clone(t *testing.T) { assert.Nil(t, nilAssistant.Clone()) } -func TestAssistant_Update(t *testing.T) { +func TestLoad_Update(t *testing.T) { // Create a test assistant ast := &Assistant{ ID: "test-id", From af30bec4f0a91012f9e94ffddaf8b5fd017d6900 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 2 Jan 2025 18:56:35 +0800 Subject: [PATCH 4/4] Enhance RAG configuration and assistant loading in Neo API - Updated the SetRAG function to include RAG settings, allowing for more detailed configuration during assistant initialization. - Refactored the LoadBuiltIn function to streamline the saving process of assistants, improving error handling and code clarity. - Modified loadPrompts and loadScript functions to return timestamps, enabling tracking of creation and update times for assistants. - Enhanced the Assistant struct with created_at and updated_at fields, improving data management and traceability. - Overall improvements to the assistant loading process, ensuring better organization and maintainability of the codebase. --- neo/assistant/assistant.go | 253 +++++++++++++++++++++++++++++++++++++ neo/assistant/load.go | 225 +++++++++------------------------ neo/assistant/types.go | 8 ++ neo/assistant/utils.go | 44 +++++++ neo/load.go | 9 +- 5 files changed, 374 insertions(+), 165 deletions(-) create mode 100644 neo/assistant/assistant.go create mode 100644 neo/assistant/utils.go diff --git a/neo/assistant/assistant.go b/neo/assistant/assistant.go new file mode 100644 index 00000000..fd24eb9f --- /dev/null +++ b/neo/assistant/assistant.go @@ -0,0 +1,253 @@ +package assistant + +import ( + "context" + "fmt" + "time" + + "github.com/fatih/color" + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou/rag/driver" + "github.com/yaoapp/kun/log" +) + +// Save save the assistant +func (ast *Assistant) Save() error { + if storage == nil { + return fmt.Errorf("storage is not set") + } + + _, err := storage.SaveAssistant(ast.Map()) + if err != nil { + return err + } + + // Update Index in background + go func() { + err := ast.UpdateIndex() + if err != nil { + log.Error("failed to update index for assistant %s: %s", ast.ID, err) + color.Red("failed to update index for assistant %s: %s", ast.ID, err) + } + }() + + return nil +} + +// UpdateIndex update the index for RAG +func (ast *Assistant) UpdateIndex() error { + + // RAG is not enabled + if rag == nil { + return nil + } + + if rag.Engine == nil { + return fmt.Errorf("engine is not set") + } + + // Update Index + index := fmt.Sprintf("%sassistants", rag.Setting.IndexPrefix) + id := fmt.Sprintf("assistant_%s", ast.ID) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + // Check if the index exists + exists, err := rag.Engine.HasIndex(ctx, index) + if err != nil { + return err + } + + // Create the index if it does not exist + if !exists { + ctxCreate, cancelCreate := context.WithTimeout(context.Background(), 2*time.Second) + defer cancelCreate() + err = rag.Engine.CreateIndex(ctxCreate, driver.IndexConfig{Name: index}) + if err != nil { + return err + } + } + + // Check if the document exists + exists, err = rag.Engine.HasDocument(ctx, index, id) + if err != nil { + return err + } + + // Check if the document is updated + if exists { + metadata, err := rag.Engine.GetMetadata(ctx, index, id) + if err != nil { + return err + } + + if v, ok := metadata["updated_at"].(string); ok { + updatedAt, err := stringToTimestamp(v) + if err != nil { + return err + } + if updatedAt >= ast.UpdatedAt { + return nil + } + } + } + + // Update the index + content, err := jsoniter.MarshalToString(ast.Map()) + if err != nil { + return err + } + + metadata := map[string]interface{}{ + "assistant_id": ast.ID, + "type": ast.Type, + "name": ast.Name, + "updated_at": fmt.Sprintf("%d", ast.UpdatedAt), + } + + return rag.Engine.IndexDoc(ctx, index, &driver.Document{ + DocID: id, + Content: content, + Metadata: metadata, + }) +} + +// Map convert the assistant to a map +func (ast *Assistant) Map() map[string]interface{} { + + if ast == nil { + return nil + } + + return map[string]interface{}{ + "assistant_id": ast.ID, + "type": ast.Type, + "name": ast.Name, + "readonly": ast.Readonly, + "avatar": ast.Avatar, + "connector": ast.Connector, + "path": ast.Path, + "built_in": ast.BuiltIn, + "sort": ast.Sort, + "description": ast.Description, + "options": ast.Options, + "prompts": ast.Prompts, + "tags": ast.Tags, + "mentionable": ast.Mentionable, + "automated": ast.Automated, + "created_at": timeToMySQLFormat(ast.CreatedAt), + "updated_at": timeToMySQLFormat(ast.UpdatedAt), + } +} + +// Validate validates the assistant configuration +func (ast *Assistant) Validate() error { + if ast.ID == "" { + return fmt.Errorf("assistant_id is required") + } + if ast.Name == "" { + return fmt.Errorf("name is required") + } + if ast.Connector == "" { + return fmt.Errorf("connector is required") + } + return nil +} + +// Clone creates a deep copy of the assistant +func (ast *Assistant) Clone() *Assistant { + if ast == nil { + return nil + } + + clone := &Assistant{ + ID: ast.ID, + Type: ast.Type, + Name: ast.Name, + Avatar: ast.Avatar, + Connector: ast.Connector, + Path: ast.Path, + BuiltIn: ast.BuiltIn, + Sort: ast.Sort, + Description: ast.Description, + Readonly: ast.Readonly, + Mentionable: ast.Mentionable, + Automated: ast.Automated, + Script: ast.Script, + API: ast.API, + } + + // Deep copy tags + if ast.Tags != nil { + clone.Tags = make([]string, len(ast.Tags)) + copy(clone.Tags, ast.Tags) + } + + // Deep copy options + if ast.Options != nil { + clone.Options = make(map[string]interface{}) + for k, v := range ast.Options { + clone.Options[k] = v + } + } + + // Deep copy prompts + if ast.Prompts != nil { + clone.Prompts = make([]Prompt, len(ast.Prompts)) + copy(clone.Prompts, ast.Prompts) + } + + // Deep copy flows + if ast.Flows != nil { + clone.Flows = make([]map[string]interface{}, len(ast.Flows)) + for i, flow := range ast.Flows { + cloneFlow := make(map[string]interface{}) + for k, v := range flow { + cloneFlow[k] = v + } + clone.Flows[i] = cloneFlow + } + } + + return clone +} + +// Update updates the assistant properties +func (ast *Assistant) Update(data map[string]interface{}) error { + if ast == nil { + return fmt.Errorf("assistant is nil") + } + + if v, ok := data["name"].(string); ok { + ast.Name = v + } + if v, ok := data["avatar"].(string); ok { + ast.Avatar = v + } + if v, ok := data["description"].(string); ok { + ast.Description = v + } + if v, ok := data["connector"].(string); ok { + ast.Connector = v + } + if v, ok := data["type"].(string); ok { + ast.Type = v + } + if v, ok := data["sort"].(int); ok { + ast.Sort = v + } + if v, ok := data["mentionable"].(bool); ok { + ast.Mentionable = v + } + if v, ok := data["automated"].(bool); ok { + ast.Automated = v + } + if v, ok := data["tags"].([]string); ok { + ast.Tags = v + } + if v, ok := data["options"].(map[string]interface{}); ok { + ast.Options = v + } + + return ast.Validate() +} diff --git a/neo/assistant/load.go b/neo/assistant/load.go index 2460a44d..734d612e 100644 --- a/neo/assistant/load.go +++ b/neo/assistant/load.go @@ -67,16 +67,15 @@ func LoadBuiltIn() error { assistant.Tags = []string{"Built-in"} } + // Save the assistant + err = assistant.Save() + if err != nil { + return err + } + sort++ loaded.Put(assistant) - // Save the assistant - if storage != nil { - _, err := storage.SaveAssistant(assistant.Map()) - if err != nil { - return err - } - } } return nil @@ -91,11 +90,12 @@ func SetStorage(s store.Store) { // e: the RAG engine // u: the RAG file uploader // v: the RAG vectorizer -func SetRAG(e driver.Engine, u driver.FileUpload, v driver.Vectorizer) { +func SetRAG(e driver.Engine, u driver.FileUpload, v driver.Vectorizer, setting RAGSetting) { rag = &RAG{ Engine: e, Uploader: u, Vectorizer: v, + Setting: setting, } } @@ -177,24 +177,30 @@ func LoadPath(path string) (*Assistant, error) { data["assistant_id"] = id data["type"] = "assistant" data["path"] = path + + updatedAt := int64(0) + // prompts promptsfile := filepath.Join(path, "prompts.yml") if has, _ := app.Exists(promptsfile); has { - prompts, err := loadPrompts(promptsfile, path) + prompts, ts, err := loadPrompts(promptsfile, path) if err != nil { return nil, err } data["prompts"] = prompts + data["updated_at"] = ts + updatedAt = ts } // load script scriptfile := filepath.Join(path, "src", "index.ts") if has, _ := app.Exists(scriptfile); has { - script, err := loadScript(scriptfile, path) + script, ts, err := loadScript(scriptfile, path) if err != nil { return nil, err } data["script"] = script + data["updated_at"] = max(updatedAt, ts) } // load functions @@ -307,19 +313,42 @@ func loadMap(data map[string]interface{}) (*Assistant, error) { } } + // created_at + if v, has := data["created_at"]; has { + ts, err := getTimestamp(v) + if err != nil { + return nil, err + } + assistant.CreatedAt = ts + } + + // updated_at + if v, has := data["updated_at"]; has { + ts, err := getTimestamp(v) + if err != nil { + return nil, err + } + assistant.UpdatedAt = ts + } + return assistant, nil } -func loadPrompts(file string, root string) (string, error) { +func loadPrompts(file string, root string) (string, int64, error) { app, err := fs.Get("app") if err != nil { - return "", err + return "", 0, err + } + + ts, err := app.ModTime(file) + if err != nil { + return "", 0, err } prompts, err := app.ReadFile(file) if err != nil { - return "", err + return "", 0, err } re := regexp.MustCompile(`@assets/([^\s]+\.(md|yml|yaml|json|txt))`) @@ -339,11 +368,27 @@ func loadPrompts(file string, root string) (string, error) { return []byte(formattedContent) }) - return string(prompts), nil + return string(prompts), ts.UnixNano(), nil } -func loadScript(file string, root string) (*v8.Script, error) { - return v8.Load(file, share.ID(root, file)) +func loadScript(file string, root string) (*v8.Script, int64, error) { + + app, err := fs.Get("app") + if err != nil { + return nil, 0, err + } + + ts, err := app.ModTime(file) + if err != nil { + return nil, 0, err + } + + script, err := v8.Load(file, share.ID(root, file)) + if err != nil { + return nil, 0, err + } + + return script, ts.UnixNano(), nil } func loadScriptSource(source string, file string) (*v8.Script, error) { @@ -353,151 +398,3 @@ func loadScriptSource(source string, file string) (*v8.Script, error) { } return script, nil } - -// Save save the assistant -func (ast *Assistant) Save() error { - if storage == nil { - return fmt.Errorf("storage is not set") - } - - _, err := storage.SaveAssistant(ast.Map()) - return err -} - -// Map convert the assistant to a map -func (ast *Assistant) Map() map[string]interface{} { - - if ast == nil { - return nil - } - - return map[string]interface{}{ - "assistant_id": ast.ID, - "type": ast.Type, - "name": ast.Name, - "readonly": ast.Readonly, - "avatar": ast.Avatar, - "connector": ast.Connector, - "path": ast.Path, - "built_in": ast.BuiltIn, - "sort": ast.Sort, - "description": ast.Description, - "options": ast.Options, - "prompts": ast.Prompts, - "tags": ast.Tags, - "mentionable": ast.Mentionable, - "automated": ast.Automated, - } -} - -// Validate validates the assistant configuration -func (ast *Assistant) Validate() error { - if ast.ID == "" { - return fmt.Errorf("assistant_id is required") - } - if ast.Name == "" { - return fmt.Errorf("name is required") - } - if ast.Connector == "" { - return fmt.Errorf("connector is required") - } - return nil -} - -// Clone creates a deep copy of the assistant -func (ast *Assistant) Clone() *Assistant { - if ast == nil { - return nil - } - - clone := &Assistant{ - ID: ast.ID, - Type: ast.Type, - Name: ast.Name, - Avatar: ast.Avatar, - Connector: ast.Connector, - Path: ast.Path, - BuiltIn: ast.BuiltIn, - Sort: ast.Sort, - Description: ast.Description, - Readonly: ast.Readonly, - Mentionable: ast.Mentionable, - Automated: ast.Automated, - Script: ast.Script, - API: ast.API, - } - - // Deep copy tags - if ast.Tags != nil { - clone.Tags = make([]string, len(ast.Tags)) - copy(clone.Tags, ast.Tags) - } - - // Deep copy options - if ast.Options != nil { - clone.Options = make(map[string]interface{}) - for k, v := range ast.Options { - clone.Options[k] = v - } - } - - // Deep copy prompts - if ast.Prompts != nil { - clone.Prompts = make([]Prompt, len(ast.Prompts)) - copy(clone.Prompts, ast.Prompts) - } - - // Deep copy flows - if ast.Flows != nil { - clone.Flows = make([]map[string]interface{}, len(ast.Flows)) - for i, flow := range ast.Flows { - cloneFlow := make(map[string]interface{}) - for k, v := range flow { - cloneFlow[k] = v - } - clone.Flows[i] = cloneFlow - } - } - - return clone -} - -// Update updates the assistant properties -func (ast *Assistant) Update(data map[string]interface{}) error { - if ast == nil { - return fmt.Errorf("assistant is nil") - } - - if v, ok := data["name"].(string); ok { - ast.Name = v - } - if v, ok := data["avatar"].(string); ok { - ast.Avatar = v - } - if v, ok := data["description"].(string); ok { - ast.Description = v - } - if v, ok := data["connector"].(string); ok { - ast.Connector = v - } - if v, ok := data["type"].(string); ok { - ast.Type = v - } - if v, ok := data["sort"].(int); ok { - ast.Sort = v - } - if v, ok := data["mentionable"].(bool); ok { - ast.Mentionable = v - } - if v, ok := data["automated"].(bool); ok { - ast.Automated = v - } - if v, ok := data["tags"].([]string); ok { - ast.Tags = v - } - if v, ok := data["options"].(map[string]interface{}); ok { - ast.Options = v - } - - return ast.Validate() -} diff --git a/neo/assistant/types.go b/neo/assistant/types.go index 17f17bf0..5cec9ba6 100644 --- a/neo/assistant/types.go +++ b/neo/assistant/types.go @@ -22,6 +22,12 @@ type RAG struct { Engine driver.Engine Uploader driver.FileUpload Vectorizer driver.Vectorizer + Setting RAGSetting +} + +// RAGSetting the RAG setting +type RAGSetting struct { + IndexPrefix string `json:"index_prefix" yaml:"index_prefix"` } // Prompt a prompt @@ -59,6 +65,8 @@ type Assistant struct { Flows []map[string]interface{} `json:"flows,omitempty"` // Assistant Flows Script *v8.Script `json:"-" yaml:"-"` // Assistant Script API API `json:"-" yaml:"-"` // Assistant API + CreatedAt int64 `json:"created_at"` // Creation timestamp + UpdatedAt int64 `json:"updated_at"` // Last update timestamp } // File the file diff --git a/neo/assistant/utils.go b/neo/assistant/utils.go new file mode 100644 index 00000000..aa7c6720 --- /dev/null +++ b/neo/assistant/utils.go @@ -0,0 +1,44 @@ +package assistant + +import ( + "fmt" + "strconv" + "time" +) + +func getTimestamp(v interface{}) (int64, error) { + switch v := v.(type) { + case int64: + return v, nil + case int: + return int64(v), nil + + case string: + if ts, err := time.Parse(time.RFC3339, v); err == nil { + return ts.UnixNano(), nil + } + + // MySQL format + if ts, err := time.Parse("2006-01-02 15:04:05", v); err == nil { + return ts.UnixNano(), nil + } + + // UnixNano format + if ts, err := strconv.ParseInt(v, 10, 64); err == nil { + return ts, nil + } + + } + return 0, fmt.Errorf("invalid timestamp type") +} + +func stringToTimestamp(v string) (int64, error) { + return strconv.ParseInt(v, 10, 64) +} + +func timeToMySQLFormat(ts int64) string { + if ts == 0 { + return "0000-00-00 00:00:00" + } + return time.Unix(ts/1e9, ts%1e9).Format("2006-01-02 15:04:05") +} diff --git a/neo/load.go b/neo/load.go index cfafc65c..2e364b6c 100644 --- a/neo/load.go +++ b/neo/load.go @@ -119,7 +119,14 @@ func (neo *DSL) initAssistant() error { // Assistant RAG if Neo.RAG != nil { - assistant.SetRAG(Neo.RAG.Engine(), Neo.RAG.FileUpload(), Neo.RAG.Vectorizer()) + assistant.SetRAG( + Neo.RAG.Engine(), + Neo.RAG.FileUpload(), + Neo.RAG.Vectorizer(), + assistant.RAGSetting{ + IndexPrefix: Neo.RAGSetting.IndexPrefix, + }, + ) } // Load Built-in Assistants