diff --git a/engine/load.go b/engine/load.go index f80e5445..f7acefe8 100644 --- a/engine/load.go +++ b/engine/load.go @@ -48,6 +48,7 @@ import ( sandbox "github.com/yaoapp/yao/sandbox/v2" "github.com/yaoapp/yao/schedule" "github.com/yaoapp/yao/script" + "github.com/yaoapp/yao/setting" "github.com/yaoapp/yao/share" "github.com/yaoapp/yao/store" sui "github.com/yaoapp/yao/sui/api" @@ -439,6 +440,14 @@ func Load(cfg config.Config, options LoadOption, progressCallback ...func(string warnings = append(warnings, Warning{Widget: "MCP Client Registry", Error: err}) } + // Initialize Setting Registry + err = loadStep("Setting Registry", func() error { + return setting.Init() + }, callback) + if err != nil { + warnings = append(warnings, Warning{Widget: "Setting Registry", Error: err}) + } + for name, hook := range LoadHooks { err = hook(cfg) if err != nil { @@ -699,6 +708,19 @@ func Reload(cfg config.Config, options LoadOption) (err error) { } } + // Reload Setting Registry + if setting.Global != nil { + err = setting.Global.Reload() + if err != nil { + printErr(cfg.Mode, "Setting Registry", err) + } + } else { + err = setting.Init() + if err != nil { + printErr(cfg.Mode, "Setting Registry", err) + } + } + // Load OpenAPI _, err = openapi.Load(cfg) if err != nil { diff --git a/llmprovider/doc.go b/llmprovider/doc.go new file mode 100644 index 00000000..86a0c23d --- /dev/null +++ b/llmprovider/doc.go @@ -0,0 +1,12 @@ +package llmprovider + +import ( + _ "embed" + + "github.com/yaoapp/gou/doc" +) + +//go:embed doc.yml +var docYAML []byte + +func init() { doc.LoadYAML(docYAML) } diff --git a/llmprovider/doc.yml b/llmprovider/doc.yml new file mode 100644 index 00000000..cf54adad --- /dev/null +++ b/llmprovider/doc.yml @@ -0,0 +1,256 @@ +group: llmprovider +type: process +desc: | + CRUD operations for the LLM Provider Registry. Manages provider connections + (OpenAI, Anthropic, Ollama, etc.) with persistence, API key encryption, and + lazy connector registration. + + Process names follow the pattern "llmprovider.". + + Provider structure (returned by get, getmasked, create, update; array elements from list): + - key (string): Unique identifier for this provider. Required on create. + - connector_id (string): Runtime connector ID, auto-generated. + Format: "s." for system owner, "u." for user owner, + "t." for team owner. BuiltIn providers retain their original ID. + - name (string): Display name (e.g. "OpenAI", "My Custom Provider"). + - type (string): Connector protocol type. + Values: "openai", "anthropic", "google", "ollama", "custom". + - api_url (string): Base API URL (e.g. "https://api.openai.com"). + - api_key (string): API key. Returned in full by "get"; masked by "getmasked" + and "list" (e.g. "sk-***test"). Encrypted at rest with AES-256-GCM. + - models (array of ModelInfo): Available models for this provider. + - enabled (bool): Whether the provider is active. + - status (string): Connection status. Values: "connected", "disconnected", "unconfigured". + - is_custom (bool, optional): Whether user manually configured (not from preset). + - preset_key (string, optional): Key of the preset this was created from (e.g. "openai"). + - require_key (bool): Whether an API key is required. + - source (string): Origin. Values: "dynamic" (registry-created), "builtin" (loaded from .yao DSL). + - owner (ProviderOwner): Ownership information. + + ModelInfo structure (elements of Provider.models): + - id (string): Model identifier (e.g. "gpt-4o", "claude-sonnet-4-20250514"). + - name (string): Human-readable name (e.g. "GPT-4o"). + - capabilities (array of string): Model capabilities. + Known values: "vision", "tool_calls", "streaming", "json", "reasoning". + - enabled (bool): Whether this model is active. + + ProviderOwner structure (Provider.owner): + - type (string): Scope level. Values: "system", "team", "user". + - team_id (string, optional): Required when type is "team". + - user_id (string, optional): Required when type is "user". + + ProviderFilter structure (optional argument for list): + - source (string, optional): Filter by source. + Values: "dynamic" (default when omitted), "builtin", "all". + - owner (ProviderOwner, optional): Filter by owner. Omit to include all owners. + - enabled (bool, optional): Filter by enabled status. Omit to include both. + - type (string, optional): Filter by provider type (e.g. "openai"). + - preset_key (string, optional): Filter by preset key. + - capabilities (array of string, optional): AND filter — matches providers that have + at least one model satisfying ALL listed capabilities. + - keyword (string, optional): Case-insensitive substring search in key and name. + + ProviderPreset structure (returned by getpresets, getpreset): + - key (string): Preset identifier (e.g. "openai", "anthropic", "ollama"). + - name (string): Display name. + - type (string): Connector type. + - api_url (string): Default API URL for UI auto-fill. + - require_key (bool): Whether API key is required. + - is_cloud (bool, optional): Whether this is a cloud-hosted service. + - url_editable (bool, optional): Whether the user can modify the URL. + - default_models (array of ModelInfo): Suggested models for UI pre-population. + +entries: + - name: get + desc: | + Get a provider by key, returning the full Provider object with plaintext API key. + Lazily ensures the runtime connector is registered on first access. + Throws 404 if the provider key does not exist. + args: + - name: key + type: string + required: true + desc: Provider key (e.g. "openai", "my-custom-provider"). + return: + type: object + desc: | + Full Provider object. See Provider structure above. + The api_key field contains the decrypted plaintext value. + Example: {"key":"openai","connector_id":"s.openai","name":"OpenAI","type":"openai", + "api_url":"https://api.openai.com","api_key":"sk-abc123...", + "models":[{"id":"gpt-4o","name":"GPT-4o","capabilities":["vision","streaming"],"enabled":true}], + "enabled":true,"status":"connected","source":"dynamic", + "owner":{"type":"system"}} + + - name: getmasked + desc: | + Get a provider by key with the API key masked for safe display. + Masking rule: keeps last 4 characters visible, replaces every preceding + character with "*". Example: "sk-abc123test" (14 chars) → "**********test". + Keys with 4 or fewer characters are fully replaced with "*" per character + (e.g. "abcd" → "****", "ab" → "**"). + Throws 404 if the provider key does not exist. + args: + - name: key + type: string + required: true + desc: Provider key. + return: + type: object + desc: | + Provider object with api_key masked. All other fields are identical to "get". + Example api_key value: "**********test" (for a 14-char key) + + - name: create + desc: | + Create a new LLM provider. Persists to __yao.store (with API key encrypted), + registers a runtime connector, and returns the complete Provider object. + The "source" field is automatically set to "dynamic". + The "connector_id" field is auto-generated based on owner type. + Throws 400 if key is empty or already exists. + args: + - name: data + type: object + required: true + desc: | + Provider data object with the following fields: + - key (string, required): Unique provider key. + - name (string): Display name. + - type (string): Connector type ("openai", "anthropic", etc.). + - api_url (string): Base API URL. + - api_key (string): API key (will be encrypted for storage). + - models (array of ModelInfo): Model list. + - enabled (bool): Active status (default false). + - require_key (bool): Whether API key is required. + - owner (ProviderOwner): Ownership. Defaults to {"type":"system"}. + - preset_key (string, optional): Preset key if created from template. + - is_custom (bool, optional): Custom flag. + Example: + {"key":"my-openai","name":"My OpenAI","type":"openai", + "api_url":"https://api.openai.com","api_key":"sk-abc123", + "models":[{"id":"gpt-4o","name":"GPT-4o","capabilities":["streaming","vision"],"enabled":true}], + "enabled":true,"require_key":true,"owner":{"type":"user","user_id":"42"}} + return: + type: object + desc: | + Created Provider object with connector_id and source="dynamic" populated. + The api_key in the response is the plaintext value (not encrypted). + + - name: update + desc: | + Update an existing provider by key. Replaces the stored provider with the + provided data, re-encrypts the API key, hot-replaces the runtime connector, + and returns the updated Provider object. + IMPORTANT: This is a full replacement, not a partial merge. You must provide + all fields you want to keep (name, type, api_url, api_key, models, enabled, etc.). + Only "key", "source", "connector_id", and "owner" are automatically preserved + from the existing record if omitted or zero-valued in the input. + Throws 400 if the provider key is not found. + args: + - name: key + type: string + required: true + desc: Provider key to update. + - name: data + type: object + required: true + desc: | + Full Provider data object. Same field structure as "create". + The "key" field inside data is ignored; the first argument determines + which provider to update. Fields not provided will be reset to zero values + (empty string, false, nil), except source, connector_id, and owner which + fall back to the existing record's values. + return: + type: object + desc: Updated Provider object with all fields. + + - name: delete + desc: | + Delete a provider by key. Removes from persistent store, clears cache, + and unregisters the runtime connector. + Throws 404 if the provider key does not exist. + args: + - name: key + type: string + required: true + desc: Provider key to delete. + return: + type: "null" + desc: Returns null on success. + + - name: list + desc: | + List providers matching a filter. Returns an array of Provider objects + with API keys masked. When no filter is provided, defaults to source="dynamic" + (only registry-created providers). Pass {"source":"all"} to include both + dynamic and built-in (.yao DSL) providers. + args: + - name: filter + type: object + required: false + desc: | + ProviderFilter object. All fields are optional: + - source (string): "dynamic" (default), "builtin", or "all". + - owner (ProviderOwner): {"type":"user","user_id":"42"}. + - enabled (bool): true or false. + - type (string): e.g. "openai". + - preset_key (string): e.g. "openai". + - capabilities (array of string): e.g. ["vision","streaming"]. + - keyword (string): Substring search in key and name. + Example: {"source":"all","type":"openai","capabilities":["vision"]} + Omit this argument entirely to list all dynamic providers. + return: + type: array + desc: | + Array of Provider objects with api_key masked. + May be empty if no providers match the filter. + + - name: getsetting + desc: | + Get the runtime connector setting map for a provider. This returns the + low-level connection parameters as used by the connector engine. + Throws 404 if the provider key does not exist. + args: + - name: key + type: string + required: true + desc: Provider key. + return: + type: object + desc: | + Key-value map of connector settings. Typical fields: + - host (string): API host URL. + - model (string): Default model ID. + - key (string): API key (plaintext). + Exact fields depend on connector type. + Example: {"host":"https://api.openai.com","model":"gpt-4o","key":"sk-abc123"} + + - name: getpresets + desc: | + Get all provider presets. Presets are static UI-only templates loaded from + the embedded presets.yml at compile time. They do not participate in runtime + logic — only used for UI form auto-filling when creating a new provider. + args: [] + return: + type: array + desc: | + Array of ProviderPreset objects. See ProviderPreset structure above. + Currently includes: openai, anthropic, ollama, azure, yaoagents. + Example element: + {"key":"openai","name":"OpenAI","type":"openai", + "api_url":"https://api.openai.com","require_key":true, + "default_models":[{"id":"gpt-4o","name":"GPT-4o", + "capabilities":["vision","tool_calls","streaming","json"],"enabled":true}]} + + - name: getpreset + desc: | + Get a single provider preset by key. + Throws 404 if the preset key does not exist. + args: + - name: key + type: string + required: true + desc: 'Preset key. Available keys: "openai", "anthropic", "ollama", "azure", "yaoagents".' + return: + type: object + desc: ProviderPreset object. See ProviderPreset structure above. diff --git a/llmprovider/process.go b/llmprovider/process.go new file mode 100644 index 00000000..aa0a9759 --- /dev/null +++ b/llmprovider/process.go @@ -0,0 +1,170 @@ +package llmprovider + +import ( + "encoding/json" + + "github.com/yaoapp/gou/process" + "github.com/yaoapp/kun/exception" +) + +func init() { + process.RegisterGroup("llmprovider", map[string]process.Handler{ + "get": ProcessGet, + "getmasked": ProcessGetMasked, + "create": ProcessCreate, + "update": ProcessUpdate, + "delete": ProcessDelete, + "list": ProcessList, + "getsetting": ProcessGetSetting, + "getpresets": ProcessGetPresets, + "getpreset": ProcessGetPreset, + }) +} + +func requireGlobal() { + if Global == nil { + exception.New("LLM Provider Registry not initialized", 500).Throw() + } +} + +// ProcessGet retrieves a provider by key. +// Args[0] string: provider key +func ProcessGet(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + key := p.ArgsString(0) + + provider, err := Global.Get(key) + if err != nil { + exception.New(err.Error(), 404).Throw() + } + return provider +} + +// ProcessGetMasked retrieves a provider with API key masked. +// Args[0] string: provider key +func ProcessGetMasked(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + key := p.ArgsString(0) + + provider, err := Global.GetMasked(key) + if err != nil { + exception.New(err.Error(), 404).Throw() + } + return provider +} + +// ProcessCreate adds a new provider. +// Args[0] map: Provider data +func ProcessCreate(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + + var provider Provider + raw, err := json.Marshal(p.Args[0]) + if err != nil { + exception.New("invalid provider data: "+err.Error(), 400).Throw() + } + if err := json.Unmarshal(raw, &provider); err != nil { + exception.New("invalid provider data: "+err.Error(), 400).Throw() + } + + result, err := Global.Create(&provider) + if err != nil { + exception.New(err.Error(), 400).Throw() + } + return result +} + +// ProcessUpdate modifies an existing provider. +// Args[0] string: provider key +// Args[1] map: Provider data +func ProcessUpdate(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(2) + key := p.ArgsString(0) + + var provider Provider + raw, err := json.Marshal(p.Args[1]) + if err != nil { + exception.New("invalid provider data: "+err.Error(), 400).Throw() + } + if err := json.Unmarshal(raw, &provider); err != nil { + exception.New("invalid provider data: "+err.Error(), 400).Throw() + } + + result, err := Global.Update(key, &provider) + if err != nil { + exception.New(err.Error(), 400).Throw() + } + return result +} + +// ProcessDelete removes a provider by key. +// Args[0] string: provider key +func ProcessDelete(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + key := p.ArgsString(0) + + if err := Global.Delete(key); err != nil { + exception.New(err.Error(), 404).Throw() + } + return nil +} + +// ProcessList returns providers matching a filter. +// Args[0] map: ProviderFilter (optional) +func ProcessList(p *process.Process) interface{} { + requireGlobal() + + var filter *ProviderFilter + if len(p.Args) > 0 && p.Args[0] != nil { + raw, err := json.Marshal(p.Args[0]) + if err == nil { + var f ProviderFilter + if json.Unmarshal(raw, &f) == nil { + filter = &f + } + } + } + + result, err := Global.List(filter) + if err != nil { + exception.New(err.Error(), 500).Throw() + } + return result +} + +// ProcessGetSetting returns the runtime connector setting map. +// Args[0] string: provider key +func ProcessGetSetting(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + key := p.ArgsString(0) + + setting, err := Global.GetSetting(key) + if err != nil { + exception.New(err.Error(), 404).Throw() + } + return setting +} + +// ProcessGetPresets returns all provider presets. +func ProcessGetPresets(p *process.Process) interface{} { + return GetPresets() +} + +// ProcessGetPreset returns a single preset by key. +// Args[0] string: preset key +func ProcessGetPreset(p *process.Process) interface{} { + p.ValidateArgNums(1) + key := p.ArgsString(0) + + preset := GetPreset(key) + if preset == nil { + exception.New("preset "+key+" not found", 404).Throw() + } + return preset +} diff --git a/llmprovider/process_test.go b/llmprovider/process_test.go new file mode 100644 index 00000000..273fceac --- /dev/null +++ b/llmprovider/process_test.go @@ -0,0 +1,164 @@ +package llmprovider_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/gou/process" +) + +func TestProcessCreate(t *testing.T) { + setupRegistry(t) + + p := process.New("llmprovider.create", map[string]interface{}{ + "key": "proc-test", + "name": "Proc Test", + "type": "openai", + "api_url": "https://api.openai.com", + "api_key": "sk-proc-test", + "enabled": true, + "require_key": true, + "models": []interface{}{map[string]interface{}{"id": "gpt-4o", "name": "GPT-4o", "capabilities": []interface{}{"streaming"}, "enabled": true}}, + "owner": map[string]interface{}{"type": "system"}, + }) + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + + m := toMapResult(t, result) + assert.Equal(t, "proc-test", m["key"]) + assert.NotEmpty(t, m["connector_id"]) + assert.Equal(t, "dynamic", m["source"]) +} + +func TestProcessGet(t *testing.T) { + setupRegistry(t) + createViaProcess(t, "proc-get") + + p := process.New("llmprovider.get", "proc-get") + result, err := p.Exec() + require.NoError(t, err) + + m := toMapResult(t, result) + assert.Equal(t, "proc-get", m["key"]) + assert.Equal(t, "sk-proc-test", m["api_key"]) +} + +func TestProcessGetMasked(t *testing.T) { + setupRegistry(t) + createViaProcess(t, "proc-masked") + + p := process.New("llmprovider.getmasked", "proc-masked") + result, err := p.Exec() + require.NoError(t, err) + + m := toMapResult(t, result) + apiKey, _ := m["api_key"].(string) + assert.NotEqual(t, "sk-proc-test", apiKey) + assert.Contains(t, apiKey, "test") +} + +func TestProcessUpdate(t *testing.T) { + setupRegistry(t) + createViaProcess(t, "proc-upd") + + p := process.New("llmprovider.update", "proc-upd", map[string]interface{}{ + "name": "Updated Name", + "api_url": "https://custom.openai.com", + "enabled": true, + "models": []interface{}{map[string]interface{}{"id": "gpt-4o", "name": "GPT-4o", "capabilities": []interface{}{"streaming"}, "enabled": true}}, + }) + result, err := p.Exec() + require.NoError(t, err) + + m := toMapResult(t, result) + assert.Equal(t, "Updated Name", m["name"]) + assert.Equal(t, "https://custom.openai.com", m["api_url"]) +} + +func TestProcessDelete(t *testing.T) { + setupRegistry(t) + createViaProcess(t, "proc-del") + + p := process.New("llmprovider.delete", "proc-del") + _, err := p.Exec() + require.NoError(t, err) + + pGet := process.New("llmprovider.get", "proc-del") + _, err = pGet.Exec() + assert.Error(t, err) +} + +func TestProcessList(t *testing.T) { + setupRegistry(t) + createViaProcess(t, "proc-list-1") + createViaProcess(t, "proc-list-2") + + p := process.New("llmprovider.list", map[string]interface{}{ + "source": "dynamic", + }) + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + t.Logf("list result type: %T", result) +} + +func TestProcessGetSetting(t *testing.T) { + setupRegistry(t) + createViaProcess(t, "proc-setting") + + p := process.New("llmprovider.getsetting", "proc-setting") + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) +} + +func TestProcessGetPresets(t *testing.T) { + p := process.New("llmprovider.getpresets") + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) +} + +func TestProcessGetPreset(t *testing.T) { + p := process.New("llmprovider.getpreset", "openai") + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + + m := toMapResult(t, result) + assert.Equal(t, "openai", m["key"]) +} + +// --- helpers --- + +func createViaProcess(t *testing.T, key string) { + t.Helper() + p := process.New("llmprovider.create", map[string]interface{}{ + "key": key, + "name": "Test " + key, + "type": "openai", + "api_url": "https://api.openai.com", + "api_key": "sk-proc-test", + "enabled": true, + "require_key": true, + "models": []interface{}{map[string]interface{}{"id": "gpt-4o", "name": "GPT-4o", "capabilities": []interface{}{"streaming"}, "enabled": true}}, + "owner": map[string]interface{}{"type": "system"}, + }) + _, err := p.Exec() + require.NoError(t, err) +} + +func toMapResult(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + if m, ok := v.(map[string]interface{}); ok { + return m + } + raw, err := json.Marshal(v) + require.NoError(t, err) + var m map[string]interface{} + require.NoError(t, json.Unmarshal(raw, &m)) + return m +} diff --git a/mcpclient/doc.go b/mcpclient/doc.go new file mode 100644 index 00000000..2f951263 --- /dev/null +++ b/mcpclient/doc.go @@ -0,0 +1,12 @@ +package mcpclient + +import ( + _ "embed" + + "github.com/yaoapp/gou/doc" +) + +//go:embed doc.yml +var docYAML []byte + +func init() { doc.LoadYAML(docYAML) } diff --git a/mcpclient/doc.yml b/mcpclient/doc.yml new file mode 100644 index 00000000..0c665655 --- /dev/null +++ b/mcpclient/doc.yml @@ -0,0 +1,189 @@ +group: mcpclient +type: process +desc: | + CRUD operations for the MCP Client Registry. Manages MCP (Model Context Protocol) + client connections with persistence and lazy runtime registration. + + Process names follow the pattern "mcpclient.". + + Client structure (returned by get, create, update; array elements from list): + Embeds all fields from ClientDSL plus registry management fields. + + Inherited from ClientDSL: + - id (string): Unique client identifier. Required on create. + - name (string): Display name (e.g. "GitHub MCP", "File System"). + - version (string, optional): Client version. + - type (string, optional): Client type. Values: "standard", "agent", "system". + - transport (string): Transport protocol. Values: "stdio", "http", "sse", "process". + + Inherited from MetaInfo (embedded in ClientDSL): + - label (string, optional): Human-readable label for display. + - description (string, optional): Description text (markdown or plain). + - tags (array of string, optional): Categorization tags. + - readonly (bool, optional): Whether this client is read-only. + - builtin (bool, optional): Whether this is a built-in client. + + For stdio transport: + - command (string): Executable command (e.g. "npx", "python"). + - arguments (array of string): Command arguments (e.g. ["-y", "@modelcontextprotocol/server-github"]). + - env (object, optional): Environment variables as key-value pairs. + + For http/sse transport: + - url (string): Server URL. + - endpoint (string, optional): API endpoint path (e.g. "/api/mcp"). + - authorization_token (string, optional): Bearer token for authentication. + - timeout (string, optional): Request timeout (e.g. "30s", "5m"). + + For process transport: + - tools (object, optional): Tool name → process name mapping. + - prompts (object, optional): Prompt name → process name mapping. + - resources (object, optional): Resource name → process name mapping. + + Client capability flags: + - enable_sampling (bool, optional): Enable sampling capability. + - enable_roots (bool, optional): Enable roots capability. + - roots_list_changed (bool, optional): Subscribe to root change notifications. + - enable_elicitation (bool, optional): Enable elicitation capability. + + Dependencies: + - dependencies (object, optional): Other MCP clients this depends on (name → version constraint). + + Registry management fields (added by the registry): + - runtime_id (string): Runtime registration ID, auto-generated. + Format: "s." for system, "u." for user, "t." for team. + BuiltIn clients retain their original ID. + - enabled (bool): Whether the client is active. + - status (string): Connection status. Values: "connected", "disconnected", "unconfigured". + - source (string): Origin. Values: "dynamic" (registry-created), "builtin" (loaded from .yao DSL). + - tool_list (array of Tool, optional): Discovered tools from the MCP server. + Each Tool has: name (string), description (string), inputSchema (object). + - owner (ClientOwner): Ownership information. + + ClientOwner structure (Client.owner): + - type (string): Scope level. Values: "system", "team", "user". + - id (string, optional): Team ID or User ID depending on type. + + ClientFilter structure (optional argument for list): + - source (string, optional): Filter by source. + Values: "dynamic" (default when omitted), "builtin", "all". + - owner (ClientOwner, optional): Filter by owner. + - enabled (bool, optional): Filter by enabled status. Omit to include both. + - transport (string, optional): Filter by transport type ("stdio", "http", "sse", "process"). + - type (string, optional): Filter by client type ("standard", "agent", "system"). + - keyword (string, optional): Case-insensitive substring search in id, name, and label. + +entries: + - name: get + desc: | + Get an MCP client by ID, returning the full Client object. + Lazily ensures the runtime MCP client is registered on first access. + Throws 404 if the client ID does not exist. + args: + - name: id + type: string + required: true + desc: Client ID (e.g. "github-mcp", "filesystem"). + return: + type: object + desc: | + Full Client object. See Client structure above. + Example: {"id":"github-mcp","name":"GitHub MCP","type":"standard", + "transport":"stdio","command":"npx", + "arguments":["-y","@modelcontextprotocol/server-github"], + "enabled":true,"status":"connected","source":"dynamic", + "runtime_id":"s.github-mcp","owner":{"type":"system"}} + + - name: create + desc: | + Create a new MCP client. Persists to __yao.store, registers the runtime + MCP client, and returns the complete Client object. + The "source" field is automatically set to "dynamic". + The "runtime_id" field is auto-generated based on owner type. + Throws 400 if id is empty or already exists. + args: + - name: data + type: object + required: true + desc: | + Client data object. Required fields depend on transport type: + + For stdio transport: + {"id":"my-mcp","name":"My MCP","type":"standard","transport":"stdio", + "command":"npx","arguments":["-y","@some/mcp-server"], + "enabled":true,"owner":{"type":"system"}} + + For http/sse transport: + {"id":"remote-mcp","name":"Remote MCP","type":"standard","transport":"sse", + "url":"https://mcp.example.com","authorization_token":"Bearer xxx", + "enabled":true,"owner":{"type":"user","id":"42"}} + + For process transport: + {"id":"local-tools","name":"Local Tools","type":"standard","transport":"process", + "tools":{"search":"scripts.search.Run","fetch":"scripts.fetch.Run"}, + "enabled":true,"owner":{"type":"system"}} + return: + type: object + desc: Created Client object with runtime_id and source="dynamic" populated. + + - name: update + desc: | + Update an existing MCP client by ID. Replaces the stored client with the + provided data, hot-replaces the runtime client, and returns the updated object. + IMPORTANT: This is a full replacement, not a partial merge. You must provide + all fields you want to keep. Only "id", "source", "runtime_id", and "owner" + are automatically preserved from the existing record if omitted or zero-valued. + Throws 400 if the client ID is not found. + args: + - name: id + type: string + required: true + desc: Client ID to update. + - name: data + type: object + required: true + desc: | + Full Client data object. Same field structure as "create". + The "id" field inside data is ignored; the first argument determines + which client to update. Fields not provided will be reset to zero values, + except source, runtime_id, and owner which fall back to the existing values. + return: + type: object + desc: Updated Client object with all fields. + + - name: delete + desc: | + Delete an MCP client by ID. Removes from persistent store, clears cache, + and unloads the runtime MCP client. + Throws 404 if the client ID does not exist. + args: + - name: id + type: string + required: true + desc: Client ID to delete. + return: + type: "null" + desc: Returns null on success. + + - name: list + desc: | + List MCP clients matching a filter. When no filter is provided, defaults to + source="dynamic" (only registry-created clients). Pass {"source":"all"} to + include both dynamic and built-in (.yao DSL) clients. + args: + - name: filter + type: object + required: false + desc: | + ClientFilter object. All fields are optional: + - source (string): "dynamic" (default), "builtin", or "all". + - owner (ClientOwner): e.g. {"type":"user","id":"42"}. + - enabled (bool): true or false. + - transport (string): "stdio", "http", "sse", or "process". + - type (string): "standard", "agent", or "system". + - keyword (string): Substring search in id, name, and label. + Example: {"source":"all","transport":"stdio"} + Omit this argument entirely to list all dynamic clients. + return: + type: array + desc: | + Array of Client objects. May be empty if no clients match the filter. diff --git a/mcpclient/process.go b/mcpclient/process.go new file mode 100644 index 00000000..7d1995bb --- /dev/null +++ b/mcpclient/process.go @@ -0,0 +1,120 @@ +package mcpclient + +import ( + "encoding/json" + + "github.com/yaoapp/gou/process" + "github.com/yaoapp/kun/exception" +) + +func init() { + process.RegisterGroup("mcpclient", map[string]process.Handler{ + "get": ProcessGet, + "create": ProcessCreate, + "update": ProcessUpdate, + "delete": ProcessDelete, + "list": ProcessList, + }) +} + +func requireGlobal() { + if Global == nil { + exception.New("MCP Client Registry not initialized", 500).Throw() + } +} + +// ProcessGet retrieves a client by ID. +// Args[0] string: client ID +func ProcessGet(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + id := p.ArgsString(0) + + client, err := Global.Get(id) + if err != nil { + exception.New(err.Error(), 404).Throw() + } + return client +} + +// ProcessCreate adds a new MCP client. +// Args[0] map: Client data +func ProcessCreate(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + + var client Client + raw, err := json.Marshal(p.Args[0]) + if err != nil { + exception.New("invalid client data: "+err.Error(), 400).Throw() + } + if err := json.Unmarshal(raw, &client); err != nil { + exception.New("invalid client data: "+err.Error(), 400).Throw() + } + + result, err := Global.Create(&client) + if err != nil { + exception.New(err.Error(), 400).Throw() + } + return result +} + +// ProcessUpdate modifies an existing MCP client. +// Args[0] string: client ID +// Args[1] map: Client data +func ProcessUpdate(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(2) + id := p.ArgsString(0) + + var client Client + raw, err := json.Marshal(p.Args[1]) + if err != nil { + exception.New("invalid client data: "+err.Error(), 400).Throw() + } + if err := json.Unmarshal(raw, &client); err != nil { + exception.New("invalid client data: "+err.Error(), 400).Throw() + } + + result, err := Global.Update(id, &client) + if err != nil { + exception.New(err.Error(), 400).Throw() + } + return result +} + +// ProcessDelete removes a client by ID. +// Args[0] string: client ID +func ProcessDelete(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + id := p.ArgsString(0) + + if err := Global.Delete(id); err != nil { + exception.New(err.Error(), 404).Throw() + } + return nil +} + +// ProcessList returns clients matching a filter. +// Args[0] map: ClientFilter (optional) +func ProcessList(p *process.Process) interface{} { + requireGlobal() + + var filter *ClientFilter + if len(p.Args) > 0 && p.Args[0] != nil { + raw, err := json.Marshal(p.Args[0]) + if err == nil { + var f ClientFilter + if json.Unmarshal(raw, &f) == nil { + filter = &f + } + } + } + + result, err := Global.List(filter) + if err != nil { + exception.New(err.Error(), 500).Throw() + } + return result +} diff --git a/mcpclient/process_test.go b/mcpclient/process_test.go new file mode 100644 index 00000000..1f83e9c2 --- /dev/null +++ b/mcpclient/process_test.go @@ -0,0 +1,120 @@ +package mcpclient_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/gou/process" +) + +func TestProcessCreate(t *testing.T) { + setupRegistry(t) + + p := process.New("mcpclient.create", map[string]interface{}{ + "id": "proc-test", + "name": "Proc Test", + "type": "standard", + "transport": "stdio", + "command": "echo", + "arguments": []interface{}{"hello"}, + "enabled": true, + "owner": map[string]interface{}{"type": "system"}, + }) + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + + m := toMapResult(t, result) + assert.Equal(t, "proc-test", m["id"]) + assert.NotEmpty(t, m["runtime_id"]) + assert.Equal(t, "dynamic", m["source"]) +} + +func TestProcessGet(t *testing.T) { + setupRegistry(t) + createClientViaProcess(t, "proc-get") + + p := process.New("mcpclient.get", "proc-get") + result, err := p.Exec() + require.NoError(t, err) + + m := toMapResult(t, result) + assert.Equal(t, "proc-get", m["id"]) +} + +func TestProcessUpdate(t *testing.T) { + setupRegistry(t) + createClientViaProcess(t, "proc-upd") + + p := process.New("mcpclient.update", "proc-upd", map[string]interface{}{ + "name": "Updated MCP", + "type": "standard", + "transport": "stdio", + "command": "cat", + "enabled": true, + }) + result, err := p.Exec() + require.NoError(t, err) + + m := toMapResult(t, result) + assert.Equal(t, "Updated MCP", m["name"]) +} + +func TestProcessDelete(t *testing.T) { + setupRegistry(t) + createClientViaProcess(t, "proc-del") + + p := process.New("mcpclient.delete", "proc-del") + _, err := p.Exec() + require.NoError(t, err) + + pGet := process.New("mcpclient.get", "proc-del") + _, err = pGet.Exec() + assert.Error(t, err) +} + +func TestProcessList(t *testing.T) { + setupRegistry(t) + createClientViaProcess(t, "proc-list-1") + createClientViaProcess(t, "proc-list-2") + + p := process.New("mcpclient.list", map[string]interface{}{ + "source": "dynamic", + }) + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + t.Logf("list result type: %T", result) +} + +// --- helpers --- + +func createClientViaProcess(t *testing.T, id string) { + t.Helper() + p := process.New("mcpclient.create", map[string]interface{}{ + "id": id, + "name": "Test " + id, + "type": "standard", + "transport": "stdio", + "command": "echo", + "arguments": []interface{}{"hello"}, + "enabled": true, + "owner": map[string]interface{}{"type": "system"}, + }) + _, err := p.Exec() + require.NoError(t, err) +} + +func toMapResult(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + if m, ok := v.(map[string]interface{}); ok { + return m + } + raw, err := json.Marshal(v) + require.NoError(t, err) + var m map[string]interface{} + require.NoError(t, json.Unmarshal(raw, &m)) + return m +} diff --git a/setting/doc.go b/setting/doc.go new file mode 100644 index 00000000..b84825de --- /dev/null +++ b/setting/doc.go @@ -0,0 +1,12 @@ +package setting + +import ( + _ "embed" + + "github.com/yaoapp/gou/doc" +) + +//go:embed doc.yml +var docYAML []byte + +func init() { doc.LoadYAML(docYAML) } diff --git a/setting/doc.yml b/setting/doc.yml new file mode 100644 index 00000000..bbad84b4 --- /dev/null +++ b/setting/doc.yml @@ -0,0 +1,163 @@ +group: setting +type: process +desc: | + Generic user personalization settings store with three-level scope hierarchy. + Stores arbitrary JSON data organized by namespace and scope, with cascading + merge support (system ← team ← user, later scope wins). + + Process names follow the pattern "setting.". + + Scoping model: + Three levels, from lowest to highest priority: + 1. system — Global defaults, shared by all users. + 2. team — Team-level overrides, shared by team members. + 3. user — Individual user preferences, highest priority. + + ScopeID structure (used as argument for get, set, delete, listnamespaces): + - scope (string, required): Scope level. Values: "system", "team", "user". + - team_id (string): Required when scope is "team". + - user_id (string): Required when scope is "user". + Examples: + System scope: {"scope":"system"} + Team scope: {"scope":"team","team_id":"99"} + User scope: {"scope":"user","user_id":"42"} + + Entry structure (returned by set): + - namespace (string): Namespace name (e.g. "preferences", "privacy", "models"). + - scope (ScopeID): The scope this entry belongs to. + - data (object): Arbitrary key-value data stored for this namespace. + - updated_at (string): ISO 8601 timestamp of last update. + + Namespace convention: + Namespaces are free-form strings chosen by the consuming module. + Typical examples: "preferences", "privacy", "models", "notifications". + Each namespace stores one JSON object (map of string → any). + The registry does not enforce any schema — the consuming module defines + the expected structure. + + Merge behavior (getmerged): + Shallow merge across three scopes: system ← team ← user. + For each top-level key, the highest-priority scope's value wins. + Example: + system: {"theme":"light","lang":"en","font_size":14} + team: {"lang":"zh-CN"} + user: {"theme":"dark"} + merged: {"theme":"dark","lang":"zh-CN","font_size":14} + If a scope has no data for the namespace, it is skipped. + Returns 404 only when no data exists at any scope. + +entries: + - name: get + desc: | + Get a namespace entry for a specific scope. Returns the raw data object + without any merging. Throws 404 if the namespace does not exist at + the given scope. + args: + - name: scope + type: object + required: true + desc: | + ScopeID object identifying the scope. + Examples: + {"scope":"system"} + {"scope":"team","team_id":"99"} + {"scope":"user","user_id":"42"} + - name: namespace + type: string + required: true + desc: 'Namespace name (e.g. "preferences", "privacy", "models").' + return: + type: object + desc: | + The namespace data as a key-value map (not wrapped in Entry). + Example: {"theme":"dark","lang":"zh-CN","font_size":14} + + - name: getmerged + desc: | + Get a namespace with three-level cascade merge: system ← team ← user. + Reads data from all three scopes and shallow-merges them, with higher-priority + scopes overriding lower ones. Pass empty string for userID or teamID to + skip that scope. Throws 404 if no data exists at any scope. + args: + - name: userID + type: string + required: true + desc: 'User ID. Pass "" (empty string) to skip user scope.' + - name: teamID + type: string + required: true + desc: 'Team ID. Pass "" (empty string) to skip team scope.' + - name: namespace + type: string + required: true + desc: Namespace name. + return: + type: object + desc: | + Shallow-merged data from all available scopes. + Example with system={"a":"sys","b":"sys"}, team={"b":"team"}, user={"a":"user"}: + Result: {"a":"user","b":"team"} + + - name: set + desc: | + Set (create or overwrite) a namespace entry for a given scope. + Persists to __yao.store and updates __yao.cache. + Overwrites any existing data for this scope + namespace combination. + args: + - name: scope + type: object + required: true + desc: 'ScopeID object. Example: {"scope":"user","user_id":"42"}' + - name: namespace + type: string + required: true + desc: Namespace name. + - name: data + type: object + required: true + desc: | + Key-value data to store. Arbitrary JSON object. + Example: {"theme":"dark","lang":"zh-CN","font_size":14} + return: + type: object + desc: | + Entry object confirming the write. Fields: + - namespace (string) + - scope (ScopeID) + - data (object): The stored data. + - updated_at (string): ISO 8601 timestamp. + Example: {"namespace":"preferences","scope":{"scope":"user","user_id":"42"}, + "data":{"theme":"dark","lang":"zh-CN"},"updated_at":"2025-01-15T10:30:00Z"} + + - name: delete + desc: | + Delete a namespace entry from a scope. Removes from __yao.store + and __yao.cache. Throws 404 if the namespace does not exist at the scope. + args: + - name: scope + type: object + required: true + desc: 'ScopeID object. Example: {"scope":"system"}' + - name: namespace + type: string + required: true + desc: Namespace name to delete. + return: + type: "null" + desc: Returns null on success. + + - name: listnamespaces + desc: | + List all namespace names stored under a scope. Returns the namespace + strings only, not the data. Use "get" to retrieve data for each namespace. + args: + - name: scope + type: object + required: true + desc: 'ScopeID object. Example: {"scope":"team","team_id":"99"}' + return: + type: array + desc: | + Array of namespace name strings. + Example: ["preferences","privacy","models"] + Returns empty array if no namespaces exist for the scope. diff --git a/setting/process.go b/setting/process.go new file mode 100644 index 00000000..289c1353 --- /dev/null +++ b/setting/process.go @@ -0,0 +1,130 @@ +package setting + +import ( + "encoding/json" + + "github.com/yaoapp/gou/process" + "github.com/yaoapp/kun/exception" +) + +func init() { + process.RegisterGroup("setting", map[string]process.Handler{ + "get": ProcessGet, + "getmerged": ProcessGetMerged, + "set": ProcessSet, + "delete": ProcessDelete, + "listnamespaces": ProcessListNamespaces, + }) +} + +func requireGlobal() { + if Global == nil { + exception.New("Setting Registry not initialized", 500).Throw() + } +} + +func parseScopeID(arg interface{}) ScopeID { + raw, err := json.Marshal(arg) + if err != nil { + exception.New("invalid scope: "+err.Error(), 400).Throw() + } + var scope ScopeID + if err := json.Unmarshal(raw, &scope); err != nil { + exception.New("invalid scope: "+err.Error(), 400).Throw() + } + return scope +} + +// ProcessGet reads a namespace entry for a given scope. +// Args[0] map: ScopeID {scope, team_id?, user_id?} +// Args[1] string: namespace +func ProcessGet(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(2) + + scope := parseScopeID(p.Args[0]) + ns := p.ArgsString(1) + + data, err := Global.Get(scope, ns) + if err != nil { + exception.New(err.Error(), 404).Throw() + } + return data +} + +// ProcessGetMerged reads a namespace with three-level cascade merge. +// Args[0] string: userID +// Args[1] string: teamID +// Args[2] string: namespace +func ProcessGetMerged(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(3) + + userID := p.ArgsString(0) + teamID := p.ArgsString(1) + ns := p.ArgsString(2) + + data, err := Global.GetMerged(userID, teamID, ns) + if err != nil { + exception.New(err.Error(), 404).Throw() + } + return data +} + +// ProcessSet writes a namespace entry for a given scope. +// Args[0] map: ScopeID +// Args[1] string: namespace +// Args[2] map: data +func ProcessSet(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(3) + + scope := parseScopeID(p.Args[0]) + ns := p.ArgsString(1) + + raw, err := json.Marshal(p.Args[2]) + if err != nil { + exception.New("invalid data: "+err.Error(), 400).Throw() + } + var data map[string]interface{} + if err := json.Unmarshal(raw, &data); err != nil { + exception.New("invalid data: "+err.Error(), 400).Throw() + } + + entry, err := Global.Set(scope, ns, data) + if err != nil { + exception.New(err.Error(), 400).Throw() + } + return entry +} + +// ProcessDelete removes a namespace entry from a given scope. +// Args[0] map: ScopeID +// Args[1] string: namespace +func ProcessDelete(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(2) + + scope := parseScopeID(p.Args[0]) + ns := p.ArgsString(1) + + if err := Global.Delete(scope, ns); err != nil { + exception.New(err.Error(), 404).Throw() + } + return nil +} + +// ProcessListNamespaces returns all namespace names under a scope. +// Args[0] map: ScopeID +func ProcessListNamespaces(p *process.Process) interface{} { + requireGlobal() + p.ValidateArgNums(1) + + scope := parseScopeID(p.Args[0]) + + ns, err := Global.ListNamespaces(scope) + if err != nil { + exception.New(err.Error(), 500).Throw() + } + return ns +} diff --git a/setting/process_test.go b/setting/process_test.go new file mode 100644 index 00000000..cf14914a --- /dev/null +++ b/setting/process_test.go @@ -0,0 +1,109 @@ +package setting_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/gou/process" +) + +var sysScope = map[string]interface{}{"scope": "system"} +var teamScopeP = map[string]interface{}{"scope": "team", "team_id": "99"} +var userScopeP = map[string]interface{}{"scope": "user", "user_id": "42"} + +func TestProcessSet(t *testing.T) { + setupRegistry(t) + + p := process.New("setting.set", sysScope, "prefs", map[string]interface{}{ + "theme": "dark", "lang": "zh-CN", + }) + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + + m := toMapR(t, result) + assert.Equal(t, "prefs", m["namespace"]) + assert.NotEmpty(t, m["updated_at"]) +} + +func TestProcessGet(t *testing.T) { + setupRegistry(t) + + process.New("setting.set", sysScope, "gettest", map[string]interface{}{ + "color": "blue", + }).Exec() + + p := process.New("setting.get", sysScope, "gettest") + result, err := p.Exec() + require.NoError(t, err) + + m := toMapR(t, result) + assert.Equal(t, "blue", m["color"]) +} + +func TestProcessGetMerged(t *testing.T) { + setupRegistry(t) + + process.New("setting.set", sysScope, "merged", map[string]interface{}{ + "a": "sys", "b": "sys", + }).Exec() + process.New("setting.set", teamScopeP, "merged", map[string]interface{}{ + "b": "team", + }).Exec() + process.New("setting.set", userScopeP, "merged", map[string]interface{}{ + "a": "user", + }).Exec() + + p := process.New("setting.getmerged", "42", "99", "merged") + result, err := p.Exec() + require.NoError(t, err) + + m := toMapR(t, result) + assert.Equal(t, "user", m["a"]) + assert.Equal(t, "team", m["b"]) +} + +func TestProcessDelete(t *testing.T) { + setupRegistry(t) + + process.New("setting.set", sysScope, "deltest", map[string]interface{}{ + "x": "y", + }).Exec() + + p := process.New("setting.delete", sysScope, "deltest") + _, err := p.Exec() + require.NoError(t, err) + + pGet := process.New("setting.get", sysScope, "deltest") + _, err = pGet.Exec() + assert.Error(t, err) +} + +func TestProcessListNamespaces(t *testing.T) { + setupRegistry(t) + + process.New("setting.set", sysScope, "ns-a", map[string]interface{}{"v": 1}).Exec() + process.New("setting.set", sysScope, "ns-b", map[string]interface{}{"v": 2}).Exec() + + p := process.New("setting.listnamespaces", sysScope) + result, err := p.Exec() + require.NoError(t, err) + require.NotNil(t, result) + t.Logf("namespaces: %v", result) +} + +// --- helpers --- + +func toMapR(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + if m, ok := v.(map[string]interface{}); ok { + return m + } + raw, err := json.Marshal(v) + require.NoError(t, err) + var m map[string]interface{} + require.NoError(t, json.Unmarshal(raw, &m)) + return m +} diff --git a/setting/registry.go b/setting/registry.go new file mode 100644 index 00000000..52c73c67 --- /dev/null +++ b/setting/registry.go @@ -0,0 +1,183 @@ +package setting + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/yaoapp/gou/store" +) + +// Global is the singleton Setting Registry. +var Global *Registry + +// Registry manages namespaced settings with three-level scope cascade. +type Registry struct { + store store.Store + cache store.Store + mu sync.RWMutex +} + +// Init initializes the global Registry. +// Must be called after store.Load (so __yao.store and __yao.cache are available). +func Init() error { + s, err := store.Get("__yao.store") + if err != nil { + return fmt.Errorf("setting.Init: %w", err) + } + c, _ := store.Get("__yao.cache") + Global = &Registry{store: s, cache: c} + return nil +} + +// Get reads the raw data for a single scope+namespace. +// If one or more dest pointers are provided, the data is also unmarshalled +// into dest[0] (like json.Unmarshal). +func (r *Registry) Get(scope ScopeID, ns string, dest ...interface{}) (map[string]interface{}, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + data, err := storeGet(r.store, r.cache, scope, ns) + if err != nil { + return nil, err + } + + if len(dest) > 0 && dest[0] != nil { + if err := bindDest(data, dest[0]); err != nil { + return data, fmt.Errorf("setting bind: %w", err) + } + } + return data, nil +} + +// GetMerged reads a namespace across all three scopes and returns a shallow-merged +// result: system <- team <- user (later wins). +// If one or more dest pointers are provided, the merged data is also unmarshalled +// into dest[0]. +func (r *Registry) GetMerged(userID, teamID, ns string, dest ...interface{}) (map[string]interface{}, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + merged := make(map[string]interface{}) + + if sys, err := storeGet(r.store, r.cache, ScopeID{Scope: ScopeSystem}, ns); err == nil { + shallowMerge(merged, sys) + } + + if teamID != "" { + if team, err := storeGet(r.store, r.cache, ScopeID{Scope: ScopeTeam, TeamID: teamID}, ns); err == nil { + shallowMerge(merged, team) + } + } + + if userID != "" { + if user, err := storeGet(r.store, r.cache, ScopeID{Scope: ScopeUser, UserID: userID}, ns); err == nil { + shallowMerge(merged, user) + } + } + + if len(merged) == 0 { + return nil, fmt.Errorf("setting %s: no data found at any scope", ns) + } + + if len(dest) > 0 && dest[0] != nil { + if err := bindDest(merged, dest[0]); err != nil { + return merged, fmt.Errorf("setting bind: %w", err) + } + } + return merged, nil +} + +// Set writes (or overwrites) a namespace entry for the given scope. +func (r *Registry) Set(scope ScopeID, ns string, data map[string]interface{}) (*Entry, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if ns == "" { + return nil, fmt.Errorf("namespace is required") + } + + if err := storeSet(r.store, r.cache, scope, ns, data); err != nil { + return nil, err + } + if err := indexAdd(r.store, r.cache, scope, ns); err != nil { + return nil, err + } + + return &Entry{ + Namespace: ns, + Scope: scope, + Data: data, + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + }, nil +} + +// Delete removes a namespace entry from a given scope. +func (r *Registry) Delete(scope ScopeID, ns string) error { + r.mu.Lock() + defer r.mu.Unlock() + + sk := storeKey(scope, ns) + if !r.store.Has(sk) { + return fmt.Errorf("setting %s/%s not found", scopePrefix(scope), ns) + } + + if err := storeDel(r.store, r.cache, scope, ns); err != nil { + return err + } + return indexRemove(r.store, r.cache, scope, ns) +} + +// ListNamespaces returns all namespace names stored under the given scope. +func (r *Registry) ListNamespaces(scope ScopeID) ([]string, error) { + r.mu.RLock() + defer r.mu.RUnlock() + return indexGet(r.store, r.cache, scope) +} + +// Reload clears the cache and re-populates it from the persistent store. +func (r *Registry) Reload() error { + r.mu.Lock() + defer r.mu.Unlock() + + if r.cache != nil { + _ = r.cache.Del(keyPrefix + "*") + } + + for _, scope := range []ScopeID{ + {Scope: ScopeSystem}, + } { + keys, err := indexGet(r.store, nil, scope) + if err != nil { + continue + } + ik := indexKey(scope) + raw, ok := r.store.Get(ik) + if ok && r.cache != nil { + r.cache.Set(ik, raw, 0) + } + for _, ns := range keys { + if data, err := storeGet(r.store, nil, scope, ns); err == nil && r.cache != nil { + r.cache.Set(storeKey(scope, ns), data, 0) + } + } + } + return nil +} + +// shallowMerge copies all keys from src into dst (overwrites existing keys). +func shallowMerge(dst, src map[string]interface{}) { + for k, v := range src { + dst[k] = v + } +} + +// bindDest marshals data to JSON and then unmarshals into the dest pointer. +func bindDest(data map[string]interface{}, dest interface{}) error { + raw, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(raw, dest) +} diff --git a/setting/registry_test.go b/setting/registry_test.go new file mode 100644 index 00000000..2d93fca9 --- /dev/null +++ b/setting/registry_test.go @@ -0,0 +1,358 @@ +package setting_test + +import ( + "fmt" + "os" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yaoapp/gou/store" + "github.com/yaoapp/yao/config" + "github.com/yaoapp/yao/setting" + "github.com/yaoapp/yao/test" +) + +func TestMain(m *testing.M) { + test.Prepare(nil, config.Conf) + defer test.Clean() + os.Exit(m.Run()) +} + +func setupRegistry(t *testing.T) *setting.Registry { + t.Helper() + test.Prepare(t, config.Conf) + + err := setting.Init() + require.NoError(t, err) + + t.Cleanup(func() { + s, _ := store.Get("__yao.store") + if s != nil { + s.Del("setting:*") + } + c, _ := store.Get("__yao.cache") + if c != nil { + c.Del("setting:*") + } + test.Clean() + }) + + return setting.Global +} + +var systemScope = setting.ScopeID{Scope: setting.ScopeSystem} +var teamScope = setting.ScopeID{Scope: setting.ScopeTeam, TeamID: "99"} +var userScope = setting.ScopeID{Scope: setting.ScopeUser, UserID: "42"} + +func TestSetAndGet(t *testing.T) { + r := setupRegistry(t) + + data := map[string]interface{}{ + "theme": "dark", + "language": "zh-CN", + "fontSize": float64(14), + } + entry, err := r.Set(systemScope, "preferences", data) + require.NoError(t, err) + assert.Equal(t, "preferences", entry.Namespace) + assert.Equal(t, systemScope, entry.Scope) + assert.NotEmpty(t, entry.UpdatedAt) + + got, err := r.Get(systemScope, "preferences") + require.NoError(t, err) + assert.Equal(t, "dark", got["theme"]) + assert.Equal(t, "zh-CN", got["language"]) + assert.Equal(t, float64(14), got["fontSize"]) +} + +func TestGetWithBind(t *testing.T) { + r := setupRegistry(t) + + data := map[string]interface{}{ + "default_chat": "gpt-4o", + "vision_model": "gpt-4o", + "embedding_enabled": true, + } + _, err := r.Set(systemScope, "models", data) + require.NoError(t, err) + + type ModelsConfig struct { + DefaultChat string `json:"default_chat"` + VisionModel string `json:"vision_model"` + EmbeddingEnabled bool `json:"embedding_enabled"` + } + + var cfg ModelsConfig + raw, err := r.Get(systemScope, "models", &cfg) + require.NoError(t, err) + + assert.Equal(t, "gpt-4o", raw["default_chat"]) + assert.Equal(t, "gpt-4o", cfg.DefaultChat) + assert.Equal(t, "gpt-4o", cfg.VisionModel) + assert.True(t, cfg.EmbeddingEnabled) +} + +func TestGetMergedWithBind(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "prefs", map[string]interface{}{ + "theme": "dark", "lang": "zh-CN", "font_size": float64(14), + }) + require.NoError(t, err) + + _, err = r.Set(teamScope, "prefs", map[string]interface{}{ + "lang": "en-US", + }) + require.NoError(t, err) + + _, err = r.Set(userScope, "prefs", map[string]interface{}{ + "theme": "light", + }) + require.NoError(t, err) + + type Prefs struct { + Theme string `json:"theme"` + Lang string `json:"lang"` + FontSize float64 `json:"font_size"` + } + + var p Prefs + _, err = r.GetMerged("42", "99", "prefs", &p) + require.NoError(t, err) + assert.Equal(t, "light", p.Theme) + assert.Equal(t, "en-US", p.Lang) + assert.Equal(t, float64(14), p.FontSize) +} + +func TestGetNotFound(t *testing.T) { + r := setupRegistry(t) + _, err := r.Get(systemScope, "nonexistent") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func TestGetMerged(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "theme", map[string]interface{}{ + "primary": "blue", "dark_mode": true, "font": "inter", + }) + require.NoError(t, err) + + _, err = r.Set( + setting.ScopeID{Scope: setting.ScopeTeam, TeamID: "t1"}, + "theme", + map[string]interface{}{"dark_mode": false}, + ) + require.NoError(t, err) + + _, err = r.Set( + setting.ScopeID{Scope: setting.ScopeUser, UserID: "u1"}, + "theme", + map[string]interface{}{"primary": "red"}, + ) + require.NoError(t, err) + + merged, err := r.GetMerged("u1", "t1", "theme") + require.NoError(t, err) + assert.Equal(t, "red", merged["primary"]) + assert.Equal(t, false, merged["dark_mode"]) + assert.Equal(t, "inter", merged["font"]) +} + +func TestGetMergedPartial(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "partial", map[string]interface{}{"a": "1", "b": "2"}) + require.NoError(t, err) + + // Only system + user, no team data + _, err = r.Set(userScope, "partial", map[string]interface{}{"b": "override"}) + require.NoError(t, err) + + merged, err := r.GetMerged("42", "", "partial") + require.NoError(t, err) + assert.Equal(t, "1", merged["a"]) + assert.Equal(t, "override", merged["b"]) +} + +func TestGetMergedNoData(t *testing.T) { + r := setupRegistry(t) + _, err := r.GetMerged("42", "99", "nothing") + assert.Error(t, err) + assert.Contains(t, err.Error(), "no data found") +} + +func TestSetOverwrite(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "overwrite", map[string]interface{}{"a": "1"}) + require.NoError(t, err) + + _, err = r.Set(systemScope, "overwrite", map[string]interface{}{"a": "2", "b": "3"}) + require.NoError(t, err) + + got, err := r.Get(systemScope, "overwrite") + require.NoError(t, err) + assert.Equal(t, "2", got["a"]) + assert.Equal(t, "3", got["b"]) +} + +func TestSetEmptyNamespace(t *testing.T) { + r := setupRegistry(t) + _, err := r.Set(systemScope, "", map[string]interface{}{"a": "1"}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "namespace is required") +} + +func TestDelete(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "to-delete", map[string]interface{}{"x": "y"}) + require.NoError(t, err) + + err = r.Delete(systemScope, "to-delete") + require.NoError(t, err) + + _, err = r.Get(systemScope, "to-delete") + assert.Error(t, err) +} + +func TestDeleteNotFound(t *testing.T) { + r := setupRegistry(t) + err := r.Delete(systemScope, "no-such-ns") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func TestListNamespaces(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "ns-a", map[string]interface{}{"v": 1}) + require.NoError(t, err) + _, err = r.Set(systemScope, "ns-b", map[string]interface{}{"v": 2}) + require.NoError(t, err) + _, err = r.Set(teamScope, "ns-c", map[string]interface{}{"v": 3}) + require.NoError(t, err) + + sysNS, err := r.ListNamespaces(systemScope) + require.NoError(t, err) + assert.Contains(t, sysNS, "ns-a") + assert.Contains(t, sysNS, "ns-b") + assert.NotContains(t, sysNS, "ns-c") + + teamNS, err := r.ListNamespaces(teamScope) + require.NoError(t, err) + assert.Contains(t, teamNS, "ns-c") +} + +func TestMultipleNamespaces(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(userScope, "alpha", map[string]interface{}{"color": "red"}) + require.NoError(t, err) + _, err = r.Set(userScope, "beta", map[string]interface{}{"color": "blue"}) + require.NoError(t, err) + + a, err := r.Get(userScope, "alpha") + require.NoError(t, err) + assert.Equal(t, "red", a["color"]) + + b, err := r.Get(userScope, "beta") + require.NoError(t, err) + assert.Equal(t, "blue", b["color"]) +} + +func TestScopeIsolation(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "shared", map[string]interface{}{"level": "system"}) + require.NoError(t, err) + _, err = r.Set(teamScope, "shared", map[string]interface{}{"level": "team"}) + require.NoError(t, err) + _, err = r.Set(userScope, "shared", map[string]interface{}{"level": "user"}) + require.NoError(t, err) + + sys, err := r.Get(systemScope, "shared") + require.NoError(t, err) + assert.Equal(t, "system", sys["level"]) + + team, err := r.Get(teamScope, "shared") + require.NoError(t, err) + assert.Equal(t, "team", team["level"]) + + user, err := r.Get(userScope, "shared") + require.NoError(t, err) + assert.Equal(t, "user", user["level"]) +} + +func TestReload(t *testing.T) { + r := setupRegistry(t) + + _, err := r.Set(systemScope, "reload-test", map[string]interface{}{"k": "v"}) + require.NoError(t, err) + + c, _ := store.Get("__yao.cache") + if c != nil { + c.Del("setting:*") + } + + err = r.Reload() + require.NoError(t, err) + + got, err := r.Get(systemScope, "reload-test") + require.NoError(t, err) + assert.Equal(t, "v", got["k"]) +} + +func TestConcurrency(t *testing.T) { + r := setupRegistry(t) + + var wg sync.WaitGroup + errCh := make(chan error, 30) + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + ns := fmt.Sprintf("conc-%d", idx) + _, err := r.Set(systemScope, ns, map[string]interface{}{"idx": idx}) + if err != nil { + errCh <- err + } + }(i) + } + wg.Wait() + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + ns := fmt.Sprintf("conc-%d", idx) + _, err := r.Get(systemScope, ns) + if err != nil { + errCh <- err + } + }(i) + } + wg.Wait() + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + ns := fmt.Sprintf("conc-%d", idx) + if err := r.Delete(systemScope, ns); err != nil { + errCh <- err + } + }(i) + } + wg.Wait() + close(errCh) + + for err := range errCh { + t.Errorf("concurrent operation error: %v", err) + } +} diff --git a/setting/store.go b/setting/store.go new file mode 100644 index 00000000..62df76f3 --- /dev/null +++ b/setting/store.go @@ -0,0 +1,178 @@ +package setting + +import ( + "encoding/json" + "fmt" + + "github.com/yaoapp/gou/store" +) + +const keyPrefix = "setting:" + +func scopePrefix(scope ScopeID) string { + switch scope.Scope { + case ScopeTeam: + return "t" + scope.TeamID + ":" + case ScopeUser: + return "u" + scope.UserID + ":" + default: + return "s:" + } +} + +func storeKey(scope ScopeID, ns string) string { + return keyPrefix + scopePrefix(scope) + ns +} + +func indexKey(scope ScopeID) string { + return keyPrefix + "idx:" + scopePrefix(scope) +} + +// storeGet reads a namespace entry from cache first, then persistent store. +func storeGet(s, c store.Store, scope ScopeID, ns string) (map[string]interface{}, error) { + sk := storeKey(scope, ns) + + if c != nil { + if val, ok := c.Get(sk); ok { + if m, ok := val.(map[string]interface{}); ok { + return m, nil + } + } + } + + val, ok := s.Get(sk) + if !ok { + return nil, fmt.Errorf("setting %s/%s not found", scopePrefix(scope), ns) + } + + m, err := toMap(val) + if err != nil { + return nil, fmt.Errorf("setting %s/%s: %w", scopePrefix(scope), ns, err) + } + + if c != nil { + c.Set(sk, m, 0) + } + return m, nil +} + +// storeSet writes a namespace entry to both persistent store and cache. +func storeSet(s, c store.Store, scope ScopeID, ns string, data map[string]interface{}) error { + sk := storeKey(scope, ns) + if err := s.Set(sk, data, 0); err != nil { + return err + } + if c != nil { + c.Set(sk, data, 0) + } + return nil +} + +// storeDel removes a namespace entry from both persistent store and cache. +func storeDel(s, c store.Store, scope ScopeID, ns string) error { + sk := storeKey(scope, ns) + if err := s.Del(sk); err != nil { + return err + } + if c != nil { + c.Del(sk) + } + return nil +} + +// indexGet returns all namespace names for a given scope. +func indexGet(s, c store.Store, scope ScopeID) ([]string, error) { + ik := indexKey(scope) + var raw interface{} + var ok bool + + if c != nil { + raw, ok = c.Get(ik) + } + if !ok { + raw, ok = s.Get(ik) + if !ok { + return nil, nil + } + if c != nil { + c.Set(ik, raw, 0) + } + } + + switch v := raw.(type) { + case []interface{}: + keys := make([]string, 0, len(v)) + for _, item := range v { + if str, ok := item.(string); ok { + keys = append(keys, str) + } + } + return keys, nil + case []string: + return v, nil + default: + return nil, fmt.Errorf("unexpected index type %T", raw) + } +} + +// indexSet writes the full namespace index. +func indexSet(s, c store.Store, scope ScopeID, keys []string) error { + ik := indexKey(scope) + iface := make([]interface{}, len(keys)) + for i, k := range keys { + iface[i] = k + } + if err := s.Set(ik, iface, 0); err != nil { + return err + } + if c != nil { + c.Set(ik, iface, 0) + } + return nil +} + +// indexAdd appends a namespace to the index if not already present. +func indexAdd(s, c store.Store, scope ScopeID, ns string) error { + keys, err := indexGet(s, c, scope) + if err != nil { + return err + } + for _, k := range keys { + if k == ns { + return nil + } + } + return indexSet(s, c, scope, append(keys, ns)) +} + +// indexRemove removes a namespace from the index. +func indexRemove(s, c store.Store, scope ScopeID, ns string) error { + keys, err := indexGet(s, c, scope) + if err != nil { + return err + } + filtered := make([]string, 0, len(keys)) + for _, k := range keys { + if k != ns { + filtered = append(filtered, k) + } + } + return indexSet(s, c, scope, filtered) +} + +// toMap normalizes a store value to map[string]interface{}. +// The xun store may return values that need re-serialization. +func toMap(val interface{}) (map[string]interface{}, error) { + if m, ok := val.(map[string]interface{}); ok { + return m, nil + } + raw, err := json.Marshal(val) + if err != nil { + return nil, err + } + var m map[string]interface{} + if err := json.Unmarshal(raw, &m); err != nil { + return nil, err + } + return m, nil +} diff --git a/setting/types.go b/setting/types.go new file mode 100644 index 00000000..8bf82567 --- /dev/null +++ b/setting/types.go @@ -0,0 +1,28 @@ +package setting + +// Scope identifies the level at which a setting is stored. +type Scope string + +const ( + ScopeSystem Scope = "system" + ScopeTeam Scope = "team" + ScopeUser Scope = "user" +) + +// ScopeID fully identifies a scope instance. +// For ScopeSystem, TeamID and UserID are ignored. +// For ScopeTeam, TeamID is required. +// For ScopeUser, UserID is required. +type ScopeID struct { + Scope Scope `json:"scope"` + TeamID string `json:"team_id,omitempty"` + UserID string `json:"user_id,omitempty"` +} + +// Entry represents a single namespace's data within a scope. +type Entry struct { + Namespace string `json:"namespace"` + Scope ScopeID `json:"scope"` + Data map[string]interface{} `json:"data"` + UpdatedAt string `json:"updated_at"` +}