diff --git a/api/api.go b/api/api.go index 14ea19cb..7f919d98 100644 --- a/api/api.go +++ b/api/api.go @@ -19,8 +19,8 @@ func LoadFrom(dir string, prefix string) { } share.Walk(dir, ".json", func(root, filename string) { - name := share.SpecName(root, filename) + name := prefix + share.SpecName(root, filename) content := share.ReadFile(filename) - gou.LoadAPI(string(content), prefix+name) + gou.LoadAPI(string(content), name) }) } diff --git a/chart/chart.go b/chart/chart.go index daf05124..15cc87ee 100644 --- a/chart/chart.go +++ b/chart/chart.go @@ -27,13 +27,24 @@ func LoadFrom(dir string, prefix string) { } share.Walk(dir, ".json", func(root, filename string) { - name := share.SpecName(root, filename) + name := prefix + share.SpecName(root, filename) content := share.ReadFile(filename) _, err := LoadChart(content, name) if err != nil { exception.New("%s 图表格式错误", 400, name).Ctx(filename).Throw() } }) + + // Load Script + share.Walk(dir, ".js", func(root, filename string) { + name := prefix + share.SpecName(root, filename) + chart := Select(name) + if chart != nil { + script := share.ScriptName(filename) + content := share.ReadFile(filename) + chart.LoadScript(string(content), script) + } + }) } // LoadChart 载入数据表格 diff --git a/engine/load.go b/engine/load.go index ef1f5dc4..33888184 100644 --- a/engine/load.go +++ b/engine/load.go @@ -12,6 +12,7 @@ import ( "github.com/yaoapp/xiang/config" "github.com/yaoapp/xiang/flow" "github.com/yaoapp/xiang/model" + "github.com/yaoapp/xiang/page" "github.com/yaoapp/xiang/plugin" "github.com/yaoapp/xiang/query" "github.com/yaoapp/xiang/share" @@ -33,6 +34,7 @@ func Load(cfg config.Config) { plugin.Load(cfg) // 加载业务插件 plugin table.Load(cfg) // 加载数据表格 table chart.Load(cfg) // 加载分析图表 chart + page.Load(cfg) // 加载页面 page // 加密密钥函数 gou.LoadCrypt(fmt.Sprintf(`{"key":"%s"}`, cfg.Database.AESKey), "AES") diff --git a/flow/flow.go b/flow/flow.go index 422cc4c3..8773669a 100644 --- a/flow/flow.go +++ b/flow/flow.go @@ -19,8 +19,19 @@ func LoadFrom(dir string, prefix string) { } share.Walk(dir, ".json", func(root, filename string) { - name := share.SpecName(root, filename) + name := prefix + share.SpecName(root, filename) content := share.ReadFile(filename) - gou.LoadFlow(string(content), prefix+name) + gou.LoadFlow(string(content), name) + }) + + // Load Script + share.Walk(dir, ".js", func(root, filename string) { + name := prefix + share.SpecName(root, filename) + flow := gou.SelectFlow(name) + if flow != nil { + script := share.ScriptName(filename) + content := share.ReadFile(filename) + flow.LoadScript(string(content), script) + } }) } diff --git a/model/model.go b/model/model.go index 37a38145..cb2e5e05 100644 --- a/model/model.go +++ b/model/model.go @@ -19,8 +19,8 @@ func LoadFrom(dir string, prefix string) { } share.Walk(dir, ".json", func(root, filename string) { - name := share.SpecName(root, filename) + name := prefix + share.SpecName(root, filename) content := share.ReadFile(filename) - gou.LoadModel(string(content), prefix+name) + gou.LoadModel(string(content), name) }) } diff --git a/page/REAME.md b/page/REAME.md new file mode 100644 index 00000000..82e08d54 --- /dev/null +++ b/page/REAME.md @@ -0,0 +1 @@ +# 数据页面 diff --git a/page/api.go b/page/api.go new file mode 100644 index 00000000..594e64b8 --- /dev/null +++ b/page/api.go @@ -0,0 +1,60 @@ +package page + +import ( + "github.com/yaoapp/xiang/share" +) + +// SetupAPIs 设定API数据 +func (page *Page) SetupAPIs() { + + defaults := map[string]share.API{ + "data": apiDataDefault(), + "setting": apiSettingDefault(), + } + + // 开发者填写的规则 + for name := range page.APIs { + if _, has := defaults[name]; !has { + delete(page.APIs, name) + continue + } + + api := defaults[name] + api.Name = name + if page.APIs[name].Process != "" { + api.Process = page.APIs[name].Process + } + + if page.APIs[name].Guard != "" { + api.Guard = page.APIs[name].Guard + } + + if page.APIs[name].Default != nil { + api.Default = page.APIs[name].Default + } + + defaults[name] = api + } + + page.APIs = defaults +} + +// apiSearchDefault data 接口默认值 +func apiDataDefault() share.API { + param := map[string]interface{}{} + return share.API{ + Name: "data", + Guard: "bearer-jwt", + Process: "xiang.page.data", + Default: []interface{}{param}, + } +} + +// apiSettingDefault setting 接口默认值 +func apiSettingDefault() share.API { + return share.API{ + Name: "setting", + Guard: "bearer-jwt", + Process: "xiang.page.setting", + } +} diff --git a/page/page.go b/page/page.go new file mode 100644 index 00000000..7a17079e --- /dev/null +++ b/page/page.go @@ -0,0 +1,86 @@ +package page + +import ( + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/exception" + "github.com/yaoapp/xiang/config" + "github.com/yaoapp/xiang/share" + "github.com/yaoapp/xiang/xlog" +) + +// Pages 已载入页面 +var Pages = map[string]*Page{} + +// Load 加载页面 +func Load(cfg config.Config) { + LoadFrom(cfg.RootPage, "") +} + +// LoadFrom 从特定目录加载 +func LoadFrom(dir string, prefix string) { + + if share.DirNotExists(dir) { + return + } + + share.Walk(dir, ".json", func(root, filename string) { + name := prefix + share.SpecName(root, filename) + content := share.ReadFile(filename) + _, err := LoadPage(content, name) + if err != nil { + exception.New("%s 页面格式错误", 400, name).Ctx(filename).Throw() + } + }) + + // Load Script + share.Walk(dir, ".js", func(root, filename string) { + name := prefix + share.SpecName(root, filename) + page := Select(name) + if page != nil { + script := share.ScriptName(filename) + content := share.ReadFile(filename) + page.LoadScript(string(content), script) + } + }) + +} + +// LoadPage 载入页面 +func LoadPage(source []byte, name string) (*Page, error) { + page := &Page{ + Flow: gou.Flow{ + Name: name, + }, + } + err := jsoniter.Unmarshal(source, page) + if err != nil { + xlog.Println(name) + xlog.Println(err.Error()) + xlog.Println(string(source)) + return nil, err + } + page.Prepare() + page.SetupAPIs() + Pages[name] = page + return page, nil +} + +// Select 读取已加载页面 +func Select(name string) *Page { + page, has := Pages[name] + if !has { + exception.New( + fmt.Sprintf("Page:%s; 尚未加载", name), + 400, + ).Throw() + } + return page +} + +// GetData 运行 flow 返回数值 +func (page Page) GetData(params map[string]interface{}) interface{} { + return page.Flow.Exec(params) +} diff --git a/page/page_test.go b/page/page_test.go new file mode 100644 index 00000000..507906c6 --- /dev/null +++ b/page/page_test.go @@ -0,0 +1,29 @@ +package page + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/yaoapp/xiang/config" + "github.com/yaoapp/xiang/model" + "github.com/yaoapp/xiang/query" + "github.com/yaoapp/xiang/share" +) + +func TestLoad(t *testing.T) { + share.DBConnect(config.Conf.Database) + model.Load(config.Conf) + query.Load(config.Conf) + + Load(config.Conf) + LoadFrom("not a path", "404.") + check(t) +} + +func check(t *testing.T) { + keys := []string{} + for key := range Pages { + keys = append(keys, key) + } + assert.Equal(t, 1, len(keys)) +} diff --git a/page/process.go b/page/process.go new file mode 100644 index 00000000..c9c016fb --- /dev/null +++ b/page/process.go @@ -0,0 +1,82 @@ +package page + +import ( + "strings" + + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/maps" +) + +// 注册处理器 +func init() { + gou.RegisterProcessHandler("xiang.page.data", ProcessData) + gou.RegisterProcessHandler("xiang.page.setting", ProcessSetting) +} + +// ProcessData xiang.page.data +// 查询数据分析页面中定义的数据 +func ProcessData(process *gou.Process) interface{} { + + process.ValidateArgNums(3) + name := process.ArgsString(0) + params := process.ArgsMap(1) + page := Select(name) + api := page.APIs["data"] + if process.NumOfArgsIs(4) && api.IsAllow(process.Args[3]) { + return nil + } + + if len(api.Default) > 0 { + if defaults, ok := api.Default[0].(map[string]interface{}); ok { + for key, value := range defaults { + if !params.Has(key) { + params.Set(key, value) + } + } + } + } + + return page.GetData(params) +} + +// ProcessSetting xiang.page.setting +// 查询数据分析页面中定义的数据 +func ProcessSetting(process *gou.Process) interface{} { + + process.ValidateArgNums(2) + name := process.ArgsString(0) + field := process.ArgsString(1) + page := Select(name) + api := page.APIs["setting"] + if process.NumOfArgsIs(2) && api.IsAllow(process.Args[1]) { + return nil + } + + fields := strings.Split(field, ",") + setting := maps.Map{ + "name": page.Name, + "label": page.Label, + "version": page.Version, + "description": page.Description, + "filters": page.Filters, + "page": page.Page, + } + + if len(fields) == 1 && setting.Has(fields[0]) { + field := strings.TrimSpace(fields[0]) + return setting.Get(field) + } + + if len(fields) > 1 { + res := maps.Map{} + for _, field := range fields { + field = strings.TrimSpace(field) + if setting.Has(field) { + res.Set(field, setting.Get(field)) + } + } + return res + } + return setting + +} diff --git a/page/process_test.go b/page/process_test.go new file mode 100644 index 00000000..973fad16 --- /dev/null +++ b/page/process_test.go @@ -0,0 +1,77 @@ +package page + +import ( + "net/url" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou" + "github.com/yaoapp/kun/any" + "github.com/yaoapp/kun/utils" + "github.com/yaoapp/xiang/config" + _ "github.com/yaoapp/xiang/helper" + "github.com/yaoapp/xiang/model" + "github.com/yaoapp/xiang/query" + "github.com/yaoapp/xiang/share" +) + +func init() { + share.DBConnect(config.Conf.Database) + model.Load(config.Conf) + query.Load(config.Conf) + Load(config.Conf) +} +func TestProcessSetting(t *testing.T) { + + args := []interface{}{ + "service.compare", + nil, + &gin.Context{}, + } + process := gou.NewProcess("xiang.page.Setting", args...) + response := ProcessSetting(process) + assert.NotNil(t, response) + res := any.Of(response).Map() + assert.True(t, res.Has("name")) + assert.True(t, res.Has("label")) + assert.True(t, res.Has("description")) + assert.True(t, res.Has("page")) + assert.True(t, res.Has("version")) + + args = []interface{}{ + "service.compare", + "page,name", + &gin.Context{}, + } + process = gou.NewProcess("xiang.page.Setting", args...) + response = ProcessSetting(process) + assert.NotNil(t, response) + + res = any.Of(response).Map() + assert.True(t, res.Has("name")) + assert.True(t, res.Has("page")) + assert.False(t, res.Has("label")) +} + +func TestProcessData(t *testing.T) { + + params := url.Values{ + "from": []string{"1981-01-01", "1990-01-01"}, + } + params.Set("to", "2049-12-31") + + args := []interface{}{ + "service.compare", + params, + &gin.Context{}, + } + process := gou.NewProcess("xiang.page.Data", args...) + response := ProcessData(process) + utils.Dump(response) + + assert.NotNil(t, response) + + res := any.Of(response).Map().Dot() + assert.Equal(t, "北京", res.Get("合并.0.城市")) +} diff --git a/page/types.go b/page/types.go new file mode 100644 index 00000000..53f21e6c --- /dev/null +++ b/page/types.go @@ -0,0 +1,14 @@ +package page + +import ( + "github.com/yaoapp/gou" + "github.com/yaoapp/xiang/share" +) + +// Page 页面格式 +type Page struct { + gou.Flow + APIs map[string]share.API `json:"apis,omitempty"` + Filters map[string]share.Filter `json:"filters,omitempty"` + Page share.Page `json:"page,omitempty"` +} diff --git a/service/watch.go b/service/watch.go index 5c0a8c55..6d90f0af 100644 --- a/service/watch.go +++ b/service/watch.go @@ -9,6 +9,7 @@ import ( "github.com/yaoapp/gou" "github.com/yaoapp/xiang/chart" "github.com/yaoapp/xiang/config" + "github.com/yaoapp/xiang/page" "github.com/yaoapp/xiang/share" "github.com/yaoapp/xiang/table" ) @@ -22,6 +23,7 @@ func Watch(cfg config.Config) { WatchPlugin(cfg.RootPlugin) WatchTable(cfg.RootTable, "") WatchChart(cfg.RootChart, "") + WatchPage(cfg.RootPage, "") } // WatchEngine 监听监听引擎内建数据变更 @@ -114,13 +116,23 @@ func WatchFlow(root string, prefix string) { if strings.HasSuffix(filename, ".js") { name := prefix + share.SpecName(root, filename) - filename = name + ".flow.json" + name = strings.ReplaceAll(name, ".", "/") + filename = filepath.Join(root, name+".flow.json") } if op == "write" || op == "create" { name := prefix + share.SpecName(root, filename) content := share.ReadFile(filename) - gou.LoadFlow(string(content), name) // Reload + flow := gou.LoadFlow(string(content), name) // Reload + if flow != nil { // Reload Script + dir := filepath.Dir(filename) + share.Walk(dir, ".js", func(root, filename string) { + script := share.ScriptName(filename) + content := share.ReadFile(filename) + flow.LoadScript(string(content), script) + }) + } + log.Printf("Flow %s 已重新加载完毕", name) } else if op == "remove" || op == "rename" { @@ -214,13 +226,22 @@ func WatchChart(root string, prefix string) { if strings.HasSuffix(filename, ".js") { name := prefix + share.SpecName(root, filename) - filename = name + ".chart.json" + name = strings.ReplaceAll(name, ".", "/") + filename = filepath.Join(root, name+".chart.json") } if op == "write" || op == "create" { name := prefix + share.SpecName(root, filename) content := share.ReadFile(filename) - chart.LoadChart(content, name) // Relaod + chart, _ := chart.LoadChart(content, name) // Relaod + if chart != nil { // Reload Script + dir := filepath.Dir(filename) + share.Walk(dir, ".js", func(root, filename string) { + script := share.ScriptName(filename) + content := share.ReadFile(filename) + chart.LoadScript(string(content), script) + }) + } api, has := gou.APIs["xiang.chart"] if has { @@ -246,3 +267,57 @@ func WatchChart(root string, prefix string) { } }) } + +// WatchPage 监听页面更新 +func WatchPage(root string, prefix string) { + if share.DirNotExists(root) { + return + } + root = share.DirAbs(root) + go share.Watch(root, func(op string, filename string) { + if !strings.HasSuffix(filename, ".json") && !strings.HasSuffix(filename, ".js") { + return + } + + if strings.HasSuffix(filename, ".js") { + name := prefix + share.SpecName(root, filename) + name = strings.ReplaceAll(name, ".", "/") + filename = filepath.Join(root, name+".page.json") + } + + if op == "write" || op == "create" { + name := prefix + share.SpecName(root, filename) + content := share.ReadFile(filename) + page, _ := page.LoadPage(content, name) // Relaod + if page != nil { // Reload Script + dir := filepath.Dir(filename) + share.Walk(dir, ".js", func(root, filename string) { + script := share.ScriptName(filename) + content := share.ReadFile(filename) + page.LoadScript(string(content), script) + }) + } + + api, has := gou.APIs["xiang.page"] + if has { + api.Reload() // 重载API + } + log.Printf("Page %s 已重新加载完毕", name) + + } else if op == "remove" || op == "rename" { + name := prefix + share.SpecName(root, filename) + if _, has := page.Pages[name]; has { + delete(page.Pages, name) + log.Printf("Page %s 已经移除", name) + } + } + + // 重启服务器 + if op == "write" || op == "create" || op == "remove" || op == "rename" { + Stop(func() { + log.Printf("服务器重启完毕") + go Start() + }) + } + }) +} diff --git a/share/utils.go b/share/utils.go index 9252024d..e5a7dbcd 100644 --- a/share/utils.go +++ b/share/utils.go @@ -38,6 +38,16 @@ func SpecName(root string, file string) string { return name } +// ScriptName 解析数据处理脚本名称 +func ScriptName(filename string) string { + filename = strings.TrimSuffix(filename, ".js") + namer := strings.Split(filename, ".") // ["foo/bar", "http", "json"] + if len(namer) < 2 { + return namer[0] + } + return namer[len(namer)-1] +} + // ReadFile 读取文件 func ReadFile(filename string) []byte { file, err := os.Open(filename) diff --git a/tests/charts/service/compare.json b/tests/charts/service/compare.chart.json similarity index 100% rename from tests/charts/service/compare.json rename to tests/charts/service/compare.chart.json diff --git a/tests/charts/service/compare.data.js b/tests/charts/service/compare.data.js index 5306cd7f..20b71204 100644 --- a/tests/charts/service/compare.data.js +++ b/tests/charts/service/compare.data.js @@ -1,3 +1,4 @@ function main(args, out, res) { console.log(args, out, res); + return out; } diff --git a/tests/pages/service/compare.data.js b/tests/pages/service/compare.data.js new file mode 100644 index 00000000..ead9aa82 --- /dev/null +++ b/tests/pages/service/compare.data.js @@ -0,0 +1,3 @@ +function main(args, out, res) { + return { args: args }; +} diff --git a/tests/pages/service/compare.page.json b/tests/pages/service/compare.page.json new file mode 100644 index 00000000..52a284a9 --- /dev/null +++ b/tests/pages/service/compare.page.json @@ -0,0 +1,111 @@ +{ + "label": "指标对比", + "version": "1.0.0", + "description": "指标对比用于测试", + "nodes": [ + { + "name": "行业", + "engine": "xiang", + "query": { + "debug": true, + "select": [ + "city", + ":COUNT(id) as 数量", + "industries[*](string 50) as industry" + ], + "from": "$service", + "wheres": [ + { "field": "created_at", ">": "?:$from.0" }, + { "field": "created_at", "<": "?:$to" } + ], + "orders": "数量 desc", + "limit": 100 + } + }, + { + "name": "计费", + "engine": "xiang", + "query": { + "select": ["city", ":COUNT(id) as 数量", "price_options[*] as option"], + "from": "$service", + "wheres": [ + { "field": "created_at", ">": "?:$from.0" }, + { "field": "created_at", "<": "?:$to" } + ], + "orders": "数量 desc", + "limit": 100 + } + }, + { + "name": "脚本", + "script": "data", + "outs": ["{{$out}}", "1"] + }, + { + "name": "合并结果", + "process": "xiang.helper.ArrayPluck", + "args": [ + ["城市", "行业", "计费"], + { + "行业": { "key": "city", "value": "数量", "items": "{{$res.行业}}" }, + "计费": { "key": "city", "value": "数量", "items": "{{$res.计费}}" } + } + ] + } + ], + "output": { + "合并": "{{$res.合并结果}}", + "计费": "{{$res.计费}}", + "行业": "{{$res.行业}}", + "参数": "{{$in}}", + "脚本": "{{$res.脚本}}", + "page": true + }, + "apis": { + "data": { + "disable": false, + "guard": "-", + "default": [{ "from": "1980-01-02", "to": "2050-12-31" }] + }, + "setting": { + "disable": false, + "guard": "-" + } + }, + "filters": { + "开始时间": { + "label": "开始时间", + "bind": "from", + "input": { + "type": "date", + "props": { + "placeholder": "请选择开始时间" + } + } + }, + "结束时间": { + "label": "结束时间", + "bind": "to", + "input": { + "type": "date", + "props": { + "placeholder": "请选择结束时间" + } + } + } + }, + "page": { + "primary": "城市", + "layout": { + "filters": [ + { "name": "开始时间", "width": 6 }, + { "name": "结束时间", "width": 6 } + ], + "charts": [ + { "type": "line", "props": {} }, + { "type": "bar", "props": {} } + ] + }, + "actions": {} + } +} diff --git a/xiang/apis/page.http.json b/xiang/apis/page.http.json new file mode 100644 index 00000000..6c2161cf --- /dev/null +++ b/xiang/apis/page.http.json @@ -0,0 +1,29 @@ +{ + "name": "页面接口", + "version": "1.0.0", + "description": "页面接口API", + "group": "xiang/page", + "guard": "in-process", + "paths": [ + { + "path": "/:name/data", + "method": "GET", + "process": "xiang.page.Data", + "in": ["$param.name", ":query"], + "out": { + "status": 200, + "type": "application/json" + } + }, + { + "path": "/:name/setting", + "method": "GET", + "process": "xiang.page.Setting", + "in": ["$param.name", "$query.select"], + "out": { + "status": 200, + "type": "application/json" + } + } + ] +}