opencode/yao
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[add] migrate importer

Browse source
This commit is contained in:
Max 2023-02-03 14:43:00 +08:00
parent a9d5e8dd42
commit 5faf62a789
6 changed files with 133 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -9,7 +9,7 @@ COMMIT := $(shell git log | head -n 1 | awk '{print substr($$2, 0, 12)}')
NOW := $(shell date +"%FT%T%z")
# ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget' | grep -vE 'examples|tests*|config|widgets')
TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget|importer' | grep -vE 'examples|tests*|config|widgets')
TESTTAGS ?= ""
# TESTWIDGETS := $(shell $(GO) list ./widgets/...)

2
config/config.go
View file

@ -22,7 +22,7 @@ var Conf Config
var LogOutput *os.File // 日志文件
// DSLExtensions the dsl file Extensions
var DSLExtensions = []string{".yao", ".json", ".jsonc"}
var DSLExtensions = []string{"*.yao", "*.json", "*.jsonc"}
func init() {
filename, _ := filepath.Abs(filepath.Join(".", ".env"))

32
importer/importer.go
View file

@ -14,6 +14,7 @@ import (
"github.com/yaoapp/kun/exception"
"github.com/yaoapp/kun/log"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/fs"
"github.com/yaoapp/yao/importer/from"
"github.com/yaoapp/yao/importer/xlsx"
"github.com/yaoapp/yao/share"
@ -22,9 +23,23 @@ import (
// Importers 导入器
var Importers = map[string]*Importer{}
// DataRoot data file root
var DataRoot string = ""
// Load 加载导入器
func Load(cfg config.Config) error {
return application.App.Walk("apis", func(root, file string, isdir bool) error {
root, err := fs.Root(cfg)
if err != nil {
return err
}
DataRoot = root
exts := []string{"*.imp.yao", "*.imp.json", "*.imp.jsonc"}
return application.App.Walk("imports", func(root, file string, isdir bool) error {
if isdir {
return nil
}
id := share.ID(root, file)
data, err := application.App.Read(file)
@ -37,10 +52,10 @@ func Load(cfg config.Config) error {
if err != nil {
return fmt.Errorf("%s 导入配置错误. %s", id, err.Error())
}
Importers[id] = &importer
return err
}, config.DSLExtensions...)
Importers[id] = &importer
return nil
}, exts...)
}
// Select 选择已加载导入器
@ -57,13 +72,8 @@ func Open(name string) from.Source {
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
switch ext {
case "xlsx":
// Refector *****
// fullpath := name
// fullpath := filepath.Join(xfs.Stor.Root, name)
// if !strings.HasPrefix(fullpath, "/") {
// fullpath = filepath.Join(xfs.Stor.Root, name)
// }
return xlsx.Open(name)
file := filepath.Join(DataRoot, name)
return xlsx.Open(file)
}
exception.New("暂不支持: %s 文件导入", 400, ext).Throw()
return nil

87
importer/importer_test.go
View file

@ -5,25 +5,41 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/yaoapp/kun/utils"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/fs"
"github.com/yaoapp/yao/importer/xlsx"
"github.com/yaoapp/yao/script"
"github.com/yaoapp/yao/test"
)
func TestLoad(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
assert.IsType(t, &Importer{}, Select("order"))
}
func TestFingerprintSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
imp := Select("order")
fingerprint := imp.Fingerprint(file)
assert.Equal(t, "3451ca87d71801687abba8993e5a69af79482914435d7cc064236fd93160f999", fingerprint)
}
func TestAutoMappingSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
@ -47,7 +63,12 @@ func TestAutoMappingSimple(t *testing.T) {
}
func TestDataGetSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
@ -68,7 +89,11 @@ func TestDataGetSimple(t *testing.T) {
}
func TestDataChunkSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
@ -82,7 +107,11 @@ func TestDataChunkSimple(t *testing.T) {
}
func TestRunSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
@ -96,7 +125,11 @@ func TestRunSimple(t *testing.T) {
}
func TestDataPreviewSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
imp := Select("order")
@ -115,7 +148,11 @@ func TestDataPreviewSimple(t *testing.T) {
}
func TestMappingPreviewSimple(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
@ -139,19 +176,47 @@ func TestMappingPreviewSimple(t *testing.T) {
}
func TestMappingSetting(t *testing.T) {
simple := filepath.Join(config.Conf.Root, "data", "assets", "simple.xlsx")
test.Prepare(t, config.Conf)
defer test.Clean()
root := prepare(t, config.Conf)
simple := filepath.Join(root, "assets", "simple.xlsx")
file := xlsx.Open(simple)
defer file.Close()
imp := Select("order")
setting := imp.MappingSetting(file)
utils.Dump(setting)
// utils.Dump(setting)
assert.NotNil(t, setting)
}
func TestDataSetting(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
imp := Select("order")
setting := imp.DataSetting()
utils.Dump(setting)
// utils.Dump(setting)
assert.NotNil(t, setting)
}
func prepare(t *testing.T, cfg config.Config) string {
err := Load(cfg)
if err != nil {
t.Fatal(err)
}
dataRoot, err := fs.Root(cfg)
if err != nil {
t.Fatal(err)
}
err = script.Load(cfg)
if err != nil {
t.Fatal(err)
}
return dataRoot
}

23
importer/init_test.go
View file

@ -1,23 +0,0 @@
package importer
import (
"os"
"testing"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/model"
"github.com/yaoapp/yao/query"
"github.com/yaoapp/yao/script"
"github.com/yaoapp/yao/share"
)
func TestMain(m *testing.M) {
share.DBConnect(config.Conf.DB)
model.Load(config.Conf)
query.Load(config.Conf)
share.Load(config.Conf)
script.Load(config.Conf)
Load(config.Conf)
code := m.Run()
os.Exit(code)
}

42
importer/process_test.go
View file

@ -6,52 +6,78 @@ import (
"github.com/stretchr/testify/assert"
"github.com/yaoapp/gou/process"
"github.com/yaoapp/yao/config"
"github.com/yaoapp/yao/test"
)
func TestProcessMapping(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
simple := filepath.Join("assets", "simple.xlsx")
args := []interface{}{"order", simple}
response := process.New("xiang.import.Mapping", args...).Run()
response := process.New("yao.import.Mapping", args...).Run()
_, ok := response.(*Mapping)
assert.True(t, ok)
}
func TestProcessMappingSetting(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
simple := filepath.Join("assets", "simple.xlsx")
args := []interface{}{"order", simple}
response := process.New("xiang.import.MappingSetting", args...).Run()
response := process.New("yao.import.MappingSetting", args...).Run()
_, ok := response.(map[string]interface{})
assert.True(t, ok)
}
func TestProcessData(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
simple := filepath.Join("assets", "simple.xlsx")
mapping := process.New("xiang.import.Mapping", "order", simple).Run()
mapping := process.New("yao.import.Mapping", "order", simple).Run()
args := []interface{}{"order", simple, 1, 2, mapping}
response := process.New("xiang.import.Data", args...).Run()
response := process.New("yao.import.Data", args...).Run()
_, ok := response.(map[string]interface{})
assert.True(t, ok)
}
func TestProcessDataSetting(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
args := []interface{}{"order"}
response := process.New("xiang.import.DataSetting", args...).Run()
response := process.New("yao.import.DataSetting", args...).Run()
_, ok := response.(map[string]interface{})
assert.True(t, ok)
}
func TestProcessSetting(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
args := []interface{}{"order"}
response := process.New("xiang.import.Setting", args...).Run()
response := process.New("yao.import.Setting", args...).Run()
_, ok := response.(map[string]interface{})
assert.True(t, ok)
}
func TestProcessRun(t *testing.T) {
test.Prepare(t, config.Conf)
defer test.Clean()
prepare(t, config.Conf)
simple := filepath.Join("assets", "simple.xlsx")
mapping := process.New("xiang.import.Mapping", "order", simple).Run()
mapping := process.New("yao.import.Mapping", "order", simple).Run()
args := []interface{}{"order", simple, mapping}
response := process.New("xiang.import.Run", args...).Run()
response := process.New("yao.import.Run", args...).Run()
_, ok := response.(map[string]int)
assert.True(t, ok)
}