From 7a99582271e72ab0204edd38117c82e6a42c7f1b Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 15 Jan 2022 11:15:08 +0800 Subject: [PATCH] + xlsx --- go.mod | 1 + go.sum | 12 ++++++ importer/from/source.go | 1 + importer/importer_test.go | 6 ++- importer/xlsx/xlsx.go | 88 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 102 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index eb9fcbce..f43f8bf7 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 + github.com/xuri/excelize/v2 v2.5.0 // indirect github.com/yaoapp/gou v0.0.0-20211120135538-e5387704eb03 github.com/yaoapp/kun v0.6.7 github.com/yaoapp/xun v0.6.0 diff --git a/go.sum b/go.sum index 8c6a027a..adb49e82 100644 --- a/go.sum +++ b/go.sum @@ -381,6 +381,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mojocn/base64Captcha v1.3.5 h1:Qeilr7Ta6eDtG4S+tQuZ5+hO+QHbiGAJdi4PfoagaA0= github.com/mojocn/base64Captcha v1.3.5/go.mod h1:/tTTXn4WTpX9CfrmipqRytCpJ27Uw3G6I7NcP2WwcmY= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -411,6 +413,10 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/richardlehane/mscfb v1.0.3 h1:rD8TBkYWkObWO0oLDFCbwMeZ4KoalxQy+QgniCj3nKI= +github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f h1:a7clxaGmmqtdNTXyvrp/lVO/Gnkzlhc/+dLs5v965GM= github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f/go.mod h1:/mK7FZ3mFYEn9zvNPhpngTyatyehSwte5bJZ4ehL5Xw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -461,6 +467,10 @@ github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 h1:EpI0bqf/eX9SdZDwlMmahKM+CDBgNbsXMhsN28XrM8o= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.5.0 h1:nDDVfX0qaDuGjAvb+5zTd0Bxxoqa1Ffv9B4kiE23PTM= +github.com/xuri/excelize/v2 v2.5.0/go.mod h1:rSu0C3papjzxQA3sdK8cU544TebhrPUoTOaGPIh0Q1A= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -509,6 +519,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d h1:RNPAfi2nHY7C2srAV8A49jpsYr0ADedCk1wq6fTMTvs= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -575,6 +586,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211006190231-62292e806868 h1:KlOXYy8wQWTUJYFgkUI40Lzr06ofg5IRXUK5C7qZt1k= diff --git a/importer/from/source.go b/importer/from/source.go index 9ca9aa84..c611e6d2 100644 --- a/importer/from/source.go +++ b/importer/from/source.go @@ -5,6 +5,7 @@ type Source interface { Data(page int, size int) []map[string]interface{} Columns() []Column Bind(mapping Mapping) + Close() error } // Column 源数据列 diff --git a/importer/importer_test.go b/importer/importer_test.go index fcfe0113..5fa45faa 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -2,9 +2,11 @@ package importer import ( "fmt" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/yaoapp/xiang/config" "github.com/yaoapp/xiang/importer/xlsx" ) @@ -13,7 +15,9 @@ func TestLoad(t *testing.T) { assert.IsType(t, &Importer{}, Select("manu")) } func TestFingerprint(t *testing.T) { - file := xlsx.Open() + simple := filepath.Join(config.Conf.Root, "imports", "assets", "simple.xlsx") + file := xlsx.Open(simple) + defer file.Close() imp := Select("manu") fp := imp.Fingerprint(file) fmt.Println(fp) diff --git a/importer/xlsx/xlsx.go b/importer/xlsx/xlsx.go index 044b8b09..32826b4b 100644 --- a/importer/xlsx/xlsx.go +++ b/importer/xlsx/xlsx.go @@ -1,13 +1,60 @@ package xlsx -import "github.com/yaoapp/xiang/importer/from" +import ( + "fmt" + + "github.com/xuri/excelize/v2" + "github.com/yaoapp/kun/exception" + "github.com/yaoapp/xiang/importer/from" + "github.com/yaoapp/xiang/xlog" +) // Xlsx xlsx file -type Xlsx struct{} +type Xlsx struct { + File *excelize.File + SheetName string + SheetIndex int + Cols *excelize.Cols + Rows *excelize.Rows +} // Open 打开 Xlsx 文件 -func Open() *Xlsx { - return &Xlsx{} +func Open(filename string) *Xlsx { + file, err := excelize.OpenFile(filename) + if err != nil { + exception.New("打开文件错误 %s", 400, err.Error()).Throw() + } + + sheetIndex := file.GetActiveSheetIndex() + sheetName := file.GetSheetName(sheetIndex) + + rows, err := file.Rows(sheetName) + if err != nil { + exception.New("读取表格行失败 %s %s", 400, sheetName, err.Error()).Throw() + } + + if rows.TotalRows() > 100000 { + exception.New("数据表 %s 超过10万行 %d", 400, sheetName, rows.TotalRows()).Throw() + } + + cols, err := file.Cols(sheetName) + if err != nil { + exception.New("读取表格列信息失败 %s %s", 400, sheetName, err.Error()).Throw() + } + if cols.TotalCols() > 1000 { + exception.New("数据表 %s 超过1000列 %d", 400, sheetName, cols.TotalCols()).Throw() + } + + return &Xlsx{File: file, Rows: rows, Cols: cols, SheetName: sheetName, SheetIndex: sheetIndex} +} + +// Close 关闭文件句柄 +func (xlsx *Xlsx) Close() error { + if err := xlsx.File.Close(); err != nil { + xlog.Println(err.Error()) + return err + } + return nil } // Data 读取数据 @@ -16,7 +63,38 @@ func (xlsx *Xlsx) Data(page int, size int) []map[string]interface{} { } // Columns 读取列 -func (xlsx *Xlsx) Columns() []from.Column { return nil } +func (xlsx *Xlsx) Columns() []from.Column { + fmt.Println(xlsx.SheetName, xlsx.Rows.TotalRows(), xlsx.Cols.TotalCols()) + + // 扫描标题位置坐标 扫描行 + pos := []int{0, 0, 0} // {行, 开始列, 结束列} + line := 0 + success := false + for xlsx.Rows.Next() { + row, err := xlsx.Rows.Columns() + if err != nil { + exception.New("数据表 %s 扫描行 %d 信息失败 %", 400, xlsx.SheetName, line, err.Error()).Throw() + } + + // 扫描列 + for i, cell := range row { + if cell != "" { + pos = []int{line, i, len(row)} + success = true + break + } + } + + if success == true { + break + } + line++ + } + + fmt.Println(pos) + + return nil +} // Bind 绑定映射表 func (xlsx *Xlsx) Bind(mapping from.Mapping) {