删除非必要字段

This commit is contained in:
Max 2022-01-18 13:05:58 +08:00
parent 269da4fc94
commit 1aba1e78cd
4 changed files with 16 additions and 22 deletions

View file

@ -17,9 +17,9 @@ const (
// Source 导入文件接口 // Source 导入文件接口
type Source interface { type Source interface {
Data(row int, size int, cols []int) [][]interface{} Data(row int, size int, axises []string) [][]interface{}
Columns() []Column Columns() []Column
Chunk(size int, cols []int, cb func(line int, data [][]interface{})) Chunk(size int, axises []string, cb func(line int, data [][]interface{}))
Inspect() Inspect Inspect() Inspect
Close() error Close() error
} }
@ -28,8 +28,6 @@ type Source interface {
type Column struct { type Column struct {
Name string Name string
Type byte Type byte
Col int
Row int
Axis string Axis string
} }

View file

@ -87,7 +87,7 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping {
if !ok { if !ok {
continue continue
} }
binding := Binding{Name: "", Axis: "", Col: -1, Rules: []string{}, Field: name, Label: imp.Columns[i].Label} binding := Binding{Name: "", Axis: "", Rules: []string{}, Field: name, Label: imp.Columns[i].Label}
for _, suggest := range imp.Columns[i].Match { for _, suggest := range imp.Columns[i].Match {
if srcCol, has := sourceColumns[suggest]; has { if srcCol, has := sourceColumns[suggest]; has {
binding = Binding{ binding = Binding{
@ -95,7 +95,6 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping {
Field: name, Field: name,
Name: srcCol.Name, Name: srcCol.Name,
Axis: srcCol.Axis, Axis: srcCol.Axis,
Col: srcCol.Col,
Rules: imp.Columns[i].Rules, Rules: imp.Columns[i].Rules,
} }
continue continue
@ -114,21 +113,21 @@ func (imp *Importer) DataGet(src from.Source, page int, size int, mapping *Mappi
if row < 0 { if row < 0 {
row = mapping.RowStart row = mapping.RowStart
} }
cols := []int{} axises := []string{}
for _, d := range mapping.Columns { for _, d := range mapping.Columns {
cols = append(cols, d.Col) axises = append(axises, d.Axis)
} }
data := src.Data(row, size, cols) data := src.Data(row, size, axises)
return imp.DataClean(data, mapping.Columns) return imp.DataClean(data, mapping.Columns)
} }
// Chunk 遍历数据 // Chunk 遍历数据
func (imp *Importer) Chunk(src from.Source, mapping *Mapping, cb func(line int, data [][]interface{})) { func (imp *Importer) Chunk(src from.Source, mapping *Mapping, cb func(line int, data [][]interface{})) {
cols := []int{} axises := []string{}
for _, d := range mapping.Columns { for _, d := range mapping.Columns {
cols = append(cols, d.Col) axises = append(axises, d.Axis)
} }
src.Chunk(imp.Option.ChunkSize, cols, cb) src.Chunk(imp.Option.ChunkSize, axises, cb)
} }
// DataClean 清洗数据 // DataClean 清洗数据

View file

@ -57,6 +57,5 @@ type Binding struct {
Field string `json:"field"` // 目标字段名称 Field string `json:"field"` // 目标字段名称
Name string `json:"name"` // 源关联字段名称 Name string `json:"name"` // 源关联字段名称
Axis string `json:"axis"` // 源关联字段坐标 Axis string `json:"axis"` // 源关联字段坐标
Col int `json:"col"` // 源关联字段列号
Rules []string `json:"rules"` // 清洗规则 Rules []string `json:"rules"` // 清洗规则
} }

View file

@ -71,10 +71,10 @@ func (xlsx *Xlsx) Inspect() from.Inspect {
} }
// Data 读取数据 // Data 读取数据
func (xlsx *Xlsx) Data(row int, size int, cols []int) [][]interface{} { func (xlsx *Xlsx) Data(row int, size int, axises []string) [][]interface{} {
data := [][]interface{}{} data := [][]interface{}{}
for line := row; line < row+size; line++ { for line := row; line < row+size; line++ {
row, end := xlsx.readLine(line, cols) row, end := xlsx.readLine(line, axises)
if end { if end {
break break
} }
@ -84,7 +84,7 @@ func (xlsx *Xlsx) Data(row int, size int, cols []int) [][]interface{} {
} }
// Chunk 遍历数据 // Chunk 遍历数据
func (xlsx *Xlsx) Chunk(size int, cols []int, cb func(line int, data [][]interface{})) { func (xlsx *Xlsx) Chunk(size int, axises []string, cb func(line int, data [][]interface{})) {
line := 0 line := 0
data := [][]interface{}{} data := [][]interface{}{}
for xlsx.Rows.Next() { for xlsx.Rows.Next() {
@ -92,7 +92,7 @@ func (xlsx *Xlsx) Chunk(size int, cols []int, cb func(line int, data [][]interfa
if line < xlsx.RowStart { if line < xlsx.RowStart {
continue continue
} }
row, end := xlsx.readLine(line, cols) row, end := xlsx.readLine(line, axises)
if end { if end {
cb(line, data) cb(line, data)
break break
@ -112,12 +112,12 @@ func (xlsx *Xlsx) Chunk(size int, cols []int, cb func(line int, data [][]interfa
} }
// readLine 读取给定行信息 // readLine 读取给定行信息
func (xlsx *Xlsx) readLine(line int, cols []int) ([]interface{}, bool) { func (xlsx *Xlsx) readLine(line int, axises []string) ([]interface{}, bool) {
row := []interface{}{} row := []interface{}{}
end := true end := true
for _, c := range cols { for _, axis := range axises {
_, c, err := axisToPosition(axis)
var value = "" var value = ""
var err error
if c >= 0 { if c >= 0 {
axis := positionToAxis(line, c) axis := positionToAxis(line, c)
value, err = xlsx.File.GetCellValue(xlsx.SheetName, axis) value, err = xlsx.File.GetCellValue(xlsx.SheetName, axis)
@ -164,8 +164,6 @@ func (xlsx *Xlsx) Columns() []from.Column {
} }
columns = append(columns, from.Column{ columns = append(columns, from.Column{
Name: cell, Name: cell,
Col: i,
Row: line,
Axis: axis, Axis: axis,
Type: byte(cellType), Type: byte(cellType),
}) })