删除非必要字段
This commit is contained in:
parent
269da4fc94
commit
1aba1e78cd
4 changed files with 16 additions and 22 deletions
|
|
@ -17,9 +17,9 @@ const (
|
||||||
|
|
||||||
// Source 导入文件接口
|
// Source 导入文件接口
|
||||||
type Source interface {
|
type Source interface {
|
||||||
Data(row int, size int, cols []int) [][]interface{}
|
Data(row int, size int, axises []string) [][]interface{}
|
||||||
Columns() []Column
|
Columns() []Column
|
||||||
Chunk(size int, cols []int, cb func(line int, data [][]interface{}))
|
Chunk(size int, axises []string, cb func(line int, data [][]interface{}))
|
||||||
Inspect() Inspect
|
Inspect() Inspect
|
||||||
Close() error
|
Close() error
|
||||||
}
|
}
|
||||||
|
|
@ -28,8 +28,6 @@ type Source interface {
|
||||||
type Column struct {
|
type Column struct {
|
||||||
Name string
|
Name string
|
||||||
Type byte
|
Type byte
|
||||||
Col int
|
|
||||||
Row int
|
|
||||||
Axis string
|
Axis string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -87,7 +87,7 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping {
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
binding := Binding{Name: "", Axis: "", Col: -1, Rules: []string{}, Field: name, Label: imp.Columns[i].Label}
|
binding := Binding{Name: "", Axis: "", Rules: []string{}, Field: name, Label: imp.Columns[i].Label}
|
||||||
for _, suggest := range imp.Columns[i].Match {
|
for _, suggest := range imp.Columns[i].Match {
|
||||||
if srcCol, has := sourceColumns[suggest]; has {
|
if srcCol, has := sourceColumns[suggest]; has {
|
||||||
binding = Binding{
|
binding = Binding{
|
||||||
|
|
@ -95,7 +95,6 @@ func (imp *Importer) AutoMapping(src from.Source) *Mapping {
|
||||||
Field: name,
|
Field: name,
|
||||||
Name: srcCol.Name,
|
Name: srcCol.Name,
|
||||||
Axis: srcCol.Axis,
|
Axis: srcCol.Axis,
|
||||||
Col: srcCol.Col,
|
|
||||||
Rules: imp.Columns[i].Rules,
|
Rules: imp.Columns[i].Rules,
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
|
|
@ -114,21 +113,21 @@ func (imp *Importer) DataGet(src from.Source, page int, size int, mapping *Mappi
|
||||||
if row < 0 {
|
if row < 0 {
|
||||||
row = mapping.RowStart
|
row = mapping.RowStart
|
||||||
}
|
}
|
||||||
cols := []int{}
|
axises := []string{}
|
||||||
for _, d := range mapping.Columns {
|
for _, d := range mapping.Columns {
|
||||||
cols = append(cols, d.Col)
|
axises = append(axises, d.Axis)
|
||||||
}
|
}
|
||||||
data := src.Data(row, size, cols)
|
data := src.Data(row, size, axises)
|
||||||
return imp.DataClean(data, mapping.Columns)
|
return imp.DataClean(data, mapping.Columns)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chunk 遍历数据
|
// Chunk 遍历数据
|
||||||
func (imp *Importer) Chunk(src from.Source, mapping *Mapping, cb func(line int, data [][]interface{})) {
|
func (imp *Importer) Chunk(src from.Source, mapping *Mapping, cb func(line int, data [][]interface{})) {
|
||||||
cols := []int{}
|
axises := []string{}
|
||||||
for _, d := range mapping.Columns {
|
for _, d := range mapping.Columns {
|
||||||
cols = append(cols, d.Col)
|
axises = append(axises, d.Axis)
|
||||||
}
|
}
|
||||||
src.Chunk(imp.Option.ChunkSize, cols, cb)
|
src.Chunk(imp.Option.ChunkSize, axises, cb)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DataClean 清洗数据
|
// DataClean 清洗数据
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,5 @@ type Binding struct {
|
||||||
Field string `json:"field"` // 目标字段名称
|
Field string `json:"field"` // 目标字段名称
|
||||||
Name string `json:"name"` // 源关联字段名称
|
Name string `json:"name"` // 源关联字段名称
|
||||||
Axis string `json:"axis"` // 源关联字段坐标
|
Axis string `json:"axis"` // 源关联字段坐标
|
||||||
Col int `json:"col"` // 源关联字段列号
|
|
||||||
Rules []string `json:"rules"` // 清洗规则
|
Rules []string `json:"rules"` // 清洗规则
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -71,10 +71,10 @@ func (xlsx *Xlsx) Inspect() from.Inspect {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Data 读取数据
|
// Data 读取数据
|
||||||
func (xlsx *Xlsx) Data(row int, size int, cols []int) [][]interface{} {
|
func (xlsx *Xlsx) Data(row int, size int, axises []string) [][]interface{} {
|
||||||
data := [][]interface{}{}
|
data := [][]interface{}{}
|
||||||
for line := row; line < row+size; line++ {
|
for line := row; line < row+size; line++ {
|
||||||
row, end := xlsx.readLine(line, cols)
|
row, end := xlsx.readLine(line, axises)
|
||||||
if end {
|
if end {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
@ -84,7 +84,7 @@ func (xlsx *Xlsx) Data(row int, size int, cols []int) [][]interface{} {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chunk 遍历数据
|
// Chunk 遍历数据
|
||||||
func (xlsx *Xlsx) Chunk(size int, cols []int, cb func(line int, data [][]interface{})) {
|
func (xlsx *Xlsx) Chunk(size int, axises []string, cb func(line int, data [][]interface{})) {
|
||||||
line := 0
|
line := 0
|
||||||
data := [][]interface{}{}
|
data := [][]interface{}{}
|
||||||
for xlsx.Rows.Next() {
|
for xlsx.Rows.Next() {
|
||||||
|
|
@ -92,7 +92,7 @@ func (xlsx *Xlsx) Chunk(size int, cols []int, cb func(line int, data [][]interfa
|
||||||
if line < xlsx.RowStart {
|
if line < xlsx.RowStart {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
row, end := xlsx.readLine(line, cols)
|
row, end := xlsx.readLine(line, axises)
|
||||||
if end {
|
if end {
|
||||||
cb(line, data)
|
cb(line, data)
|
||||||
break
|
break
|
||||||
|
|
@ -112,12 +112,12 @@ func (xlsx *Xlsx) Chunk(size int, cols []int, cb func(line int, data [][]interfa
|
||||||
}
|
}
|
||||||
|
|
||||||
// readLine 读取给定行信息
|
// readLine 读取给定行信息
|
||||||
func (xlsx *Xlsx) readLine(line int, cols []int) ([]interface{}, bool) {
|
func (xlsx *Xlsx) readLine(line int, axises []string) ([]interface{}, bool) {
|
||||||
row := []interface{}{}
|
row := []interface{}{}
|
||||||
end := true
|
end := true
|
||||||
for _, c := range cols {
|
for _, axis := range axises {
|
||||||
|
_, c, err := axisToPosition(axis)
|
||||||
var value = ""
|
var value = ""
|
||||||
var err error
|
|
||||||
if c >= 0 {
|
if c >= 0 {
|
||||||
axis := positionToAxis(line, c)
|
axis := positionToAxis(line, c)
|
||||||
value, err = xlsx.File.GetCellValue(xlsx.SheetName, axis)
|
value, err = xlsx.File.GetCellValue(xlsx.SheetName, axis)
|
||||||
|
|
@ -164,8 +164,6 @@ func (xlsx *Xlsx) Columns() []from.Column {
|
||||||
}
|
}
|
||||||
columns = append(columns, from.Column{
|
columns = append(columns, from.Column{
|
||||||
Name: cell,
|
Name: cell,
|
||||||
Col: i,
|
|
||||||
Row: line,
|
|
||||||
Axis: axis,
|
Axis: axis,
|
||||||
Type: byte(cellType),
|
Type: byte(cellType),
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue