pax_global_header 0000666 0000000 0000000 00000000064 14257146113 0014516 g ustar 00root root 0000000 0000000 52 comment=2b7b86bab664a79d0e2e5c3d285c195df32aadc9
csvutil-1.7.1/ 0000775 0000000 0000000 00000000000 14257146113 0013215 5 ustar 00root root 0000000 0000000 csvutil-1.7.1/.github/ 0000775 0000000 0000000 00000000000 14257146113 0014555 5 ustar 00root root 0000000 0000000 csvutil-1.7.1/.github/PULL_REQUEST_TEMPLATE.md 0000664 0000000 0000000 00000000414 14257146113 0020355 0 ustar 00root root 0000000 0000000 ### Description
Please explain the changes that have been made in this area.
### Checklist
- [ ] Code compiles without errors
- [ ] Added new tests for the provided functionality
- [ ] All tests are passing
- [ ] Updated the README and/or documentation, if necessary
csvutil-1.7.1/.github/workflows/ 0000775 0000000 0000000 00000000000 14257146113 0016612 5 ustar 00root root 0000000 0000000 csvutil-1.7.1/.github/workflows/go.yml 0000664 0000000 0000000 00000002203 14257146113 0017737 0 ustar 00root root 0000000 0000000 name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
test:
strategy:
matrix:
go-version: [1.17, 1.18]
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Test
run: go test -race -v ./... -coverprofile=coverage.txt -covermode=atomic
- name: Codecov
uses: codecov/codecov-action@v1.2.1
with:
files: coverage.txt
backward-compatibility-test:
runs-on: ubuntu-latest
strategy:
matrix:
go-version: [1.8, 1.9, "1.10"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
path: go/src/github.com/jszwec/csvutil
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Test
run: go test -race -v ./...
env:
GOPATH: /home/runner/work/csvutil/csvutil/go
working-directory: go/src/github.com/jszwec/csvutil
csvutil-1.7.1/.gitignore 0000664 0000000 0000000 00000002236 14257146113 0015210 0 ustar 00root root 0000000 0000000 # Created by https://www.gitignore.io/api/osx,go,windows,linux
### OSX ###
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### Go ###
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
### Windows ###
# Windows image file caches
Thumbs.db
ehthumbs.db
# Folder config file
Desktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msm
*.msp
# Windows shortcuts
*.lnk
### Linux ###
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
csvutil-1.7.1/LICENSE 0000664 0000000 0000000 00000002054 14257146113 0014223 0 ustar 00root root 0000000 0000000 MIT License
Copyright (c) 2017 Jacek Szwec
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
csvutil-1.7.1/README.md 0000664 0000000 0000000 00000047211 14257146113 0014501 0 ustar 00root root 0000000 0000000 csvutil [](https://pkg.go.dev/github.com/jszwec/csvutil?tab=doc)  [](https://goreportcard.com/report/github.com/jszwec/csvutil) [](https://codecov.io/gh/jszwec/csvutil)
=================
Package csvutil provides fast, idiomatic, and dependency free mapping between CSV and Go (golang) values.
This package is not a CSV parser, it is based on the [Reader](https://godoc.org/github.com/jszwec/csvutil#Reader) and [Writer](https://godoc.org/github.com/jszwec/csvutil#Writer)
interfaces which are implemented by eg. std Go (golang) [csv package](https://golang.org/pkg/encoding/csv). This gives a possibility
of choosing any other CSV writer or reader which may be more performant.
Installation
------------
go get github.com/jszwec/csvutil
Requirements
-------------
* Go1.8+
Index
------
1. [Examples](#examples)
1. [Unmarshal](#examples_unmarshal)
2. [Marshal](#examples_marshal)
3. [Unmarshal and metadata](#examples_unmarshal_and_metadata)
4. [But my CSV file has no header...](#examples_but_my_csv_has_no_header)
5. [Decoder.Map - data normalization](#examples_decoder_map)
6. [Different separator/delimiter](#examples_different_separator)
7. [Custom Types](#examples_custom_types)
8. [Custom time.Time format](#examples_time_format)
9. [Custom struct tags](#examples_struct_tags)
10. [Slice and Map fields](#examples_slice_and_map_field)
11. [Nested/Embedded structs](#examples_nested_structs)
12. [Inline tag](#examples_inlined_structs)
2. [Performance](#performance)
1. [Unmarshal](#performance_unmarshal)
2. [Marshal](#performance_marshal)
Example
--------
### Unmarshal
Nice and easy Unmarshal is using the Go std [csv.Reader](https://golang.org/pkg/encoding/csv/#Reader) with its default options. Use [Decoder](https://godoc.org/github.com/jszwec/csvutil#Decoder) for streaming and more advanced use cases.
```go
var csvInput = []byte(`
name,age,CreatedAt
jacek,26,2012-04-01T15:00:00Z
john,,0001-01-01T00:00:00Z`,
)
type User struct {
Name string `csv:"name"`
Age int `csv:"age,omitempty"`
CreatedAt time.Time
}
var users []User
if err := csvutil.Unmarshal(csvInput, &users); err != nil {
fmt.Println("error:", err)
}
for _, u := range users {
fmt.Printf("%+v\n", u)
}
// Output:
// {Name:jacek Age:26 CreatedAt:2012-04-01 15:00:00 +0000 UTC}
// {Name:john Age:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC}
```
### Marshal
Marshal is using the Go std [csv.Writer](https://golang.org/pkg/encoding/csv/#Writer) with its default options. Use [Encoder](https://godoc.org/github.com/jszwec/csvutil#Encoder) for streaming or to use a different Writer.
```go
type Address struct {
City string
Country string
}
type User struct {
Name string
Address
Age int `csv:"age,omitempty"`
CreatedAt time.Time
}
users := []User{
{
Name: "John",
Address: Address{"Boston", "USA"},
Age: 26,
CreatedAt: time.Date(2010, 6, 2, 12, 0, 0, 0, time.UTC),
},
{
Name: "Alice",
Address: Address{"SF", "USA"},
},
}
b, err := csvutil.Marshal(users)
if err != nil {
fmt.Println("error:", err)
}
fmt.Println(string(b))
// Output:
// Name,City,Country,age,CreatedAt
// John,Boston,USA,26,2010-06-02T12:00:00Z
// Alice,SF,USA,,0001-01-01T00:00:00Z
```
### Unmarshal and metadata
It may happen that your CSV input will not always have the same header. In addition
to your base fields you may get extra metadata that you would still like to store.
[Decoder](https://godoc.org/github.com/jszwec/csvutil#Decoder) provides
[Unused](https://godoc.org/github.com/jszwec/csvutil#Decoder.Unused) method, which after each call to
[Decode](https://godoc.org/github.com/jszwec/csvutil#Decoder.Decode) can report which header indexes
were not used during decoding. Based on that, it is possible to handle and store all these extra values.
```go
type User struct {
Name string `csv:"name"`
City string `csv:"city"`
Age int `csv:"age"`
OtherData map[string]string `csv:"-"`
}
csvReader := csv.NewReader(strings.NewReader(`
name,age,city,zip
alice,25,la,90005
bob,30,ny,10005`))
dec, err := csvutil.NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
header := dec.Header()
var users []User
for {
u := User{OtherData: make(map[string]string)}
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
for _, i := range dec.Unused() {
u.OtherData[header[i]] = dec.Record()[i]
}
users = append(users, u)
}
fmt.Println(users)
// Output:
// [{alice la 25 map[zip:90005]} {bob ny 30 map[zip:10005]}]
```
### But my CSV file has no header...
Some CSV files have no header, but if you know how it should look like, it is
possible to define a struct and generate it. All that is left to do, is to pass
it to a decoder.
```go
type User struct {
ID int
Name string
Age int `csv:",omitempty"`
City string
}
csvReader := csv.NewReader(strings.NewReader(`
1,John,27,la
2,Bob,,ny`))
// in real application this should be done once in init function.
userHeader, err := csvutil.Header(User{}, "csv")
if err != nil {
log.Fatal(err)
}
dec, err := csvutil.NewDecoder(csvReader, userHeader...)
if err != nil {
log.Fatal(err)
}
var users []User
for {
var u User
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
users = append(users, u)
}
fmt.Printf("%+v", users)
// Output:
// [{ID:1 Name:John Age:27 City:la} {ID:2 Name:Bob Age:0 City:ny}]
```
### Decoder.Map - data normalization
The Decoder's [Map](https://godoc.org/github.com/jszwec/csvutil#Decoder.Map) function is a powerful tool that can help clean up or normalize
the incoming data before the actual decoding takes place.
Lets say we want to decode some floats and the csv input contains some NaN values, but these values are represented by the 'n/a' string. An attempt to decode 'n/a' into float will end up with error, because strconv.ParseFloat expects 'NaN'. Knowing that, we can implement a Map function that will normalize our 'n/a' string and turn it to 'NaN' only for float types.
```go
dec, err := NewDecoder(r)
if err != nil {
log.Fatal(err)
}
dec.Map = func(field, column string, v interface{}) string {
if _, ok := v.(float64); ok && field == "n/a" {
return "NaN"
}
return field
}
```
Now our float64 fields will be decoded properly into NaN. What about float32, float type aliases and other NaN formats? Look at the full example [here](https://gist.github.com/jszwec/2bb94f8f3612e0162eb16003701f727e).
### Different separator/delimiter
Some files may use different value separators, for example TSV files would use `\t`. The following examples show how to set up a Decoder and Encoder for such use case.
#### Decoder:
```go
csvReader := csv.NewReader(r)
csvReader.Comma = '\t'
dec, err := NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
var users []User
for {
var u User
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
users = append(users, u)
}
```
#### Encoder:
```go
var buf bytes.Buffer
w := csv.NewWriter(&buf)
w.Comma = '\t'
enc := csvutil.NewEncoder(w)
for _, u := range users {
if err := enc.Encode(u); err != nil {
log.Fatal(err)
}
}
w.Flush()
if err := w.Error(); err != nil {
log.Fatal(err)
}
```
### Custom Types and Overrides
There are multiple ways to customize or override your type's behavior.
1. a type implements [csvutil.Marshaler](https://pkg.go.dev/github.com/jszwec/csvutil#Marshaler) and/or [csvutil.Unmarshaler](https://pkg.go.dev/github.com/jszwec/csvutil#Unmarshaler)
```go
type Foo int64
func (f Foo) MarshalCSV() ([]byte, error) {
return strconv.AppendInt(nil, int64(f), 16), nil
}
func (f *Foo) UnmarshalCSV(data []byte) error {
i, err := strconv.ParseInt(string(data), 16, 64)
if err != nil {
return err
}
*f = Foo(i)
return nil
}
```
2. a type implements [encoding.TextUnmarshaler](https://golang.org/pkg/encoding/#TextUnmarshaler) and/or [encoding.TextMarshaler](https://golang.org/pkg/encoding/#TextMarshaler)
```go
type Foo int64
func (f Foo) MarshalText() ([]byte, error) {
return strconv.AppendInt(nil, int64(f), 16), nil
}
func (f *Foo) UnmarshalText(data []byte) error {
i, err := strconv.ParseInt(string(data), 16, 64)
if err != nil {
return err
}
*f = Foo(i)
return nil
}
```
3. a type is registered using [Encoder.Register](https://pkg.go.dev/github.com/jszwec/csvutil#Encoder.Register) and/or [Decoder.Register](https://pkg.go.dev/github.com/jszwec/csvutil#Decoder.Register)
```go
type Foo int64
enc.Register(func(f Foo) ([]byte, error) {
return strconv.AppendInt(nil, int64(f), 16), nil
})
dec.Register(func(data []byte, f *Foo) error {
v, err := strconv.ParseInt(string(data), 16, 64)
if err != nil {
return err
}
*f = Foo(v)
return nil
})
```
4. a type implements an interface that was registered using [Encoder.Register](https://pkg.go.dev/github.com/jszwec/csvutil#Encoder.Register) and/or [Decoder.Register](https://pkg.go.dev/github.com/jszwec/csvutil#Decoder.Register)
```go
type Foo int64
func (f Foo) String() string {
return strconv.FormatInt(int64(f), 16)
}
func (f *Foo) Scan(state fmt.ScanState, verb rune) error {
// too long; look here: https://github.com/jszwec/csvutil/blob/master/example_decoder_register_test.go#L19
}
enc.Register(func(s fmt.Stringer) ([]byte, error) {
return []byte(s.String()), nil
})
dec.Register(func(data []byte, s fmt.Scanner) error {
_, err := fmt.Sscan(string(data), s)
return err
})
```
The order of precedence for both Encoder and Decoder is:
1. type is registered
2. type implements an interface that was registered
3. csvutil.{Un,M}arshaler
4. encoding.Text{Un,M}arshaler
For more examples look [here](https://pkg.go.dev/github.com/jszwec/csvutil?readme=expanded#pkg-examples)
### Custom time.Time format
Type [time.Time](https://golang.org/pkg/time/#Time) can be used as is in the struct fields by both Decoder and Encoder
due to the fact that both have builtin support for [encoding.TextUnmarshaler](https://golang.org/pkg/encoding/#TextUnmarshaler) and [encoding.TextMarshaler](https://golang.org/pkg/encoding/#TextMarshaler). This means that by default
Time has a specific format; look at [MarshalText](https://golang.org/pkg/time/#Time.MarshalText) and [UnmarshalText](https://golang.org/pkg/time/#Time.UnmarshalText). There are two ways to override it, which one you choose depends on your use case:
1. Via Register func (based on encoding/json)
```go
const format = "2006/01/02 15:04:05"
marshalTime := func(t time.Time) ([]byte, error) {
return t.AppendFormat(nil, format), nil
}
unmarshalTime := func(data []byte, t *time.Time) error {
tt, err := time.Parse(format, string(data))
if err != nil {
return err
}
*t = tt
return nil
}
enc := csvutil.NewEncoder(w)
enc.Register(marshalTime)
dec, err := csvutil.NewDecoder(r)
if err != nil {
return err
}
dec.Register(unmarshalTime)
```
2. With custom type:
```go
type Time struct {
time.Time
}
const format = "2006/01/02 15:04:05"
func (t Time) MarshalCSV() ([]byte, error) {
var b [len(format)]byte
return t.AppendFormat(b[:0], format), nil
}
func (t *Time) UnmarshalCSV(data []byte) error {
tt, err := time.Parse(format, string(data))
if err != nil {
return err
}
*t = Time{Time: tt}
return nil
}
```
### Custom struct tags
Like in other Go encoding packages struct field tags can be used to set
custom names or options. By default encoders and decoders are looking at `csv` tag.
However, this can be overriden by manually setting the Tag field.
```go
type Foo struct {
Bar int `custom:"bar"`
}
```
```go
dec, err := csvutil.NewDecoder(r)
if err != nil {
log.Fatal(err)
}
dec.Tag = "custom"
```
```go
enc := csvutil.NewEncoder(w)
enc.Tag = "custom"
```
### Slice and Map fields
There is no default encoding/decoding support for slice and map fields because there is no CSV spec for such values.
In such case, it is recommended to create a custom type alias and implement Marshaler and Unmarshaler interfaces.
Please note that slice and map aliases behave differently than aliases of other types - there is no need for type casting.
```go
type Strings []string
func (s Strings) MarshalCSV() ([]byte, error) {
return []byte(strings.Join(s, ",")), nil // strings.Join takes []string but it will also accept Strings
}
type StringMap map[string]string
func (sm StringMap) MarshalCSV() ([]byte, error) {
return []byte(fmt.Sprint(sm)), nil
}
func main() {
b, err := csvutil.Marshal([]struct {
Strings Strings `csv:"strings"`
Map StringMap `csv:"map"`
}{
{[]string{"a", "b"}, map[string]string{"a": "1"}}, // no type casting is required for slice and map aliases
{Strings{"c", "d"}, StringMap{"b": "1"}},
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", b)
// Output:
// strings,map
// "a,b",map[a:1]
// "c,d",map[b:1]
}
```
### Nested/Embedded structs
Both Encoder and Decoder support nested or embedded structs.
Playground: https://play.golang.org/p/ZySjdVkovbf
```go
package main
import (
"fmt"
"github.com/jszwec/csvutil"
)
type Address struct {
Street string `csv:"street"`
City string `csv:"city"`
}
type User struct {
Name string `csv:"name"`
Address
}
func main() {
users := []User{
{
Name: "John",
Address: Address{
Street: "Boylston",
City: "Boston",
},
},
}
b, err := csvutil.Marshal(users)
if err != nil {
panic(err)
}
fmt.Printf("%s\n", b)
var out []User
if err := csvutil.Unmarshal(b, &out); err != nil {
panic(err)
}
fmt.Printf("%+v\n", out)
// Output:
//
// name,street,city
// John,Boylston,Boston
//
// [{Name:John Address:{Street:Boylston City:Boston}}]
}
```
### Inline tag
Fields with inline tag behave similarly to embedded struct fields. However,
it gives a possibility to specify the prefix for all underlying fields. This
can be useful when one structure can define multiple CSV columns because they
are different from each other only by a certain prefix. Look at the example below.
Playground: https://play.golang.org/p/jyEzeskSnj7
```go
package main
import (
"fmt"
"github.com/jszwec/csvutil"
)
func main() {
type Address struct {
Street string `csv:"street"`
City string `csv:"city"`
}
type User struct {
Name string `csv:"name"`
Address Address `csv:",inline"`
HomeAddress Address `csv:"home_address_,inline"`
WorkAddress Address `csv:"work_address_,inline"`
Age int `csv:"age,omitempty"`
}
users := []User{
{
Name: "John",
Address: Address{"Washington", "Boston"},
HomeAddress: Address{"Boylston", "Boston"},
WorkAddress: Address{"River St", "Cambridge"},
Age: 26,
},
}
b, err := csvutil.Marshal(users)
if err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%s\n", b)
// Output:
// name,street,city,home_address_street,home_address_city,work_address_street,work_address_city,age
// John,Washington,Boston,Boylston,Boston,River St,Cambridge,26
}
```
Performance
------------
csvutil provides the best encoding and decoding performance with small memory usage.
### Unmarshal
[benchmark code](https://gist.github.com/jszwec/e8515e741190454fa3494bcd3e1f100f)
#### csvutil:
```
BenchmarkUnmarshal/csvutil.Unmarshal/1_record-12 280696 4516 ns/op 7332 B/op 26 allocs/op
BenchmarkUnmarshal/csvutil.Unmarshal/10_records-12 95750 11517 ns/op 8356 B/op 35 allocs/op
BenchmarkUnmarshal/csvutil.Unmarshal/100_records-12 14997 83146 ns/op 18532 B/op 125 allocs/op
BenchmarkUnmarshal/csvutil.Unmarshal/1000_records-12 1485 750143 ns/op 121094 B/op 1025 allocs/op
BenchmarkUnmarshal/csvutil.Unmarshal/10000_records-12 154 7587205 ns/op 1136662 B/op 10025 allocs/op
BenchmarkUnmarshal/csvutil.Unmarshal/100000_records-12 14 76126616 ns/op 11808744 B/op 100025 allocs/op
```
#### gocsv:
```
BenchmarkUnmarshal/gocsv.Unmarshal/1_record-12 141330 7499 ns/op 7795 B/op 97 allocs/op
BenchmarkUnmarshal/gocsv.Unmarshal/10_records-12 54252 21664 ns/op 13891 B/op 307 allocs/op
BenchmarkUnmarshal/gocsv.Unmarshal/100_records-12 6920 159662 ns/op 72644 B/op 2380 allocs/op
BenchmarkUnmarshal/gocsv.Unmarshal/1000_records-12 752 1556083 ns/op 650248 B/op 23083 allocs/op
BenchmarkUnmarshal/gocsv.Unmarshal/10000_records-12 72 17086623 ns/op 7017469 B/op 230092 allocs/op
BenchmarkUnmarshal/gocsv.Unmarshal/100000_records-12 7 163610749 ns/op 75004923 B/op 2300105 allocs/op
```
#### easycsv:
```
BenchmarkUnmarshal/easycsv.ReadAll/1_record-12 101527 10662 ns/op 8855 B/op 81 allocs/op
BenchmarkUnmarshal/easycsv.ReadAll/10_records-12 23325 51437 ns/op 24072 B/op 391 allocs/op
BenchmarkUnmarshal/easycsv.ReadAll/100_records-12 2402 447296 ns/op 170538 B/op 3454 allocs/op
BenchmarkUnmarshal/easycsv.ReadAll/1000_records-12 272 4370854 ns/op 1595683 B/op 34057 allocs/op
BenchmarkUnmarshal/easycsv.ReadAll/10000_records-12 24 47502457 ns/op 18861808 B/op 340068 allocs/op
BenchmarkUnmarshal/easycsv.ReadAll/100000_records-12 3 468974170 ns/op 189427066 B/op 3400082 allocs/op
```
### Marshal
[benchmark code](https://gist.github.com/jszwec/31980321e1852ebb5615a44ccf374f17)
#### csvutil:
```
BenchmarkMarshal/csvutil.Marshal/1_record-12 279558 4390 ns/op 9952 B/op 12 allocs/op
BenchmarkMarshal/csvutil.Marshal/10_records-12 82478 15608 ns/op 10800 B/op 21 allocs/op
BenchmarkMarshal/csvutil.Marshal/100_records-12 10275 117288 ns/op 28208 B/op 112 allocs/op
BenchmarkMarshal/csvutil.Marshal/1000_records-12 1075 1147473 ns/op 168508 B/op 1014 allocs/op
BenchmarkMarshal/csvutil.Marshal/10000_records-12 100 11985382 ns/op 1525973 B/op 10017 allocs/op
BenchmarkMarshal/csvutil.Marshal/100000_records-12 9 113640813 ns/op 22455873 B/op 100021 allocs/op
```
#### gocsv:
```
BenchmarkMarshal/gocsv.Marshal/1_record-12 203052 6077 ns/op 5914 B/op 81 allocs/op
BenchmarkMarshal/gocsv.Marshal/10_records-12 50132 24585 ns/op 9284 B/op 360 allocs/op
BenchmarkMarshal/gocsv.Marshal/100_records-12 5480 212008 ns/op 51916 B/op 3151 allocs/op
BenchmarkMarshal/gocsv.Marshal/1000_records-12 514 2053919 ns/op 444506 B/op 31053 allocs/op
BenchmarkMarshal/gocsv.Marshal/10000_records-12 52 21066666 ns/op 4332377 B/op 310064 allocs/op
BenchmarkMarshal/gocsv.Marshal/100000_records-12 5 207408929 ns/op 51169419 B/op 3100077 allocs/op
```
csvutil-1.7.1/_config.yml 0000664 0000000 0000000 00000000051 14257146113 0015340 0 ustar 00root root 0000000 0000000 theme: jekyll-theme-cayman
markdown: GFM
csvutil-1.7.1/cache.go 0000664 0000000 0000000 00000006160 14257146113 0014612 0 ustar 00root root 0000000 0000000 package csvutil
import (
"reflect"
"sort"
)
type field struct {
name string
baseType reflect.Type
typ reflect.Type
tag tag
index []int
}
type fields []field
func (fs fields) Len() int { return len(fs) }
func (fs fields) Swap(i, j int) { fs[i], fs[j] = fs[j], fs[i] }
func (fs fields) Less(i, j int) bool {
for k, n := range fs[i].index {
if n != fs[j].index[k] {
return n < fs[j].index[k]
}
}
return len(fs[i].index) < len(fs[j].index)
}
type typeKey struct {
tag string
reflect.Type
}
type fieldMap map[string]fields
func (m fieldMap) insert(f field) {
fs, ok := m[f.name]
if !ok {
m[f.name] = append(fs, f)
return
}
// insert only fields with the shortest path.
if len(fs[0].index) != len(f.index) {
return
}
// fields that are tagged have priority.
if !f.tag.empty {
m[f.name] = append([]field{f}, fs...)
return
}
m[f.name] = append(fs, f)
}
func (m fieldMap) fields() fields {
out := make(fields, 0, len(m))
for _, v := range m {
for i, f := range v {
if f.tag.empty != v[0].tag.empty {
v = v[:i]
break
}
}
if len(v) > 1 {
continue
}
out = append(out, v[0])
}
sort.Sort(out)
return out
}
func buildFields(k typeKey) fields {
type key struct {
reflect.Type
tag
}
q := fields{{typ: k.Type}}
visited := make(map[key]struct{})
fm := make(fieldMap)
for len(q) > 0 {
f := q[0]
q = q[1:]
key := key{f.typ, f.tag}
if _, ok := visited[key]; ok {
continue
}
visited[key] = struct{}{}
depth := len(f.index)
numField := f.typ.NumField()
for i := 0; i < numField; i++ {
sf := f.typ.Field(i)
if sf.PkgPath != "" && !sf.Anonymous {
// unexported field
continue
}
if sf.Anonymous {
t := sf.Type
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
if sf.PkgPath != "" && t.Kind() != reflect.Struct {
// ignore embedded unexported non-struct fields.
continue
}
}
tag := parseTag(k.tag, sf)
if tag.ignore {
continue
}
if f.tag.prefix != "" {
tag.prefix = f.tag.prefix + tag.prefix
}
ft := sf.Type
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
newf := field{
name: tag.prefix + tag.name,
baseType: sf.Type,
typ: ft,
tag: tag,
index: makeIndex(f.index, i),
}
if sf.Anonymous && ft.Kind() == reflect.Struct && tag.empty {
q = append(q, newf)
continue
}
if tag.inline && ft.Kind() == reflect.Struct {
q = append(q, newf)
continue
}
fm.insert(newf)
// look for duplicate nodes on the same level. Nodes won't be
// revisited, so write all fields for the current type now.
for _, v := range q {
if len(v.index) != depth {
break
}
if v.typ == f.typ && v.tag.prefix == tag.prefix {
// other nodes can have different path.
fm.insert(field{
name: tag.prefix + tag.name,
baseType: sf.Type,
typ: ft,
tag: tag,
index: makeIndex(v.index, i),
})
}
}
}
}
return fm.fields()
}
func makeIndex(index []int, v int) []int {
out := make([]int, len(index), len(index)+1)
copy(out, index)
return append(out, v)
}
csvutil-1.7.1/cache_go17.go 0000664 0000000 0000000 00000000650 14257146113 0015445 0 ustar 00root root 0000000 0000000 // +build !go1.9
package csvutil
import (
"sync"
)
var fieldCache = struct {
mtx sync.RWMutex
m map[typeKey][]field
}{m: make(map[typeKey][]field)}
func cachedFields(k typeKey) fields {
fieldCache.mtx.RLock()
fields, ok := fieldCache.m[k]
fieldCache.mtx.RUnlock()
if ok {
return fields
}
fields = buildFields(k)
fieldCache.mtx.Lock()
fieldCache.m[k] = fields
fieldCache.mtx.Unlock()
return fields
}
csvutil-1.7.1/cache_go19.go 0000664 0000000 0000000 00000000422 14257146113 0015444 0 ustar 00root root 0000000 0000000 // +build go1.9
package csvutil
import (
"sync"
)
var fieldCache sync.Map // map[typeKey][]field
func cachedFields(k typeKey) fields {
if v, ok := fieldCache.Load(k); ok {
return v.(fields)
}
v, _ := fieldCache.LoadOrStore(k, buildFields(k))
return v.(fields)
}
csvutil-1.7.1/csvutil.go 0000664 0000000 0000000 00000012277 14257146113 0015246 0 ustar 00root root 0000000 0000000 package csvutil
import (
"bytes"
"encoding/csv"
"io"
"reflect"
)
const defaultTag = "csv"
var (
_bytes = reflect.TypeOf(([]byte)(nil))
_error = reflect.TypeOf((*error)(nil)).Elem()
)
// Unmarshal parses the CSV-encoded data and stores the result in the slice or
// the array pointed to by v. If v is nil or not a pointer to a struct slice or
// struct array, Unmarshal returns an InvalidUnmarshalError.
//
// Unmarshal uses the std encoding/csv.Reader for parsing and csvutil.Decoder
// for populating the struct elements in the provided slice. For exact decoding
// rules look at the Decoder's documentation.
//
// The first line in data is treated as a header. Decoder will use it to map
// csv columns to struct's fields.
//
// In case of success the provided slice will be reinitialized and its content
// fully replaced with decoded data.
func Unmarshal(data []byte, v interface{}) error {
val := reflect.ValueOf(v)
if val.Kind() != reflect.Ptr || val.IsNil() {
return &InvalidUnmarshalError{Type: reflect.TypeOf(v)}
}
switch val.Type().Elem().Kind() {
case reflect.Slice, reflect.Array:
default:
return &InvalidUnmarshalError{Type: val.Type()}
}
typ := val.Type().Elem()
if walkType(typ.Elem()).Kind() != reflect.Struct {
return &InvalidUnmarshalError{Type: val.Type()}
}
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err == io.EOF {
return nil
} else if err != nil {
return err
}
// for the array just call decodeArray directly; for slice values call the
// optimized code for better performance.
if typ.Kind() == reflect.Array {
return dec.decodeArray(val.Elem())
}
c := countRecords(data)
slice := reflect.MakeSlice(typ, c, c)
var i int
for ; ; i++ {
// just in case countRecords counts it wrong.
if i >= c && i >= slice.Len() {
slice = reflect.Append(slice, reflect.New(typ.Elem()).Elem())
}
if err := dec.Decode(slice.Index(i).Addr().Interface()); err == io.EOF {
break
} else if err != nil {
return err
}
}
val.Elem().Set(slice.Slice3(0, i, i))
return nil
}
// Marshal returns the CSV encoding of slice or array v. If v is not a slice or
// elements are not structs then Marshal returns InvalidMarshalError.
//
// Marshal uses the std encoding/csv.Writer with its default settings for csv
// encoding.
//
// Marshal will always encode the CSV header even for the empty slice.
//
// For the exact encoding rules look at Encoder.Encode method.
func Marshal(v interface{}) ([]byte, error) {
val := walkValue(reflect.ValueOf(v))
if !val.IsValid() {
return nil, &InvalidMarshalError{}
}
switch val.Kind() {
case reflect.Array, reflect.Slice:
default:
return nil, &InvalidMarshalError{Type: reflect.ValueOf(v).Type()}
}
typ := walkType(val.Type().Elem())
if typ.Kind() != reflect.Struct {
return nil, &InvalidMarshalError{Type: reflect.ValueOf(v).Type()}
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
if err := enc.encodeHeader(typ); err != nil {
return nil, err
}
if err := enc.encodeArray(val); err != nil {
return nil, err
}
w.Flush()
if err := w.Error(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func countRecords(s []byte) (n int) {
var prev byte
inQuote := false
for {
if len(s) == 0 && prev != '"' {
return n
}
i := bytes.IndexAny(s, "\n\"")
if i == -1 {
return n + 1
}
switch s[i] {
case '\n':
if !inQuote && (i > 0 || prev == '"') {
n++
}
case '"':
inQuote = !inQuote
}
prev = s[i]
s = s[i+1:]
}
}
// Header scans the provided struct type and generates a CSV header for it.
//
// Field names are written in the same order as struct fields are defined.
// Embedded struct's fields are treated as if they were part of the outer struct.
// Fields that are embedded types and that are tagged are treated like any
// other field.
//
// Unexported fields and fields with tag "-" are ignored.
//
// Tagged fields have the priority over non tagged fields with the same name.
//
// Following the Go visibility rules if there are multiple fields with the same
// name (tagged or not tagged) on the same level and choice between them is
// ambiguous, then all these fields will be ignored.
//
// It is a good practice to call Header once for each type. The suitable place
// for calling it is init function. Look at Decoder.DecodingDataWithNoHeader
// example.
//
// If tag is left empty the default "csv" will be used.
//
// Header will return UnsupportedTypeError if the provided value is nil or is
// not a struct.
func Header(v interface{}, tag string) ([]string, error) {
typ, err := valueType(v)
if err != nil {
return nil, err
}
if tag == "" {
tag = defaultTag
}
fields := cachedFields(typeKey{tag, typ})
h := make([]string, len(fields))
for i, f := range fields {
h[i] = f.name
}
return h, nil
}
func valueType(v interface{}) (reflect.Type, error) {
val := reflect.ValueOf(v)
if !val.IsValid() {
return nil, &UnsupportedTypeError{}
}
loop:
for {
switch val.Kind() {
case reflect.Ptr, reflect.Interface:
el := val.Elem()
if !el.IsValid() {
break loop
}
val = el
default:
break loop
}
}
typ := walkType(val.Type())
if typ.Kind() != reflect.Struct {
return nil, &UnsupportedTypeError{Type: typ}
}
return typ, nil
}
csvutil-1.7.1/csvutil_go110_test.go 0000664 0000000 0000000 00000001550 14257146113 0017204 0 ustar 00root root 0000000 0000000 //go:build !go1.17 && go1.10
// +build !go1.17,go1.10
package csvutil
import (
"encoding/csv"
"reflect"
"testing"
)
var testUnmarshalInvalidFirstLineErr = &csv.ParseError{
StartLine: 1,
Line: 1,
Column: 1,
Err: csv.ErrQuote,
}
var testUnmarshalInvalidSecondLineErr = &csv.ParseError{
StartLine: 2,
Line: 2,
Column: 1,
Err: csv.ErrQuote,
}
var ptrUnexportedEmbeddedDecodeErr = errPtrUnexportedStruct(reflect.TypeOf(new(embedded)))
func TestUnmarshalGo110(t *testing.T) {
t.Run("unmarshal type error message", func(t *testing.T) {
expected := `csvutil: cannot unmarshal "field" into Go value of type int: field "X"`
err := Unmarshal([]byte("Y,X\n1,1\n2,field"), &[]A{})
if err == nil {
t.Fatal("want err not to be nil")
}
if err.Error() != expected {
t.Errorf("want=%s; got %s", expected, err.Error())
}
})
}
csvutil-1.7.1/csvutil_go113_test.go 0000664 0000000 0000000 00000000634 14257146113 0017211 0 ustar 00root root 0000000 0000000 // +build go1.13
package csvutil
import (
"errors"
"fmt"
"reflect"
"testing"
)
func TestMarshalerError(t *testing.T) {
testErr := errors.New("error")
err := fmt.Errorf("some custom error: %w", &MarshalerError{
Type: reflect.TypeOf(1),
MarshalerType: "csvutil.Marshaler",
Err: testErr,
})
if !errors.Is(err, testErr) {
t.Errorf("expected errors.Is to return testErr")
}
}
csvutil-1.7.1/csvutil_go117_test.go 0000664 0000000 0000000 00000001726 14257146113 0017220 0 ustar 00root root 0000000 0000000 //go:build go1.17
// +build go1.17
package csvutil
import (
"encoding/csv"
"reflect"
"testing"
)
// In Go1.17 csv.ParseError.Column became 1-indexed instead of 0-indexed.
// so we need this file for Go 1.17+.
var testUnmarshalInvalidFirstLineErr = &csv.ParseError{
StartLine: 1,
Line: 1,
Column: 2,
Err: csv.ErrQuote,
}
var testUnmarshalInvalidSecondLineErr = &csv.ParseError{
StartLine: 2,
Line: 2,
Column: 2,
Err: csv.ErrQuote,
}
var ptrUnexportedEmbeddedDecodeErr = errPtrUnexportedStruct(reflect.TypeOf(new(embedded)))
func TestUnmarshalGo117(t *testing.T) {
t.Run("unmarshal type error message", func(t *testing.T) {
expected := `csvutil: cannot unmarshal "field" into Go value of type int: field "X" line 3 column 3`
err := Unmarshal([]byte("Y,X\n1,1\n2,field"), &[]A{})
if err == nil {
t.Fatal("want err not to be nil")
}
if err.Error() != expected {
t.Errorf("want=%s; got %s", expected, err.Error())
}
})
}
csvutil-1.7.1/csvutil_go17.go 0000664 0000000 0000000 00000000215 14257146113 0016070 0 ustar 00root root 0000000 0000000 // +build !go1.9
package csvutil
import (
"encoding/csv"
"io"
)
func newCSVReader(r io.Reader) *csv.Reader {
return csv.NewReader(r)
}
csvutil-1.7.1/csvutil_go17_test.go 0000664 0000000 0000000 00000001351 14257146113 0017131 0 ustar 00root root 0000000 0000000 //go:build !go1.10
// +build !go1.10
package csvutil
import (
"encoding/csv"
"testing"
)
var testUnmarshalInvalidFirstLineErr = &csv.ParseError{
Line: 1,
Column: 1,
Err: csv.ErrQuote,
}
var testUnmarshalInvalidSecondLineErr = &csv.ParseError{
Line: 2,
Column: 1,
Err: csv.ErrQuote,
}
var ptrUnexportedEmbeddedDecodeErr error
func TestUnmarshalGo17(t *testing.T) {
t.Run("unmarshal type error message", func(t *testing.T) {
expected := `csvutil: cannot unmarshal "field" into Go value of type int: field "X"`
err := Unmarshal([]byte("Y,X\n1,1\n2,field"), &[]A{})
if err == nil {
t.Fatal("want err not to be nil")
}
if err.Error() != expected {
t.Errorf("want=%s; got %s", expected, err.Error())
}
})
}
csvutil-1.7.1/csvutil_go19.go 0000664 0000000 0000000 00000000255 14257146113 0016076 0 ustar 00root root 0000000 0000000 // +build go1.9
package csvutil
import (
"encoding/csv"
"io"
)
func newCSVReader(r io.Reader) *csv.Reader {
rr := csv.NewReader(r)
rr.ReuseRecord = true
return rr
}
csvutil-1.7.1/csvutil_race_test.go 0000664 0000000 0000000 00000003663 14257146113 0017276 0 ustar 00root root 0000000 0000000 // +build race
package csvutil
import (
"bytes"
"encoding/csv"
"io"
"sync"
"testing"
)
func TestCacheDataRaces(t *testing.T) {
const routines = 50
const rows = 1000
v := TypeF{
Int: 1,
Pint: pint(2),
Int8: 3,
Pint8: pint8(4),
Int16: 5,
Pint16: pint16(6),
Int32: 7,
Pint32: pint32(8),
Int64: 9,
Pint64: pint64(10),
UInt: 11,
Puint: puint(12),
Uint8: 13,
Puint8: puint8(14),
Uint16: 15,
Puint16: puint16(16),
Uint32: 17,
Puint32: puint32(18),
Uint64: 19,
Puint64: puint64(20),
Float32: 21,
Pfloat32: pfloat32(22),
Float64: 23,
Pfloat64: pfloat64(24),
String: "25",
PString: pstring("26"),
Bool: true,
Pbool: pbool(true),
V: ppint(100),
Pv: pinterface(ppint(200)),
Binary: Binary,
PBinary: &Binary,
}
t.Run("encoding", func(t *testing.T) {
var wg sync.WaitGroup
for i := 0; i < routines; i++ {
tag := "csv"
if i%2 == 0 {
tag = "custom"
}
wg.Add(1)
go func() {
defer wg.Done()
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.Tag = tag
for i := 0; i < rows; i++ {
if err := enc.Encode(v); err != nil {
panic(err)
}
}
w.Flush()
}()
}
wg.Wait()
})
t.Run("decoding", func(t *testing.T) {
vs := make([]*TypeF, 0, rows)
for i := 0; i < rows; i++ {
vs = append(vs, &v)
}
data, err := Marshal(vs)
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
for i := 0; i < routines; i++ {
tag := "csv"
if i%2 == 0 {
tag = "custom"
}
wg.Add(1)
go func() {
defer wg.Done()
dec, err := NewDecoder(csv.NewReader(bytes.NewReader(data)))
if err != nil {
t.Fatal(err)
}
dec.Tag = tag
for {
var val TypeF
if err := dec.Decode(&val); err == io.EOF {
break
} else if err != nil {
panic(err)
}
}
}()
}
wg.Wait()
})
}
csvutil-1.7.1/csvutil_test.go 0000664 0000000 0000000 00000044707 14257146113 0016310 0 ustar 00root root 0000000 0000000 package csvutil
import (
"reflect"
"testing"
)
func TestUnmarshal(t *testing.T) {
fixture := []struct {
desc string
src []byte
in interface{}
out interface{}
}{
{
desc: "type with two records",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: new([]TypeI),
out: &[]TypeI{
{"string1", 1},
{"string2", 2},
},
},
{
desc: "pointer types with two records",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: &[]*TypeI{},
out: &[]*TypeI{
{"string1", 1},
{"string2", 2},
},
},
{
desc: "array - two records",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: new([2]TypeI),
out: &[2]TypeI{
{"string1", 1},
{"string2", 2},
},
},
{
desc: "array - pointer type with two records",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: &[2]*TypeI{},
out: &[2]*TypeI{
{"string1", 1},
{"string2", 2},
},
},
{
desc: "array - pointer type with two records size three",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: &[3]*TypeI{},
out: &[3]*TypeI{
{"string1", 1},
{"string2", 2},
nil,
},
},
{
desc: "array - pointer type with two records size three - initialized",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: &[3]*TypeI{{}, {}, {}},
out: &[3]*TypeI{
{"string1", 1},
{"string2", 2},
nil,
},
},
{
desc: "array - two records size three",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: new([3]TypeI),
out: &[3]TypeI{
{"string1", 1},
{"string2", 2},
{},
},
},
{
desc: "array - two records size one",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: new([1]TypeI),
out: &[1]TypeI{
{"string1", 1},
},
},
{
desc: "array - two records size zero",
src: []byte("String,int\nstring1,1\nstring2,2"),
in: new([0]TypeI),
out: &[0]TypeI{},
},
{
desc: "quoted input",
src: []byte("\n\n\n\"String\",\"int\"\n\"string1,\n\",\"1\"\n\n\n\n\"string2\",\"2\""),
in: &[]TypeI{},
out: &[]TypeI{
{"string1,\n", 1},
{"string2", 2},
},
},
{
desc: "quoted input - with endline",
src: []byte("\n\n\n\"String\",\"int\"\n\"string1,\n\",\"1\"\n\"string2\",\"2\"\n\n\n"),
in: &[]TypeI{},
out: &[]TypeI{
{"string1,\n", 1},
{"string2", 2},
},
},
{
desc: "header only",
src: []byte("String,int\n"),
in: &[]TypeI{},
out: &[]TypeI{},
},
{
desc: "no data",
src: []byte(""),
in: &[]TypeI{},
out: &[]TypeI{},
},
}
for _, f := range fixture {
t.Run(f.desc, func(t *testing.T) {
if err := Unmarshal(f.src, f.in); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
if !reflect.DeepEqual(f.in, f.out) {
t.Errorf("want out=%v; got %v", f.out, f.in)
}
out := reflect.ValueOf(f.out).Elem()
in := reflect.ValueOf(f.in).Elem()
if cout, cin := out.Cap(), in.Cap(); cout != cin {
t.Errorf("want cap=%d; got %d", cout, cin)
}
})
}
t.Run("invalid data", func(t *testing.T) {
type A struct{}
fixtures := []struct {
desc string
data []byte
err error
}{
{
desc: "invalid first line",
data: []byte(`"`),
err: testUnmarshalInvalidFirstLineErr,
},
{
desc: "invalid second line",
data: []byte("line\n\""),
err: testUnmarshalInvalidSecondLineErr,
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
var a []A
if err := Unmarshal(f.data, &a); !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
})
}
})
t.Run("test invalid arguments", func(t *testing.T) {
n := 1
var fixtures = []struct {
desc string
v interface{}
expected string
}{
{"nil interface", interface{}(nil), "csvutil: Unmarshal(nil)"},
{"nil", nil, "csvutil: Unmarshal(nil)"},
{"non pointer struct", struct{}{}, "csvutil: Unmarshal(non-pointer struct {})"},
{"invalid type double pointer int", (**int)(nil), "csvutil: Unmarshal(invalid type **int)"},
{"invalid type int", (*int)(nil), "csvutil: Unmarshal(invalid type *int)"},
{"invalid initialized type int", &n, "csvutil: Unmarshal(invalid type *int)"},
{"invalid type array of slice", (*[2][]TypeI)(nil), "csvutil: Unmarshal(invalid type *[2][]csvutil.TypeI)"},
{"double array", &[2][1]TypeI{}, "csvutil: Unmarshal(invalid type *[2][1]csvutil.TypeI)"},
{"double slice", &[][]TypeI{}, "csvutil: Unmarshal(invalid type *[][]csvutil.TypeI)"},
{"triple slice", &[][][]TypeI{}, "csvutil: Unmarshal(invalid type *[][][]csvutil.TypeI)"},
{"double ptr slice", &[]*[]TypeI{}, "csvutil: Unmarshal(invalid type *[]*[]csvutil.TypeI)"},
{"int slice", &[]int{}, "csvutil: Unmarshal(invalid type *[]int)"},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
err := Unmarshal([]byte(""), f.v)
if err == nil {
t.Fatalf("want err != nil")
}
if got := err.Error(); got != f.expected {
t.Errorf("want err=%s; got %s", f.expected, got)
}
})
}
})
}
func TestCountLines(t *testing.T) {
fixtures := []struct {
desc string
data []byte
out int
}{
{
desc: "three lines no endline",
data: []byte(`line1,line1
line2,line2,
line3,line3`),
out: 3,
},
{
desc: "three lines",
data: []byte(`line1,line1
line2,line2
line3,line3
`),
out: 3,
},
{
desc: "no data",
data: []byte(``),
out: 0,
},
{
desc: "endline in a quoted string",
data: []byte(`"line
""1""",line1
line2,"line
2"""
`),
out: 2,
},
{
desc: "empty lines",
data: []byte("\n\nline1,line1\n\n\n\nline2,line2\n\n"),
out: 2,
},
{
desc: "1 line ending with quote",
data: []byte(`"line1","line2"`),
out: 1,
},
{
desc: "1 line ending with quote - with endline",
data: []byte(`"line1","line2"
`),
out: 1,
},
{
desc: "2 lines ending with quote",
data: []byte(`"line1","line2"
line2,"line2"`),
out: 2,
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
if out := countRecords(f.data); out != f.out {
t.Errorf("want=%d; got %d", f.out, out)
}
})
}
}
func TestMarshal(t *testing.T) {
fixtures := []struct {
desc string
v interface{}
out [][]string
err error
}{
{
desc: "slice with basic type",
v: []TypeI{
{String: "string", Int: 10},
{String: "", Int: 0},
},
out: [][]string{
{"String", "int"},
{"string", "10"},
{"", ""},
},
},
{
desc: "array with basic type",
v: [2]TypeI{
{String: "string", Int: 10},
{String: "", Int: 0},
},
out: [][]string{
{"String", "int"},
{"string", "10"},
{"", ""},
},
},
{
desc: "slice with pointer type",
v: []*TypeI{
{String: "string", Int: 10},
{String: "", Int: 0},
},
out: [][]string{
{"String", "int"},
{"string", "10"},
{"", ""},
},
},
{
desc: "array with pointer type",
v: [2]*TypeI{
{String: "string", Int: 10},
{String: "", Int: 0},
},
out: [][]string{
{"String", "int"},
{"string", "10"},
{"", ""},
},
},
{
desc: "slice pointer",
v: &[]*TypeI{
{String: "string", Int: 10},
},
out: [][]string{
{"String", "int"},
{"string", "10"},
},
},
{
desc: "array pointer",
v: &[1]*TypeI{
{String: "string", Int: 10},
},
out: [][]string{
{"String", "int"},
{"string", "10"},
},
},
{
desc: "slice pointer wrapped in interface",
v: func() (v interface{}) {
v = &[]*TypeI{
{String: "string", Int: 10},
}
return v
}(),
out: [][]string{
{"String", "int"},
{"string", "10"},
},
},
{
desc: "array pointer wrapped in interface",
v: func() (v interface{}) {
v = &[1]*TypeI{
{String: "string", Int: 10},
}
return v
}(),
out: [][]string{
{"String", "int"},
{"string", "10"},
},
},
{
desc: "not a slice or array",
v: int64(1),
err: &InvalidMarshalError{Type: reflect.TypeOf(int64(1))},
},
{
desc: "slice of non structs",
v: []int64{1},
err: &InvalidMarshalError{Type: reflect.TypeOf([]int64{})},
},
{
desc: "array of non pointers",
v: [1]int64{1},
err: &InvalidMarshalError{Type: reflect.TypeOf([1]int64{})},
},
{
desc: "nil value",
v: nilIface,
err: &InvalidMarshalError{Type: reflect.TypeOf(nilIface)},
},
{
desc: "nil ptr value",
v: nilPtr,
err: &InvalidMarshalError{},
},
{
desc: "nil interface ptr value",
v: nilIfacePtr,
err: &InvalidMarshalError{},
},
{
desc: "marshal empty slice",
v: []TypeI{},
out: [][]string{
{"String", "int"},
},
},
{
desc: "marshal nil slice",
v: []TypeI(nil),
out: [][]string{
{"String", "int"},
},
},
{
desc: "marshal invalid struct type",
v: []InvalidType(nil),
err: &UnsupportedTypeError{Type: reflect.TypeOf(struct{}{})},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
b, err := Marshal(f.v)
if f.err != nil {
if !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
return
} else if err != nil {
t.Errorf("want err=nil; got %v", err)
}
if expected := encodeCSV(t, f.out); string(b) != expected {
t.Errorf("want %s; got %s", expected, string(b))
}
})
}
t.Run("invalid marshal error message", func(t *testing.T) {
fixtures := []struct {
desc string
expected string
v interface{}
}{
{
desc: "int64",
expected: "csvutil: Marshal(invalid type int64)",
v: int64(1),
},
{
desc: "*int64",
expected: "csvutil: Marshal(invalid type *int64)",
v: pint64(1),
},
{
desc: "[]int64",
expected: "csvutil: Marshal(non struct slice []int64)",
v: []int64{},
},
{
desc: "[]int64",
expected: "csvutil: Marshal(non struct slice *[]int64)",
v: &[]int64{},
},
{
desc: "[2]int64",
expected: "csvutil: Marshal(non struct array *[2]int64)",
v: &[2]int64{},
},
{
desc: "[2]*int64",
expected: "csvutil: Marshal(non struct array *[2]*int64)",
v: &[2]*int64{},
},
{
desc: "*[][]*TypeI",
expected: "csvutil: Marshal(non struct slice *[][]*csvutil.TypeI)",
v: &[][]*TypeI{{}},
},
{
desc: "*[]*[]*TypeI",
expected: "csvutil: Marshal(non struct slice *[]*[]*csvutil.TypeI)",
v: &[]*[]*TypeI{{}},
},
{
desc: "*[1][2]*TypeI",
expected: "csvutil: Marshal(non struct array *[1][2]*csvutil.TypeI)",
v: &[1][2]*TypeI{{}},
},
{
desc: "*[1]*[2]*TypeI",
expected: "csvutil: Marshal(non struct array *[1]*[2]*csvutil.TypeI)",
v: &[1]*[2]*TypeI{{}},
},
{
desc: "[1][2]TypeI",
expected: "csvutil: Marshal(non struct array [1][2]csvutil.TypeI)",
v: [1][2]TypeI{{}},
},
{
desc: "nil interface",
expected: "csvutil: Marshal(nil)",
v: nilIface,
},
{
desc: "nil ptr value",
expected: "csvutil: Marshal(nil)",
v: nilPtr,
},
{
desc: "nil interface ptr value",
expected: "csvutil: Marshal(nil)",
v: nilIfacePtr,
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
_, err := Marshal(f.v)
if err == nil {
t.Fatal("want err not to be nil")
}
if err.Error() != f.expected {
t.Errorf("want=%s; got %s", f.expected, err.Error())
}
})
}
})
}
type TypeJ struct {
String string `csv:"STR" json:"string"`
Int string `csv:"int" json:"-"`
Embedded16
Float string `csv:"float"`
}
type Embedded16 struct {
Bool bool `json:"bool"`
Uint uint `csv:"-"`
Uint8 uint8 `json:"-"`
}
func TestHeader(t *testing.T) {
fixture := []struct {
desc string
v interface{}
tag string
header []string
err error
}{
{
desc: "simple type with default tag",
v: TypeG{},
tag: "",
header: []string{"String", "Int"},
},
{
desc: "simple type",
v: TypeG{},
tag: "csv",
header: []string{"String", "Int"},
},
{
desc: "simple type with ptr value",
v: &TypeG{},
tag: "csv",
header: []string{"String", "Int"},
},
{
desc: "embedded types with conflict",
v: &TypeA{},
tag: "csv",
header: []string{"string", "bool", "int"},
},
{
desc: "embedded type with tag",
v: &TypeB{},
tag: "csv",
header: []string{"json", "string"},
},
{
desc: "embedded ptr type with tag",
v: &TypeD{},
tag: "csv",
header: []string{"json", "string"},
},
{
desc: "embedded ptr type no tag",
v: &TypeC{},
tag: "csv",
header: []string{"float", "string"},
},
{
desc: "type with omitempty tags",
v: TypeI{},
tag: "csv",
header: []string{"String", "int"},
},
{
desc: "embedded with different json tag",
v: TypeJ{},
tag: "json",
header: []string{"string", "bool", "Uint", "Float"},
},
{
desc: "embedded with default csv tag",
v: TypeJ{},
tag: "csv",
header: []string{"STR", "int", "Bool", "Uint8", "float"},
},
{
desc: "not a struct",
v: int(10),
tag: "csv",
err: &UnsupportedTypeError{Type: reflect.TypeOf(int(0))},
},
{
desc: "slice",
v: []TypeJ{{}},
tag: "csv",
err: &UnsupportedTypeError{Type: reflect.TypeOf([]TypeJ{})},
},
{
desc: "nil interface",
v: nilIface,
tag: "csv",
err: &UnsupportedTypeError{},
},
{
desc: "circular reference type",
v: &A{},
tag: "csv",
header: []string{"Y", "X"},
},
{
desc: "conflicting fields",
v: &Embedded10{},
tag: "csv",
header: []string{"Y"},
},
{
desc: "inline - simple",
v: &Inline{},
tag: "csv",
header: []string{
"int",
"Bool",
"Uint8",
"float",
"prefix-STR",
"prefix-int",
"prefix-Bool",
"prefix-Uint8",
"prefix-float",
"top-string",
"STR",
},
},
{
desc: "inline - chain",
v: &Inline5{},
tag: "csv",
header: []string{"AS", "AAA", "S", "A"},
},
{
desc: "inline - top level",
v: &Inline8{},
tag: "csv",
header: []string{"AA"},
},
{
desc: "nil ptr of TypeF",
v: nilPtr,
tag: "csv",
header: []string{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
{
desc: "ptr to nil interface ptr of TypeF",
v: &nilIfacePtr,
tag: "csv",
header: []string{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
{
desc: "nil interface ptr of TypeF",
v: nilIfacePtr,
tag: "csv",
header: []string{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
{
desc: "ptr to nil interface",
v: &nilIface,
err: &UnsupportedTypeError{Type: reflect.ValueOf(&nilIface).Type().Elem()},
},
}
for _, f := range fixture {
t.Run(f.desc, func(t *testing.T) {
h, err := Header(f.v, f.tag)
if !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
if !reflect.DeepEqual(h, f.header) {
t.Errorf("want header=%v; got %v", f.header, h)
}
})
}
t.Run("test nil value error message", func(t *testing.T) {
const expected = "csvutil: unsupported type: nil"
h, err := Header(nilIface, "")
if h != nil {
t.Errorf("want h=nil; got %v", h)
}
if err.Error() != expected {
t.Errorf("want err=%s; got %s", expected, err.Error())
}
})
}
func TestParity(t *testing.T) {
type A struct {
Int int
Pint *int
OmitInt int `csv:",omitempty"`
OmitPint *int `csv:",omitempty"`
}
in := []A{
{
Int: 0,
Pint: pint(0),
OmitInt: 0,
OmitPint: pint(0),
},
{
Int: 1,
Pint: pint(1),
OmitInt: 1,
OmitPint: pint(1),
},
{
Int: 0,
Pint: nil,
OmitInt: 0,
OmitPint: nil,
},
}
b, err := Marshal(in)
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
var out []A
if err := Unmarshal(b, &out); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
if !reflect.DeepEqual(in, out) {
t.Errorf("want out=%v; got %v", in, out)
}
}
func checkErr(expected, err error) bool {
if expected == err {
return true
}
eVal := reflect.New(reflect.TypeOf(expected))
if !asError(err, eVal.Interface()) {
return false
}
return reflect.DeepEqual(eVal.Elem().Interface(), expected)
}
// asError is a copy of errors.As to support older Go versions.
//
// This copy exists because we want to avoid dependencies like:
// "golang.org/x/xerrors"
func asError(err error, target interface{}) bool {
if target == nil {
panic("errors: target cannot be nil")
}
val := reflect.ValueOf(target)
typ := val.Type()
if typ.Kind() != reflect.Ptr || val.IsNil() {
panic("errors: target must be a non-nil pointer")
}
targetType := typ.Elem()
if targetType.Kind() != reflect.Interface && !targetType.Implements(errorType) {
panic("errors: *target must be interface or implement error")
}
for err != nil {
if reflect.TypeOf(err).AssignableTo(targetType) {
val.Elem().Set(reflect.ValueOf(err))
return true
}
if x, ok := err.(interface{ As(interface{}) bool }); ok && x.As(target) {
return true
}
err = unwrap(err)
}
return false
}
var errorType = reflect.TypeOf((*error)(nil)).Elem()
// unwrap is a copy of errors.Unwrap for older Go versions and to avoid
// dependencies.
func unwrap(err error) error {
u, ok := err.(interface {
Unwrap() error
})
if !ok {
return nil
}
return u.Unwrap()
}
csvutil-1.7.1/decode.go 0000664 0000000 0000000 00000013307 14257146113 0014773 0 ustar 00root root 0000000 0000000 package csvutil
import (
"encoding"
"encoding/base64"
"reflect"
"strconv"
)
var (
textUnmarshaler = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
csvUnmarshaler = reflect.TypeOf((*Unmarshaler)(nil)).Elem()
)
var intDecoders = map[int]decodeFunc{
8: decodeIntN(8),
16: decodeIntN(16),
32: decodeIntN(32),
64: decodeIntN(64),
}
var uintDecoders = map[int]decodeFunc{
8: decodeUintN(8),
16: decodeUintN(16),
32: decodeUintN(32),
64: decodeUintN(64),
}
var (
decodeFloat32 = decodeFloatN(32)
decodeFloat64 = decodeFloatN(64)
)
type decodeFunc func(s string, v reflect.Value) error
func decodeFuncValue(f reflect.Value) decodeFunc {
isIface := f.Type().In(1).Kind() == reflect.Interface
return func(s string, v reflect.Value) error {
if isIface && v.Type().Kind() == reflect.Interface && v.IsNil() {
return &UnmarshalTypeError{Value: s, Type: v.Type()}
}
out := f.Call([]reflect.Value{
reflect.ValueOf([]byte(s)),
v,
})
err, _ := out[0].Interface().(error)
return err
}
}
func decodeFuncValuePtr(f reflect.Value) decodeFunc {
return func(s string, v reflect.Value) error {
out := f.Call([]reflect.Value{
reflect.ValueOf([]byte(s)),
v.Addr(),
})
err, _ := out[0].Interface().(error)
return err
}
}
func decodeString(s string, v reflect.Value) error {
v.SetString(s)
return nil
}
func decodeIntN(bits int) decodeFunc {
return func(s string, v reflect.Value) error {
n, err := strconv.ParseInt(s, 10, bits)
if err != nil {
return &UnmarshalTypeError{Value: s, Type: v.Type()}
}
v.SetInt(n)
return nil
}
}
func decodeUintN(bits int) decodeFunc {
return func(s string, v reflect.Value) error {
n, err := strconv.ParseUint(s, 10, bits)
if err != nil {
return &UnmarshalTypeError{Value: s, Type: v.Type()}
}
v.SetUint(n)
return nil
}
}
func decodeFloatN(bits int) decodeFunc {
return func(s string, v reflect.Value) error {
n, err := strconv.ParseFloat(s, bits)
if err != nil {
return &UnmarshalTypeError{Value: s, Type: v.Type()}
}
v.SetFloat(n)
return nil
}
}
func decodeBool(s string, v reflect.Value) error {
b, err := strconv.ParseBool(s)
if err != nil {
return &UnmarshalTypeError{Value: s, Type: v.Type()}
}
v.SetBool(b)
return nil
}
func decodePtrTextUnmarshaler(s string, v reflect.Value) error {
return decodeTextUnmarshaler(s, v.Addr())
}
func decodeTextUnmarshaler(s string, v reflect.Value) error {
return v.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(s))
}
func decodePtrFieldUnmarshaler(s string, v reflect.Value) error {
return decodeFieldUnmarshaler(s, v.Addr())
}
func decodeFieldUnmarshaler(s string, v reflect.Value) error {
return v.Interface().(Unmarshaler).UnmarshalCSV([]byte(s))
}
func decodePtr(typ reflect.Type, funcMap map[reflect.Type]reflect.Value, ifaceFuncs []reflect.Value) (decodeFunc, error) {
next, err := decodeFn(typ.Elem(), funcMap, ifaceFuncs)
if err != nil {
return nil, err
}
return func(s string, v reflect.Value) error {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
return next(s, v.Elem())
}, nil
}
func decodeInterface(funcMap map[reflect.Type]reflect.Value, ifaceFuncs []reflect.Value) decodeFunc {
return func(s string, v reflect.Value) error {
if v.NumMethod() != 0 {
return &UnmarshalTypeError{
Value: s,
Type: v.Type(),
}
}
if v.IsNil() {
v.Set(reflect.ValueOf(s))
return nil
}
el := walkValue(v)
if !el.CanSet() {
if el.IsValid() {
// we may get a value receiver unmarshalers or registered funcs
// underneath the interface in which case we should call
// Unmarshal/Registered func.
typ := el.Type()
if f, ok := funcMap[typ]; ok {
return decodeFuncValue(f)(s, el)
}
for _, f := range ifaceFuncs {
if typ.AssignableTo(f.Type().In(1)) {
return decodeFuncValue(f)(s, el)
}
}
if typ.Implements(csvUnmarshaler) {
return decodeFieldUnmarshaler(s, el)
}
if typ.Implements(textUnmarshaler) {
return decodeTextUnmarshaler(s, el)
}
}
v.Set(reflect.ValueOf(s))
return nil
}
fn, err := decodeFn(el.Type(), funcMap, ifaceFuncs)
if err != nil {
return err
}
return fn(s, el)
}
}
func decodeBytes(s string, v reflect.Value) error {
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return err
}
v.SetBytes(b)
return nil
}
func decodeFn(typ reflect.Type, funcMap map[reflect.Type]reflect.Value, ifaceFuncs []reflect.Value) (decodeFunc, error) {
if f, ok := funcMap[typ]; ok {
return decodeFuncValue(f), nil
}
if f, ok := funcMap[reflect.PtrTo(typ)]; ok {
return decodeFuncValuePtr(f), nil
}
for _, f := range ifaceFuncs {
argType := f.Type().In(1)
if typ.AssignableTo(argType) {
return decodeFuncValue(f), nil
}
if reflect.PtrTo(typ).AssignableTo(argType) {
return decodeFuncValuePtr(f), nil
}
}
if reflect.PtrTo(typ).Implements(csvUnmarshaler) {
return decodePtrFieldUnmarshaler, nil
}
if reflect.PtrTo(typ).Implements(textUnmarshaler) {
return decodePtrTextUnmarshaler, nil
}
switch typ.Kind() {
case reflect.Ptr:
return decodePtr(typ, funcMap, ifaceFuncs)
case reflect.Interface:
return decodeInterface(funcMap, ifaceFuncs), nil
case reflect.String:
return decodeString, nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return intDecoders[typ.Bits()], nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return uintDecoders[typ.Bits()], nil
case reflect.Float32:
return decodeFloat32, nil
case reflect.Float64:
return decodeFloat64, nil
case reflect.Bool:
return decodeBool, nil
case reflect.Slice:
if typ.Elem().Kind() == reflect.Uint8 {
return decodeBytes, nil
}
}
return nil, &UnsupportedTypeError{Type: typ}
}
csvutil-1.7.1/decoder.go 0000664 0000000 0000000 00000033766 14257146113 0015170 0 ustar 00root root 0000000 0000000 package csvutil
import (
"errors"
"io"
"reflect"
)
type decField struct {
columnIndex int
field
decodeFunc
zero interface{}
}
// A Decoder reads and decodes string records into structs.
type Decoder struct {
// Tag defines which key in the struct field's tag to scan for names and
// options (Default: 'csv').
Tag string
// If true, Decoder will return a MissingColumnsError if it discovers
// that any of the columns are missing. This means that a CSV input
// will be required to contain all columns that were defined in the
// provided struct.
DisallowMissingColumns bool
// If not nil, Map is a function that is called for each field in the csv
// record before decoding the data. It allows mapping certain string values
// for specific columns or types to a known format. Decoder calls Map with
// the current column name (taken from header) and a zero non-pointer value
// of a type to which it is going to decode data into. Implementations
// should use type assertions to recognize the type.
//
// The good example of use case for Map is if NaN values are represented by
// eg 'n/a' string, implementing a specific Map function for all floats
// could map 'n/a' back into 'NaN' to allow successful decoding.
//
// Use Map with caution. If the requirements of column or type are not met
// Map should return 'field', since it is the original value that was
// read from the csv input, this would indicate no change.
//
// If struct field is an interface v will be of type string, unless the
// struct field contains a settable pointer value - then v will be a zero
// value of that type.
//
// Map must be set before the first call to Decode and not changed after it.
Map func(field, col string, v interface{}) string
r Reader
typeKey typeKey
hmap map[string]int
header []string
record []string
cache []decField
unused []int
funcMap map[reflect.Type]reflect.Value
ifaceFuncs []reflect.Value
}
// NewDecoder returns a new decoder that reads from r.
//
// Decoder will match struct fields according to the given header.
//
// If header is empty NewDecoder will read one line and treat it as a header.
//
// Records coming from r must be of the same length as the header.
//
// NewDecoder may return io.EOF if there is no data in r and no header was
// provided by the caller.
func NewDecoder(r Reader, header ...string) (dec *Decoder, err error) {
if len(header) == 0 {
header, err = r.Read()
if err != nil {
return nil, err
}
}
h := make([]string, len(header))
copy(h, header)
header = h
m := make(map[string]int, len(header))
for i, h := range header {
m[h] = i
}
return &Decoder{
r: r,
header: header,
hmap: m,
unused: make([]int, 0, len(header)),
}, nil
}
// Decode reads the next string record or records from its input and stores it
// in the value pointed to by v which must be a pointer to a struct, struct slice
// or struct array.
//
// Decode matches all exported struct fields based on the header. Struct fields
// can be adjusted by using tags.
//
// The "omitempty" option specifies that the field should be omitted from
// the decoding if record's field is an empty string.
//
// Examples of struct field tags and their meanings:
// // Decode matches this field with "myName" header column.
// Field int `csv:"myName"`
//
// // Decode matches this field with "Field" header column.
// Field int
//
// // Decode matches this field with "myName" header column and decoding is not
// // called if record's field is an empty string.
// Field int `csv:"myName,omitempty"`
//
// // Decode matches this field with "Field" header column and decoding is not
// // called if record's field is an empty string.
// Field int `csv:",omitempty"`
//
// // Decode ignores this field.
// Field int `csv:"-"`
//
// // Decode treats this field exactly as if it was an embedded field and
// // matches header columns that start with "my_prefix_" to all fields of this
// // type.
// Field Struct `csv:"my_prefix_,inline"`
//
// // Decode treats this field exactly as if it was an embedded field.
// Field Struct `csv:",inline"`
//
// By default decode looks for "csv" tag, but this can be changed by setting
// Decoder.Tag field.
//
// To Decode into a custom type v must implement csvutil.Unmarshaler or
// encoding.TextUnmarshaler.
//
// Anonymous struct fields with tags are treated like normal fields and they
// must implement csvutil.Unmarshaler or encoding.TextUnmarshaler unless inline
// tag is specified.
//
// Anonymous struct fields without tags are populated just as if they were
// part of the main struct. However, fields in the main struct have bigger
// priority and they are populated first. If main struct and anonymous struct
// field have the same fields, the main struct's fields will be populated.
//
// Fields of type []byte expect the data to be base64 encoded strings.
//
// Float fields are decoded to NaN if a string value is 'NaN'. This check
// is case insensitive.
//
// Interface fields are decoded to strings unless they contain settable pointer
// value.
//
// Pointer fields are decoded to nil if a string value is empty.
//
// If v is a slice, Decode resets it and reads the input until EOF, storing all
// decoded values in the given slice. Decode returns nil on EOF.
//
// If v is an array, Decode reads the input until EOF or until it decodes all
// corresponding array elements. If the input contains less elements than the
// array, the additional Go array elements are set to zero values. Decode
// returns nil on EOF unless there were no records decoded.
//
// Fields with inline tags that have a non-empty prefix must not be cyclic
// structures. Passing such values to Decode will result in an infinite loop.
func (d *Decoder) Decode(v interface{}) (err error) {
val := reflect.ValueOf(v)
if val.Kind() != reflect.Ptr || val.IsNil() {
return &InvalidDecodeError{Type: reflect.TypeOf(v)}
}
elem := indirect(val.Elem())
switch elem.Kind() {
case reflect.Struct:
return d.decodeStruct(elem)
case reflect.Slice:
return d.decodeSlice(elem)
case reflect.Array:
return d.decodeArray(elem)
case reflect.Interface, reflect.Invalid:
elem = walkValue(elem)
if elem.Kind() != reflect.Invalid {
return &InvalidDecodeError{Type: elem.Type()}
}
return &InvalidDecodeError{Type: val.Type()}
default:
return &InvalidDecodeError{Type: reflect.PtrTo(elem.Type())}
}
}
// Record returns the most recently read record. The slice is valid until the
// next call to Decode.
func (d *Decoder) Record() []string {
return d.record
}
// Header returns the first line that came from the reader, or returns the
// defined header by the caller.
func (d *Decoder) Header() []string {
header := make([]string, len(d.header))
copy(header, d.header)
return header
}
// NormalizeHeader applies f to every column in the header. It returns error
// if calling f results in conflicting header columns.
//
// NormalizeHeader must be called before Decode.
func (d *Decoder) NormalizeHeader(f func(string) string) error {
set := make(map[string]int, len(d.header))
for i, s := range d.header {
set[f(s)] = i
}
if len(set) != len(d.header) {
return errors.New("csvutil: normalize header results in conflicting columns")
}
for s, i := range set {
d.header[i] = s
}
d.hmap = set
return nil
}
// Unused returns a list of column indexes that were not used during decoding
// due to lack of matching struct field.
func (d *Decoder) Unused() []int {
if len(d.unused) == 0 {
return nil
}
indices := make([]int, len(d.unused))
copy(indices, d.unused)
return indices
}
// Register registers a custom decoding function for a concrete type or interface.
// The argument f must be of type:
// func([]byte, T) error
//
// T must be a concrete type such as *time.Time, or interface that has at least one
// method.
//
// During decoding, fields are matched by the concrete type first. If match is not
// found then Decoder looks if field implements any of the registered interfaces
// in order they were registered.
//
// Register panics if:
// - f does not match the right signature
// - f is an empty interface
// - f was already registered
//
// Register is based on the encoding/json proposal:
// https://github.com/golang/go/issues/5901.
func (d *Decoder) Register(f interface{}) {
v := reflect.ValueOf(f)
typ := v.Type()
if typ.Kind() != reflect.Func ||
typ.NumIn() != 2 || typ.NumOut() != 1 ||
typ.In(0) != _bytes || typ.Out(0) != _error {
panic("csvutil: func must be of type func([]byte, T) error")
}
argType := typ.In(1)
if argType.Kind() == reflect.Interface && argType.NumMethod() == 0 {
panic("csvutil: func argument type must not be an empty interface")
}
if d.funcMap == nil {
d.funcMap = make(map[reflect.Type]reflect.Value)
}
if _, ok := d.funcMap[argType]; ok {
panic("csvutil: func " + typ.String() + " already registered")
}
d.funcMap[argType] = v
if argType.Kind() == reflect.Interface {
d.ifaceFuncs = append(d.ifaceFuncs, v)
}
}
func (d *Decoder) decodeSlice(slice reflect.Value) error {
typ := slice.Type().Elem()
if walkType(typ).Kind() != reflect.Struct {
return &InvalidDecodeError{Type: reflect.PtrTo(slice.Type())}
}
slice.SetLen(0)
var c int
for ; ; c++ {
v := reflect.New(typ)
err := d.decodeStruct(indirect(v))
if err == io.EOF {
if c == 0 {
return io.EOF
}
break
}
// we want to ensure that we append this element to the slice even if it
// was partially decoded due to error. This is how JSON pkg does it.
slice.Set(reflect.Append(slice, v.Elem()))
if err != nil {
return err
}
}
slice.Set(slice.Slice3(0, c, c))
return nil
}
func (d *Decoder) decodeArray(v reflect.Value) error {
if walkType(v.Type().Elem()).Kind() != reflect.Struct {
return &InvalidDecodeError{Type: reflect.PtrTo(v.Type())}
}
l := v.Len()
var i int
for ; i < l; i++ {
if err := d.decodeStruct(indirect(v.Index(i))); err == io.EOF {
if i == 0 {
return io.EOF
}
break
} else if err != nil {
return err
}
}
zero := reflect.Zero(v.Type().Elem())
for i := i; i < l; i++ {
v.Index(i).Set(zero)
}
return nil
}
func (d *Decoder) decodeStruct(v reflect.Value) (err error) {
d.record, err = d.r.Read()
if err != nil {
return err
}
if len(d.record) != len(d.header) {
return ErrFieldCount
}
return d.unmarshal(d.record, v)
}
func (d *Decoder) unmarshal(record []string, v reflect.Value) error {
fields, err := d.fields(typeKey{d.tag(), v.Type()})
if err != nil {
return err
}
fieldLoop:
for _, f := range fields {
isBlank := record[f.columnIndex] == ""
if f.tag.omitEmpty && isBlank {
continue
}
fv := v
for n, i := range f.index {
fv = fv.Field(i)
if fv.Kind() == reflect.Ptr {
if fv.IsNil() {
if isBlank && n == len(f.index)-1 { // ensure we are on the leaf.
continue fieldLoop
}
// this can happen if a field is an unexported embedded
// pointer type. In Go prior to 1.10 it was possible to
// set such value because of a bug in the reflect package
// https://github.com/golang/go/issues/21353
if !fv.CanSet() {
return errPtrUnexportedStruct(fv.Type())
}
fv.Set(reflect.New(fv.Type().Elem()))
}
if isBlank && n == len(f.index)-1 { // ensure we are on the leaf.
fv.Set(reflect.Zero(fv.Type()))
continue fieldLoop
}
if n != len(f.index)-1 {
fv = fv.Elem() // walk pointer until we are on the the leaf.
}
}
}
s := record[f.columnIndex]
if d.Map != nil && f.zero != nil {
zero := f.zero
if fv := walkPtr(fv); fv.Kind() == reflect.Interface && !fv.IsNil() {
if v := walkValue(fv); v.CanSet() {
zero = reflect.Zero(v.Type()).Interface()
}
}
s = d.Map(s, d.header[f.columnIndex], zero)
}
if err := f.decodeFunc(s, fv); err != nil {
return wrapDecodeError(d.r, d.header[f.columnIndex], f.columnIndex, err)
}
}
return nil
}
// wrapDecodeError provides the given error with more context such as:
// - column name (field)
// - line number
// - column within record
//
// Line and Column info is available only if the used Reader supports 'FieldPos'
// that is available e.g. in csv.Reader (since Go1.17).
//
// The caller should use errors.As in order to fetch the original error.
func wrapDecodeError(r Reader, field string, fieldIndex int, err error) error {
fp, ok := r.(interface {
FieldPos(fieldIndex int) (line, column int)
})
if !ok {
return &DecodeError{
Field: field,
Err: err,
}
}
l, c := fp.FieldPos(fieldIndex)
return &DecodeError{
Field: field,
Line: l,
Column: c,
Err: err,
}
}
func (d *Decoder) fields(k typeKey) ([]decField, error) {
if k == d.typeKey {
return d.cache, nil
}
var (
fields = cachedFields(k)
decFields = make([]decField, 0, len(fields))
used = make([]bool, len(d.header))
missingCols []string
)
for _, f := range fields {
i, ok := d.hmap[f.name]
if !ok {
if d.DisallowMissingColumns {
missingCols = append(missingCols, f.name)
}
continue
}
fn, err := decodeFn(f.baseType, d.funcMap, d.ifaceFuncs)
if err != nil {
return nil, err
}
df := decField{
columnIndex: i,
field: f,
decodeFunc: fn,
}
if d.Map != nil {
switch f.typ.Kind() {
case reflect.Interface:
df.zero = "" // interface values are decoded to strings
default:
df.zero = reflect.Zero(walkType(f.typ)).Interface()
}
}
decFields = append(decFields, df)
used[i] = true
}
if len(missingCols) > 0 {
return nil, &MissingColumnsError{
Columns: missingCols,
}
}
d.unused = d.unused[:0]
for i, b := range used {
if !b {
d.unused = append(d.unused, i)
}
}
d.cache, d.typeKey = decFields, k
return d.cache, nil
}
func (d *Decoder) tag() string {
if d.Tag == "" {
return defaultTag
}
return d.Tag
}
func indirect(v reflect.Value) reflect.Value {
for {
switch v.Kind() {
case reflect.Interface:
if v.IsNil() {
return v
}
e := v.Elem()
if e.Kind() == reflect.Ptr && !e.IsNil() {
v = e
continue
}
return v
case reflect.Ptr:
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
v = v.Elem()
default:
return v
}
}
}
csvutil-1.7.1/decoder_test.go 0000664 0000000 0000000 00000176513 14257146113 0016225 0 ustar 00root root 0000000 0000000 package csvutil
import (
"bytes"
"encoding/base64"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"reflect"
"strconv"
"strings"
"testing"
"unicode"
)
var Binary = []byte("binary-data")
var EncodedBinary = base64.StdEncoding.EncodeToString(Binary)
var BinaryLarge = bytes.Repeat([]byte("1"), 128*1024)
var EncodedBinaryLarge = base64.StdEncoding.EncodeToString(BinaryLarge)
type Float float64
type Enum uint8
const (
EnumDefault = iota
EnumFirst
EnumSecond
)
func (e Enum) MarshalCSV() ([]byte, error) {
switch e {
case EnumFirst:
return []byte("first"), nil
case EnumSecond:
return []byte("second"), nil
default:
return []byte("default"), nil
}
}
func (e *Enum) UnmarshalCSV(data []byte) error {
s := string(data)
switch s {
case "first":
*e = EnumFirst
case "second":
*e = EnumSecond
default:
*e = EnumDefault
}
return nil
}
type ValueRecUnmarshaler struct {
S *string
}
func (u ValueRecUnmarshaler) UnmarshalCSV(data []byte) error {
*u.S = string(data)
return nil
}
func (u ValueRecUnmarshaler) Scan(data []byte) error {
*u.S = "scan: "
*u.S += string(data)
return nil
}
type ValueRecTextUnmarshaler struct {
S *string
}
func (u ValueRecTextUnmarshaler) UnmarshalText(text []byte) error {
*u.S = string(text)
return nil
}
type IntStruct struct {
Value int
}
func (i *IntStruct) Scan(state fmt.ScanState, verb rune) error {
switch verb {
case 'd', 'v':
default:
return errors.New("unsupported verb")
}
t, err := state.Token(false, unicode.IsDigit)
if err != nil {
return err
}
n, err := strconv.Atoi(string(t))
if err != nil {
return err
}
*i = IntStruct{Value: n}
return nil
}
type EnumType struct {
Enum Enum `csv:"enum"`
}
type Embedded1 struct {
String string `csv:"string"`
Float float64 `csv:"float"`
}
type Embedded2 struct {
Float float64 `csv:"float"`
Bool bool `csv:"bool"`
}
type Embedded3 map[string]string
func (e *Embedded3) UnmarshalCSV(s []byte) error {
return json.Unmarshal(s, e)
}
func (e Embedded3) MarshalCSV() ([]byte, error) {
return json.Marshal(e)
}
type Embedded4 interface{}
type Embedded5 struct {
Embedded6
Embedded7
Embedded8
}
type Embedded6 struct {
X int
}
type Embedded7 Embedded6
type Embedded8 struct {
Embedded9
}
type Embedded9 struct {
X int
Y int
}
type Embedded10 struct {
Embedded11
Embedded12
Embedded13
}
type Embedded11 struct {
Embedded6
}
type Embedded12 struct {
Embedded6
}
type Embedded13 struct {
Embedded8
}
type Embedded17 struct {
*Embedded18
}
type Embedded18 struct {
X *float64
Y *float64
}
type TypeA struct {
Embedded1
String string `csv:"string"`
Embedded2
Int int `csv:"int"`
}
type TypeB struct {
Embedded3 `csv:"json"`
String string `csv:"string"`
}
type TypeC struct {
*Embedded1
String string `csv:"string"`
}
type TypeD struct {
*Embedded3 `csv:"json"`
String string `csv:"string"`
}
type TypeE struct {
String **string `csv:"string"`
Int *int `csv:"int"`
}
type TypeF struct {
Int int `csv:"int" custom:"int"`
Pint *int `csv:"pint" custom:"pint"`
Int8 int8 `csv:"int8" custom:"int8"`
Pint8 *int8 `csv:"pint8" custom:"pint8"`
Int16 int16 `csv:"int16" custom:"int16"`
Pint16 *int16 `csv:"pint16" custom:"pint16"`
Int32 int32 `csv:"int32" custom:"int32"`
Pint32 *int32 `csv:"pint32" custom:"pint32"`
Int64 int64 `csv:"int64" custom:"int64"`
Pint64 *int64 `csv:"pint64" custom:"pint64"`
UInt uint `csv:"uint" custom:"uint"`
Puint *uint `csv:"puint" custom:"puint"`
Uint8 uint8 `csv:"uint8" custom:"uint8"`
Puint8 *uint8 `csv:"puint8" custom:"puint8"`
Uint16 uint16 `csv:"uint16" custom:"uint16"`
Puint16 *uint16 `csv:"puint16" custom:"puint16"`
Uint32 uint32 `csv:"uint32" custom:"uint32"`
Puint32 *uint32 `csv:"puint32" custom:"puint32"`
Uint64 uint64 `csv:"uint64" custom:"uint64"`
Puint64 *uint64 `csv:"puint64" custom:"puint64"`
Float32 float32 `csv:"float32" custom:"float32"`
Pfloat32 *float32 `csv:"pfloat32" custom:"pfloat32"`
Float64 float64 `csv:"float64" custom:"float64"`
Pfloat64 *float64 `csv:"pfloat64" custom:"pfloat64"`
String string `csv:"string" custom:"string"`
PString *string `csv:"pstring" custom:"pstring"`
Bool bool `csv:"bool" custom:"bool"`
Pbool *bool `csv:"pbool" custom:"pbool"`
V interface{} `csv:"interface" custom:"interface"`
Pv *interface{} `csv:"pinterface" custom:"pinterface"`
Binary []byte `csv:"binary" custom:"binary"`
PBinary *[]byte `csv:"pbinary" custom:"pbinary"`
}
type TypeG struct {
String string
Int int
Float float64 `csv:"-"`
unexported1 int
unexported2 int `csv:"unexported2"`
}
type TypeI struct {
String string `csv:",omitempty"`
Int int `csv:"int,omitempty"`
}
type TypeK struct {
*TypeL
}
type TypeL struct {
String string
Int int `csv:",omitempty"`
}
type Unmarshalers struct {
CSVUnmarshaler CSVUnmarshaler `csv:"csv"`
PCSVUnmarshaler *CSVUnmarshaler `csv:"pcsv"`
TextUnmarshaler TextUnmarshaler `csv:"text"`
PTextUnmarshaler *TextUnmarshaler `csv:"ptext"`
CSVTextUnmarshaler CSVTextUnmarshaler `csv:"csv-text"`
PCSVTextUnmarshaler *CSVTextUnmarshaler `csv:"pcsv-text"`
}
type EmbeddedUnmarshalers struct {
CSVUnmarshaler `csv:"csv"`
TextUnmarshaler `csv:"text"`
CSVTextUnmarshaler `csv:"csv-text"`
}
type EmbeddedPtrUnmarshalers struct {
*CSVUnmarshaler `csv:"csv"`
*TextUnmarshaler `csv:"text"`
*CSVTextUnmarshaler `csv:"csv-text"`
}
type CSVUnmarshaler struct {
String string `csv:"string"`
}
func (t *CSVUnmarshaler) UnmarshalCSV(s []byte) error {
t.String = "unmarshalCSV:" + string(s)
return nil
}
type TextUnmarshaler struct {
String string `csv:"string"`
}
func (t *TextUnmarshaler) UnmarshalText(text []byte) error {
t.String = "unmarshalText:" + string(text)
return nil
}
type CSVTextUnmarshaler struct {
String string `csv:"string"`
}
func (t *CSVTextUnmarshaler) UnmarshalCSV(s []byte) error {
t.String = "unmarshalCSV:" + string(s)
return nil
}
func (t *CSVTextUnmarshaler) UnmarshalText(text []byte) error {
t.String = "unmarshalText:" + string(text)
return nil
}
type TypeWithInvalidField struct {
String TypeI `csv:"string"`
}
type InvalidType struct {
String struct{}
}
type TagPriority struct {
Foo int
Bar int `csv:"Foo"`
}
type embedded struct {
Foo int `csv:"foo"`
bar int `csv:"bar"`
}
type UnexportedEmbedded struct {
embedded
}
type UnexportedEmbeddedPtr struct {
*embedded
}
type A struct {
B
X int
}
type B struct {
*A
Y int
}
var Int = 10
var String = "string"
var PString = &String
var TypeISlice []TypeI
func pint(n int) *int { return &n }
func pint8(n int8) *int8 { return &n }
func pint16(n int16) *int16 { return &n }
func pint32(n int32) *int32 { return &n }
func pint64(n int64) *int64 { return &n }
func puint(n uint) *uint { return &n }
func puint8(n uint8) *uint8 { return &n }
func puint16(n uint16) *uint16 { return &n }
func puint32(n uint32) *uint32 { return &n }
func puint64(n uint64) *uint64 { return &n }
func pfloat32(f float32) *float32 { return &f }
func pfloat64(f float64) *float64 { return &f }
func pstring(s string) *string { return &s }
func pbool(b bool) *bool { return &b }
func pinterface(v interface{}) *interface{} { return &v }
func ppint(n int) **int { p := pint(n); return &p }
func pppint(n int) ***int { p := ppint(n); return &p }
func ppTypeI(v TypeI) **TypeI { p := &v; return &p }
func TestDecoder(t *testing.T) {
fixtures := []struct {
desc string
in string
regFuncs []interface{}
out interface{}
expected interface{}
expectedRecord []string
inheader []string
header []string
unused []int
err error
}{
{
desc: "embedded type - no tag - conflicting float tag",
in: "string,int,float,bool\nstring,5,2.5,t",
out: &TypeA{},
expected: &TypeA{
Embedded1: Embedded1{},
Embedded2: Embedded2{Bool: true},
String: "string",
Int: 5,
},
unused: []int{2},
expectedRecord: []string{"string", "5", "2.5", "t"},
header: []string{"string", "int", "float", "bool"},
},
{
desc: "embedded type - with tag",
in: `string,json
string,"{""key"":""value""}"
`,
out: &TypeB{},
expected: &TypeB{
Embedded3: Embedded3{"key": "value"},
String: "string",
},
expectedRecord: []string{"string", `{"key":"value"}`},
header: []string{"string", "json"},
},
{
desc: "embedded pointer type - no tag - type with conflicting tag",
in: "string,float\nstring,2.5",
out: &TypeC{},
expected: &TypeC{
Embedded1: &Embedded1{Float: 2.5},
String: "string",
},
expectedRecord: []string{"string", "2.5"},
header: []string{"string", "float"},
},
{
desc: "embedded pointer type - with tag ",
in: `string,json
string,"{""key"":""value""}"
`,
out: &TypeD{},
expected: &TypeD{
Embedded3: &Embedded3{"key": "value"},
String: "string",
},
expectedRecord: []string{"string", `{"key":"value"}`},
header: []string{"string", "json"},
},
{
desc: "pointer types",
in: "string,int\nstring,10",
out: &TypeE{},
expected: &TypeE{
String: &PString,
Int: &Int,
},
expectedRecord: []string{"string", "10"},
header: []string{"string", "int"},
},
{
desc: "basic types",
in: "int,pint,int8,pint8,int16,pint16,int32,pint32,int64,pint64,uint," +
"puint,uint8,puint8,uint16,puint16,uint32,puint32,uint64,puint64,float32," +
"pfloat32,float64,pfloat64,string,pstring,bool,pbool,interface,pinterface,binary,pbinary\n" +
"1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,true,true,true,1," +
EncodedBinary + "," + EncodedBinaryLarge,
out: &TypeF{},
expected: &TypeF{
Int: 1,
Pint: pint(2),
Int8: 3,
Pint8: pint8(4),
Int16: 5,
Pint16: pint16(6),
Int32: 7,
Pint32: pint32(8),
Int64: 9,
Pint64: pint64(10),
UInt: 11,
Puint: puint(12),
Uint8: 13,
Puint8: puint8(14),
Uint16: 15,
Puint16: puint16(16),
Uint32: 17,
Puint32: puint32(18),
Uint64: 19,
Puint64: puint64(20),
Float32: 21,
Pfloat32: pfloat32(22),
Float64: 23,
Pfloat64: pfloat64(24),
String: "25",
PString: pstring("26"),
Bool: true,
Pbool: pbool(true),
V: "true",
Pv: pinterface("1"),
Binary: Binary,
PBinary: &BinaryLarge,
},
expectedRecord: []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12",
"13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26",
"true", "true", "true", "1", EncodedBinary, EncodedBinaryLarge},
header: []string{"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
{
desc: "tags and unexported fields",
in: "String,int,Float64,unexported1,unexported2\nstring,10,2.5,1,1",
out: &TypeG{},
expected: &TypeG{
String: "string",
},
expectedRecord: []string{"string", "10", "2.5", "1", "1"},
unused: []int{1, 2, 3, 4},
header: []string{"String", "int", "Float64", "unexported1", "unexported2"},
},
{
desc: "omitempty tag",
in: "String,int\n,",
out: &TypeI{},
expected: &TypeI{},
expectedRecord: []string{"", ""},
header: []string{"String", "int"},
},
{
desc: "empty struct",
in: "String\n1",
out: &struct{}{},
expected: &struct{}{},
expectedRecord: []string{"1"},
header: []string{"String"},
unused: []int{0},
},
{
desc: "decode value receiver unmarshalers",
in: "1,2,3\n1,2,3",
out: &struct {
ValueRecUnmarshaler ValueRecUnmarshaler `csv:"1"`
PtrValueRecUnmarshaler *ValueRecUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
}{
ValueRecUnmarshaler{new(string)},
&ValueRecUnmarshaler{new(string)},
ValueRecUnmarshaler{new(string)},
},
expected: &struct {
ValueRecUnmarshaler ValueRecUnmarshaler `csv:"1"`
PtrValueRecUnmarshaler *ValueRecUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
}{
ValueRecUnmarshaler{pstring("1")},
&ValueRecUnmarshaler{pstring("2")},
ValueRecUnmarshaler{pstring("3")},
},
expectedRecord: []string{"1", "2", "3"},
header: []string{"1", "2", "3"},
},
{
desc: "decode value receiver registered func",
in: "1,2,3,4\n1,2,3,4",
out: &struct {
ValueRecUnmarshaler ValueRecUnmarshaler `csv:"1"`
PtrValueRecUnmarshaler *ValueRecUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
Iface2 interface{} `csv:"4"`
}{
ValueRecUnmarshaler{new(string)},
&ValueRecUnmarshaler{new(string)},
ValueRecUnmarshaler{new(string)},
&ValueRecUnmarshaler{new(string)},
},
expected: &struct {
ValueRecUnmarshaler ValueRecUnmarshaler `csv:"1"`
PtrValueRecUnmarshaler *ValueRecUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
Iface2 interface{} `csv:"4"`
}{
ValueRecUnmarshaler{pstring("scan: 1")},
&ValueRecUnmarshaler{pstring("scan: 2")},
ValueRecUnmarshaler{pstring("scan: 3")},
&ValueRecUnmarshaler{pstring("scan: 4")},
},
regFuncs: []interface{}{
func(data []byte, v ValueRecUnmarshaler) error {
return v.Scan(data)
},
},
expectedRecord: []string{"1", "2", "3", "4"},
header: []string{"1", "2", "3", "4"},
},
{
desc: "decode value receiver registered func - T is interface",
in: "1,2,3,4\n1,2,3,4",
out: &struct {
ValueRecUnmarshaler ValueRecUnmarshaler `csv:"1"`
PtrValueRecUnmarshaler *ValueRecUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
Iface2 interface{} `csv:"4"`
}{
ValueRecUnmarshaler{new(string)},
&ValueRecUnmarshaler{new(string)},
ValueRecUnmarshaler{new(string)},
&ValueRecUnmarshaler{new(string)},
},
expected: &struct {
ValueRecUnmarshaler ValueRecUnmarshaler `csv:"1"`
PtrValueRecUnmarshaler *ValueRecUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
Iface2 interface{} `csv:"4"`
}{
ValueRecUnmarshaler{pstring("scan: 1")},
&ValueRecUnmarshaler{pstring("scan: 2")},
ValueRecUnmarshaler{pstring("scan: 3")},
&ValueRecUnmarshaler{pstring("scan: 4")},
},
regFuncs: []interface{}{
func(data []byte, v interface{ Scan([]byte) error }) error {
return v.Scan(data)
},
},
expectedRecord: []string{"1", "2", "3", "4"},
header: []string{"1", "2", "3", "4"},
},
{
desc: "decode value receiver textmarshaler",
in: "1,2,3,4\n1,2,3,4",
out: &struct {
ValueRecTextUnmarshaler ValueRecTextUnmarshaler `csv:"1"`
PtrValueRecTextUnmarshaler *ValueRecTextUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
Iface2 interface{} `csv:"4"`
}{
ValueRecTextUnmarshaler{new(string)},
&ValueRecTextUnmarshaler{new(string)},
ValueRecTextUnmarshaler{new(string)},
&ValueRecTextUnmarshaler{new(string)},
},
expected: &struct {
ValueRecTextUnmarshaler ValueRecTextUnmarshaler `csv:"1"`
PtrValueRecTextUnmarshaler *ValueRecTextUnmarshaler `csv:"2"`
Iface interface{} `csv:"3"`
Iface2 interface{} `csv:"4"`
}{
ValueRecTextUnmarshaler{pstring("1")},
&ValueRecTextUnmarshaler{pstring("2")},
ValueRecTextUnmarshaler{pstring("3")},
&ValueRecTextUnmarshaler{pstring("4")},
},
expectedRecord: []string{"1", "2", "3", "4"},
header: []string{"1", "2", "3", "4"},
},
{
desc: "decode unmarshalers",
in: "csv,pcsv,text,ptext,csv-text,pcsv-text\nfield,field,field,field,field,field",
out: &Unmarshalers{},
expected: &Unmarshalers{
CSVUnmarshaler: CSVUnmarshaler{"unmarshalCSV:field"},
PCSVUnmarshaler: &CSVUnmarshaler{"unmarshalCSV:field"},
TextUnmarshaler: TextUnmarshaler{"unmarshalText:field"},
PTextUnmarshaler: &TextUnmarshaler{"unmarshalText:field"},
CSVTextUnmarshaler: CSVTextUnmarshaler{"unmarshalCSV:field"},
PCSVTextUnmarshaler: &CSVTextUnmarshaler{"unmarshalCSV:field"},
},
expectedRecord: []string{"field", "field", "field", "field", "field", "field"},
header: []string{"csv", "pcsv", "text", "ptext", "csv-text", "pcsv-text"},
},
{
desc: "decode embedded tagged unmarshalers",
in: "csv,text,csv-text\nfield,field,field",
out: &EmbeddedUnmarshalers{},
expected: &EmbeddedUnmarshalers{
CSVUnmarshaler: CSVUnmarshaler{"unmarshalCSV:field"},
TextUnmarshaler: TextUnmarshaler{"unmarshalText:field"},
CSVTextUnmarshaler: CSVTextUnmarshaler{"unmarshalCSV:field"},
},
expectedRecord: []string{"field", "field", "field"},
header: []string{"csv", "text", "csv-text"},
},
{
desc: "decode pointer embedded tagged unmarshalers",
in: "csv,text,csv-text\nfield,field,field",
out: &EmbeddedPtrUnmarshalers{},
expected: &EmbeddedPtrUnmarshalers{
CSVUnmarshaler: &CSVUnmarshaler{"unmarshalCSV:field"},
TextUnmarshaler: &TextUnmarshaler{"unmarshalText:field"},
CSVTextUnmarshaler: &CSVTextUnmarshaler{"unmarshalCSV:field"},
},
expectedRecord: []string{"field", "field", "field"},
header: []string{"csv", "text", "csv-text"},
},
{
desc: "custom header",
in: "string,10",
out: &TypeI{},
expected: &TypeI{
String: "string",
Int: 10,
},
expectedRecord: []string{"string", "10"},
inheader: []string{"String", "int"},
header: []string{"String", "int"},
},
{
desc: "tag priority over field",
in: "Foo\n1",
out: &TagPriority{},
expected: &TagPriority{
Foo: 0,
Bar: 1,
},
expectedRecord: []string{"1"},
header: []string{"Foo"},
},
{
desc: "decode into unexported embedded field",
in: "foo,bar\n1,1",
out: &UnexportedEmbedded{},
expected: &UnexportedEmbedded{
embedded{
Foo: 1,
bar: 0,
},
},
expectedRecord: []string{"1", "1"},
header: []string{"foo", "bar"},
unused: []int{1},
},
{
desc: "decode into ptr unexported embedded field",
in: "foo,bar\n1,1",
out: &UnexportedEmbeddedPtr{},
expected: &UnexportedEmbeddedPtr{
&embedded{
Foo: 1,
bar: 0,
},
},
expectedRecord: []string{"1", "1"},
header: []string{"foo", "bar"},
unused: []int{1},
// this test will fail starting go1.10
err: ptrUnexportedEmbeddedDecodeErr,
},
{
desc: "embedded field conflict #1",
in: "X,Y\n1,2",
out: &Embedded5{},
expected: &Embedded5{
Embedded8: Embedded8{
Embedded9: Embedded9{Y: 2},
},
},
expectedRecord: []string{"1", "2"},
header: []string{"X", "Y"},
unused: []int{0},
},
{
desc: "embedded field conflict #2",
in: "X,Y\n1,2",
out: &Embedded10{},
expected: &Embedded10{
Embedded13: Embedded13{
Embedded8: Embedded8{
Embedded9: Embedded9{Y: 2},
},
},
},
expectedRecord: []string{"1", "2"},
header: []string{"X", "Y"},
unused: []int{0},
},
{
desc: "circular reference",
in: "X,Y\n1,2",
out: &A{},
expected: &A{X: 1, B: B{Y: 2}},
expectedRecord: []string{"1", "2"},
header: []string{"X", "Y"},
},
{
desc: "primitive type alias with Unmarshaler",
in: "enum\nfirst",
out: &EnumType{},
expected: &EnumType{Enum: EnumFirst},
expectedRecord: []string{"first"},
header: []string{"enum"},
},
{
desc: "alias type",
in: "Float\n3.14",
out: &struct{ Float float64 }{},
expected: &struct{ Float float64 }{3.14},
expectedRecord: []string{"3.14"},
header: []string{"Float"},
},
{
desc: "empty base64 string",
in: "Binary,foo\n,1\n",
out: &struct{ Binary []byte }{},
expected: &struct{ Binary []byte }{[]byte{}},
expectedRecord: []string{"", "1"},
header: []string{"Binary", "foo"},
unused: []int{1},
},
{
desc: "inline fields",
in: "int,Bool,Uint8,float,prefix-STR,prefix-int,prefix-Bool,prefix-Uint8,prefix-float,top-string,STR\n" +
"1,true,1,1,j2,2,true,2,2,top-level-str,STR",
out: &Inline{},
expected: &Inline{
J1: TypeJ{
Int: "1",
Float: "1",
Embedded16: Embedded16{Bool: true, Uint8: 1},
},
J2: TypeJ{
String: "j2",
Int: "2",
Float: "2",
Embedded16: Embedded16{Bool: true, Uint8: 2},
},
String: "top-level-str",
String2: "STR",
},
expectedRecord: []string{"1", "true", "1", "1", "j2", "2", "true", "2", "2", "top-level-str", "STR"},
header: []string{"int", "Bool", "Uint8", "float", "prefix-STR", "prefix-int", "prefix-Bool", "prefix-Uint8", "prefix-float", "top-string", "STR"},
},
{
desc: "inline chain",
in: "AS,AAA,AA,S,A\n1,11,34,2,22",
out: &Inline5{},
expected: &Inline5{
A: Inline2{
S: "1",
A: Inline3{
Inline4: Inline4{A: "11"},
},
},
B: Inline2{
S: "2",
B: Inline3{
Inline4: Inline4{A: "22"},
},
},
},
unused: []int{2},
expectedRecord: []string{"1", "11", "34", "2", "22"},
header: []string{"AS", "AAA", "AA", "S", "A"},
},
{
desc: "cyclic inline - no prefix",
in: "X\n1",
out: &Inline6{},
expected: &Inline6{
A: Inline7{
A: nil,
X: 1,
},
},
expectedRecord: []string{"1"},
header: []string{"X"},
},
{
desc: "inline visibility rules",
in: "AA\n1",
out: &Inline8{},
expected: &Inline8{
AA: 1,
},
expectedRecord: []string{"1"},
header: []string{"AA"},
},
{
desc: "initialized interface",
in: "Int,Float,String,Bool,Unmarshaler,NilPtr\n10,3.14,string,true,lol,nil",
out: &struct{ Int, Float, String, Bool, Unmarshaler, NilPtr interface{} }{
Int: int(0),
Float: float64(0),
String: "",
Bool: false,
Unmarshaler: CSVUnmarshaler{},
NilPtr: (*int)(nil),
},
expected: &struct{ Int, Float, String, Bool, Unmarshaler, NilPtr interface{} }{
Int: "10",
Float: "3.14",
String: "string",
Bool: "true",
Unmarshaler: "lol",
NilPtr: "nil",
},
expectedRecord: []string{"10", "3.14", "string", "true", "lol", "nil"},
header: []string{"Int", "Float", "String", "Bool", "Unmarshaler", "NilPtr"},
},
{
desc: "initialized ptr interface",
in: "Int,Float,String,Bool,Unmarshaler,DoublePtr\n10,3.14,string,true,lol,100",
out: &struct{ Int, Float, String, Bool, Unmarshaler, DoublePtr interface{} }{
Int: pint(0),
Float: pfloat64(0),
String: pstring(""),
Bool: pbool(false),
Unmarshaler: &CSVUnmarshaler{},
DoublePtr: ppint(0),
},
expected: &struct{ Int, Float, String, Bool, Unmarshaler, DoublePtr interface{} }{
Int: pint(10),
Float: pfloat64(3.14),
String: pstring("string"),
Bool: pbool(true),
Unmarshaler: &CSVUnmarshaler{
String: "unmarshalCSV:lol",
},
DoublePtr: ppint(100),
},
expectedRecord: []string{"10", "3.14", "string", "true", "lol", "100"},
header: []string{"Int", "Float", "String", "Bool", "Unmarshaler", "DoublePtr"},
},
{
desc: "initialized ptr interface fields",
in: "Int,Float,String,Bool,Unmarshaler\n10,3.14,string,true,lol",
out: &struct{ Int, Float, String, Bool, Unmarshaler *interface{} }{
Int: pinterface(int(0)),
Float: pinterface(float64(0)),
String: pinterface(""),
Bool: pinterface(false),
Unmarshaler: pinterface(CSVUnmarshaler{}),
},
expected: &struct{ Int, Float, String, Bool, Unmarshaler *interface{} }{
Int: pinterface("10"),
Float: pinterface("3.14"),
String: pinterface("string"),
Bool: pinterface("true"),
Unmarshaler: pinterface("lol"),
},
expectedRecord: []string{"10", "3.14", "string", "true", "lol"},
header: []string{"Int", "Float", "String", "Bool", "Unmarshaler"},
},
{
desc: "initialized ptr interface fields to ptr values",
in: "Int,Float,String,Bool,Unmarshaler,DoublePtr\n10,3.14,string,true,lol,30",
out: &struct{ Int, Float, String, Bool, Unmarshaler, DoublePtr *interface{} }{
Int: pinterface(pint(0)),
Float: pinterface(pfloat64(0)),
String: pinterface(pstring("")),
Bool: pinterface(pbool(false)),
Unmarshaler: pinterface(&CSVUnmarshaler{}),
DoublePtr: pinterface(ppint(0)),
},
expected: &struct{ Int, Float, String, Bool, Unmarshaler, DoublePtr *interface{} }{
Int: pinterface(pint(10)),
Float: pinterface(pfloat64(3.14)),
String: pinterface(pstring("string")),
Bool: pinterface(pbool(true)),
Unmarshaler: pinterface(&CSVUnmarshaler{
String: "unmarshalCSV:lol",
}),
DoublePtr: pinterface(ppint(30)),
},
expectedRecord: []string{"10", "3.14", "string", "true", "lol", "30"},
header: []string{"Int", "Float", "String", "Bool", "Unmarshaler", "DoublePtr"},
},
{
desc: "nil slice of structs",
in: "String,int\nfirst,1\nsecond,2",
out: &TypeISlice,
expected: &[]TypeI{
{String: "first", Int: 1},
{String: "second", Int: 2},
},
expectedRecord: nil,
header: []string{"String", "int"},
},
{
desc: "slice of structs",
in: "String,int\nfirst,1\nsecond,2",
out: &[]TypeI{},
expected: &[]TypeI{
{String: "first", Int: 1},
{String: "second", Int: 2},
},
expectedRecord: nil,
header: []string{"String", "int"},
},
{
desc: "slice of structs - pre-allocated",
in: "String,int\nfirst,1\nsecond,2",
out: &[]TypeI{0: {Int: 200}, 1024: {Int: 100}},
expected: &[]TypeI{
{String: "first", Int: 1},
{String: "second", Int: 2},
},
expectedRecord: nil,
header: []string{"String", "int"},
},
{
desc: "slice of pointer structs",
in: "String,int\nfirst,1\nsecond,2",
out: &[]*TypeI{},
expected: &[]*TypeI{
{String: "first", Int: 1},
{String: "second", Int: 2},
},
expectedRecord: nil,
header: []string{"String", "int"},
},
{
desc: "slice of double pointer structs",
in: "String,int\nfirst,1\nsecond,2",
out: &[]**TypeI{},
expected: &[]**TypeI{
ppTypeI(TypeI{String: "first", Int: 1}),
ppTypeI(TypeI{String: "second", Int: 2}),
},
expectedRecord: nil,
header: []string{"String", "int"},
},
{
desc: "invalid slice of interfaces",
in: "String,int\nfirst,1",
out: &[]interface{}{},
err: &InvalidDecodeError{
Type: reflect.TypeOf(&[]interface{}{}),
},
},
{
desc: "invalid slice of ints",
in: "String,int\nfirst,1",
out: &[]int{},
err: &InvalidDecodeError{
Type: reflect.TypeOf(&[]int{}),
},
},
{
desc: "invalid non pointer slice of ints",
in: "String,int\nfirst,1",
out: []int{},
err: &InvalidDecodeError{
Type: reflect.TypeOf([]int{}),
},
},
{
desc: "array of structs",
in: "String,int\nfirst,1\nsecond,2",
out: &[2]TypeI{},
expected: &[2]TypeI{
{String: "first", Int: 1},
{String: "second", Int: 2},
},
expectedRecord: []string{"second", "2"},
header: []string{"String", "int"},
},
{
desc: "array of pointer structs",
in: "String,int\nfirst,1\nsecond,2",
out: &[2]*TypeI{},
expected: &[2]*TypeI{
{String: "first", Int: 1},
{String: "second", Int: 2},
},
expectedRecord: []string{"second", "2"},
header: []string{"String", "int"},
},
{
desc: "array of double pointer structs",
in: "String,int\nfirst,1\nsecond,2",
out: &[2]**TypeI{},
expected: &[2]**TypeI{
ppTypeI(TypeI{String: "first", Int: 1}),
ppTypeI(TypeI{String: "second", Int: 2}),
},
expectedRecord: []string{"second", "2"},
header: []string{"String", "int"},
},
{
desc: "array of structs - bigger than the data set",
in: "String,int\nfirst,1\nsecond,2",
out: &[4]TypeI{
3: {String: "I should be zeroed out", Int: 1024},
},
expected: &[4]TypeI{
0: {String: "first", Int: 1},
1: {String: "second", Int: 2},
},
expectedRecord: nil,
header: []string{"String", "int"},
},
{
desc: "array of structs - smaller than the data set",
in: "String,int\nfirst,1\nsecond,2",
out: &[1]TypeI{},
expected: &[1]TypeI{
0: {String: "first", Int: 1},
},
expectedRecord: []string{"first", "1"},
header: []string{"String", "int"},
},
{
desc: "invalid array of interfaces",
in: "String,int\nfirst,1",
out: &[1]interface{}{},
err: &InvalidDecodeError{
Type: reflect.TypeOf(&[1]interface{}{}),
},
},
{
desc: "invalid array of ints",
in: "String,int\nfirst,1",
out: &[1]int{},
err: &InvalidDecodeError{
Type: reflect.TypeOf(&[1]int{}),
},
},
{
desc: "invalid non pointer array of ints",
in: "String,int\nfirst,1",
out: [1]int{},
err: &InvalidDecodeError{
Type: reflect.TypeOf([1]int{}),
},
},
{
desc: "unsupported type",
in: "string,int\ns,1",
out: &TypeWithInvalidField{},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(TypeI{}),
},
},
{
desc: "unsupported double ptr type",
in: "A\n1",
out: &struct {
A **struct{}
}{},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(struct{}{}),
},
},
{
desc: "invalid int",
in: "Int,Foo\n,",
out: &struct{ Int int }{},
err: &UnmarshalTypeError{Value: "", Type: reflect.TypeOf(int(0))},
},
{
desc: "int overflow",
in: "Int\n1024",
out: &struct{ Int int8 }{},
err: &UnmarshalTypeError{Value: "1024", Type: reflect.TypeOf(int8(0))},
},
{
desc: "invalid int pointer",
in: "Int,Foo\nbar,",
out: &struct{ Int *int }{},
err: &UnmarshalTypeError{Value: "bar", Type: reflect.TypeOf(int(0))},
},
{
desc: "invalid type pointer",
in: "Int,Foo\n,",
out: &struct{ Int *struct{} }{},
err: &UnsupportedTypeError{Type: reflect.TypeOf(struct{}{})},
},
{
desc: "invalid uint",
in: "Uint,Foo\n,",
out: &struct{ Uint uint }{},
err: &UnmarshalTypeError{Value: "", Type: reflect.TypeOf(uint(0))},
},
{
desc: "uint overflow",
in: "Uint\n1024",
out: &struct{ Uint uint8 }{},
err: &UnmarshalTypeError{Value: "1024", Type: reflect.TypeOf(uint8(0))},
},
{
desc: "invalid uint pointer",
in: "Uint\na",
out: &struct{ Uint *uint }{},
err: &UnmarshalTypeError{Value: "a", Type: reflect.TypeOf(uint(0))},
},
{
desc: "invalid float",
in: "Float,Foo\n,",
out: &struct{ Float float64 }{},
err: &UnmarshalTypeError{Value: "", Type: reflect.TypeOf(float64(0))},
},
{
desc: "invalid float pointer",
in: "Float\na",
out: &struct{ Float *float64 }{},
err: &UnmarshalTypeError{Value: "a", Type: reflect.TypeOf(float64(0))},
},
{
desc: "invalid bool",
in: "Bool,Foo\n,",
out: &struct{ Bool bool }{},
err: &UnmarshalTypeError{Value: "", Type: reflect.TypeOf(bool(false))},
},
{
desc: "invalid interface",
in: "Interface,Foo\n,",
out: &struct{ Interface Unmarshaler }{},
err: &UnmarshalTypeError{Value: "", Type: csvUnmarshaler},
},
{
desc: "invalid interface pointer",
in: "Interface,Foo\nbar,",
out: &struct{ Interface *Unmarshaler }{},
err: &UnmarshalTypeError{Value: "bar", Type: csvUnmarshaler},
},
{
desc: "invalid field in embedded type",
in: "String,int\n1,1",
out: &struct{ InvalidType }{},
err: &UnsupportedTypeError{Type: reflect.TypeOf(struct{}{})},
},
{
desc: "not a struct in decode",
in: "string,int\n1,1",
out: &Int,
err: &InvalidDecodeError{Type: reflect.TypeOf(&Int)},
},
{
desc: "not a struct in decode - non ptr",
in: "string,int\n1,1",
out: Int,
err: &InvalidDecodeError{Type: reflect.TypeOf(Int)},
},
{
desc: "invalid base64 string",
in: "Binary\n1",
out: &struct{ Binary []byte }{},
err: base64.CorruptInputError(0),
},
{
desc: "invalid int under interface value",
in: "Int,Foo\n,",
out: &struct{ Int interface{} }{Int: pint(0)},
err: &UnmarshalTypeError{Value: "", Type: reflect.TypeOf(int(0))},
},
{
desc: "unsupported type under interface value",
in: "Invalid\n1",
out: &struct{ Invalid interface{} }{Invalid: &InvalidType{}},
err: &UnsupportedTypeError{Type: reflect.TypeOf(InvalidType{})},
},
{
desc: "no panic on embedded pointer fields with blank value",
in: "X,Y\n,",
out: &Embedded17{},
expected: &Embedded17{
Embedded18: &Embedded18{},
},
expectedRecord: []string{"", ""},
header: []string{"X", "Y"},
},
{
desc: "set blank values to nil on pointers",
in: "X,Y\n1,",
out: &Embedded17{
Embedded18: &Embedded18{
X: pfloat64(10),
Y: pfloat64(20),
},
},
expected: &Embedded17{
Embedded18: &Embedded18{
X: pfloat64(1),
Y: nil,
},
},
expectedRecord: []string{"1", ""},
header: []string{"X", "Y"},
},
{
desc: "no panic on embedded pointer fields with blank value 2",
in: "X,Y\n1,",
out: &Embedded17{},
expected: &Embedded17{
Embedded18: &Embedded18{X: pfloat64(1)},
},
expectedRecord: []string{"1", ""},
header: []string{"X", "Y"},
},
{
desc: "fails on blank non float string with ptr embedded",
in: "string,float\n,",
out: &TypeC{},
err: &UnmarshalTypeError{Type: reflect.TypeOf(float64(0)), Value: ""},
},
{
desc: "blank values on embedded pointers",
in: "String,Int\n,",
out: &TypeK{},
expected: &TypeK{
&TypeL{String: "", Int: 0},
},
expectedRecord: []string{"", ""},
header: []string{"String", "Int"},
},
{
desc: "blank values on pointers decode to nil",
in: "int,pint,int8,pint8,int16,pint16,int32,pint32,int64,pint64,uint," +
"puint,uint8,puint8,uint16,puint16,uint32,puint32,uint64,puint64,float32," +
"pfloat32,float64,pfloat64,string,pstring,bool,pbool,interface,pinterface,binary,pbinary\n" +
"1,,3,,5,,7,,9,,11,,13,,15,,17,,19,,21,,23,,25,,true,,true,," +
EncodedBinary + "," + "",
out: &TypeF{
Pint: pint(10),
},
expected: &TypeF{
Int: 1,
Pint: nil,
Int8: 3,
Pint8: nil,
Int16: 5,
Pint16: nil,
Int32: 7,
Pint32: nil,
Int64: 9,
Pint64: nil,
UInt: 11,
Puint: nil,
Uint8: 13,
Puint8: nil,
Uint16: 15,
Puint16: nil,
Uint32: 17,
Puint32: nil,
Uint64: 19,
Puint64: nil,
Float32: 21,
Pfloat32: nil,
Float64: 23,
Pfloat64: nil,
String: "25",
PString: nil,
Bool: true,
Pbool: nil,
V: "true",
Pv: nil,
Binary: Binary,
PBinary: nil,
},
expectedRecord: []string{"1", "", "3", "", "5", "", "7", "", "9", "", "11", "",
"13", "", "15", "", "17", "", "19", "", "21", "", "23", "", "25", "",
"true", "", "true", "", EncodedBinary, ""},
header: []string{"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
{
desc: "registered func",
in: "Int,Pint,Iface,Piface\na,b,c,d",
out: &struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{Iface: pint(10), Piface: pinterface(pint(10))},
expected: &struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Int: 10,
Pint: pint(11),
Iface: pint(12),
Piface: pinterface(pint(13)),
},
regFuncs: []interface{}{
func(data []byte, n *int) error {
x, err := strconv.ParseInt(string(data), 16, 64)
if err != nil {
return err
}
*n = int(x)
return nil
},
},
expectedRecord: []string{"a", "b", "c", "d"},
header: []string{"Int", "Pint", "Iface", "Piface"},
},
{
desc: "registered func - initialized interface ptr",
in: "Iface,Piface\na,b",
out: &struct {
Iface interface{}
Piface *interface{}
}{Iface: 10, Piface: pinterface(10)},
expected: &struct {
Iface interface{}
Piface *interface{}
}{
Iface: "a",
Piface: pinterface("b"),
},
regFuncs: []interface{}{
func(data []byte, n *int) error {
x, err := strconv.ParseInt(string(data), 16, 64)
if err != nil {
return err
}
*n = int(x)
return nil
},
},
expectedRecord: []string{"a", "b"},
header: []string{"Iface", "Piface"},
},
{
desc: "registered func - interfaces",
in: "Int,Pint,Iface,Piface,Scanner,PScanner\n10,20,30,40,50,60",
out: &struct {
Int IntStruct
Pint *IntStruct
Iface interface{}
Piface *interface{}
Scanner fmt.Scanner
PScanner *fmt.Scanner
}{
Iface: &IntStruct{},
Piface: pinterface(&IntStruct{}),
Scanner: &IntStruct{},
PScanner: &[]fmt.Scanner{&IntStruct{}}[0],
},
expected: &struct {
Int IntStruct
Pint *IntStruct
Iface interface{}
Piface *interface{}
Scanner fmt.Scanner
PScanner *fmt.Scanner
}{
Int: IntStruct{Value: 10},
Pint: &IntStruct{Value: 20},
Iface: &IntStruct{Value: 30},
Piface: pinterface(&IntStruct{Value: 40}),
Scanner: &IntStruct{Value: 50},
PScanner: &[]fmt.Scanner{&IntStruct{Value: 60}}[0],
},
regFuncs: []interface{}{
func(data []byte, scanner fmt.Scanner) error {
_, err := fmt.Sscan(string(data), scanner)
return err
},
},
expectedRecord: []string{"10", "20", "30", "40", "50", "60"},
header: []string{"Int", "Pint", "Iface", "Piface", "Scanner", "PScanner"},
},
{
desc: "registered func - invalid interface",
in: "Foo\n1",
regFuncs: []interface{}{
func(data []byte, scanner fmt.Scanner) error {
_, err := fmt.Sscan(string(data), scanner)
return err
},
},
out: &struct{ Foo fmt.Scanner }{},
err: &UnmarshalTypeError{Value: "1", Type: reflect.TypeOf((*fmt.Scanner)(nil)).Elem()},
},
{
desc: "registered func - invalid *interface",
in: "Foo\n1",
regFuncs: []interface{}{
func(data []byte, scanner fmt.Scanner) error {
_, err := fmt.Sscan(string(data), scanner)
return err
},
},
out: &struct{ Foo *fmt.Scanner }{},
err: &UnmarshalTypeError{Value: "1", Type: reflect.TypeOf((*fmt.Scanner)(nil)).Elem()},
},
{
desc: "registered func - non ptr interface",
in: "Foo\n1",
regFuncs: []interface{}{
func(data []byte, scanner fmt.Scanner) error {
_, err := fmt.Sscan(string(data), scanner)
return err
},
},
out: &struct{ Foo interface{} }{Foo: (fmt.Scanner)(nil)},
expected: &struct{ Foo interface{} }{Foo: "1"},
expectedRecord: []string{"1"},
header: []string{"Foo"},
},
{
desc: "registered func - ptr interface",
in: "Foo\n1",
regFuncs: []interface{}{
func(data []byte, scanner fmt.Scanner) error {
_, err := fmt.Sscan(string(data), scanner)
return err
},
},
out: &struct{ Foo interface{} }{Foo: (*fmt.Scanner)(nil)},
expected: &struct{ Foo interface{} }{Foo: "1"},
expectedRecord: []string{"1"},
header: []string{"Foo"},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
r, err := NewDecoder(newCSVReader(strings.NewReader(f.in)), f.inheader...)
if err != nil {
t.Fatal(err)
}
for _, fn := range f.regFuncs {
r.Register(fn)
}
err = r.Decode(&f.out)
if f.err != nil {
if !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
return
}
if err != nil {
t.Errorf("want err=nil; got %v", err)
}
if !reflect.DeepEqual(r.Record(), f.expectedRecord) {
t.Errorf("want rec=%q; got %q", f.expectedRecord, r.Record())
}
if !reflect.DeepEqual(f.out, f.expected) {
t.Errorf("want %#v; got %#v", f.expected, f.out)
}
if !reflect.DeepEqual(r.Unused(), f.unused) {
t.Errorf("want unused=%v; got %v", f.unused, r.Unused())
}
if !reflect.DeepEqual(r.Header(), f.header) {
t.Errorf("want header=%v; got %v", f.header, r.Header())
}
})
}
t.Run("decode with custom tag", func(t *testing.T) {
type Type struct {
String string `customtag:"string"`
Int int `customtag:"int"`
}
dec, err := NewDecoder(NewReader([]string{"string", "10"}), "string", "int")
if err != nil {
t.Fatal(err)
}
dec.Tag = "customtag"
var tt Type
if err := dec.Decode(&tt); err != nil {
t.Errorf("want err=nil; got %v", err)
}
expected := Type{"string", 10}
if !reflect.DeepEqual(tt, expected) {
t.Errorf("want tt=%v; got %v", expected, tt)
}
})
t.Run("decode with disallow missing columns", func(t *testing.T) {
type Type struct {
String string
Int int
Float float64
}
t.Run("all present", func(t *testing.T) {
dec, err := NewDecoder(NewReader(
[]string{"String", "Int", "Float"},
[]string{"lol", "1", "2.0"},
))
if err != nil {
t.Fatal(err)
}
dec.DisallowMissingColumns = true
var tt Type
if err := dec.Decode(&tt); err != nil {
t.Fatalf("expected err to be nil; got %v", err)
}
if expected := (Type{"lol", 1, 2}); !reflect.DeepEqual(tt, expected) {
t.Errorf("want=%v; got %v", expected, tt)
}
})
fixtures := []struct {
desc string
recs [][]string
missingCols []string
msg string
}{
{
desc: "one missing",
recs: [][]string{
{"String", "Int"},
{"lol", "1"},
},
missingCols: []string{"Float"},
msg: `csvutil: missing columns: "Float"`,
},
{
desc: "two missing",
recs: [][]string{
{"String"},
{"lol"},
},
missingCols: []string{"Int", "Float"},
msg: `csvutil: missing columns: "Int", "Float"`,
},
{
desc: "all missing",
recs: [][]string{
{"w00t"},
{"lol"},
},
missingCols: []string{"String", "Int", "Float"},
msg: `csvutil: missing columns: "String", "Int", "Float"`,
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
dec, err := NewDecoder(NewReader(f.recs...))
if err != nil {
t.Fatal(err)
}
dec.DisallowMissingColumns = true
var tt Type
err = dec.Decode(&tt)
if err == nil {
t.Fatal("expected err != nil")
}
mcerr, ok := err.(*MissingColumnsError)
if !ok {
t.Fatalf("expected err to be of *MissingColumnErr; got %[1]T (%[1]v)", err)
}
if !reflect.DeepEqual(mcerr.Columns, f.missingCols) {
t.Errorf("expected missing columns to be %v; got %v", f.missingCols, mcerr.Columns)
}
if err.Error() != f.msg {
t.Errorf("expected err message to be %q; got %q", f.msg, err.Error())
}
})
}
})
t.Run("invalid unmarshal tests", func(t *testing.T) {
var fixtures = []struct {
v interface{}
expected string
}{
{nil, "csvutil: Decode(nil)"},
{nilIface, "csvutil: Decode(nil)"},
{struct{}{}, "csvutil: Decode(non-pointer struct {})"},
{int(1), "csvutil: Decode(non-pointer int)"},
{[]int{}, "csvutil: Decode(non-pointer []int)"},
{(*int)(nil), "csvutil: Decode(invalid type *int)"},
{(*[]int)(nil), "csvutil: Decode(invalid type *[]int)"},
{(*[]*int)(nil), "csvutil: Decode(invalid type *[]*int)"},
{(*[1]*int)(nil), "csvutil: Decode(invalid type *[1]*int)"},
{&nilIface, "csvutil: Decode(invalid type *interface {})"},
{(*TypeA)(nil), "csvutil: Decode(nil *csvutil.TypeA)"},
}
for _, f := range fixtures {
r, err := NewDecoder(newCSVReader(strings.NewReader("string\ns")))
if err != nil {
t.Fatal(err)
}
err = r.Decode(f.v)
if err == nil {
t.Errorf("Decode expecting error, got nil")
continue
}
if got := err.Error(); got != f.expected {
t.Errorf("want Decode=%q; got %q", f.expected, got)
}
}
})
t.Run("header and field length mismatch", func(t *testing.T) {
type Foo struct {
Col1 string `csv:"col1"`
Col2 string `csv:"col2"`
}
data := []byte("1,1,1")
r, err := NewDecoder(newCSVReader(bytes.NewReader(data)), "col1", "col2")
if err != nil {
t.Fatal(err)
}
var foo Foo
if err := r.Decode(&foo); err != ErrFieldCount {
t.Errorf("want err=%v; got %v", ErrFieldCount, err)
}
})
t.Run("decode different types", func(t *testing.T) {
data := []byte(`
String,Int,Float,Bool
s,1,3.14,true
s,1,3.14,true
s,1,3.14,true
`)
type A struct {
String string
Foo string
}
type B struct {
Int int
Foo int
}
type C struct {
Bool bool
Float float64
Int int
String string
Foo int
}
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Errorf("want err=nil; got %v", err)
}
fixtures := []struct {
out interface{}
expected interface{}
unused []int
}{
{
out: &A{},
expected: &A{String: "s"},
unused: []int{1, 2, 3},
},
{
out: &B{},
expected: &B{Int: 1},
unused: []int{0, 2, 3},
},
{
out: &C{},
expected: &C{
Bool: true,
Float: 3.14,
Int: 1,
String: "s",
},
},
}
for _, f := range fixtures {
if err := dec.Decode(f.out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if !reflect.DeepEqual(f.out, f.expected) {
t.Errorf("want %v; got %v", f.expected, f.out)
}
if !reflect.DeepEqual(dec.Unused(), f.unused) {
t.Errorf("want %v; got %v", f.unused, dec.Unused())
}
}
})
t.Run("decode NaN", func(t *testing.T) {
data := []byte("F1,F2,F3,F4,F5\nNaN,nan,NAN,nAn,NaN")
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
v := struct {
F1, F2, F3, F4 float64
F5 Float // aliased type
}{}
if err := dec.Decode(&v); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
for _, f := range []float64{v.F1, v.F2, v.F3, v.F4, float64(v.F5)} {
if !math.IsNaN(f) {
t.Errorf("want f=NaN; got %v", f)
}
}
})
t.Run("map", func(t *testing.T) {
t.Run("receives non-pointer and non-interface zero values", func(t *testing.T) {
data := []byte("int,pint,int8,pint8,int16,pint16,int32,pint32,int64,pint64,uint," +
"puint,uint8,puint8,uint16,puint16,uint32,puint32,uint64,puint64,float32," +
"pfloat32,float64,pfloat64,string,pstring,bool,pbool,interface,pinterface,binary,pbinary\n" +
"1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,true,true,true,1," +
EncodedBinary + "," + EncodedBinaryLarge)
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
var out TypeF
var counter int
m := func(field, col string, v interface{}) string {
switch v.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64,
float32, float64, string, bool, []byte:
counter++ // interface values are passed as strings.
}
return field
}
dec.Map = m
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if numField := reflect.TypeOf(out).NumField(); counter != numField {
t.Errorf("expected counter=%d; got %d", numField, counter)
}
})
t.Run("replaced value", func(t *testing.T) {
m := func(field, col string, v interface{}) string {
if _, ok := v.(float64); ok && field == "n/a" {
return "NaN"
}
return field
}
data := []byte("F1,F2,F3\nn/a,n/a,n/a")
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
dec.Map = m
var out struct {
F1 float64
F2 *float64
F3 **float64
}
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if !math.IsNaN(out.F1) {
t.Errorf("want F1 to be NaN but is %f", out.F1)
}
if out.F2 == nil {
t.Error("want F2 to not be nil")
}
if !math.IsNaN(*out.F2) {
t.Errorf("want F2 to be NaN but is %f", *out.F2)
}
if out.F3 == nil {
t.Error("want F3 to not be nil")
}
if *out.F3 == nil {
t.Error("want *F3 to not be nil")
}
if !math.IsNaN(**out.F3) {
t.Errorf("want F3 to be NaN but is %f", **out.F3)
}
})
t.Run("unmarshaler types", func(t *testing.T) {
m := func(field, col string, v interface{}) string {
if _, ok := v.(CSVUnmarshaler); ok && field == "" {
return "csv_unmarshaler"
}
if _, ok := v.(TextUnmarshaler); ok && field == "" {
return "text_unmarshaler"
}
return field
}
data := []byte("csv,pcsv,text,ptext\n,,,")
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
dec.Map = m
var out Unmarshalers
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
expected := Unmarshalers{
CSVUnmarshaler: CSVUnmarshaler{String: "unmarshalCSV:csv_unmarshaler"},
PCSVUnmarshaler: nil,
TextUnmarshaler: TextUnmarshaler{String: "unmarshalText:text_unmarshaler"},
PTextUnmarshaler: nil,
}
if !reflect.DeepEqual(out, expected) {
t.Errorf("want out=%v; got %v", expected, out)
}
})
t.Run("interface types", func(t *testing.T) {
m := func(field, col string, v interface{}) string {
if _, ok := v.(string); ok {
return strings.ToUpper(field)
}
if _, ok := v.(int); ok {
return "100"
}
t.Fatalf("expected v to be a string, was %T", v)
return field
}
data := []byte("F1,F2,F3,F4,F5,F6\na,b,3,4,5,6")
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
dec.Map = m
var out = struct {
F1 interface{}
F2 *interface{}
F3 interface{}
F4 interface{}
F5 interface{}
F6 *interface{}
}{
F3: int(0), // initialize an interface with a different type
F4: pint(0),
F5: (*int)(nil),
F6: pinterface(ppint(0)),
}
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if out.F1 != "A" {
t.Errorf("expected F1=A got: %v", out.F1)
}
if *out.F2 != "B" {
t.Errorf("expected F2=B got: %v", *out.F2)
}
if out.F3 != "3" {
t.Errorf("expected F3=\"3\" got: %v", out.F3)
}
f4, ok := out.F4.(*int)
if !ok || f4 == nil {
t.Error("expected F4 to be non nil int ptr")
return
}
if *f4 != 100 {
t.Errorf("expected F4=100 got: %v", f4)
}
if out.F5 != "5" {
t.Errorf("expected F5=\"5\" got: %v", out.F5)
}
f6, ok := (*out.F6).(**int)
if !ok || f4 == nil {
t.Error("expected F6 to be non nil int ptr")
return
}
if **f6 != 100 {
t.Errorf("expected F4=100 got: %v", f6)
}
})
t.Run("receives a proper column name", func(t *testing.T) {
const val = "magic_column"
m := func(field, col string, v interface{}) string {
if col == "F2" {
return val
}
return field
}
data := []byte("F1,F2\na,b")
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
dec.Map = m
var out = struct {
F1 string
F2 string
}{}
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if out.F1 != "a" {
t.Errorf("expected F1=a got: %v", out.F1)
}
if out.F2 != val {
t.Errorf("expected F2=%s got: %v", val, out.F1)
}
})
})
t.Run("decoding into specific values", func(t *testing.T) {
setup := func(t *testing.T) *Decoder {
data := []byte("String,Int\na,1")
dec, err := NewDecoder(newCSVReader(bytes.NewReader(data)))
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
return dec
}
t.Run("wrapped in interfaces", func(t *testing.T) {
dec := setup(t)
var out *TypeG
var ii interface{} = &out
var i interface{} = ii
if err := dec.Decode(&i); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if out == nil {
t.Fatal("want out to not be nil")
}
if expected := (TypeG{String: "a", Int: 1}); *out != expected {
t.Errorf("want expected=%v; got %v", expected, *out)
}
})
t.Run("wrapped in interfaces #2", func(t *testing.T) {
dec := setup(t)
var out *TypeG
var ii interface{} = &out
var i interface{} = &ii
if err := dec.Decode(&i); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if out == nil {
t.Fatal("want out to not be nil")
}
if expected := (TypeG{String: "a", Int: 1}); *out != expected {
t.Errorf("want expected=%v; got %v", expected, *out)
}
})
t.Run("wrapped in interfaces not ptr", func(t *testing.T) {
dec := setup(t)
var out *TypeG
var ii interface{} = out
var i interface{} = ii
expected := &InvalidDecodeError{Type: reflect.TypeOf(&TypeG{})}
if err := dec.Decode(i); !reflect.DeepEqual(err, expected) {
t.Errorf("want err=%v; got %v", expected, err)
}
})
t.Run("wrapped in interface non ptr value", func(t *testing.T) {
dec := setup(t)
var out TypeG
var ii interface{} = &out
var i interface{} = ii
if err := dec.Decode(&i); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if expected := (TypeG{String: "a", Int: 1}); out != expected {
t.Errorf("want expected=%v; got %v", expected, out)
}
})
t.Run("interface to interface", func(t *testing.T) {
dec := setup(t)
var ii interface{}
var i interface{} = &ii
expected := &InvalidDecodeError{Type: reflect.TypeOf((*interface{})(nil))}
if err := dec.Decode(&i); !reflect.DeepEqual(err, expected) {
t.Errorf("want err=%v; got %v", expected, err)
}
})
t.Run("interface to nil interface", func(t *testing.T) {
dec := setup(t)
var ii *interface{}
var i interface{} = ii
expected := &InvalidDecodeError{Type: reflect.TypeOf((*interface{})(nil))}
if err := dec.Decode(&i); !reflect.DeepEqual(err, expected) {
t.Errorf("want err=%v; got %v", expected, err)
}
})
t.Run("nil ptr value", func(t *testing.T) {
dec := setup(t)
var out *TypeG
expected := &InvalidDecodeError{Type: reflect.TypeOf(&TypeG{})}
if err := dec.Decode(out); !reflect.DeepEqual(err, expected) {
t.Errorf("want err=%v; got %v", expected, err)
}
})
t.Run("nil ptr value ptr", func(t *testing.T) {
dec := setup(t)
var out *TypeG
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if out == nil {
t.Fatal("want out to not be nil")
}
if expected := (TypeG{String: "a", Int: 1}); *out != expected {
t.Errorf("want expected=%v; got %v", expected, *out)
}
})
t.Run("nil double ptr value", func(t *testing.T) {
dec := setup(t)
var out **TypeG
expected := &InvalidDecodeError{Type: reflect.TypeOf(out)}
if err := dec.Decode(out); !reflect.DeepEqual(err, expected) {
t.Errorf("want err=%v; got %v", expected, err)
}
})
t.Run("nil double ptr value ptr", func(t *testing.T) {
dec := setup(t)
var out **TypeG
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if out == nil {
t.Fatal("want out to not be nil")
}
if expected := (TypeG{String: "a", Int: 1}); **out != expected {
t.Errorf("want expected=%v; got %v", expected, **out)
}
})
t.Run("non ptr value ptr", func(t *testing.T) {
dec := setup(t)
var out TypeG
if err := dec.Decode(&out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if expected := (TypeG{String: "a", Int: 1}); out != expected {
t.Errorf("want expected=%v; got %v", expected, out)
}
})
})
t.Run("decode slice", func(t *testing.T) {
csvr := csv.NewReader(strings.NewReader("String,int\nfirst,1\nsecond,2"))
dec, err := NewDecoder(csvr)
if err != nil {
t.Fatalf("want err == nil; got %v", err)
}
var data []TypeI
if err := dec.Decode(&data); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if len(data) != 2 {
t.Fatalf("want len=2; got %d", len(data))
}
if err := dec.Decode(&data); err != io.EOF {
t.Errorf("want err=EOF; got %v", err)
}
})
t.Run("decode slice - error", func(t *testing.T) {
csvr := csv.NewReader(strings.NewReader("String,int\nfirst,1\nsecond,notint\nthird,3"))
dec, err := NewDecoder(csvr)
if err != nil {
t.Fatalf("want err == nil; got %v", err)
}
var data []TypeI
if err := dec.Decode(&data); err == nil {
t.Errorf("want err!=nil; got %v", err)
}
if len(data) != 2 {
t.Errorf("want len=2; got %d", len(data))
}
if data[1].String != "second" {
t.Errorf("want String=second; got %s", data[1].String)
}
if data[1].Int != 0 {
t.Errorf("want Int=0; got %d", data[1].Int)
}
})
t.Run("decode array", func(t *testing.T) {
csvr := csv.NewReader(strings.NewReader("String,int\nfirst,1\nsecond,2"))
dec, err := NewDecoder(csvr)
if err != nil {
t.Fatalf("want err == nil; got %v", err)
}
var data [1]TypeI
if err := dec.Decode(&data); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if expected := (TypeI{String: "first", Int: 1}); data[0] != expected {
t.Errorf("want %v; got %v", expected, data[0])
}
if err := dec.Decode(&data); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if expected := (TypeI{String: "second", Int: 2}); data[0] != expected {
t.Errorf("want %v; got %v", expected, data[0])
}
if err := dec.Decode(&data); err != io.EOF {
t.Errorf("want err=EOF; got %v", err)
}
})
t.Run("decode array - error", func(t *testing.T) {
csvr := csv.NewReader(strings.NewReader("String,int\nfirst,1\nsecond,notint"))
dec, err := NewDecoder(csvr)
if err != nil {
t.Fatalf("want err == nil; got %v", err)
}
var data [2]TypeI
if err := dec.Decode(&data); err == nil {
t.Errorf("want err!=nil; got %v", err)
}
if data[1].String != "second" {
t.Errorf("want String=second; got %s", data[1].String)
}
if data[1].Int != 0 {
t.Errorf("want Int=0; got %d", data[1].Int)
}
})
t.Run("register panics", func(t *testing.T) {
dec, err := NewDecoder(csv.NewReader(nil), "foo")
if err != nil {
panic(err)
}
fixtures := []struct {
desc string
arg interface{}
}{
{
desc: "not a func",
arg: 1,
},
{
desc: "nil",
arg: nil,
},
{
desc: "T == empty interface",
arg: func([]byte, interface{}) error { return nil },
},
{
desc: "first in not bytes",
arg: func(int, int) error { return nil },
},
{
desc: "out not error",
arg: func([]byte, *int) int { return 0 },
},
{
desc: "func with one in value",
arg: func(int) error { return nil },
},
{
desc: "func with no returns",
arg: func([]byte, int) {},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
var e interface{}
func() {
defer func() {
e = recover()
}()
dec.Register(f.arg)
}()
if e == nil {
t.Error("Register was supposed to panic but it didnt")
}
t.Log(e)
})
}
t.Run("already registered", func(t *testing.T) {
f := func([]byte, int) error { return nil }
dec.Register(f)
var e interface{}
func() {
defer func() {
e = recover()
}()
dec.Register(f)
}()
if e == nil {
t.Error("Register was supposed to panic but it didnt")
}
t.Log(e)
})
})
t.Run("normalize header", func(t *testing.T) {
csvr := csv.NewReader(strings.NewReader("STRING,INT\nfirst,1"))
dec, err := NewDecoder(csvr)
if err != nil {
t.Fatalf("want err == nil; got %v", err)
}
if err := dec.NormalizeHeader(strings.ToLower); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
var data struct {
String string `csv:"string"`
Int int `csv:"int"`
}
if err := dec.Decode(&data); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
if data.String != "first" {
t.Errorf("want String=first; got %s", data.String)
}
if data.Int != 1 {
t.Errorf("want Int=1; got %d", data.Int)
}
})
t.Run("normalize header - duplicate error", func(t *testing.T) {
csvr := csv.NewReader(strings.NewReader("STRING,string\nfirst,1"))
dec, err := NewDecoder(csvr)
if err != nil {
t.Fatalf("want err == nil; got %v", err)
}
if err := dec.NormalizeHeader(strings.ToLower); err == nil {
t.Fatal("want err not to be nil")
}
})
}
func BenchmarkDecode(b *testing.B) {
type A struct {
A int `csv:"a"`
B float64 `csv:"b"`
C string `csv:"c"`
D int64 `csv:"d"`
E int8 `csv:"e"`
F float32 `csv:"f"`
G float32 `csv:"g"`
H float32 `csv:"h"`
I string `csv:"i"`
J int `csv:"j"`
}
header := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}
record := []string{"1", "2.5", "xD", "6", "7", "8", "9", "10", "lol", "10"}
fixtures := []struct {
desc string
len int
}{
{"10 field struct 1 record", 1},
{"10 field struct 10 records", 10},
{"10 field struct 100 records", 100},
{"10 field struct 1000 records", 1000},
{"10 field struct 10000 records", 10000},
}
for _, f := range fixtures {
var records [][]string
for i := 0; i < f.len; i++ {
records = append(records, record)
}
b.Run(f.desc, func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
dec, err := NewDecoder(NewReader(records...), header...)
if err != nil {
b.Fatal(err)
}
var a A
b.StartTimer()
for {
if err := dec.Decode(&a); err == io.EOF {
break
} else if err != nil {
b.Fatal(err)
}
}
}
})
}
b.Run("10 field struct first decode", func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
dec, err := NewDecoder(NewReader(record), header...)
if err != nil {
b.Fatal(err)
}
var a A
b.StartTimer()
if err := dec.Decode(&a); err != nil {
b.Fatal(err)
}
}
})
b.Run("10 field struct second decode", func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
r, err := NewDecoder(NewReader(record, record), header...)
if err != nil {
b.Fatal(err)
}
var a A
if err := r.Decode(&a); err != nil {
b.Fatal(err)
}
a = A{}
b.StartTimer()
if err := r.Decode(&a); err != nil {
b.Fatal(err)
}
}
})
}
type reader struct {
records [][]string
i int
}
func NewReader(records ...[]string) Reader {
return &reader{records, 0}
}
func (r *reader) Read() ([]string, error) {
if r.i >= len(r.records) {
return nil, io.EOF
}
r.i++
return r.records[r.i-1], nil
}
csvutil-1.7.1/doc.go 0000664 0000000 0000000 00000000541 14257146113 0014311 0 ustar 00root root 0000000 0000000 // Package csvutil provides fast and idiomatic mapping between CSV and Go values.
//
// This package does not provide a CSV parser itself, it is based on the Reader and Writer
// interfaces which are implemented by eg. std csv package. This gives a possibility
// of choosing any other CSV writer or reader which may be more performant.
package csvutil
csvutil-1.7.1/encode.go 0000664 0000000 0000000 00000014112 14257146113 0015000 0 ustar 00root root 0000000 0000000 package csvutil
import (
"encoding"
"encoding/base64"
"reflect"
"strconv"
)
var (
textMarshaler = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
csvMarshaler = reflect.TypeOf((*Marshaler)(nil)).Elem()
)
var (
encodeFloat32 = encodeFloatN(32)
encodeFloat64 = encodeFloatN(64)
)
type encodeFunc func(buf []byte, v reflect.Value, omitempty bool) ([]byte, error)
func nopEncode(buf []byte, _ reflect.Value, _ bool) ([]byte, error) {
return buf, nil
}
func encodeFuncValue(fn reflect.Value) encodeFunc {
return func(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
out := fn.Call([]reflect.Value{v})
err, _ := out[1].Interface().(error)
if err != nil {
return nil, err
}
return append(buf, out[0].Bytes()...), nil
}
}
func encodeFuncValuePtr(fn reflect.Value) encodeFunc {
return func(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
if !v.CanAddr() {
fallback, err := encodeFn(v.Type(), false, nil, nil)
if err != nil {
return nil, err
}
return fallback(buf, v, omitempty)
}
out := fn.Call([]reflect.Value{v.Addr()})
err, _ := out[1].Interface().(error)
if err != nil {
return nil, err
}
return append(buf, out[0].Bytes()...), nil
}
}
func encodeString(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
return append(buf, v.String()...), nil
}
func encodeInt(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
n := v.Int()
if n == 0 && omitempty {
return buf, nil
}
return strconv.AppendInt(buf, n, 10), nil
}
func encodeUint(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
n := v.Uint()
if n == 0 && omitempty {
return buf, nil
}
return strconv.AppendUint(buf, n, 10), nil
}
func encodeFloatN(bits int) encodeFunc {
return func(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
f := v.Float()
if f == 0 && omitempty {
return buf, nil
}
return strconv.AppendFloat(buf, f, 'G', -1, bits), nil
}
}
func encodeBool(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
t := v.Bool()
if !t && omitempty {
return buf, nil
}
return strconv.AppendBool(buf, t), nil
}
func encodeInterface(funcMap map[reflect.Type]reflect.Value, funcs []reflect.Value) encodeFunc {
return func(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
if !v.IsValid() || v.IsNil() || !v.Elem().IsValid() {
return buf, nil
}
v = v.Elem()
canAddr := v.Kind() == reflect.Ptr
switch v.Kind() {
case reflect.Ptr, reflect.Interface:
if v.IsNil() {
return buf, nil
}
default:
}
enc, err := encodeFn(v.Type(), canAddr, funcMap, funcs)
if err != nil {
return nil, err
}
return enc(buf, v, omitempty)
}
}
func encodePtrMarshaler(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
if v.CanAddr() {
return encodeMarshaler(buf, v.Addr(), omitempty)
}
fallback, err := encodeFn(v.Type(), false, nil, nil)
if err != nil {
return nil, err
}
return fallback(buf, v, omitempty)
}
func encodeTextMarshaler(buf []byte, v reflect.Value, _ bool) ([]byte, error) {
if v.Kind() == reflect.Ptr && v.IsNil() {
return buf, nil
}
b, err := v.Interface().(encoding.TextMarshaler).MarshalText()
if err != nil {
return nil, &MarshalerError{Type: v.Type(), MarshalerType: "MarshalText", Err: err}
}
return append(buf, b...), nil
}
func encodePtrTextMarshaler(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
if v.CanAddr() {
return encodeTextMarshaler(buf, v.Addr(), omitempty)
}
fallback, err := encodeFn(v.Type(), false, nil, nil)
if err != nil {
return nil, err
}
return fallback(buf, v, omitempty)
}
func encodeMarshaler(buf []byte, v reflect.Value, _ bool) ([]byte, error) {
if v.Kind() == reflect.Ptr && v.IsNil() {
return buf, nil
}
b, err := v.Interface().(Marshaler).MarshalCSV()
if err != nil {
return nil, &MarshalerError{Type: v.Type(), MarshalerType: "MarshalCSV", Err: err}
}
return append(buf, b...), nil
}
func encodePtr(typ reflect.Type, canAddr bool, funcMap map[reflect.Type]reflect.Value, funcs []reflect.Value) (encodeFunc, error) {
next, err := encodeFn(typ.Elem(), canAddr, funcMap, funcs)
if err != nil {
return nil, err
}
return func(buf []byte, v reflect.Value, omitempty bool) ([]byte, error) {
if v.IsNil() {
return buf, nil
}
return next(buf, v.Elem(), omitempty)
}, nil
}
func encodeBytes(buf []byte, v reflect.Value, _ bool) ([]byte, error) {
data := v.Bytes()
l := len(buf)
buf = append(buf, make([]byte, base64.StdEncoding.EncodedLen(len(data)))...)
base64.StdEncoding.Encode(buf[l:], data)
return buf, nil
}
func encodeFn(typ reflect.Type, canAddr bool, funcMap map[reflect.Type]reflect.Value, funcs []reflect.Value) (encodeFunc, error) {
if v, ok := funcMap[typ]; ok {
return encodeFuncValue(v), nil
}
if v, ok := funcMap[reflect.PtrTo(typ)]; ok && canAddr {
return encodeFuncValuePtr(v), nil
}
for _, v := range funcs {
argType := v.Type().In(0)
if typ.AssignableTo(argType) {
return encodeFuncValue(v), nil
}
if canAddr && reflect.PtrTo(typ).AssignableTo(argType) {
return encodeFuncValuePtr(v), nil
}
}
if typ.Implements(csvMarshaler) {
return encodeMarshaler, nil
}
if canAddr && reflect.PtrTo(typ).Implements(csvMarshaler) {
return encodePtrMarshaler, nil
}
if typ.Implements(textMarshaler) {
return encodeTextMarshaler, nil
}
if canAddr && reflect.PtrTo(typ).Implements(textMarshaler) {
return encodePtrTextMarshaler, nil
}
switch typ.Kind() {
case reflect.String:
return encodeString, nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return encodeInt, nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return encodeUint, nil
case reflect.Float32:
return encodeFloat32, nil
case reflect.Float64:
return encodeFloat64, nil
case reflect.Bool:
return encodeBool, nil
case reflect.Interface:
return encodeInterface(funcMap, funcs), nil
case reflect.Ptr:
return encodePtr(typ, canAddr, funcMap, funcs)
case reflect.Slice:
if typ.Elem().Kind() == reflect.Uint8 {
return encodeBytes, nil
}
}
return nil, &UnsupportedTypeError{Type: typ}
}
csvutil-1.7.1/encoder.go 0000664 0000000 0000000 00000026446 14257146113 0015177 0 ustar 00root root 0000000 0000000 package csvutil
import (
"reflect"
"sort"
)
const defaultBufSize = 4096
type encField struct {
field
encodeFunc
}
type encCache struct {
fields []encField
buf []byte
index []int
record []string
}
func newEncCache(k typeKey, funcMap map[reflect.Type]reflect.Value, funcs []reflect.Value, header []string) (_ *encCache, err error) {
fields := cachedFields(k)
encFields := make([]encField, 0, len(fields))
// if header is not empty, we are going to track columns in a set and we will
// track which columns are covered by type fields.
set := make(map[string]bool, len(header))
for _, s := range header {
set[s] = false
}
for _, f := range fields {
if _, ok := set[f.name]; len(header) > 0 && !ok {
continue
}
set[f.name] = true
fn, err := encodeFn(f.baseType, true, funcMap, funcs)
if err != nil {
return nil, err
}
encFields = append(encFields, encField{
field: f,
encodeFunc: fn,
})
}
if len(header) > 0 {
// look for columns that were defined in a header but are not present
// in the provided data type. In case we find any, we will set it to
// a no-op encoder that always produces an empty column.
for k, b := range set {
if b {
continue
}
encFields = append(encFields, encField{
field: field{
name: k,
},
encodeFunc: nopEncode,
})
}
sortEncFields(header, encFields)
}
return &encCache{
fields: encFields,
buf: make([]byte, 0, defaultBufSize),
index: make([]int, len(encFields)),
record: make([]string, len(encFields)),
}, nil
}
// sortEncFields sorts the provided fields according to the given header.
// at this stage header expects to contain matching fields, so both slices
// are expected to be of the same length.
func sortEncFields(header []string, fields []encField) {
set := make(map[string]int, len(header))
for i, s := range header {
set[s] = i
}
sort.Slice(fields, func(i, j int) bool {
return set[fields[i].name] < set[fields[j].name]
})
}
// Encoder writes structs CSV representations to the output stream.
type Encoder struct {
// Tag defines which key in the struct field's tag to scan for names and
// options (Default: 'csv').
Tag string
// If AutoHeader is true, a struct header is encoded during the first call
// to Encode automatically (Default: true).
AutoHeader bool
w Writer
c *encCache
header []string
noHeader bool
typeKey typeKey
funcMap map[reflect.Type]reflect.Value
ifaceFuncs []reflect.Value
}
// NewEncoder returns a new encoder that writes to w.
func NewEncoder(w Writer) *Encoder {
return &Encoder{
w: w,
noHeader: true,
AutoHeader: true,
}
}
// Register registers a custom encoding function for a concrete type or interface.
// The argument f must be of type:
// func(T) ([]byte, error)
//
// T must be a concrete type such as Foo or *Foo, or interface that has at
// least one method.
//
// During encoding, fields are matched by the concrete type first. If match is not
// found then Encoder looks if field implements any of the registered interfaces
// in order they were registered.
//
// Register panics if:
// - f does not match the right signature
// - f is an empty interface
// - f was already registered
//
// Register is based on the encoding/json proposal:
// https://github.com/golang/go/issues/5901.
func (e *Encoder) Register(f interface{}) {
v := reflect.ValueOf(f)
typ := v.Type()
if typ.Kind() != reflect.Func ||
typ.NumIn() != 1 || typ.NumOut() != 2 ||
typ.Out(0) != _bytes || typ.Out(1) != _error {
panic("csvutil: func must be of type func(T) ([]byte, error)")
}
argType := typ.In(0)
if argType.Kind() == reflect.Interface && argType.NumMethod() == 0 {
panic("csvutil: func argument type must not be an empty interface")
}
if e.funcMap == nil {
e.funcMap = make(map[reflect.Type]reflect.Value)
}
if _, ok := e.funcMap[argType]; ok {
panic("csvutil: func " + typ.String() + " already registered")
}
e.funcMap[argType] = v
if argType.Kind() == reflect.Interface {
e.ifaceFuncs = append(e.ifaceFuncs, v)
}
}
// SetHeader overrides the provided data type's default header. Fields are
// encoded in the order of the provided header. If a column specified in the
// header doesn't exist in the provided type, it will be encoded as an empty
// column. Fields that are not part of the provided header are ignored.
// Encoder can't guarantee the right order if the provided header contains
// duplicate column names.
//
// SetHeader must be called before EncodeHeader and/or Encode in order to take
// effect.
func (enc *Encoder) SetHeader(header []string) {
cp := make([]string, len(header))
copy(cp, header)
enc.header = cp
}
// Encode writes the CSV encoding of v to the output stream. The provided
// argument v must be a struct, struct slice or struct array.
//
// Only the exported fields will be encoded.
//
// First call to Encode will write a header unless EncodeHeader was called first
// or AutoHeader is false. Header names can be customized by using tags
// ('csv' by default), otherwise original Field names are used.
//
// If header was provided through SetHeader then it overrides the provided data
// type's default header. Fields are encoded in the order of the provided header.
// If a column specified in the header doesn't exist in the provided type, it will
// be encoded as an empty column. Fields that are not part of the provided header
// are ignored. Encoder can't guarantee the right order if the provided header
// contains duplicate column names.
//
// Header and fields are written in the same order as struct fields are defined.
// Embedded struct's fields are treated as if they were part of the outer struct.
// Fields that are embedded types and that are tagged are treated like any
// other field, but they have to implement Marshaler or encoding.TextMarshaler
// interfaces.
//
// Marshaler interface has the priority over encoding.TextMarshaler.
//
// Tagged fields have the priority over non tagged fields with the same name.
//
// Following the Go visibility rules if there are multiple fields with the same
// name (tagged or not tagged) on the same level and choice between them is
// ambiguous, then all these fields will be ignored.
//
// Nil values will be encoded as empty strings. Same will happen if 'omitempty'
// tag is set, and the value is a default value like 0, false or nil interface.
//
// Bool types are encoded as 'true' or 'false'.
//
// Float types are encoded using strconv.FormatFloat with precision -1 and 'G'
// format. NaN values are encoded as 'NaN' string.
//
// Fields of type []byte are being encoded as base64-encoded strings.
//
// Fields can be excluded from encoding by using '-' tag option.
//
// Examples of struct tags:
//
// // Field appears as 'myName' header in CSV encoding.
// Field int `csv:"myName"`
//
// // Field appears as 'Field' header in CSV encoding.
// Field int
//
// // Field appears as 'myName' header in CSV encoding and is an empty string
// // if Field is 0.
// Field int `csv:"myName,omitempty"`
//
// // Field appears as 'Field' header in CSV encoding and is an empty string
// // if Field is 0.
// Field int `csv:",omitempty"`
//
// // Encode ignores this field.
// Field int `csv:"-"`
//
// // Encode treats this field exactly as if it was an embedded field and adds
// // "my_prefix_" to each field's name.
// Field Struct `csv:"my_prefix_,inline"`
//
// // Encode treats this field exactly as if it was an embedded field.
// Field Struct `csv:",inline"`
//
// Fields with inline tags that have a non-empty prefix must not be cyclic
// structures. Passing such values to Encode will result in an infinite loop.
//
// Encode doesn't flush data. The caller is responsible for calling Flush() if
// the used Writer supports it.
func (e *Encoder) Encode(v interface{}) error {
return e.encode(reflect.ValueOf(v))
}
// EncodeHeader writes the CSV header of the provided struct value to the output
// stream. The provided argument v must be a struct value.
//
// The first Encode method call will not write header if EncodeHeader was called
// before it. This method can be called in cases when a data set could be
// empty, but header is desired.
//
// EncodeHeader is like Header function, but it works with the Encoder and writes
// directly to the output stream. Look at Header documentation for the exact
// header encoding rules.
func (e *Encoder) EncodeHeader(v interface{}) error {
typ, err := valueType(v)
if err != nil {
return err
}
return e.encodeHeader(typ)
}
func (e *Encoder) encode(v reflect.Value) error {
val := walkValue(v)
if !val.IsValid() {
return &InvalidEncodeError{}
}
switch val.Kind() {
case reflect.Struct:
return e.encodeStruct(val)
case reflect.Array, reflect.Slice:
if walkType(val.Type().Elem()).Kind() != reflect.Struct {
return &InvalidEncodeError{v.Type()}
}
return e.encodeArray(val)
default:
return &InvalidEncodeError{v.Type()}
}
}
func (e *Encoder) encodeStruct(v reflect.Value) error {
if e.AutoHeader && e.noHeader {
if err := e.encodeHeader(v.Type()); err != nil {
return err
}
}
return e.marshal(v)
}
func (e *Encoder) encodeArray(v reflect.Value) error {
l := v.Len()
for i := 0; i < l; i++ {
if err := e.encodeStruct(walkValue(v.Index(i))); err != nil {
return err
}
}
return nil
}
func (e *Encoder) encodeHeader(typ reflect.Type) error {
fields, _, _, record, err := e.cache(typ)
if err != nil {
return err
}
for i, f := range fields {
record[i] = f.name
}
if err := e.w.Write(record); err != nil {
return err
}
e.noHeader = false
return nil
}
func (e *Encoder) marshal(v reflect.Value) error {
fields, buf, index, record, err := e.cache(v.Type())
if err != nil {
return err
}
for i, f := range fields {
v := walkIndex(v, f.index)
omitempty := f.tag.omitEmpty
if v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {
// We should disable omitempty for pointer and interface values,
// because if it's nil we will automatically encode it as an empty
// string. However, the initialized pointer should not be affected,
// even if it's a default value.
omitempty = false
}
if !v.IsValid() {
index[i] = 0
continue
}
b, err := f.encodeFunc(buf, v, omitempty)
if err != nil {
return err
}
index[i], buf = len(b)-len(buf), b
}
out := string(buf)
for i, n := range index {
record[i], out = out[:n], out[n:]
}
e.c.buf = buf[:0]
return e.w.Write(record)
}
func (e *Encoder) tag() string {
if e.Tag == "" {
return defaultTag
}
return e.Tag
}
func (e *Encoder) cache(typ reflect.Type) ([]encField, []byte, []int, []string, error) {
if k := (typeKey{e.tag(), typ}); k != e.typeKey {
c, err := newEncCache(k, e.funcMap, e.ifaceFuncs, e.header)
if err != nil {
return nil, nil, nil, nil, err
}
e.c, e.typeKey = c, k
}
return e.c.fields, e.c.buf[:0], e.c.index, e.c.record, nil
}
func walkIndex(v reflect.Value, index []int) reflect.Value {
for _, i := range index {
v = walkPtr(v)
if !v.IsValid() {
return reflect.Value{}
}
v = v.Field(i)
}
return v
}
func walkPtr(v reflect.Value) reflect.Value {
for v.Kind() == reflect.Ptr {
v = v.Elem()
}
return v
}
func walkValue(v reflect.Value) reflect.Value {
for {
switch v.Kind() {
case reflect.Ptr, reflect.Interface:
v = v.Elem()
default:
return v
}
}
}
func walkType(typ reflect.Type) reflect.Type {
for typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
return typ
}
csvutil-1.7.1/encoder_test.go 0000664 0000000 0000000 00000130343 14257146113 0016226 0 ustar 00root root 0000000 0000000 package csvutil
import (
"bytes"
"encoding"
"encoding/csv"
"encoding/json"
"errors"
"math"
"reflect"
"testing"
)
var Error = errors.New("error")
var nilIface interface{}
var nilPtr *TypeF
var nilIfacePtr interface{} = nilPtr
type embeddedMap map[string]string
type Embedded14 Embedded3
func (e *Embedded14) MarshalCSV() ([]byte, error) {
return json.Marshal(e)
}
type Embedded15 Embedded3
func (e *Embedded15) MarshalText() ([]byte, error) {
return json.Marshal(Embedded3(*e))
}
type CSVMarshaler struct {
Err error
}
func (m CSVMarshaler) MarshalCSV() ([]byte, error) {
if m.Err != nil {
return nil, m.Err
}
return []byte("csvmarshaler"), nil
}
type PtrRecCSVMarshaler int
func (m *PtrRecCSVMarshaler) MarshalCSV() ([]byte, error) {
return []byte("ptrreccsvmarshaler"), nil
}
func (m *PtrRecCSVMarshaler) CSV() ([]byte, error) {
return []byte("ptrreccsvmarshaler.CSV"), nil
}
type PtrRecTextMarshaler int
func (m *PtrRecTextMarshaler) MarshalText() ([]byte, error) {
return []byte("ptrrectextmarshaler"), nil
}
type TextMarshaler struct {
Err error
}
func (m TextMarshaler) MarshalText() ([]byte, error) {
if m.Err != nil {
return nil, m.Err
}
return []byte("textmarshaler"), nil
}
type CSVTextMarshaler struct {
CSVMarshaler
TextMarshaler
}
type Inline struct {
J1 TypeJ `csv:",inline"`
J2 TypeJ `csv:"prefix-,inline"`
String string `csv:"top-string"`
String2 string `csv:"STR"`
}
type Inline2 struct {
S string
A Inline3 `csv:"A,inline"`
B Inline3 `csv:",inline"`
}
type Inline3 struct {
Inline4 `csv:",inline"`
}
type Inline4 struct {
A string
}
type Inline5 struct {
A Inline2 `csv:"A,inline"`
B Inline2 `csv:",inline"`
}
type Inline6 struct {
A Inline7 `csv:",inline"`
}
type Inline7 struct {
A *Inline6 `csv:",inline"`
X int
}
type Inline8 struct {
F *Inline4 `csv:"A,inline"`
AA int
}
type TypeH struct {
Int int `csv:"int,omitempty"`
Int8 int8 `csv:"int8,omitempty"`
Int16 int16 `csv:"int16,omitempty"`
Int32 int32 `csv:"int32,omitempty"`
Int64 int64 `csv:"int64,omitempty"`
UInt uint `csv:"uint,omitempty"`
Uint8 uint8 `csv:"uint8,omitempty"`
Uint16 uint16 `csv:"uint16,omitempty"`
Uint32 uint32 `csv:"uint32,omitempty"`
Uint64 uint64 `csv:"uint64,omitempty"`
Float32 float32 `csv:"float32,omitempty"`
Float64 float64 `csv:"float64,omitempty"`
String string `csv:"string,omitempty"`
Bool bool `csv:"bool,omitempty"`
V interface{} `csv:"interface,omitempty"`
}
type TypeM struct {
*TextMarshaler `csv:"text"`
}
func TestEncoder(t *testing.T) {
fixtures := []struct {
desc string
in []interface{}
regFunc []interface{}
out [][]string
err error
}{
{
desc: "test all types",
in: []interface{}{
TypeF{
Int: 1,
Pint: pint(2),
Int8: 3,
Pint8: pint8(4),
Int16: 5,
Pint16: pint16(6),
Int32: 7,
Pint32: pint32(8),
Int64: 9,
Pint64: pint64(10),
UInt: 11,
Puint: puint(12),
Uint8: 13,
Puint8: puint8(14),
Uint16: 15,
Puint16: puint16(16),
Uint32: 17,
Puint32: puint32(18),
Uint64: 19,
Puint64: puint64(20),
Float32: 21,
Pfloat32: pfloat32(22),
Float64: 23,
Pfloat64: pfloat64(24),
String: "25",
PString: pstring("26"),
Bool: true,
Pbool: pbool(true),
V: "true",
Pv: pinterface("1"),
Binary: Binary,
PBinary: &BinaryLarge,
},
TypeF{},
},
out: [][]string{
{
"int", "pint", "int8", "pint8", "int16", "pint16", "int32",
"pint32", "int64", "pint64", "uint", "puint", "uint8", "puint8",
"uint16", "puint16", "uint32", "puint32", "uint64", "puint64",
"float32", "pfloat32", "float64", "pfloat64", "string", "pstring",
"bool", "pbool", "interface", "pinterface", "binary", "pbinary",
},
{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11",
"12", "13", "14", "15", "16", "17", "18", "19", "20", "21",
"22", "23", "24", "25", "26", "true", "true", "true", "1",
EncodedBinary, EncodedBinaryLarge,
},
{"0", "", "0", "", "0", "", "0", "", "0", "", "0", "",
"0", "", "0", "", "0", "", "0", "", "0", "", "0", "", "", "",
"false", "", "", "", "", "",
},
},
},
{
desc: "tags and unexported fields",
in: []interface{}{
TypeG{
String: "string",
Int: 1,
Float: 3.14,
unexported1: 100,
unexported2: 200,
},
},
out: [][]string{
{"String", "Int"},
{"string", "1"},
},
},
{
desc: "omitempty tags",
in: []interface{}{
TypeH{},
},
out: [][]string{
{"int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16",
"uint32", "uint64", "float32", "float64", "string", "bool", "interface",
},
{"", "", "", "", "", "", "", "", "", "", "", "", "", "", ""},
},
},
{
desc: "omitempty tags on pointers - non nil default values",
in: []interface{}{
struct {
Pint *int `csv:",omitempty"`
PPint **int `csv:",omitempty"`
PPint2 **int `csv:",omitempty"`
PString *string `csv:",omitempty"`
PBool *bool `csv:",omitempty"`
Iint *interface{} `csv:",omitempty"`
}{
pint(0),
ppint(0),
new(*int),
pstring(""),
pbool(false),
pinterface(0),
},
},
out: [][]string{
{"Pint", "PPint", "PPint2", "PString", "PBool", "Iint"},
{"0", "0", "", "", "false", "0"},
},
},
{
desc: "omitempty tags on pointers - nil ptrs",
in: []interface{}{
struct {
Pint *int `csv:",omitempty"`
PPint **int `csv:",omitempty"`
PString *string `csv:",omitempty"`
PBool *bool `csv:",omitempty"`
Iint *interface{} `csv:",omitempty"`
}{},
},
out: [][]string{
{"Pint", "PPint", "PString", "PBool", "Iint"},
{"", "", "", "", ""},
},
},
{
desc: "omitempty tags on interfaces - non nil default values",
in: []interface{}{
struct {
Iint interface{} `csv:",omitempty"`
IPint interface{} `csv:",omitempty"`
}{
0,
pint(0),
},
struct {
Iint interface{} `csv:",omitempty"`
IPint interface{} `csv:",omitempty"`
}{
1,
pint(1),
},
},
out: [][]string{
{"Iint", "IPint"},
{"0", "0"},
{"1", "1"},
},
},
{
desc: "omitempty tags on interfaces - nil",
in: []interface{}{
struct {
Iint interface{} `csv:",omitempty"`
IPint interface{} `csv:",omitempty"`
}{
nil,
nil,
},
struct {
Iint interface{} `csv:",omitempty"`
IPint interface{} `csv:",omitempty"`
}{
(*int)(nil),
pinterface((*int)(nil)),
},
},
out: [][]string{
{"Iint", "IPint"},
{"", ""},
{"", ""},
},
},
{
desc: "embedded types #1",
in: []interface{}{
TypeA{
Embedded1: Embedded1{
String: "string1",
Float: 1,
},
String: "string",
Embedded2: Embedded2{
Float: 2,
Bool: true,
},
Int: 10,
},
},
out: [][]string{
{"string", "bool", "int"},
{"string", "true", "10"},
},
},
{
desc: "embedded non struct tagged types",
in: []interface{}{
TypeB{
Embedded3: Embedded3{"key": "val"},
String: "string1",
},
},
out: [][]string{
{"json", "string"},
{`{"key":"val"}`, "string1"},
},
},
{
desc: "embedded non struct tagged types with pointer receiver MarshalCSV",
in: []interface{}{
&struct {
Embedded14 `csv:"json"`
A Embedded14 `csv:"json2"`
}{
Embedded14: Embedded14{"key": "val"},
A: Embedded14{"key1": "val1"},
},
struct {
*Embedded14 `csv:"json"`
A *Embedded14 `csv:"json2"`
}{
Embedded14: &Embedded14{"key": "val"},
A: &Embedded14{"key1": "val1"},
},
},
out: [][]string{
{"json", "json2"},
{`{"key":"val"}`, `{"key1":"val1"}`},
{`{"key":"val"}`, `{"key1":"val1"}`},
},
},
{
desc: "embedded non struct tagged types with pointer receiver MarshalText",
in: []interface{}{
&struct {
Embedded15 `csv:"json"`
A Embedded15 `csv:"json2"`
}{
Embedded15: Embedded15{"key": "val"},
A: Embedded15{"key1": "val1"},
},
struct {
*Embedded15 `csv:"json"`
A *Embedded15 `csv:"json2"`
}{
Embedded15: &Embedded15{"key": "val"},
A: &Embedded15{"key1": "val1"},
},
},
out: [][]string{
{"json", "json2"},
{`{"key":"val"}`, `{"key1":"val1"}`},
{`{"key":"val"}`, `{"key1":"val1"}`},
},
},
{
desc: "embedded pointer types",
in: []interface{}{
TypeC{
Embedded1: &Embedded1{
String: "string2",
Float: 1,
},
String: "string1",
},
},
out: [][]string{
{"float", "string"},
{`1`, "string1"},
},
},
{
desc: "embedded pointer types with nil values",
in: []interface{}{
TypeC{
Embedded1: nil,
String: "string1",
},
},
out: [][]string{
{"float", "string"},
{``, "string1"},
},
},
{
desc: "embedded non struct tagged pointer types",
in: []interface{}{
TypeD{
Embedded3: &Embedded3{"key": "val"},
String: "string1",
},
},
out: [][]string{
{"json", "string"},
{`{"key":"val"}`, "string1"},
},
},
{
desc: "embedded non struct tagged pointer types with nil value - textmarshaler",
in: []interface{}{
TypeM{
TextMarshaler: nil,
},
},
out: [][]string{
{"text"},
{""},
},
},
{
desc: "embedded non struct tagged pointer types with nil value - csvmarshaler",
in: []interface{}{
TypeD{
Embedded3: nil,
String: "string1",
},
},
out: [][]string{
{"json", "string"},
{"", "string1"},
},
},
{
desc: "tagged fields priority",
in: []interface{}{
TagPriority{Foo: 1, Bar: 2},
},
out: [][]string{
{"Foo"},
{"2"},
},
},
{
desc: "conflicting embedded fields #1",
in: []interface{}{
Embedded5{
Embedded6: Embedded6{X: 60},
Embedded7: Embedded7{X: 70},
Embedded8: Embedded8{
Embedded9: Embedded9{
X: 90,
Y: 91,
},
},
},
},
out: [][]string{
{"Y"},
{"91"},
},
},
{
desc: "conflicting embedded fields #2",
in: []interface{}{
Embedded10{
Embedded11: Embedded11{
Embedded6: Embedded6{X: 60},
},
Embedded12: Embedded12{
Embedded6: Embedded6{X: 60},
},
Embedded13: Embedded13{
Embedded8: Embedded8{
Embedded9: Embedded9{
X: 90,
Y: 91,
},
},
},
},
},
out: [][]string{
{"Y"},
{"91"},
},
},
{
desc: "double pointer",
in: []interface{}{
TypeE{
String: &PString,
Int: &Int,
},
},
out: [][]string{
{"string", "int"},
{"string", "10"},
},
},
{
desc: "nil double pointer",
in: []interface{}{
TypeE{},
},
out: [][]string{
{"string", "int"},
{"", ""},
},
},
{
desc: "unexported non-struct embedded",
in: []interface{}{
struct {
A int
embeddedMap
}{1, make(embeddedMap)},
},
out: [][]string{
{"A"},
{"1"},
},
},
{
desc: "cyclic reference",
in: []interface{}{
A{
B: B{Y: 2, A: &A{}},
X: 1,
},
},
out: [][]string{
{"Y", "X"},
{"2", "1"},
},
},
{
desc: "ptr receiver csv marshaler",
in: []interface{}{
&struct {
A PtrRecCSVMarshaler
}{},
struct {
A PtrRecCSVMarshaler
}{},
struct {
A *PtrRecCSVMarshaler
}{new(PtrRecCSVMarshaler)},
&struct {
A *PtrRecCSVMarshaler
}{new(PtrRecCSVMarshaler)},
&struct {
A *PtrRecCSVMarshaler
}{},
},
out: [][]string{
{"A"},
{"ptrreccsvmarshaler"},
{"0"},
{"ptrreccsvmarshaler"},
{"ptrreccsvmarshaler"},
{""},
},
},
{
desc: "ptr receiver text marshaler",
in: []interface{}{
&struct {
A PtrRecTextMarshaler
}{},
struct {
A PtrRecTextMarshaler
}{},
struct {
A *PtrRecTextMarshaler
}{new(PtrRecTextMarshaler)},
&struct {
A *PtrRecTextMarshaler
}{new(PtrRecTextMarshaler)},
&struct {
A *PtrRecTextMarshaler
}{},
},
out: [][]string{
{"A"},
{"ptrrectextmarshaler"},
{"0"},
{"ptrrectextmarshaler"},
{"ptrrectextmarshaler"},
{""},
},
},
{
desc: "text marshaler",
in: []interface{}{
struct {
A CSVMarshaler
}{},
struct {
A TextMarshaler
}{},
struct {
A struct {
TextMarshaler
CSVMarshaler
}
}{},
},
out: [][]string{
{"A"},
{"csvmarshaler"},
{"textmarshaler"},
{"csvmarshaler"},
},
},
{
desc: "primitive type alias implementing Marshaler",
in: []interface{}{
EnumType{Enum: EnumFirst},
EnumType{Enum: EnumSecond},
},
out: [][]string{
{"enum"},
{"first"},
{"second"},
},
},
{
desc: "aliased type",
in: []interface{}{
struct{ Float float64 }{3.14},
},
out: [][]string{
{"Float"},
{"3.14"},
},
},
{
desc: "embedded tagged marshalers",
in: []interface{}{
struct {
CSVMarshaler `csv:"csv"`
TextMarshaler `csv:"text"`
}{},
},
out: [][]string{
{"csv", "text"},
{"csvmarshaler", "textmarshaler"},
},
},
{
desc: "embedded pointer tagged marshalers",
in: []interface{}{
struct {
*CSVMarshaler `csv:"csv"`
*TextMarshaler `csv:"text"`
}{&CSVMarshaler{}, &TextMarshaler{}},
},
out: [][]string{
{"csv", "text"},
{"csvmarshaler", "textmarshaler"},
},
},
{
desc: "inline fields",
in: []interface{}{
Inline{
J1: TypeJ{
String: "j1",
Int: "1",
Float: "1",
Embedded16: Embedded16{Bool: true, Uint8: 1},
},
J2: TypeJ{
String: "j2",
Int: "2",
Float: "2",
Embedded16: Embedded16{Bool: true, Uint8: 2},
},
String: "top-level-str",
String2: "STR",
},
},
out: [][]string{
{"int", "Bool", "Uint8", "float", "prefix-STR", "prefix-int", "prefix-Bool", "prefix-Uint8", "prefix-float", "top-string", "STR"},
{"1", "true", "1", "1", "j2", "2", "true", "2", "2", "top-level-str", "STR"},
},
},
{
desc: "inline chain",
in: []interface{}{
Inline5{
A: Inline2{
S: "1",
A: Inline3{
Inline4: Inline4{A: "11"},
},
B: Inline3{
Inline4: Inline4{A: "12"},
},
},
B: Inline2{
S: "2",
A: Inline3{
Inline4: Inline4{A: "21"},
},
B: Inline3{
Inline4: Inline4{A: "22"},
},
},
},
},
out: [][]string{
{"AS", "AAA", "S", "A"},
{"1", "11", "2", "22"},
},
},
{
desc: "cyclic inline - no prefix",
in: []interface{}{
Inline6{
A: Inline7{
A: &Inline6{A: Inline7{
A: &Inline6{},
X: 10,
}},
X: 1,
},
},
},
out: [][]string{
{"X"},
{"1"},
},
},
{
desc: "embedded with inline tag",
in: []interface{}{
struct {
Inline7 `csv:"A,inline"`
}{
Inline7: Inline7{
A: &Inline6{A: Inline7{
A: &Inline6{},
X: 10,
}},
X: 1,
},
},
},
out: [][]string{
{"AX"},
{"1"},
},
},
{
desc: "embedded with empty inline tag",
in: []interface{}{
struct {
Inline7 `csv:",inline"`
}{
Inline7: Inline7{
A: &Inline6{A: Inline7{
A: &Inline6{},
X: 10,
}},
X: 1,
},
},
},
out: [][]string{
{"X"},
{"1"},
},
},
{
desc: "embedded with ptr inline tag",
in: []interface{}{
struct {
*Inline7 `csv:"A,inline"`
}{
Inline7: &Inline7{
A: &Inline6{A: Inline7{
A: &Inline6{},
X: 10,
}},
X: 1,
},
},
},
out: [][]string{
{"AX"},
{"1"},
},
},
{
desc: "inline visibility rules - top field first",
in: []interface{}{
struct {
AA string
F Inline4 `csv:"A,inline"`
}{
AA: "1",
F: Inline4{A: "10"},
},
},
out: [][]string{
{"AA"},
{"1"},
},
},
{
desc: "inline visibility rules - top field last",
in: []interface{}{
Inline8{
F: &Inline4{A: "10"},
AA: 1,
},
},
out: [][]string{
{"AA"},
{"1"},
},
},
{
desc: "ignore inline tag on non struct",
in: []interface{}{
struct {
X int `csv:",inline"`
Y int `csv:"y,inline"`
}{
X: 1,
Y: 2,
},
},
out: [][]string{
{"X", "y"},
{"1", "2"},
},
},
{
desc: "registered func - non ptr elem",
in: []interface{}{
struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Pint: pint(0),
Iface: 34,
Piface: pinterface(34),
},
},
regFunc: []interface{}{
func(int) ([]byte, error) { return []byte("int"), nil },
},
out: [][]string{
{"Int", "Pint", "Iface", "Piface"},
{"int", "int", "int", "int"},
},
},
{
desc: "registered func - ptr elem",
in: []interface{}{
&struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Pint: pint(0),
Iface: 34,
Piface: pinterface(34),
},
},
regFunc: []interface{}{
func(int) ([]byte, error) { return []byte("int"), nil },
},
out: [][]string{
{"Int", "Pint", "Iface", "Piface"},
{"int", "int", "int", "int"},
},
},
{
desc: "registered func - ptr type - non ptr elem",
in: []interface{}{
struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Pint: pint(0),
Iface: 34,
Piface: pinterface(pint(34)),
},
},
regFunc: []interface{}{
func(*int) ([]byte, error) { return []byte("int"), nil },
},
out: [][]string{
{"Int", "Pint", "Iface", "Piface"},
{"0", "int", "34", "int"},
},
},
{
desc: "registered func - ptr type - ptr elem",
in: []interface{}{
&struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Pint: pint(0),
Iface: 34,
Piface: pinterface(pint(34)),
},
},
regFunc: []interface{}{
func(*int) ([]byte, error) { return []byte("int"), nil },
},
out: [][]string{
{"Int", "Pint", "Iface", "Piface"},
{"int", "int", "34", "int"},
},
},
{
desc: "registered func - mixed types - non ptr elem",
in: []interface{}{
struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Pint: pint(0),
Iface: 34,
Piface: pinterface(pint(34)),
},
},
regFunc: []interface{}{
func(int) ([]byte, error) { return []byte("int"), nil },
func(*int) ([]byte, error) { return []byte("*int"), nil },
},
out: [][]string{
{"Int", "Pint", "Iface", "Piface"},
{"int", "*int", "int", "*int"},
},
},
{
desc: "registered func - mixed types - ptr elem",
in: []interface{}{
&struct {
Int int
Pint *int
Iface interface{}
Piface *interface{}
}{
Pint: pint(0),
Iface: 34,
Piface: pinterface(pint(34)),
},
},
regFunc: []interface{}{
func(int) ([]byte, error) { return []byte("int"), nil },
func(*int) ([]byte, error) { return []byte("*int"), nil },
},
out: [][]string{
{"Int", "Pint", "Iface", "Piface"},
{"int", "*int", "int", "*int"},
},
},
{
desc: "registered func - interfaces",
in: []interface{}{
&struct {
CSVMarshaler Marshaler
Marshaler CSVMarshaler
PMarshaler *CSVMarshaler
CSVTextMarshaler CSVTextMarshaler
PCSVTextMarshaler *CSVTextMarshaler
PtrRecCSVMarshaler PtrRecCSVMarshaler
PtrRecTextMarshaler PtrRecTextMarshaler
}{
PMarshaler: &CSVMarshaler{},
PCSVTextMarshaler: &CSVTextMarshaler{},
},
},
regFunc: []interface{}{
func(Marshaler) ([]byte, error) { return []byte("registered.marshaler"), nil },
func(encoding.TextMarshaler) ([]byte, error) { return []byte("registered.textmarshaler"), nil },
},
out: [][]string{
{"CSVMarshaler", "Marshaler", "PMarshaler", "CSVTextMarshaler", "PCSVTextMarshaler", "PtrRecCSVMarshaler", "PtrRecTextMarshaler"},
{"registered.marshaler", "registered.marshaler", "registered.marshaler", "registered.marshaler", "registered.marshaler", "registered.marshaler", "registered.textmarshaler"},
},
},
{
desc: "registered func - interface order",
in: []interface{}{
&struct {
CSVTextMarshaler CSVTextMarshaler
PCSVTextMarshaler *CSVTextMarshaler
}{
PCSVTextMarshaler: &CSVTextMarshaler{},
},
},
regFunc: []interface{}{
func(encoding.TextMarshaler) ([]byte, error) { return []byte("registered.textmarshaler"), nil },
func(Marshaler) ([]byte, error) { return []byte("registered.marshaler"), nil },
},
out: [][]string{
{"CSVTextMarshaler", "PCSVTextMarshaler"},
{"registered.textmarshaler", "registered.textmarshaler"},
},
},
{
desc: "registered func - method",
in: []interface{}{
&struct {
PtrRecCSVMarshaler PtrRecCSVMarshaler
}{},
struct {
PtrRecCSVMarshaler PtrRecCSVMarshaler
}{},
},
regFunc: []interface{}{
(*PtrRecCSVMarshaler).CSV,
},
out: [][]string{
{"PtrRecCSVMarshaler"},
{"ptrreccsvmarshaler.CSV"},
{"0"},
},
},
{
desc: "registered func - fallback error",
in: []interface{}{
struct {
Embedded14
}{},
},
regFunc: []interface{}{
(*Embedded14).MarshalCSV,
},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(Embedded14{}),
},
},
{
desc: "registered interface func - returning error",
in: []interface{}{
&struct {
Embedded14 Embedded14
}{},
},
regFunc: []interface{}{
func(Marshaler) ([]byte, error) { return nil, Error },
},
err: Error,
},
{
desc: "registered func - returning error",
in: []interface{}{
&struct {
A InvalidType
}{},
},
regFunc: []interface{}{
func(*InvalidType) ([]byte, error) { return nil, Error },
},
err: Error,
},
{
desc: "registered func - fallback error on interface",
in: []interface{}{
struct {
Embedded14
}{},
},
regFunc: []interface{}{
func(m Marshaler) ([]byte, error) { return nil, nil },
},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(Embedded14{}),
},
},
{
desc: "marshaler fallback error",
in: []interface{}{
struct {
Embedded14
}{},
},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(Embedded14{}),
},
},
{
desc: "encode different types",
// This doesnt mean the output csv is valid. Generally this is an invalid
// use. However, we need to make sure that the encoder is doing what it is
// asked to... correctly.
in: []interface{}{
struct {
A int
}{},
struct {
A int
B string
}{},
struct {
A int
}{},
struct{}{},
},
out: [][]string{
{"A"},
{"0"},
{"0", ""},
{"0"},
{},
},
},
{
desc: "encode interface values",
in: []interface{}{
struct {
V interface{}
}{1},
struct {
V interface{}
}{pint(10)},
struct {
V interface{}
}{ppint(100)},
struct {
V interface{}
}{pppint(1000)},
struct {
V *interface{}
}{pinterface(ppint(10000))},
struct {
V *interface{}
}{func() *interface{} {
var v interface{} = pppint(100000)
var vv interface{} = v
return &vv
}()},
struct {
V interface{}
}{func() interface{} {
var v interface{} = &CSVMarshaler{}
var vv interface{} = v
return &vv
}()},
struct {
V interface{}
}{func() interface{} {
var v interface{} = CSVMarshaler{}
var vv interface{} = v
return &vv
}()},
struct {
V interface{}
}{func() interface{} {
var v interface{} = &CSVMarshaler{}
var vv interface{} = v
return vv
}()},
struct {
V interface{}
}{
V: func() interface{} {
return PtrRecCSVMarshaler(5)
}(),
},
struct {
V interface{}
}{
V: func() interface{} {
m := PtrRecCSVMarshaler(5)
return &m
}(),
},
struct {
V interface{}
}{func() interface{} {
var v interface{}
var vv interface{} = v
return &vv
}()},
},
out: [][]string{
{"V"},
{"1"},
{"10"},
{"100"},
{"1000"},
{"10000"},
{"100000"},
{"csvmarshaler"},
{"csvmarshaler"},
{"csvmarshaler"},
{"5"},
{"ptrreccsvmarshaler"},
{""},
},
},
{
desc: "encode NaN",
in: []interface{}{
struct {
Float float64
}{math.NaN()},
},
out: [][]string{
{"Float"},
{"NaN"},
},
},
{
desc: "encode NaN with aliased type",
in: []interface{}{
struct {
Float Float
}{Float(math.NaN())},
},
out: [][]string{
{"Float"},
{"NaN"},
},
},
{
desc: "empty struct",
in: []interface{}{
struct{}{},
},
out: [][]string{{}, {}},
},
{
desc: "value wrapped in interfaces and pointers",
in: []interface{}{
func() (v interface{}) { v = &struct{ A int }{5}; return v }(),
},
out: [][]string{{"A"}, {"5"}},
},
{
desc: "csv marshaler error",
in: []interface{}{
struct {
A CSVMarshaler
}{
A: CSVMarshaler{Err: Error},
},
},
err: &MarshalerError{Type: reflect.TypeOf(CSVMarshaler{}), MarshalerType: "MarshalCSV", Err: Error},
},
{
desc: "csv marshaler error as registered error",
in: []interface{}{
struct {
A CSVMarshaler
}{
A: CSVMarshaler{Err: Error},
},
},
regFunc: []interface{}{
CSVMarshaler.MarshalCSV,
},
err: Error,
},
{
desc: "text marshaler error",
in: []interface{}{
struct {
A TextMarshaler
}{
A: TextMarshaler{Err: Error},
},
},
err: &MarshalerError{Type: reflect.TypeOf(TextMarshaler{}), MarshalerType: "MarshalText", Err: Error},
},
{
desc: "text marshaler fallback error - ptr reciever",
in: []interface{}{
struct {
A Embedded15
}{},
},
err: &UnsupportedTypeError{Type: reflect.TypeOf(Embedded15{})},
},
{
desc: "text marshaler error as registered func",
in: []interface{}{
struct {
A TextMarshaler
}{
A: TextMarshaler{Err: Error},
},
},
regFunc: []interface{}{
TextMarshaler.MarshalText,
},
err: Error,
},
{
desc: "unsupported type",
in: []interface{}{
InvalidType{},
},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(struct{}{}),
},
},
{
desc: "unsupported double pointer type",
in: []interface{}{
struct {
A **struct{}
}{},
},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(struct{}{}),
},
},
{
desc: "unsupported interface type",
in: []interface{}{
TypeF{V: TypeA{}},
},
err: &UnsupportedTypeError{
Type: reflect.TypeOf(TypeA{}),
},
},
{
desc: "encode not a struct",
in: []interface{}{int(1)},
err: &InvalidEncodeError{
Type: reflect.TypeOf(int(1)),
},
},
{
desc: "encode nil interface",
in: []interface{}{nilIface},
err: &InvalidEncodeError{
Type: reflect.TypeOf(nilIface),
},
},
{
desc: "encode nil ptr",
in: []interface{}{nilPtr},
err: &InvalidEncodeError{},
},
{
desc: "encode nil interface pointer",
in: []interface{}{nilIfacePtr},
err: &InvalidEncodeError{},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
for _, f := range f.regFunc {
enc.Register(f)
}
for _, v := range f.in {
err := enc.Encode(v)
if f.err != nil {
if !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
return
} else if err != nil {
t.Errorf("want err=nil; got %v", err)
}
}
w.Flush()
if err := w.Error(); err != nil {
t.Errorf("want err=nil; got %v", err)
}
var out bytes.Buffer
if err := csv.NewWriter(&out).WriteAll(f.out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if buf.String() != out.String() {
t.Errorf("want=%s; got %s", out.String(), buf.String())
}
})
}
t.Run("test decoder tags", func(t *testing.T) {
type Test struct {
A int `custom:"1"`
B string `custom:"2"`
C float64 `custom:"-"`
}
test := &Test{
A: 1,
B: "b",
C: 2.5,
}
var bufs [4]bytes.Buffer
for i := 0; i < 4; i += 2 {
encode(t, &bufs[i], test, "")
encode(t, &bufs[i+1], test, "custom")
}
if b1, b2 := bufs[0].String(), bufs[2].String(); b1 != b2 {
t.Errorf("buffers are not equal: %s vs %s", b1, b2)
}
if b1, b2 := bufs[1].String(), bufs[3].String(); b1 != b2 {
t.Errorf("buffers are not equal: %s vs %s", b1, b2)
}
expected1 := [][]string{
{"A", "B", "C"},
{"1", "b", "2.5"},
}
expected2 := [][]string{
{"1", "2"},
{"1", "b"},
}
if b1, b2 := bufs[0].String(), encodeCSV(t, expected1); b1 != b2 {
t.Errorf("want buf=%s; got %s", b2, b1)
}
if b1, b2 := bufs[1].String(), encodeCSV(t, expected2); b1 != b2 {
t.Errorf("want buf=%s; got %s", b2, b1)
}
})
t.Run("error messages", func(t *testing.T) {
fixtures := []struct {
desc string
expected string
v interface{}
}{
{
desc: "invalid encode error message",
expected: "csvutil: Encode(int64)",
v: int64(1),
},
{
desc: "invalid encode error message with nil interface",
expected: "csvutil: Encode(nil)",
v: nilIface,
},
{
desc: "invalid encode error message with nil value",
expected: "csvutil: Encode(nil)",
v: nilPtr,
},
{
desc: "unsupported type error message",
expected: "csvutil: unsupported type: struct {}",
v: struct{ InvalidType }{},
},
{
desc: "marshaler error message",
expected: "csvutil: error calling MarshalText for type csvutil.TextMarshaler: " + Error.Error(),
v: struct{ M TextMarshaler }{TextMarshaler{Error}},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
err := NewEncoder(csv.NewWriter(bytes.NewBuffer(nil))).Encode(f.v)
if err == nil {
t.Fatal("want err not to be nil")
}
if err.Error() != f.expected {
t.Errorf("want=%s; got %s", f.expected, err.Error())
}
})
}
})
t.Run("EncodeHeader", func(t *testing.T) {
t.Run("no double header with encode", func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
if err := enc.EncodeHeader(TypeI{}); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if err := enc.Encode(TypeI{}); err != nil {
t.Errorf("want err=nil; got %v", err)
}
w.Flush()
expected := encodeCSV(t, [][]string{
{"String", "int"},
{"", ""},
})
if buf.String() != expected {
t.Errorf("want out=%s; got %s", expected, buf.String())
}
})
t.Run("encode writes header if EncodeHeader fails", func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
if err := enc.EncodeHeader(InvalidType{}); err == nil {
t.Errorf("expected not nil error")
}
if err := enc.Encode(TypeI{}); err != nil {
t.Errorf("want err=nil; got %v", err)
}
w.Flush()
expected := encodeCSV(t, [][]string{
{"String", "int"},
{"", ""},
})
if buf.String() != expected {
t.Errorf("want out=%s; got %s", expected, buf.String())
}
})
fixtures := []struct {
desc string
in interface{}
tag string
out [][]string
err error
}{
{
desc: "conflicting fields",
in: &Embedded10{},
out: [][]string{
{"Y"},
},
},
{
desc: "custom tag",
in: TypeJ{},
tag: "json",
out: [][]string{
{"string", "bool", "Uint", "Float"},
},
},
{
desc: "nil interface ptr value",
in: nilIfacePtr,
out: [][]string{
{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
},
{
desc: "ptr to nil interface ptr value",
in: &nilIfacePtr,
out: [][]string{
{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
},
{
desc: "nil ptr value",
in: nilPtr,
out: [][]string{
{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
},
{
desc: "ptr to nil ptr value",
in: &nilPtr,
out: [][]string{
{
"int",
"pint",
"int8",
"pint8",
"int16",
"pint16",
"int32",
"pint32",
"int64",
"pint64",
"uint",
"puint",
"uint8",
"puint8",
"uint16",
"puint16",
"uint32",
"puint32",
"uint64",
"puint64",
"float32",
"pfloat32",
"float64",
"pfloat64",
"string",
"pstring",
"bool",
"pbool",
"interface",
"pinterface",
"binary",
"pbinary",
},
},
},
{
desc: "ptr to nil interface",
in: &nilIface,
err: &UnsupportedTypeError{Type: reflect.ValueOf(&nilIface).Type().Elem()},
},
{
desc: "nil value",
err: &UnsupportedTypeError{},
},
{
desc: "ptr - not a struct",
in: &[]int{},
err: &UnsupportedTypeError{Type: reflect.TypeOf([]int{})},
},
{
desc: "not a struct",
in: int(1),
err: &UnsupportedTypeError{Type: reflect.TypeOf(int(0))},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.Tag = f.tag
err := enc.EncodeHeader(f.in)
w.Flush()
if !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
if f.err != nil {
return
}
if expected := encodeCSV(t, f.out); buf.String() != expected {
t.Errorf("want out=%s; got %s", expected, buf.String())
}
})
}
})
t.Run("AutoHeader false", func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.AutoHeader = false
if err := enc.Encode(TypeG{
String: "s",
Int: 10,
}); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
w.Flush()
expected := encodeCSV(t, [][]string{{"s", "10"}})
if expected != buf.String() {
t.Errorf("want %s; got %s", expected, buf.String())
}
})
t.Run("fail on type encoding without header", func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.AutoHeader = false
err := enc.Encode(struct {
Invalid InvalidType
}{})
expected := &UnsupportedTypeError{Type: reflect.TypeOf(InvalidType{})}
if !reflect.DeepEqual(err, expected) {
t.Errorf("want %v; got %v", expected, err)
}
})
t.Run("fail while writing header", func(t *testing.T) {
Error := errors.New("error")
enc := NewEncoder(failingWriter{Err: Error})
if err := enc.EncodeHeader(TypeA{}); err != Error {
t.Errorf("want %v; got %v", Error, err)
}
})
t.Run("slice and array", func(t *testing.T) {
fixtures := []struct {
desc string
in interface{}
out [][]string
err error
}{
{
desc: "slice",
in: []TypeI{
{"1", 1},
{"2", 2},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"2", "2"},
},
},
{
desc: "ptr slice",
in: &[]TypeI{
{"1", 1},
{"2", 2},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"2", "2"},
},
},
{
desc: "ptr slice with ptr elements",
in: &[]*TypeI{
{"1", 1},
{"2", 2},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"2", "2"},
},
},
{
desc: "array",
in: [2]TypeI{
{"1", 1},
{"2", 2},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"2", "2"},
},
},
{
desc: "ptr array",
in: &[2]TypeI{
{"1", 1},
{"2", 2},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"2", "2"},
},
},
{
desc: "ptr array with ptr elements",
in: &[2]*TypeI{
{"1", 1},
{"2", 2},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"2", "2"},
},
},
{
desc: "array with default val",
in: [2]TypeI{
{"1", 1},
},
out: [][]string{
{"String", "int"},
{"1", "1"},
{"", ""},
},
},
{
desc: "no auto header on empty slice",
in: []TypeI{},
out: [][]string{},
},
{
desc: "no auto header on empty array",
in: [0]TypeI{},
out: [][]string{},
},
{
desc: "disallow double slice",
in: [][]TypeI{
{
{"1", 1},
},
},
err: &InvalidEncodeError{Type: reflect.TypeOf([][]TypeI{})},
},
{
desc: "disallow double ptr slice",
in: &[][]TypeI{
{
{"1", 1},
},
},
err: &InvalidEncodeError{Type: reflect.TypeOf(&[][]TypeI{})},
},
{
desc: "disallow double ptr slice with ptr slice",
in: &[]*[]TypeI{
{
{"1", 1},
},
},
err: &InvalidEncodeError{Type: reflect.TypeOf(&[]*[]TypeI{})},
},
{
desc: "disallow double array",
in: [2][2]TypeI{
{
{"1", 1},
},
},
err: &InvalidEncodeError{Type: reflect.TypeOf([2][2]TypeI{})},
},
{
desc: "disallow double ptr array",
in: &[2][2]TypeI{
{
{"1", 1},
},
},
err: &InvalidEncodeError{Type: reflect.TypeOf(&[2][2]TypeI{})},
},
{
desc: "disallow interface slice",
in: []interface{}{
TypeI{"1", 1},
},
err: &InvalidEncodeError{Type: reflect.TypeOf([]interface{}{})},
},
{
desc: "disallow interface array",
in: [1]interface{}{
TypeI{"1", 1},
},
err: &InvalidEncodeError{Type: reflect.TypeOf([1]interface{}{})},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
err := NewEncoder(w).Encode(f.in)
if f.err != nil {
if !checkErr(f.err, err) {
t.Errorf("want err=%v; got %v", f.err, err)
}
return
}
if err != nil {
t.Fatalf("want err=nil; got %v", err)
}
w.Flush()
if err := w.Error(); err != nil {
t.Errorf("want err=nil; got %v", err)
}
var out bytes.Buffer
if err := csv.NewWriter(&out).WriteAll(f.out); err != nil {
t.Errorf("want err=nil; got %v", err)
}
if buf.String() != out.String() {
t.Errorf("want=%s; got %s", out.String(), buf.String())
}
})
}
})
t.Run("with header", func(t *testing.T) {
t.Run("all present and sorted", func(t *testing.T) {
fixtures := []struct {
desc string
autoHeader bool
out [][]string
}{
{
desc: "with autoheader",
autoHeader: true,
out: [][]string{
{"C", "B", "D"},
{"c", "b", "d"},
},
},
{
desc: "without autoheader",
autoHeader: false,
out: [][]string{
{"c", "b", "d"},
},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
type Embedded struct {
D string
}
type Foo struct {
A string
Embedded
B string
C string
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.SetHeader([]string{"C", "B", "D"})
enc.AutoHeader = f.autoHeader
enc.Encode(Foo{
A: "a",
Embedded: Embedded{
D: "d",
},
B: "b",
C: "c",
})
w.Flush()
expected := encodeCSV(t, f.out)
if expected != buf.String() {
t.Errorf("want=%s; got %s", expected, buf.String())
}
})
}
})
t.Run("missing fields", func(t *testing.T) {
fixtures := []struct {
desc string
autoHeader bool
out [][]string
}{
{
desc: "with autoheader",
autoHeader: true,
out: [][]string{
{"C", "X", "A", "Z"},
{"c", "", "a", ""},
},
},
{
desc: "without autoheader",
autoHeader: false,
out: [][]string{
{"c", "", "a", ""},
},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
type Foo struct {
A string
B string
C string
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.SetHeader([]string{"C", "X", "A", "Z"})
enc.AutoHeader = f.autoHeader
enc.Encode(Foo{
A: "a",
B: "b",
C: "c",
})
w.Flush()
expected := encodeCSV(t, f.out)
if expected != buf.String() {
t.Errorf("want=%q; got %q", expected, buf.String())
}
})
}
})
t.Run("duplicates", func(t *testing.T) {
type Foo struct {
A string
B string
C string
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := NewEncoder(w)
enc.SetHeader([]string{"C", "X", "C", "A", "X", "Z", "A"})
enc.Encode(Foo{
A: "a",
B: "b",
C: "c",
})
w.Flush()
expected := encodeCSV(t, [][]string{
{"C", "X", "Z", "A"},
{"c", "", "", "a"},
})
if expected != buf.String() {
t.Errorf("want=%q; got %q", expected, buf.String())
}
})
})
t.Run("register panics", func(t *testing.T) {
var buf bytes.Buffer
r := csv.NewWriter(&buf)
enc := NewEncoder(r)
fixtures := []struct {
desc string
arg interface{}
}{
{
desc: "not a func",
arg: 1,
},
{
desc: "nil",
arg: nil,
},
{
desc: "T == empty interface",
arg: func(interface{}) ([]byte, error) { return nil, nil },
},
{
desc: "first out not bytes",
arg: func(int) (int, error) { return 0, nil },
},
{
desc: "second out not error",
arg: func(int) (int, int) { return 0, 0 },
},
{
desc: "func with one out value",
arg: func(int) error { return nil },
},
{
desc: "func with no returns",
arg: func(int) {},
},
}
for _, f := range fixtures {
t.Run(f.desc, func(t *testing.T) {
var e interface{}
func() {
defer func() {
e = recover()
}()
enc.Register(f.arg)
}()
if e == nil {
t.Error("Register was supposed to panic but it didnt")
}
t.Log(e)
})
}
t.Run("already registered", func(t *testing.T) {
f := func(int) ([]byte, error) { return nil, nil }
enc.Register(f)
var e interface{}
func() {
defer func() {
e = recover()
}()
enc.Register(f)
}()
if e == nil {
t.Error("Register was supposed to panic but it didnt")
}
t.Log(e)
})
})
}
func encode(t *testing.T, buf *bytes.Buffer, v interface{}, tag string) {
w := csv.NewWriter(buf)
enc := NewEncoder(w)
enc.Tag = tag
if err := enc.Encode(v); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
w.Flush()
if err := w.Error(); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
}
func encodeCSV(t *testing.T, recs [][]string) string {
var buf bytes.Buffer
if err := csv.NewWriter(&buf).WriteAll(recs); err != nil {
t.Fatalf("want err=nil; got %v", err)
}
return buf.String()
}
type failingWriter struct {
Err error
}
func (w failingWriter) Write([]string) error {
return w.Err
}
csvutil-1.7.1/error.go 0000664 0000000 0000000 00000012325 14257146113 0014700 0 ustar 00root root 0000000 0000000 package csvutil
import (
"bytes"
"errors"
"fmt"
"reflect"
"strconv"
)
// ErrFieldCount is returned when header's length doesn't match the length of
// the read record.
var ErrFieldCount = errors.New("wrong number of fields in record")
// An UnmarshalTypeError describes a string value that was not appropriate for
// a value of a specific Go type.
type UnmarshalTypeError struct {
Value string // string value
Type reflect.Type // type of Go value it could not be assigned to
}
func (e *UnmarshalTypeError) Error() string {
return "csvutil: cannot unmarshal " + strconv.Quote(e.Value) + " into Go value of type " + e.Type.String()
}
// An UnsupportedTypeError is returned when attempting to encode or decode
// a value of an unsupported type.
type UnsupportedTypeError struct {
Type reflect.Type
}
func (e *UnsupportedTypeError) Error() string {
if e.Type == nil {
return "csvutil: unsupported type: nil"
}
return "csvutil: unsupported type: " + e.Type.String()
}
// An InvalidDecodeError describes an invalid argument passed to Decode.
// (The argument to Decode must be a non-nil struct pointer)
type InvalidDecodeError struct {
Type reflect.Type
}
func (e *InvalidDecodeError) Error() string {
if e.Type == nil {
return "csvutil: Decode(nil)"
}
if e.Type.Kind() != reflect.Ptr {
return "csvutil: Decode(non-pointer " + e.Type.String() + ")"
}
typ := walkType(e.Type)
switch typ.Kind() {
case reflect.Struct:
case reflect.Slice, reflect.Array:
if typ.Elem().Kind() != reflect.Struct {
return "csvutil: Decode(invalid type " + e.Type.String() + ")"
}
default:
return "csvutil: Decode(invalid type " + e.Type.String() + ")"
}
return "csvutil: Decode(nil " + e.Type.String() + ")"
}
// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal.
// (The argument to Unmarshal must be a non-nil slice of structs pointer)
type InvalidUnmarshalError struct {
Type reflect.Type
}
func (e *InvalidUnmarshalError) Error() string {
if e.Type == nil {
return "csvutil: Unmarshal(nil)"
}
if e.Type.Kind() != reflect.Ptr {
return "csvutil: Unmarshal(non-pointer " + e.Type.String() + ")"
}
return "csvutil: Unmarshal(invalid type " + e.Type.String() + ")"
}
// InvalidEncodeError is returned by Encode when the provided value was invalid.
type InvalidEncodeError struct {
Type reflect.Type
}
func (e *InvalidEncodeError) Error() string {
if e.Type == nil {
return "csvutil: Encode(nil)"
}
return "csvutil: Encode(" + e.Type.String() + ")"
}
// InvalidMarshalError is returned by Marshal when the provided value was invalid.
type InvalidMarshalError struct {
Type reflect.Type
}
func (e *InvalidMarshalError) Error() string {
if e.Type == nil {
return "csvutil: Marshal(nil)"
}
if walkType(e.Type).Kind() == reflect.Slice {
return "csvutil: Marshal(non struct slice " + e.Type.String() + ")"
}
if walkType(e.Type).Kind() == reflect.Array {
return "csvutil: Marshal(non struct array " + e.Type.String() + ")"
}
return "csvutil: Marshal(invalid type " + e.Type.String() + ")"
}
// MarshalerError is returned by Encoder when MarshalCSV or MarshalText returned
// an error.
type MarshalerError struct {
Type reflect.Type
MarshalerType string
Err error
}
func (e *MarshalerError) Error() string {
return "csvutil: error calling " + e.MarshalerType + " for type " + e.Type.String() + ": " + e.Err.Error()
}
// Unwrap implements Unwrap interface for errors package in Go1.13+.
func (e *MarshalerError) Unwrap() error {
return e.Err
}
func errPtrUnexportedStruct(typ reflect.Type) error {
return fmt.Errorf("csvutil: cannot decode into a pointer to unexported struct: %s", typ)
}
// MissingColumnsError is returned by Decoder only when DisallowMissingColumns
// option was set to true. It contains a list of all missing columns.
type MissingColumnsError struct {
Columns []string
}
func (e *MissingColumnsError) Error() string {
var b bytes.Buffer
b.WriteString("csvutil: missing columns: ")
for i, c := range e.Columns {
if i > 0 {
b.WriteString(", ")
}
fmt.Fprintf(&b, "%q", c)
}
return b.String()
}
// DecodeError provides context to decoding errors if available.
//
// The caller should use errors.As in order to fetch the underlying error if
// needed.
//
// Some of the DecodeError's fields are only populated if the Reader supports
// PosField method. Specifically Line and Column. FieldPos is available in
// csv.Reader since Go1.17.
type DecodeError struct {
// Field describes the struct's tag or field name on which the error happened.
Field string
// Line is 1-indexed line number taken from FieldPost method. It is only
// available if the used Reader supports FieldPos method.
Line int
// Column is 1-indexed column index taken from FieldPost method. It is only
// available if the used Reader supports FieldPos method.
Column int
// Error is the actual error that was returned while attempting to decode
// a field.
Err error
}
func (e *DecodeError) Error() string {
if e.Line > 0 && e.Column > 0 {
// Lines and Columns are 1-indexed so this check is fine.
return fmt.Sprintf("%s: field %q line %d column %d", e.Err, e.Field, e.Line, e.Column)
}
return fmt.Sprintf("%s: field %q", e.Err, e.Field)
}
func (e *DecodeError) Unwrap() error {
return e.Err
}
csvutil-1.7.1/example_decoder_interface_test.go 0000664 0000000 0000000 00000003574 14257146113 0021754 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"bytes"
"encoding/csv"
"fmt"
"io"
"log"
"github.com/jszwec/csvutil"
)
// Value defines one record in the csv input. In this example it is important
// that Type field is defined before Value. Decoder reads headers and values
// in the same order as struct fields are defined.
type Value struct {
Type string `csv:"type"`
Value interface{} `csv:"value"`
}
func ExampleDecoder_interfaceValues() {
// lets say our csv input defines variables with their types and values.
data := []byte(`
type,value
string,string_value
int,10
`)
dec, err := csvutil.NewDecoder(csv.NewReader(bytes.NewReader(data)))
if err != nil {
log.Fatal(err)
}
// we would like to read every variable and store their already parsed values
// in the interface field. We can use Decoder.Map function to initialize
// interface with proper values depending on the input.
var value Value
dec.Map = func(field, column string, v interface{}) string {
if column == "type" {
switch field {
case "int": // csv input tells us that this variable contains an int.
var n int
value.Value = &n // lets initialize interface with an initialized int pointer.
default:
return field
}
}
return field
}
for {
value = Value{}
if err := dec.Decode(&value); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
if value.Type == "int" {
// our variable type is int, Map func already initialized our interface
// as int pointer, so we can safely cast it and use it.
n, ok := value.Value.(*int)
if !ok {
log.Fatal("expected value to be *int")
}
fmt.Printf("value_type: %s; value: (%T) %d\n", value.Type, value.Value, *n)
} else {
fmt.Printf("value_type: %s; value: (%T) %v\n", value.Type, value.Value, value.Value)
}
}
// Output:
// value_type: string; value: (string) string_value
// value_type: int; value: (*int) 10
}
csvutil-1.7.1/example_decoder_no_header_test.go 0000664 0000000 0000000 00000001654 14257146113 0021735 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"bytes"
"encoding/csv"
"fmt"
"io"
"log"
"github.com/jszwec/csvutil"
)
type User struct {
ID int
Name string
Age int `csv:",omitempty"`
State int `csv:"-"`
City string
ZIP string `csv:"zip_code"`
}
var userHeader []string
func init() {
h, err := csvutil.Header(User{}, "csv")
if err != nil {
log.Fatal(err)
}
userHeader = h
}
func ExampleDecoder_decodingDataWithNoHeader() {
data := []byte(`
1,John,27,la,90005
2,Bob,,ny,10005`)
r := csv.NewReader(bytes.NewReader(data))
dec, err := csvutil.NewDecoder(r, userHeader...)
if err != nil {
log.Fatal(err)
}
var users []User
for {
var u User
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
users = append(users, u)
}
fmt.Printf("%+v", users)
// Output:
// [{ID:1 Name:John Age:27 State:0 City:la ZIP:90005} {ID:2 Name:Bob Age:0 State:0 City:ny ZIP:10005}]
}
csvutil-1.7.1/example_decoder_register_test.go 0000664 0000000 0000000 00000003247 14257146113 0021635 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"encoding/csv"
"errors"
"fmt"
"strconv"
"strings"
"time"
"unicode"
"github.com/jszwec/csvutil"
)
type IntStruct struct {
Value int
}
func (i *IntStruct) Scan(state fmt.ScanState, verb rune) error {
switch verb {
case 'd', 'v':
default:
return errors.New("unsupported verb")
}
t, err := state.Token(false, unicode.IsDigit)
if err != nil {
return err
}
n, err := strconv.Atoi(string(t))
if err != nil {
return err
}
*i = IntStruct{Value: n}
return nil
}
func ExampleDecoder_Register() {
type Foo struct {
Time time.Time `csv:"time"`
Hex int `csv:"hex"`
PtrHex *int `csv:"ptr_hex"`
IntStruct IntStruct `csv:"int_struct"`
}
unmarshalInt := func(data []byte, n *int) error {
v, err := strconv.ParseInt(string(data), 16, 64)
if err != nil {
return err
}
*n = int(v)
return nil
}
unmarshalTime := func(data []byte, t *time.Time) error {
tt, err := time.Parse(time.Kitchen, string(data))
if err != nil {
return err
}
*t = tt
return nil
}
unmarshalScanner := func(data []byte, s fmt.Scanner) error {
_, err := fmt.Sscan(string(data), s)
return err
}
const data = `time,hex,ptr_hex,int_struct
12:00PM,f,a,34`
r := csv.NewReader(strings.NewReader(data))
dec, err := csvutil.NewDecoder(r)
if err != nil {
panic(err)
}
dec.Register(unmarshalInt)
dec.Register(unmarshalTime)
dec.Register(unmarshalScanner)
var foos []Foo
if err := dec.Decode(&foos); err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%s,%d,%d,%+v",
foos[0].Time.Format(time.Kitchen),
foos[0].Hex,
*foos[0].PtrHex,
foos[0].IntStruct,
)
// Output:
// 12:00PM,15,10,{Value:34}
}
csvutil-1.7.1/example_decoder_test.go 0000664 0000000 0000000 00000010101 14257146113 0017714 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"encoding/csv"
"fmt"
"io"
"log"
"strings"
"github.com/jszwec/csvutil"
)
func ExampleDecoder_Decode() {
type User struct {
ID *int `csv:"id,omitempty"`
Name string `csv:"name"`
City string `csv:"city"`
Age int `csv:"age"`
}
csvReader := csv.NewReader(strings.NewReader(`
id,name,age,city
,alice,25,la
,bob,30,ny`))
dec, err := csvutil.NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
var users []User
for {
var u User
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
users = append(users, u)
}
fmt.Println(users)
// Output:
// [{ alice la 25} { bob ny 30}]
}
func ExampleDecoder_Decode_slice() {
type User struct {
ID *int `csv:"id,omitempty"`
Name string `csv:"name"`
City string `csv:"city"`
Age int `csv:"age"`
}
csvReader := csv.NewReader(strings.NewReader(`
id,name,age,city
,alice,25,la
,bob,30,ny`))
dec, err := csvutil.NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
var users []User
if err := dec.Decode(&users); err != nil {
log.Fatal(err)
}
fmt.Println(users)
// Output:
// [{ alice la 25} { bob ny 30}]
}
func ExampleDecoder_Decode_array() {
type User struct {
ID *int `csv:"id,omitempty"`
Name string `csv:"name"`
City string `csv:"city"`
Age int `csv:"age"`
}
csvReader := csv.NewReader(strings.NewReader(`
id,name,age,city
,alice,25,la
,bob,30,ny
,john,29,ny`))
dec, err := csvutil.NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
var users [2]User
if err := dec.Decode(&users); err != nil {
log.Fatal(err)
}
fmt.Println(users)
// Output:
// [{ alice la 25} { bob ny 30}]
}
func ExampleDecoder_Unused() {
type User struct {
Name string `csv:"name"`
City string `csv:"city"`
Age int `csv:"age"`
OtherData map[string]string `csv:"-"`
}
csvReader := csv.NewReader(strings.NewReader(`
name,age,city,zip
alice,25,la,90005
bob,30,ny,10005`))
dec, err := csvutil.NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
header := dec.Header()
var users []User
for {
u := User{OtherData: make(map[string]string)}
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
for _, i := range dec.Unused() {
u.OtherData[header[i]] = dec.Record()[i]
}
users = append(users, u)
}
fmt.Println(users)
// Output:
// [{alice la 25 map[zip:90005]} {bob ny 30 map[zip:10005]}]
}
func ExampleDecoder_decodeEmbedded() {
type Address struct {
ID int `csv:"id"` // same field as in User - this one will be empty
City string `csv:"city"`
State string `csv:"state"`
}
type User struct {
Address
ID int `csv:"id"` // same field as in Address - this one wins
Name string `csv:"name"`
Age int `csv:"age"`
}
csvReader := csv.NewReader(strings.NewReader(
"id,name,age,city,state\n" +
"1,alice,25,la,ca\n" +
"2,bob,30,ny,ny"))
dec, err := csvutil.NewDecoder(csvReader)
if err != nil {
log.Fatal(err)
}
var users []User
for {
var u User
if err := dec.Decode(&u); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
users = append(users, u)
}
fmt.Println(users)
// Output:
// [{{0 la ca} 1 alice 25} {{0 ny ny} 2 bob 30}]
}
func ExampleDecoder_Decode_inline() {
type Address struct {
Street string `csv:"street"`
City string `csv:"city"`
}
type User struct {
Name string `csv:"name"`
Address Address `csv:",inline"`
HomeAddress Address `csv:"home_address_,inline"`
WorkAddress Address `csv:"work_address_,inline"`
Age int `csv:"age,omitempty"`
}
data := []byte(
"name,street,city,home_address_street,home_address_city,work_address_street,work_address_city,age\n" +
"John,Washington,Boston,Boylston,Boston,River St,Cambridge,26",
)
var users []User
if err := csvutil.Unmarshal(data, &users); err != nil {
fmt.Println("error:", err)
}
fmt.Println(users)
// Output:
// [{John {Washington Boston} {Boylston Boston} {River St Cambridge} 26}]
}
csvutil-1.7.1/example_decoder_unmashaler_test.go 0000664 0000000 0000000 00000001043 14257146113 0022140 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"fmt"
"strconv"
"github.com/jszwec/csvutil"
)
type Bar int
func (b *Bar) UnmarshalCSV(data []byte) error {
n, err := strconv.Atoi(string(data))
*b = Bar(n)
return err
}
type Foo struct {
Int int `csv:"int"`
Bar Bar `csv:"bar"`
}
func ExampleDecoder_customUnmarshalCSV() {
var csvInput = []byte(`
int,bar
5,10
6,11`)
var foos []Foo
if err := csvutil.Unmarshal(csvInput, &foos); err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%+v", foos)
// Output:
// [{Int:5 Bar:10} {Int:6 Bar:11}]
}
csvutil-1.7.1/example_encoder_test.go 0000664 0000000 0000000 00000010341 14257146113 0017734 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"bytes"
"encoding/csv"
"fmt"
"strconv"
"time"
"github.com/jszwec/csvutil"
)
func ExampleEncoder_Encode_streaming() {
type Address struct {
City string
Country string
}
type User struct {
Name string
Address
Age int `csv:"age,omitempty"`
}
users := []User{
{Name: "John", Address: Address{"Boston", "USA"}, Age: 26},
{Name: "Bob", Address: Address{"LA", "USA"}, Age: 27},
{Name: "Alice", Address: Address{"SF", "USA"}},
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := csvutil.NewEncoder(w)
for _, u := range users {
if err := enc.Encode(u); err != nil {
fmt.Println("error:", err)
}
}
w.Flush()
if err := w.Error(); err != nil {
fmt.Println("error:", err)
}
fmt.Println(buf.String())
// Output:
// Name,City,Country,age
// John,Boston,USA,26
// Bob,LA,USA,27
// Alice,SF,USA,
}
func ExampleEncoder_Encode_all() {
type Address struct {
City string
Country string
}
type User struct {
Name string
Address
Age int `csv:"age,omitempty"`
}
users := []User{
{Name: "John", Address: Address{"Boston", "USA"}, Age: 26},
{Name: "Bob", Address: Address{"LA", "USA"}, Age: 27},
{Name: "Alice", Address: Address{"SF", "USA"}},
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
if err := csvutil.NewEncoder(w).Encode(users); err != nil {
fmt.Println("error:", err)
}
w.Flush()
if err := w.Error(); err != nil {
fmt.Println("error:", err)
}
fmt.Println(buf.String())
// Output:
// Name,City,Country,age
// John,Boston,USA,26
// Bob,LA,USA,27
// Alice,SF,USA,
}
func ExampleEncoder_EncodeHeader() {
type User struct {
Name string
Age int `csv:"age,omitempty"`
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := csvutil.NewEncoder(w)
if err := enc.EncodeHeader(User{}); err != nil {
fmt.Println("error:", err)
}
w.Flush()
if err := w.Error(); err != nil {
fmt.Println("error:", err)
}
fmt.Println(buf.String())
// Output:
// Name,age
}
func ExampleEncoder_Encode_inline() {
type Owner struct {
Name string `csv:"name"`
}
type Address struct {
Street string `csv:"street"`
City string `csv:"city"`
Owner Owner `csv:"owner_,inline"`
}
type User struct {
Name string `csv:"name"`
Address Address `csv:",inline"`
HomeAddress Address `csv:"home_address_,inline"`
WorkAddress Address `csv:"work_address_,inline"`
Age int `csv:"age,omitempty"`
}
users := []User{
{
Name: "John",
Address: Address{"Washington", "Boston", Owner{"Steve"}},
HomeAddress: Address{"Boylston", "Boston", Owner{"Steve"}},
WorkAddress: Address{"River St", "Cambridge", Owner{"Steve"}},
Age: 26,
},
}
b, err := csvutil.Marshal(users)
if err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%s\n", b)
// Output:
// name,street,city,owner_name,home_address_street,home_address_city,home_address_owner_name,work_address_street,work_address_city,work_address_owner_name,age
// John,Washington,Boston,Steve,Boylston,Boston,Steve,River St,Cambridge,Steve,26
}
func ExampleEncoder_Register() {
type Foo struct {
Time time.Time `csv:"time"`
Hex int `csv:"hex"`
PtrHex *int `csv:"ptr_hex"`
Buffer *bytes.Buffer `csv:"buffer"`
}
foos := []Foo{
{
Time: time.Date(2020, 6, 20, 12, 0, 0, 0, time.UTC),
Hex: 15,
Buffer: bytes.NewBufferString("hello"),
},
}
marshalInt := func(n *int) ([]byte, error) {
if n == nil {
return []byte("NULL"), nil
}
return strconv.AppendInt(nil, int64(*n), 16), nil
}
marshalTime := func(t time.Time) ([]byte, error) {
return t.AppendFormat(nil, time.Kitchen), nil
}
// all fields which implement String method will use this, unless their
// concrete type was already overriden.
marshalStringer := func(s fmt.Stringer) ([]byte, error) {
return []byte(s.String()), nil
}
var buf bytes.Buffer
w := csv.NewWriter(&buf)
enc := csvutil.NewEncoder(w)
enc.Register(marshalInt)
enc.Register(marshalTime)
enc.Register(marshalStringer)
if err := enc.Encode(foos); err != nil {
fmt.Println("error:", err)
}
w.Flush()
if err := w.Error(); err != nil {
fmt.Println("error:", err)
}
fmt.Println(buf.String())
// Output:
// time,hex,ptr_hex,buffer
// 12:00PM,f,NULL,hello
}
csvutil-1.7.1/example_header_test.go 0000664 0000000 0000000 00000000623 14257146113 0017547 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"fmt"
"log"
"github.com/jszwec/csvutil"
)
func ExampleHeader() {
type User struct {
ID int
Name string
Age int `csv:",omitempty"`
State int `csv:"-"`
City string
ZIP string `csv:"zip_code"`
}
header, err := csvutil.Header(User{}, "csv")
if err != nil {
log.Fatal(err)
}
fmt.Println(header)
// Output:
// [ID Name Age City zip_code]
}
csvutil-1.7.1/example_marshal_marshaler_test.go 0000664 0000000 0000000 00000001200 14257146113 0021774 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"fmt"
"github.com/jszwec/csvutil"
)
type Status uint8
const (
Unknown = iota
Success
Failure
)
func (s Status) MarshalCSV() ([]byte, error) {
switch s {
case Success:
return []byte("success"), nil
case Failure:
return []byte("failure"), nil
default:
return []byte("unknown"), nil
}
}
type Job struct {
ID int
Status Status
}
func ExampleMarshal_customMarshalCSV() {
jobs := []Job{
{1, Success},
{2, Failure},
}
b, err := csvutil.Marshal(jobs)
if err != nil {
fmt.Println("error:", err)
}
fmt.Println(string(b))
// Output:
// ID,Status
// 1,success
// 2,failure
}
csvutil-1.7.1/example_marshal_slice_map_test.go 0000664 0000000 0000000 00000001470 14257146113 0021763 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"fmt"
"log"
"strings"
"github.com/jszwec/csvutil"
)
type Strings []string
func (s Strings) MarshalCSV() ([]byte, error) {
return []byte(strings.Join(s, ",")), nil // strings.Join takes []string but it will also accept Strings
}
type StringMap map[string]string
func (sm StringMap) MarshalCSV() ([]byte, error) {
return []byte(fmt.Sprint(sm)), nil
}
func ExampleMarshal_sliceMap() {
b, err := csvutil.Marshal([]struct {
Strings Strings `csv:"strings"`
Map StringMap `csv:"map"`
}{
{[]string{"a", "b"}, map[string]string{"a": "1"}}, // no type casting is required for slice and map aliases
{Strings{"c", "d"}, StringMap{"b": "1"}},
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", b)
// Output:
// strings,map
// "a,b",map[a:1]
// "c,d",map[b:1]
}
csvutil-1.7.1/example_marshal_test.go 0000664 0000000 0000000 00000001354 14257146113 0017750 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"fmt"
"time"
"github.com/jszwec/csvutil"
)
func ExampleMarshal() {
type Address struct {
City string
Country string
}
type User struct {
Name string
Address
Age int `csv:"age,omitempty"`
CreatedAt time.Time
}
users := []User{
{
Name: "John",
Address: Address{"Boston", "USA"},
Age: 26,
CreatedAt: time.Date(2010, 6, 2, 12, 0, 0, 0, time.UTC),
},
{
Name: "Alice",
Address: Address{"SF", "USA"},
},
}
b, err := csvutil.Marshal(users)
if err != nil {
fmt.Println("error:", err)
}
fmt.Println(string(b))
// Output:
// Name,City,Country,age,CreatedAt
// John,Boston,USA,26,2010-06-02T12:00:00Z
// Alice,SF,USA,,0001-01-01T00:00:00Z
}
csvutil-1.7.1/example_unmarshal_test.go 0000664 0000000 0000000 00000001177 14257146113 0020316 0 ustar 00root root 0000000 0000000 package csvutil_test
import (
"fmt"
"time"
"github.com/jszwec/csvutil"
)
func ExampleUnmarshal() {
var csvInput = []byte(`
name,age,CreatedAt
jacek,26,2012-04-01T15:00:00Z
john,,0001-01-01T00:00:00Z`,
)
type User struct {
Name string `csv:"name"`
Age int `csv:"age,omitempty"`
CreatedAt time.Time
}
var users []User
if err := csvutil.Unmarshal(csvInput, &users); err != nil {
fmt.Println("error:", err)
}
for _, u := range users {
fmt.Printf("%+v\n", u)
}
// Output:
// {Name:jacek Age:26 CreatedAt:2012-04-01 15:00:00 +0000 UTC}
// {Name:john Age:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC}
}
csvutil-1.7.1/go.mod 0000664 0000000 0000000 00000000052 14257146113 0014320 0 ustar 00root root 0000000 0000000 module github.com/jszwec/csvutil
go 1.13
csvutil-1.7.1/interface.go 0000664 0000000 0000000 00000001353 14257146113 0015506 0 ustar 00root root 0000000 0000000 package csvutil
// Reader provides the interface for reading a single CSV record.
//
// If there is no data left to be read, Read returns (nil, io.EOF).
//
// It is implemented by csv.Reader.
type Reader interface {
Read() ([]string, error)
}
// Writer provides the interface for writing a single CSV record.
//
// It is implemented by csv.Writer.
type Writer interface {
Write([]string) error
}
// Unmarshaler is the interface implemented by types that can unmarshal
// a single record's field description of themselves.
type Unmarshaler interface {
UnmarshalCSV([]byte) error
}
// Marshaler is the interface implemented by types that can marshal themselves
// into valid string.
type Marshaler interface {
MarshalCSV() ([]byte, error)
}
csvutil-1.7.1/tag.go 0000664 0000000 0000000 00000001335 14257146113 0014321 0 ustar 00root root 0000000 0000000 package csvutil
import (
"reflect"
"strings"
)
type tag struct {
name string
prefix string
empty bool
omitEmpty bool
ignore bool
inline bool
}
func parseTag(tagname string, field reflect.StructField) (t tag) {
tags := strings.Split(field.Tag.Get(tagname), ",")
if len(tags) == 1 && tags[0] == "" {
t.name = field.Name
t.empty = true
return
}
switch tags[0] {
case "-":
t.ignore = true
return
case "":
t.name = field.Name
default:
t.name = tags[0]
}
for _, tagOpt := range tags[1:] {
switch tagOpt {
case "omitempty":
t.omitEmpty = true
case "inline":
if walkType(field.Type).Kind() == reflect.Struct {
t.inline = true
t.prefix = tags[0]
}
}
}
return
}