pax_global_header00006660000000000000000000000064147255446360014532gustar00rootroot0000000000000052 comment=c0009dfe73ca014ae99309ab3f00ddf33e251906 msgp-1.2.5/000077500000000000000000000000001472554463600125055ustar00rootroot00000000000000msgp-1.2.5/.github/000077500000000000000000000000001472554463600140455ustar00rootroot00000000000000msgp-1.2.5/.github/workflows/000077500000000000000000000000001472554463600161025ustar00rootroot00000000000000msgp-1.2.5/.github/workflows/test.yml000066400000000000000000000011321472554463600176010ustar00rootroot00000000000000name: test concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: workflow_dispatch: push: branches: - 'master' - 'main' tags: - 'v*' pull_request: permissions: contents: read jobs: test: strategy: matrix: go-version: [1.21.x, 1.22.x, 1.23.x] os: [ubuntu-latest] runs-on: ${{ matrix.os }} timeout-minutes: 10 steps: - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - uses: actions/checkout@v4 - name: test run: make ci msgp-1.2.5/.github/workflows/validate.yml000066400000000000000000000014231472554463600204160ustar00rootroot00000000000000name: validate concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: workflow_dispatch: push: branches: - 'master' - 'main' tags: - 'v*' pull_request: permissions: contents: read jobs: linters: strategy: matrix: go-version: [1.23.x] os: [ubuntu-latest] runs-on: ${{ matrix.os }} timeout-minutes: 10 steps: - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - uses: actions/checkout@v4 - name: prepare generated code run: make prepare - name: lint uses: golangci/golangci-lint-action@v6 with: version: v1.60.3 args: --print-resources-usage --timeout=10m --verbose msgp-1.2.5/.gitignore000066400000000000000000000003421472554463600144740ustar00rootroot00000000000000_generated/generated.go _generated/generated_test.go _generated/*_gen.go _generated/*_gen_test.go _generated/embeddedStruct/*_gen.go _generated/embeddedStruct/*_gen_test.go msgp/defgen_test.go msgp/cover.out *~ *.coverprofile msgp-1.2.5/.golangci.yml000066400000000000000000000002111472554463600150630ustar00rootroot00000000000000linters: disable: - errcheck linters-settings: staticcheck: checks: - all - '-SA1019' # We use the ast package. msgp-1.2.5/LICENSE000066400000000000000000000021711472554463600135130ustar00rootroot00000000000000Copyright (c) 2014 Philip Hofer Portions Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.msgp-1.2.5/Makefile000066400000000000000000000023771472554463600141560ustar00rootroot00000000000000 # NOTE: This Makefile is only necessary if you # plan on developing the msgp tool and library. # Installation can still be performed with a # normal `go install`. # generated integration test files GGEN = ./_generated/generated.go ./_generated/generated_test.go # generated unit test files MGEN = ./msgp/defgen_test.go SHELL := /bin/bash BIN = $(GOBIN)/msgp .PHONY: clean wipe install get-deps bench all ci prepare $(BIN): */*.go @go install ./... install: $(BIN) $(GGEN): ./_generated/def.go go generate ./_generated $(MGEN): ./msgp/defs_test.go go generate ./msgp test: all go test ./... ./_generated bench: all go test -bench ./... clean: $(RM) $(GGEN) $(MGEN) wipe: clean $(RM) $(BIN) get-deps: go get -d -t ./... all: install $(GGEN) $(MGEN) # Prepare generated code to be used for linting and testing in CI prepare: go install . go generate ./msgp go generate ./_generated # CI enters here ci: prepare arch if [ `arch` == 'x86_64' ]; then \ sudo apt-get -y -q update; \ sudo apt-get -y -q install build-essential; \ wget -q https://github.com/tinygo-org/tinygo/releases/download/v0.33.0/tinygo_0.33.0_amd64.deb; \ sudo dpkg -i tinygo_0.33.0_amd64.deb; \ export PATH=$$PATH:/usr/local/tinygo/bin; \ fi go test -v ./... ./_generated msgp-1.2.5/README.md000066400000000000000000000135561472554463600137760ustar00rootroot00000000000000MessagePack Code Generator ======= [![Go Reference](https://pkg.go.dev/badge/github.com/tinylib/msgp.svg)](https://pkg.go.dev/github.com/tinylib/msgp) [![test](https://github.com/tinylib/msgp/actions/workflows/test.yml/badge.svg)](https://github.com/tinylib/msgp/actions/workflows/test.yml) [![validate](https://github.com/tinylib/msgp/actions/workflows/validate.yml/badge.svg)](https://github.com/tinylib/msgp/actions/workflows/validate.yml) This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). You can read more about MessagePack [in the wiki](http://github.com/tinylib/msgp/wiki), or at [msgpack.org](http://msgpack.org). ### Why? - Use Go as your schema language - Performance - [JSON interop](https://pkg.go.dev/github.com/tinylib/msgp/msgp#CopyToJSON) - [User-defined extensions](http://github.com/tinylib/msgp/wiki/Using-Extensions) - Type safety - Encoding flexibility ### Quickstart First install the `msgp` generator command. Using Go this is done with `go install github.com/tinylib/msgp@latest` In a source file, include the following directive: ```go //go:generate msgp ``` The `msgp` command will generate serialization methods for all exported type declarations in the file. You can [read more about the code generation options here](http://github.com/tinylib/msgp/wiki/Using-the-Code-Generator). ### Use Field names can be set in much the same way as the `encoding/json` package. For example: ```go type Person struct { Name string `msg:"name"` Address string `msg:"address"` Age int `msg:"age"` Hidden string `msg:"-"` // this field is ignored unexported bool // this field is also ignored } ``` By default, the code generator will satisfy `msgp.Sizer`, `msgp.Encodable`, `msgp.Decodable`, `msgp.Marshaler`, and `msgp.Unmarshaler`. Carefully-designed applications can use these methods to do marshalling/unmarshalling with zero heap allocations. While `msgp.Marshaler` and `msgp.Unmarshaler` are quite similar to the standard library's `json.Marshaler` and `json.Unmarshaler`, `msgp.Encodable` and `msgp.Decodable` are useful for stream serialization. (`*msgp.Writer` and `*msgp.Reader` are essentially protocol-aware versions of `*bufio.Writer` and `*bufio.Reader`, respectively.) An important thing to note is that msgp operates on *individual files*. This means if your structs include types defined in other files, these must be processed as well. ### Features - Extremely fast generated code - Test and benchmark generation - JSON interoperability (see `msgp.CopyToJSON() and msgp.UnmarshalAsJSON()`) - Support for complex type declarations - Native support for Go's `time.Time`, `complex64`, and `complex128` types - Generation of both `[]byte`-oriented and `io.Reader/io.Writer`-oriented methods - Support for arbitrary type system extensions - [Preprocessor directives](http://github.com/tinylib/msgp/wiki/Preprocessor-Directives) - File-based dependency model means fast codegen regardless of source tree size. Consider the following: ```go const Eight = 8 type MyInt int type Data []byte type Struct struct { Which map[string]*MyInt `msg:"which"` Other Data `msg:"other"` Nums [Eight]float64 `msg:"nums"` } ``` As long as the declarations of `MyInt` and `Data` are in the same file as `Struct`, the parser will determine that the type information for `MyInt` and `Data` can be passed into the definition of `Struct` before its methods are generated. #### Extensions MessagePack supports defining your own types through "extensions," which are just a tuple of the data "type" (`int8`) and the raw binary. You [can see a worked example in the wiki.](http://github.com/tinylib/msgp/wiki/Using-Extensions) ### Status Mostly stable, in that no breaking changes have been made to the `/msgp` library in more than a year. Newer versions of the code may generate different code than older versions for performance reasons. I (@philhofer) am aware of a number of stability-critical commercial applications that use this code with good results. But, caveat emptor. You can read more about how `msgp` maps MessagePack types onto Go types [in the wiki](http://github.com/tinylib/msgp/wiki). Here some of the known limitations/restrictions: - Identifiers from outside the processed source file are assumed (optimistically) to satisfy the generator's interfaces. If this isn't the case, your code will fail to compile. - Like most serializers, `chan` and `func` fields are ignored, as well as non-exported fields. - Encoding of `interface{}` is limited to built-ins or types that have explicit encoding methods. - _Maps must have `string` keys._ This is intentional (as it preserves JSON interop.) Although non-string map keys are not forbidden by the MessagePack standard, many serializers impose this restriction. (It also means *any* well-formed `struct` can be de-serialized into a `map[string]interface{}`.) The only exception to this rule is that the deserializers will allow you to read map keys encoded as `bin` types, due to the fact that some legacy encodings permitted this. (However, those values will still be cast to Go `string`s, and they will be converted to `str` types when re-encoded. It is the responsibility of the user to ensure that map keys are UTF-8 safe in this case.) The same rules hold true for JSON translation. If the output compiles, then there's a pretty good chance things are fine. (Plus, we generate tests for you.) *Please, please, please* file an issue if you think the generator is writing broken code. ### Performance If you like benchmarks, see [here](http://bravenewgeek.com/so-you-wanna-go-fast/) and [here](https://github.com/alecthomas/go_serialization_benchmarks). As one might expect, the generated methods that deal with `[]byte` are faster for small objects, but the `io.Reader/Writer` methods are generally more memory-efficient (and, at some point, faster) for large (> 2KB) objects. msgp-1.2.5/_generated/000077500000000000000000000000001472554463600146025ustar00rootroot00000000000000msgp-1.2.5/_generated/allownil.go000066400000000000000000000202431472554463600167530ustar00rootroot00000000000000package _generated import "time" //go:generate msgp type AllowNil0 struct { ABool bool `msg:"abool,allownil"` AInt int `msg:"aint,allownil"` AInt8 int8 `msg:"aint8,allownil"` AInt16 int16 `msg:"aint16,allownil"` AInt32 int32 `msg:"aint32,allownil"` AInt64 int64 `msg:"aint64,allownil"` AUint uint `msg:"auint,allownil"` AUint8 uint8 `msg:"auint8,allownil"` AUint16 uint16 `msg:"auint16,allownil"` AUint32 uint32 `msg:"auint32,allownil"` AUint64 uint64 `msg:"auint64,allownil"` AFloat32 float32 `msg:"afloat32,allownil"` AFloat64 float64 `msg:"afloat64,allownil"` AComplex64 complex64 `msg:"acomplex64,allownil"` AComplex128 complex128 `msg:"acomplex128,allownil"` ANamedBool bool `msg:"anamedbool,allownil"` ANamedInt int `msg:"anamedint,allownil"` ANamedFloat64 float64 `msg:"anamedfloat64,allownil"` AMapStrStr map[string]string `msg:"amapstrstr,allownil"` APtrNamedStr *NamedString `msg:"aptrnamedstr,allownil"` AString string `msg:"astring,allownil"` ANamedString string `msg:"anamedstring,allownil"` AByteSlice []byte `msg:"abyteslice,allownil"` ASliceString []string `msg:"aslicestring,allownil"` ASliceNamedString []NamedString `msg:"aslicenamedstring,allownil"` ANamedStruct NamedStructAN `msg:"anamedstruct,allownil"` APtrNamedStruct *NamedStructAN `msg:"aptrnamedstruct,allownil"` AUnnamedStruct struct { A string `msg:"a,allownil"` } `msg:"aunnamedstruct,allownil"` // allownil not supported on unnamed struct EmbeddableStructAN `msg:",flatten,allownil"` // embed flat EmbeddableStruct2AN `msg:"embeddablestruct2,allownil"` // embed non-flat AArrayInt [5]int `msg:"aarrayint,allownil"` // not supported ATime time.Time `msg:"atime,allownil"` } type AllowNil1 struct { ABool []bool `msg:"abool,allownil"` AInt []int `msg:"aint,allownil"` AInt8 []int8 `msg:"aint8,allownil"` AInt16 []int16 `msg:"aint16,allownil"` AInt32 []int32 `msg:"aint32,allownil"` AInt64 []int64 `msg:"aint64,allownil"` AUint []uint `msg:"auint,allownil"` AUint8 []uint8 `msg:"auint8,allownil"` AUint16 []uint16 `msg:"auint16,allownil"` AUint32 []uint32 `msg:"auint32,allownil"` AUint64 []uint64 `msg:"auint64,allownil"` AFloat32 []float32 `msg:"afloat32,allownil"` AFloat64 []float64 `msg:"afloat64,allownil"` AComplex64 []complex64 `msg:"acomplex64,allownil"` AComplex128 []complex128 `msg:"acomplex128,allownil"` ANamedBool []bool `msg:"anamedbool,allownil"` ANamedInt []int `msg:"anamedint,allownil"` ANamedFloat64 []float64 `msg:"anamedfloat64,allownil"` AMapStrStr map[string]string `msg:"amapstrstr,allownil"` APtrNamedStr *NamedString `msg:"aptrnamedstr,allownil"` AString []string `msg:"astring,allownil"` ANamedString []string `msg:"anamedstring,allownil"` AByteSlice []byte `msg:"abyteslice,allownil"` ASliceString []string `msg:"aslicestring,allownil"` ASliceNamedString []NamedString `msg:"aslicenamedstring,allownil"` ANamedStruct NamedStructAN `msg:"anamedstruct,allownil"` APtrNamedStruct *NamedStructAN `msg:"aptrnamedstruct,allownil"` AUnnamedStruct struct { A []string `msg:"a,allownil"` } `msg:"aunnamedstruct,allownil"` *EmbeddableStructAN `msg:",flatten,allownil"` // embed flat *EmbeddableStruct2AN `msg:"embeddablestruct2,allownil"` // embed non-flat AArrayInt [5]int `msg:"aarrayint,allownil"` // not supported ATime *time.Time `msg:"atime,allownil"` } type EmbeddableStructAN struct { SomeEmbed []string `msg:"someembed,allownil"` } type EmbeddableStruct2AN struct { SomeEmbed2 []string `msg:"someembed2,allownil"` } type NamedStructAN struct { A []string `msg:"a,allownil"` B []string `msg:"b,allownil"` } type AllowNilHalfFull struct { Field00 []string `msg:"field00,allownil"` Field01 []string `msg:"field01"` Field02 []string `msg:"field02,allownil"` Field03 []string `msg:"field03"` } type AllowNilLotsOFields struct { Field00 []string `msg:"field00,allownil"` Field01 []string `msg:"field01,allownil"` Field02 []string `msg:"field02,allownil"` Field03 []string `msg:"field03,allownil"` Field04 []string `msg:"field04,allownil"` Field05 []string `msg:"field05,allownil"` Field06 []string `msg:"field06,allownil"` Field07 []string `msg:"field07,allownil"` Field08 []string `msg:"field08,allownil"` Field09 []string `msg:"field09,allownil"` Field10 []string `msg:"field10,allownil"` Field11 []string `msg:"field11,allownil"` Field12 []string `msg:"field12,allownil"` Field13 []string `msg:"field13,allownil"` Field14 []string `msg:"field14,allownil"` Field15 []string `msg:"field15,allownil"` Field16 []string `msg:"field16,allownil"` Field17 []string `msg:"field17,allownil"` Field18 []string `msg:"field18,allownil"` Field19 []string `msg:"field19,allownil"` Field20 []string `msg:"field20,allownil"` Field21 []string `msg:"field21,allownil"` Field22 []string `msg:"field22,allownil"` Field23 []string `msg:"field23,allownil"` Field24 []string `msg:"field24,allownil"` Field25 []string `msg:"field25,allownil"` Field26 []string `msg:"field26,allownil"` Field27 []string `msg:"field27,allownil"` Field28 []string `msg:"field28,allownil"` Field29 []string `msg:"field29,allownil"` Field30 []string `msg:"field30,allownil"` Field31 []string `msg:"field31,allownil"` Field32 []string `msg:"field32,allownil"` Field33 []string `msg:"field33,allownil"` Field34 []string `msg:"field34,allownil"` Field35 []string `msg:"field35,allownil"` Field36 []string `msg:"field36,allownil"` Field37 []string `msg:"field37,allownil"` Field38 []string `msg:"field38,allownil"` Field39 []string `msg:"field39,allownil"` Field40 []string `msg:"field40,allownil"` Field41 []string `msg:"field41,allownil"` Field42 []string `msg:"field42,allownil"` Field43 []string `msg:"field43,allownil"` Field44 []string `msg:"field44,allownil"` Field45 []string `msg:"field45,allownil"` Field46 []string `msg:"field46,allownil"` Field47 []string `msg:"field47,allownil"` Field48 []string `msg:"field48,allownil"` Field49 []string `msg:"field49,allownil"` Field50 []string `msg:"field50,allownil"` Field51 []string `msg:"field51,allownil"` Field52 []string `msg:"field52,allownil"` Field53 []string `msg:"field53,allownil"` Field54 []string `msg:"field54,allownil"` Field55 []string `msg:"field55,allownil"` Field56 []string `msg:"field56,allownil"` Field57 []string `msg:"field57,allownil"` Field58 []string `msg:"field58,allownil"` Field59 []string `msg:"field59,allownil"` Field60 []string `msg:"field60,allownil"` Field61 []string `msg:"field61,allownil"` Field62 []string `msg:"field62,allownil"` Field63 []string `msg:"field63,allownil"` Field64 []string `msg:"field64,allownil"` Field65 []string `msg:"field65,allownil"` Field66 []string `msg:"field66,allownil"` Field67 []string `msg:"field67,allownil"` Field68 []string `msg:"field68,allownil"` Field69 []string `msg:"field69,allownil"` } type AllowNil10 struct { Field00 []string `msg:"field00,allownil"` Field01 []string `msg:"field01,allownil"` Field02 []string `msg:"field02,allownil"` Field03 []string `msg:"field03,allownil"` Field04 []string `msg:"field04,allownil"` Field05 []string `msg:"field05,allownil"` Field06 []string `msg:"field06,allownil"` Field07 []string `msg:"field07,allownil"` Field08 []string `msg:"field08,allownil"` Field09 []string `msg:"field09,allownil"` } type NotAllowNil10 struct { Field00 []string `msg:"field00"` Field01 []string `msg:"field01"` Field02 []string `msg:"field02"` Field03 []string `msg:"field03"` Field04 []string `msg:"field04"` Field05 []string `msg:"field05"` Field06 []string `msg:"field06"` Field07 []string `msg:"field07"` Field08 []string `msg:"field08"` Field09 []string `msg:"field09"` } type AllowNilOmitEmpty struct { Field00 []string `msg:"field00,allownil,omitempty"` Field01 []string `msg:"field01,allownil"` } type AllowNilOmitEmpty2 struct { Field00 []string `msg:"field00,allownil,omitempty"` Field01 []string `msg:"field01,allownil,omitempty"` } // Primitive types cannot have allownil for now. type NoAllowNil []byte msgp-1.2.5/_generated/allownil_test.go000066400000000000000000000020131472554463600200050ustar00rootroot00000000000000package _generated import ( "bytes" "reflect" "testing" "github.com/tinylib/msgp/msgp" ) func TestAllownil(t *testing.T) { tt := &NamedStructAN{ A: []string{}, B: nil, } var buf bytes.Buffer err := msgp.Encode(&buf, tt) if err != nil { t.Fatal(err) } in := buf.Bytes() for _, tnew := range []*NamedStructAN{{}, {A: []string{}}, {B: []string{}}} { err = msgp.Decode(bytes.NewBuffer(in), tnew) if err != nil { t.Error(err) } if !reflect.DeepEqual(tt, tnew) { t.Logf("in: %#v", tt) t.Logf("out: %#v", tnew) t.Fatal("objects not equal") } } in, err = tt.MarshalMsg(nil) if err != nil { t.Fatal(err) } for _, tanother := range []*NamedStructAN{{}, {A: []string{}}, {B: []string{}}} { var left []byte left, err = tanother.UnmarshalMsg(in) if err != nil { t.Error(err) } if len(left) > 0 { t.Errorf("%d bytes left", len(left)) } if !reflect.DeepEqual(tt, tanother) { t.Logf("in: %#v", tt) t.Logf("out: %#v", tanother) t.Fatal("objects not equal") } } } msgp-1.2.5/_generated/clearomitted.go000066400000000000000000000060041472554463600176050ustar00rootroot00000000000000package _generated import ( "encoding/json" "time" ) //go:generate msgp //msgp:clearomitted // check some specific cases for omitzero type ClearOmitted0 struct { AStruct ClearOmittedA `msg:"astruct,omitempty"` // leave this one omitempty BStruct ClearOmittedA `msg:"bstruct,omitzero"` // and compare to this AStructPtr *ClearOmittedA `msg:"astructptr,omitempty"` // a pointer case omitempty BStructPtr *ClearOmittedA `msg:"bstructptr,omitzero"` // a pointer case omitzero AExt OmitZeroExt `msg:"aext,omitzero"` // external type case // more APtrNamedStr *NamedStringCO `msg:"aptrnamedstr,omitzero"` ANamedStruct NamedStructCO `msg:"anamedstruct,omitzero"` APtrNamedStruct *NamedStructCO `msg:"aptrnamedstruct,omitzero"` EmbeddableStructCO `msg:",flatten,omitzero"` // embed flat EmbeddableStructCO2 `msg:"embeddablestruct2,omitzero"` // embed non-flat ATime time.Time `msg:"atime,omitzero"` ASlice []int `msg:"aslice,omitempty"` AMap map[string]int `msg:"amap,omitempty"` ABin []byte `msg:"abin,omitempty"` AInt int `msg:"aint,omitempty"` AString string `msg:"atring,omitempty"` Adur time.Duration `msg:"adur,omitempty"` AJSON json.Number `msg:"ajson,omitempty"` ClearOmittedTuple ClearOmittedTuple `msg:"ozt"` // the inside of a tuple should ignore both omitempty and omitzero } type ClearOmittedA struct { A string `msg:"a,omitempty"` B NamedStringCO `msg:"b,omitzero"` C NamedStringCO `msg:"c,omitzero"` } func (o *ClearOmittedA) IsZero() bool { if o == nil { return true } return *o == (ClearOmittedA{}) } type NamedStructCO struct { A string `msg:"a,omitempty"` B string `msg:"b,omitempty"` } func (ns *NamedStructCO) IsZero() bool { if ns == nil { return true } return *ns == (NamedStructCO{}) } type NamedStringCO string func (ns *NamedStringCO) IsZero() bool { if ns == nil { return true } return *ns == "" } type EmbeddableStructCO struct { SomeEmbed string `msg:"someembed2,omitempty"` } func (es EmbeddableStructCO) IsZero() bool { return es == (EmbeddableStructCO{}) } type EmbeddableStructCO2 struct { SomeEmbed2 string `msg:"someembed2,omitempty"` } func (es EmbeddableStructCO2) IsZero() bool { return es == (EmbeddableStructCO2{}) } //msgp:tuple ClearOmittedTuple // ClearOmittedTuple is flagged for tuple output, it should ignore all omitempty and omitzero functionality // since it's fundamentally incompatible. type ClearOmittedTuple struct { FieldA string `msg:"fielda,omitempty"` FieldB NamedStringCO `msg:"fieldb,omitzero"` FieldC NamedStringCO `msg:"fieldc,omitzero"` } type ClearOmitted1 struct { T1 ClearOmittedTuple `msg:"t1"` } msgp-1.2.5/_generated/clearomitted_test.go000066400000000000000000000044241472554463600206500ustar00rootroot00000000000000package _generated import ( "bytes" "encoding/json" "reflect" "testing" "time" "github.com/tinylib/msgp/msgp" ) func TestClearOmitted(t *testing.T) { cleared := ClearOmitted0{} encoded, err := cleared.MarshalMsg(nil) if err != nil { t.Fatal(err) } vPtr := NamedStringCO("value") filled := ClearOmitted0{ AStruct: ClearOmittedA{A: "something"}, BStruct: ClearOmittedA{A: "somthing"}, AStructPtr: &ClearOmittedA{A: "something"}, AExt: OmitZeroExt{25}, APtrNamedStr: &vPtr, ANamedStruct: NamedStructCO{A: "value"}, APtrNamedStruct: &NamedStructCO{A: "sdf"}, EmbeddableStructCO: EmbeddableStructCO{"value"}, EmbeddableStructCO2: EmbeddableStructCO2{"value"}, ATime: time.Now(), ASlice: []int{1, 2, 3}, AMap: map[string]int{"1": 1}, ABin: []byte{1, 2, 3}, ClearOmittedTuple: ClearOmittedTuple{FieldA: "value"}, AInt: 42, AString: "value", Adur: time.Second, AJSON: json.Number(`43.0000000000002`), } dst := filled _, err = dst.UnmarshalMsg(encoded) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, cleared) { t.Errorf("\n got=%#v\nwant=%#v", dst, cleared) } // Reset dst = filled err = dst.DecodeMsg(msgp.NewReader(bytes.NewReader(encoded))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, cleared) { t.Errorf("\n got=%#v\nwant=%#v", dst, cleared) } // Check that fields aren't accidentally zeroing fields. wantJson, err := json.Marshal(filled) if err != nil { t.Fatal(err) } encoded, err = filled.MarshalMsg(nil) if err != nil { t.Fatal(err) } dst = ClearOmitted0{} _, err = dst.UnmarshalMsg(encoded) if err != nil { t.Fatal(err) } got, err := json.Marshal(dst) if err != nil { t.Fatal(err) } if !bytes.Equal(got, wantJson) { t.Errorf("\n got=%#v\nwant=%#v", string(got), string(wantJson)) } // Reset dst = ClearOmitted0{} err = dst.DecodeMsg(msgp.NewReader(bytes.NewReader(encoded))) if err != nil { t.Fatal(err) } got, err = json.Marshal(dst) if err != nil { t.Fatal(err) } if !bytes.Equal(got, wantJson) { t.Errorf("\n got=%#v\nwant=%#v", string(got), string(wantJson)) } t.Log("OK - got", string(got)) } msgp-1.2.5/_generated/compactfloats.go000066400000000000000000000004201472554463600177640ustar00rootroot00000000000000package _generated //go:generate msgp //msgp:compactfloats //msgp:ignore F64 type F64 float64 //msgp:replace F64 with:float64 type Floats struct { A float64 B float32 Slice []float64 Map map[string]float64 F F64 OE float64 `msg:",omitempty"` } msgp-1.2.5/_generated/compactfloats_test.go000066400000000000000000000035661472554463600210410ustar00rootroot00000000000000package _generated import ( "bytes" "reflect" "testing" "github.com/tinylib/msgp/msgp" ) func TestCompactFloats(t *testing.T) { // Constant that can be represented in f32 without loss const f32ok = -1e2 allF32 := Floats{ A: f32ok, B: f32ok, Slice: []float64{f32ok, f32ok}, Map: map[string]float64{"a": f32ok}, F: f32ok, OE: f32ok, } asF32 := float32(f32ok) wantF32 := map[string]any{"A": asF32, "B": asF32, "F": asF32, "Map": map[string]any{"a": asF32}, "OE": asF32, "Slice": []any{asF32, asF32}} enc, err := allF32.MarshalMsg(nil) if err != nil { t.Error(err) } i, _, _ := msgp.ReadIntfBytes(enc) got := i.(map[string]any) if !reflect.DeepEqual(got, wantF32) { t.Errorf("want: %v, got: %v (diff may be types)", wantF32, got) } var buf bytes.Buffer en := msgp.NewWriter(&buf) allF32.EncodeMsg(en) en.Flush() enc = buf.Bytes() i, _, _ = msgp.ReadIntfBytes(enc) got = i.(map[string]any) if !reflect.DeepEqual(got, wantF32) { t.Errorf("want: %v, got: %v (diff may be types)", wantF32, got) } const f64ok = -10e64 allF64 := Floats{ A: f64ok, B: f32ok, Slice: []float64{f64ok, f64ok}, Map: map[string]float64{"a": f64ok}, F: f64ok, OE: f64ok, } asF64 := float64(f64ok) wantF64 := map[string]any{"A": asF64, "B": asF32, "F": asF64, "Map": map[string]any{"a": asF64}, "OE": asF64, "Slice": []any{asF64, asF64}} enc, err = allF64.MarshalMsg(nil) if err != nil { t.Error(err) } i, _, _ = msgp.ReadIntfBytes(enc) got = i.(map[string]any) if !reflect.DeepEqual(got, wantF64) { t.Errorf("want: %v, got: %v (diff may be types)", wantF64, got) } buf.Reset() en = msgp.NewWriter(&buf) allF64.EncodeMsg(en) en.Flush() enc = buf.Bytes() i, _, _ = msgp.ReadIntfBytes(enc) got = i.(map[string]any) if !reflect.DeepEqual(got, wantF64) { t.Errorf("want: %v, got: %v (diff may be types)", wantF64, got) } } msgp-1.2.5/_generated/convert.go000066400000000000000000000032261472554463600166140ustar00rootroot00000000000000package _generated import "errors" //go:generate msgp //msgp:shim ConvertStringVal as:string using:fromConvertStringVal/toConvertStringVal mode:convert //msgp:ignore ConvertStringVal func fromConvertStringVal(v ConvertStringVal) (string, error) { return string(v), nil } func toConvertStringVal(s string) (ConvertStringVal, error) { return ConvertStringVal(s), nil } type ConvertStringVal string type ConvertString struct { String ConvertStringVal } type ConvertStringSlice struct { Strings []ConvertStringVal } type ConvertStringMapValue struct { Strings map[string]ConvertStringVal } //msgp:shim ConvertIntfVal as:interface{} using:fromConvertIntfVal/toConvertIntfVal mode:convert //msgp:ignore ConvertIntfVal func fromConvertIntfVal(v ConvertIntfVal) (interface{}, error) { return v.Test, nil } func toConvertIntfVal(s interface{}) (ConvertIntfVal, error) { return ConvertIntfVal{Test: s.(string)}, nil } type ConvertIntfVal struct { Test string } type ConvertIntf struct { Intf ConvertIntfVal } //msgp:shim ConvertErrVal as:string using:fromConvertErrVal/toConvertErrVal mode:convert //msgp:ignore ConvertErrVal var ( errConvertFrom = errors.New("error: convert from") errConvertTo = errors.New("error: convert to") ) const ( fromFailStr = "fromfail" toFailStr = "tofail" ) func fromConvertErrVal(v ConvertErrVal) (string, error) { s := string(v) if s == fromFailStr { return "", errConvertFrom } return s, nil } func toConvertErrVal(s string) (ConvertErrVal, error) { if s == toFailStr { return ConvertErrVal(""), errConvertTo } return ConvertErrVal(s), nil } type ConvertErrVal string type ConvertErr struct { Err ConvertErrVal } msgp-1.2.5/_generated/convert_test.go000066400000000000000000000024451472554463600176550ustar00rootroot00000000000000package _generated import ( "bytes" "testing" "github.com/tinylib/msgp/msgp" ) func TestConvertFromEncodeError(t *testing.T) { e := ConvertErr{ConvertErrVal(fromFailStr)} var buf bytes.Buffer w := msgp.NewWriter(&buf) err := e.EncodeMsg(w) if msgp.Cause(err) != errConvertFrom { t.Fatalf("expected conversion error, found '%v'", err.Error()) } } func TestConvertToEncodeError(t *testing.T) { var in, out ConvertErr in = ConvertErr{ConvertErrVal(toFailStr)} var buf bytes.Buffer w := msgp.NewWriter(&buf) err := in.EncodeMsg(w) if err != nil { t.FailNow() } w.Flush() r := msgp.NewReader(&buf) err = (&out).DecodeMsg(r) if msgp.Cause(err) != errConvertTo { t.Fatalf("expected conversion error, found %v", err.Error()) } } func TestConvertFromMarshalError(t *testing.T) { e := ConvertErr{ConvertErrVal(fromFailStr)} var b []byte _, err := e.MarshalMsg(b) if msgp.Cause(err) != errConvertFrom { t.Fatalf("expected conversion error, found %v", err.Error()) } } func TestConvertToMarshalError(t *testing.T) { var in, out ConvertErr in = ConvertErr{ConvertErrVal(toFailStr)} b, err := in.MarshalMsg(nil) if err != nil { t.FailNow() } _, err = (&out).UnmarshalMsg(b) if msgp.Cause(err) != errConvertTo { t.Fatalf("expected conversion error, found %v", err.Error()) } } msgp-1.2.5/_generated/custom_tag.go000066400000000000000000000002271472554463600172770ustar00rootroot00000000000000package _generated //go:generate msgp //msgp:tag mytag type CustomTag struct { Foo string `mytag:"foo_custom_name"` Bar int `mytag:"bar1234"` } msgp-1.2.5/_generated/custom_tag_test.go000066400000000000000000000033551472554463600203430ustar00rootroot00000000000000package _generated import ( "encoding/json" "fmt" "reflect" "testing" "bytes" "github.com/tinylib/msgp/msgp" ) func TestCustomTag(t *testing.T) { t.Run("File Scope", func(t *testing.T) { ts := CustomTag{ Foo: "foostring13579", Bar: 999_999} encDecCustomTag(t, ts, "mytag") }) } func encDecCustomTag(t *testing.T, testStruct msgp.Encodable, tag string) { var b bytes.Buffer msgp.Encode(&b, testStruct) // Check tag names using JSON as an intermediary layer // TODO: is there a way to avoid the JSON layer? We'd need to directly decode raw msgpack -> map[string]any refJSON, err := json.Marshal(testStruct) if err != nil { t.Error(fmt.Sprintf("error encoding struct as JSON: %v", err)) } ref := make(map[string]any) // Encoding and decoding the original struct via JSON is necessary // for field comparisons to work, since JSON -> map[string]any // relies on type inferences such as all numbers being float64s json.Unmarshal(refJSON, &ref) var encJSON bytes.Buffer msgp.UnmarshalAsJSON(&encJSON, b.Bytes()) encoded := make(map[string]any) json.Unmarshal(encJSON.Bytes(), &encoded) tsType := reflect.TypeOf(testStruct) for i := 0; i < tsType.NumField(); i++ { // Check encoded field name field := tsType.Field(i) encodedValue, ok := encoded[field.Tag.Get(tag)] if !ok { t.Error("missing encoded value for field", field.Name) continue } // Check encoded field value (against original value post-JSON enc + dec) jsonName, ok := field.Tag.Lookup("json") if !ok { jsonName = field.Name } refValue := ref[jsonName] if !reflect.DeepEqual(refValue, encodedValue) { t.Error(fmt.Sprintf("incorrect encoded value for field %s. reference: %v, encoded: %v", field.Name, refValue, encodedValue)) } } } msgp-1.2.5/_generated/def.go000066400000000000000000000162231472554463600156730ustar00rootroot00000000000000package _generated import ( "encoding/json" "os" "time" "github.com/tinylib/msgp/msgp" ) //go:generate msgp -v -o generated.go // All of the struct // definitions in this // file are fed to the code // generator when `make test` is // called, followed by an // invocation of `go test -v` in this // directory. A simple way of testing // a struct definition is // by adding it to this file. type Block [32]byte // tests edge-cases with // compiling size compilation. type X struct { Values [32]byte // should compile to 32*msgp.ByteSize; encoded as Bin ValuesPtr *[32]byte // check (*)[:] deref More Block // should be identical to the above Others [][32]int32 // should compile to len(x.Others)*32*msgp.Int32Size Matrix [][]int32 // should not optimize ManyFixed []Fixed WeirdTag string `msg:"\x0b"` } // test fixed-size struct // size compilation type Fixed struct { A float64 B bool } type TestType struct { F *float64 `msg:"float"` Els map[string]string `msg:"elements"` Obj struct { // test anonymous struct ValueA string `msg:"value_a"` ValueB []byte `msg:"value_b"` } `msg:"object"` Child *TestType `msg:"child"` Time time.Time `msg:"time"` Any interface{} `msg:"any"` Appended msgp.Raw `msg:"appended"` Num msgp.Number `msg:"num"` Byte byte Rune rune RunePtr *rune RunePtrPtr **rune RuneSlice []rune Slice1 []string Slice2 []string SlicePtr *[]string MapStringEmpty map[string]struct{} MapStringEmpty2 map[string]EmptyStruct } //msgp:tuple Object type Object struct { ObjectNo string `msg:"objno"` Slice1 []string `msg:"slice1"` Slice2 []string `msg:"slice2"` MapMap map[string]map[string]string MapStringEmpty map[string]struct{} MapStringEmpty2 map[string]EmptyStruct } //msgp:tuple TestBench type TestBench struct { Name string BirthDay time.Time Phone string Siblings int Spouse bool Money float64 } //msgp:tuple TestFast type TestFast struct { Lat, Long, Alt float64 // test inline decl Data []byte } // Test nested aliases type ( FastAlias TestFast AliasContainer struct { Fast FastAlias } ) // Test dependency resolution type ( IntA int IntB IntA IntC IntB ) type TestHidden struct { A string B []float64 Bad func(string) bool // This results in a warning: field "Bad" unsupported } type Embedded struct { *Embedded // test embedded field Children []Embedded PtrChildren []*Embedded Other string } const eight = 8 type Things struct { Cmplx complex64 `msg:"complex"` // test slices Vals []int32 `msg:"values"` Arr [msgp.ExtensionPrefixSize]float64 `msg:"arr"` // test const array and *ast.SelectorExpr as array size Arr2 [4]float64 `msg:"arr2"` // test basic lit array Ext *msgp.RawExtension `msg:"ext,extension"` // test extension Oext msgp.RawExtension `msg:"oext,extension"` // test extension reference } //msgp:shim SpecialID as:[]byte using:toBytes/fromBytes type ( SpecialID string TestObj struct{ ID1, ID2 SpecialID } ) func toBytes(id SpecialID) []byte { return []byte(string(id)) } func fromBytes(id []byte) SpecialID { return SpecialID(string(id)) } type MyEnum byte const ( A MyEnum = iota B C D invalid ) // test shim directive (below) //msgp:shim MyEnum as:string using:(MyEnum).String/myenumStr //msgp:shim *os.File as:string using:filetostr/filefromstr func filetostr(f *os.File) string { return f.Name() } func filefromstr(s string) *os.File { f, _ := os.Open(s) return f } func (m MyEnum) String() string { switch m { case A: return "A" case B: return "B" case C: return "C" case D: return "D" default: return "" } } func myenumStr(s string) MyEnum { switch s { case "A": return A case "B": return B case "C": return C case "D": return D default: return invalid } } // test pass-specific directive //msgp:decode ignore Insane type Insane [3]map[string]struct{ A, B CustomInt } type Custom struct { Bts CustomBytes `msg:"bts"` Mp map[string]*Embedded `msg:"mp"` Enums []MyEnum `msg:"enums"` // test explicit enum shim Some FileHandle `msg:"file_handle"` } type Files []*os.File type FileHandle struct { Relevant Files `msg:"files"` Name string `msg:"name"` } type ( CustomInt int CustomBytes []byte ) type Wrapper struct { Tree *Tree } type Tree struct { Children []Tree Element int Parent *Wrapper } // Ensure all different widths of integer can be used as constant keys. const ( ConstantInt int = 8 ConstantInt8 int8 = 8 ConstantInt16 int16 = 8 ConstantInt32 int32 = 8 ConstantInt64 int64 = 8 ConstantUint uint = 8 ConstantUint8 uint8 = 8 ConstantUint16 uint16 = 8 ConstantUint32 uint32 = 8 ConstantUint64 uint64 = 8 ) type ArrayConstants struct { ConstantInt [ConstantInt]string ConstantInt8 [ConstantInt8]string ConstantInt16 [ConstantInt16]string ConstantInt32 [ConstantInt32]string ConstantInt64 [ConstantInt64]string ConstantUint [ConstantUint]string ConstantUint8 [ConstantUint8]string ConstantUint16 [ConstantUint16]string ConstantUint32 [ConstantUint32]string ConstantUint64 [ConstantUint64]string ConstantHex [0x16]string ConstantOctal [0o7]string } // Ensure non-msg struct tags work: // https://github.com/tinylib/msgp/issues/201 type NonMsgStructTags struct { A []string `json:"fooJSON" msg:"fooMsgp"` B string `json:"barJSON"` C []string `json:"bazJSON" msg:"-"` Nested []struct { A []string `json:"a"` B string `json:"b"` C []string `json:"c"` VeryNested []struct { A []string `json:"a"` B []string `msg:"bbbb" xml:"-"` C map[string]struct{} `msg:"cccc"` } } } type EmptyStruct struct{} type StructByteSlice struct { ABytes []byte `msg:",allownil"` AString []string `msg:",allownil"` ABool []bool `msg:",allownil"` AInt []int `msg:",allownil"` AInt8 []int8 `msg:",allownil"` AInt16 []int16 `msg:",allownil"` AInt32 []int32 `msg:",allownil"` AInt64 []int64 `msg:",allownil"` AUint []uint `msg:",allownil"` AUint8 []uint8 `msg:",allownil"` AUint16 []uint16 `msg:",allownil"` AUint32 []uint32 `msg:",allownil"` AUint64 []uint64 `msg:",allownil"` AFloat32 []float32 `msg:",allownil"` AFloat64 []float64 `msg:",allownil"` AComplex64 []complex64 `msg:",allownil"` AComplex128 []complex128 `msg:",allownil"` AStruct []Fixed `msg:",allownil"` } type NumberJSONSample struct { Single json.Number Array []json.Number Map map[string]json.Number OE json.Number `msg:",omitempty"` } type Flobbity struct { A Flobs `msg:"a,omitempty"` B Flobs `msg:"b,omitempty"` } type Flobs []Flob type Flob struct { X Numberwang `msg:"x"` Y int8 `msg:"y"` Z int8 `msg:"z"` W int32 `msg:"w"` } type Numberwang int8 msgp-1.2.5/_generated/def_test.go000066400000000000000000000113151472554463600167270ustar00rootroot00000000000000package _generated import ( "bytes" "encoding/json" "reflect" "testing" "github.com/tinylib/msgp/msgp" ) func TestRuneEncodeDecode(t *testing.T) { tt := &TestType{} r := 'r' rp := &r tt.Rune = r tt.RunePtr = &r tt.RunePtrPtr = &rp tt.RuneSlice = []rune{'a', 'b', '😳'} var buf bytes.Buffer wrt := msgp.NewWriter(&buf) if err := tt.EncodeMsg(wrt); err != nil { t.Errorf("%v", err) } wrt.Flush() var out TestType rdr := msgp.NewReader(&buf) if err := (&out).DecodeMsg(rdr); err != nil { t.Errorf("%v", err) } if r != out.Rune { t.Errorf("rune mismatch: expected %c found %c", r, out.Rune) } if r != *out.RunePtr { t.Errorf("rune ptr mismatch: expected %c found %c", r, *out.RunePtr) } if r != **out.RunePtrPtr { t.Errorf("rune ptr ptr mismatch: expected %c found %c", r, **out.RunePtrPtr) } if !reflect.DeepEqual(tt.RuneSlice, out.RuneSlice) { t.Errorf("rune slice mismatch") } } func TestRuneMarshalUnmarshal(t *testing.T) { tt := &TestType{} r := 'r' rp := &r tt.Rune = r tt.RunePtr = &r tt.RunePtrPtr = &rp tt.RuneSlice = []rune{'a', 'b', '😳'} bts, err := tt.MarshalMsg(nil) if err != nil { t.Errorf("%v", err) } var out TestType if _, err := (&out).UnmarshalMsg(bts); err != nil { t.Errorf("%v", err) } if r != out.Rune { t.Errorf("rune mismatch: expected %c found %c", r, out.Rune) } if r != *out.RunePtr { t.Errorf("rune ptr mismatch: expected %c found %c", r, *out.RunePtr) } if r != **out.RunePtrPtr { t.Errorf("rune ptr ptr mismatch: expected %c found %c", r, **out.RunePtrPtr) } if !reflect.DeepEqual(tt.RuneSlice, out.RuneSlice) { t.Errorf("rune slice mismatch") } } func TestJSONNumber(t *testing.T) { test := NumberJSONSample{ Single: "-42", Array: []json.Number{"0", "-0", "1", "-1", "0.1", "-0.1", "1234", "-1234", "12.34", "-12.34", "12E0", "12E1", "12e34", "12E-0", "12e+1", "12e-34", "-12E0", "-12E1", "-12e34", "-12E-0", "-12e+1", "-12e-34", "1.2E0", "1.2E1", "1.2e34", "1.2E-0", "1.2e+1", "1.2e-34", "-1.2E0", "-1.2E1", "-1.2e34", "-1.2E-0", "-1.2e+1", "-1.2e-34", "0E0", "0E1", "0e34", "0E-0", "0e+1", "0e-34", "-0E0", "-0E1", "-0e34", "-0E-0", "-0e+1", "-0e-34"}, Map: map[string]json.Number{ "a": json.Number("50.2"), }, } // This is not guaranteed to be symmetric encoded, err := test.MarshalMsg(nil) if err != nil { t.Errorf("%v", err) } var v NumberJSONSample _, err = v.UnmarshalMsg(encoded) if err != nil { t.Errorf("%v", err) } // Test two values if v.Single != "-42" { t.Errorf("want %v, got %v", "-42", v.Single) } if v.Map["a"] != "50.2" { t.Errorf("want %v, got %v", "50.2", v.Map["a"]) } var jsBuf bytes.Buffer remain, err := msgp.UnmarshalAsJSON(&jsBuf, encoded) if err != nil { t.Errorf("%v", err) } if len(remain) != 0 { t.Errorf("remain should be empty") } wantjs := `{"Single":-42,"Array":[0,0,1,-1,0.1,-0.1,1234,-1234,12.34,-12.34,12,120,120000000000000000000000000000000000,12,120,0.0000000000000000000000000000000012,-12,-120,-120000000000000000000000000000000000,-12,-120,-0.0000000000000000000000000000000012,1.2,12,12000000000000000000000000000000000,1.2,12,0.00000000000000000000000000000000012,-1.2,-12,-12000000000000000000000000000000000,-1.2,-12,-0.00000000000000000000000000000000012,0,0,0,0,0,0,-0,-0,-0,-0,-0,-0],"Map":{"a":50.2}}` if jsBuf.String() != wantjs { t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) } // Test encoding var buf bytes.Buffer en := msgp.NewWriter(&buf) err = test.EncodeMsg(en) if err != nil { t.Errorf("%v", err) } en.Flush() encoded = buf.Bytes() dc := msgp.NewReader(&buf) err = v.DecodeMsg(dc) if err != nil { t.Errorf("%v", err) } // Test two values if v.Single != "-42" { t.Errorf("want %s, got %s", "-42", v.Single) } if v.Map["a"] != "50.2" { t.Errorf("want %s, got %s", "50.2", v.Map["a"]) } jsBuf.Reset() remain, err = msgp.UnmarshalAsJSON(&jsBuf, encoded) if err != nil { t.Errorf("%v", err) } if len(remain) != 0 { t.Errorf("remain should be empty") } if jsBuf.String() != wantjs { t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) } // Try interface encoder jd := json.NewDecoder(&jsBuf) jd.UseNumber() var jsIntf map[string]any err = jd.Decode(&jsIntf) if err != nil { t.Errorf("%v", err) } // Ensure we encode correctly _ = (jsIntf["Single"]).(json.Number) fromInt, err := msgp.AppendIntf(nil, jsIntf) if err != nil { t.Errorf("%v", err) } // Take the value from the JSON interface encoder and unmarshal back into our struct. v = NumberJSONSample{} _, err = v.UnmarshalMsg(fromInt) if err != nil { t.Errorf("%v", err) } if v.Single != "-42" { t.Errorf("want %s, got %s", "-42", v.Single) } if v.Map["a"] != "50.2" { t.Errorf("want %s, got %s", "50.2", v.Map["a"]) } } msgp-1.2.5/_generated/embedded_struct.go000066400000000000000000000005651472554463600202740ustar00rootroot00000000000000package _generated //go:generate msgp type GetUserRequestWithEmbeddedStruct struct { Common `msg:",flatten"` UserID uint32 `msg:"user_id"` } type GetUserRequest struct { RequestID uint32 `msg:"request_id"` Token string `msg:"token"` UserID uint32 `msg:"user_id"` } type Common struct { RequestID uint32 `msg:"request_id"` Token string `msg:"token"` } msgp-1.2.5/_generated/embedded_struct_test.go000066400000000000000000000030441472554463600213260ustar00rootroot00000000000000package _generated import ( "testing" ) func TestConvertDataFromAEmbeddedStructToANonEmbeddedStruct(t *testing.T) { getUserRequestWithEmbeddedStruct := GetUserRequestWithEmbeddedStruct{ Common: Common{ RequestID: 10, Token: "token", }, UserID: 1000, } bytes, err := getUserRequestWithEmbeddedStruct.MarshalMsg(nil) if err != nil { t.Fatal(err) } getUserRequest := GetUserRequest{} _, err = getUserRequest.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } if getUserRequest.RequestID != getUserRequestWithEmbeddedStruct.RequestID { t.Fatal("not same request id") } if getUserRequest.UserID != getUserRequestWithEmbeddedStruct.UserID { t.Fatal("not same user id") } if getUserRequest.Token != getUserRequestWithEmbeddedStruct.Token { t.Fatal("not same token") } return } func TestConvertDataFromANonEmbeddedStructToAEmbeddedStruct(t *testing.T) { getUserRequest := GetUserRequest{ RequestID: 10, Token: "token", UserID: 1000, } bytes, err := getUserRequest.MarshalMsg(nil) if err != nil { t.Fatal(err) } getUserRequestWithEmbeddedStruct := GetUserRequestWithEmbeddedStruct{} _, err = getUserRequestWithEmbeddedStruct.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } if getUserRequest.RequestID != getUserRequestWithEmbeddedStruct.RequestID { t.Fatal("not same request id") } if getUserRequest.UserID != getUserRequestWithEmbeddedStruct.UserID { t.Fatal("not same user id") } if getUserRequest.Token != getUserRequestWithEmbeddedStruct.Token { t.Fatal("not same token") } return } msgp-1.2.5/_generated/errorwrap.go000066400000000000000000000026701472554463600171610ustar00rootroot00000000000000package _generated //go:generate msgp // The leaves of interest in this crazy structs are strings. The test case // looks for strings in the serialised msgpack and makes them unreadable. type ErrorCtxMapChild struct { Val string } type ErrorCtxMapChildNotInline struct { Val1, Val2, Val3, Val4, Val5 string } type ErrorCtxAsMap struct { Val string Child *ErrorCtxMapChild Children []*ErrorCtxMapChild ComplexChild *ErrorCtxMapChildNotInline Map map[string]string Nest struct { Val string Child *ErrorCtxMapChild Children []*ErrorCtxMapChild Map map[string]string Nest struct { Val string Child *ErrorCtxMapChild Children []*ErrorCtxMapChild Map map[string]string } } } //msgp:tuple ErrorCtxTupleChild type ErrorCtxTupleChild struct { Val string } //msgp:tuple ErrorCtxTupleChildNotInline type ErrorCtxTupleChildNotInline struct { Val1, Val2, Val3, Val4, Val5 string } //msgp:tuple ErrorCtxAsTuple type ErrorCtxAsTuple struct { Val string Child *ErrorCtxTupleChild Children []*ErrorCtxTupleChild ComplexChild *ErrorCtxTupleChildNotInline Map map[string]string Nest struct { Val string Child *ErrorCtxTupleChild Children []*ErrorCtxTupleChild Map map[string]string Nest struct { Val string Child *ErrorCtxTupleChild Children []*ErrorCtxTupleChild Map map[string]string } } } msgp-1.2.5/_generated/errorwrap_test.go000066400000000000000000000162271472554463600202230ustar00rootroot00000000000000package _generated import ( "bytes" "io" "testing" "github.com/tinylib/msgp/msgp" ) func fillErrorCtxAsMap() *ErrorCtxAsMap { v := &ErrorCtxAsMap{} v.Val = "foo" v.ComplexChild = &ErrorCtxMapChildNotInline{Val1: "a", Val2: "b", Val3: "c", Val4: "d", Val5: "e"} v.Child = &ErrorCtxMapChild{Val: "foo"} v.Children = []*ErrorCtxMapChild{{Val: "foo"}, {Val: "bar"}} v.Map = map[string]string{"foo": "bar", "baz": "qux"} v.Nest.Val = "foo" v.Nest.Child = &ErrorCtxMapChild{Val: "foo"} v.Nest.Children = []*ErrorCtxMapChild{{Val: "foo"}, {Val: "bar"}} v.Nest.Map = map[string]string{"foo": "bar", "baz": "qux"} v.Nest.Nest.Val = "foo" v.Nest.Nest.Child = &ErrorCtxMapChild{Val: "foo"} v.Nest.Nest.Children = []*ErrorCtxMapChild{{Val: "foo"}, {Val: "bar"}} v.Nest.Nest.Map = map[string]string{"foo": "bar", "baz": "qux"} return v } func fillErrorCtxAsTuple() *ErrorCtxAsTuple { v := &ErrorCtxAsTuple{} v.Val = "foo" v.ComplexChild = &ErrorCtxTupleChildNotInline{Val1: "a", Val2: "b", Val3: "c", Val4: "d", Val5: "e"} v.Child = &ErrorCtxTupleChild{Val: "foo"} v.Children = []*ErrorCtxTupleChild{{Val: "foo"}, {Val: "bar"}} v.Map = map[string]string{"foo": "bar", "baz": "qux"} v.Nest.Val = "foo" v.Nest.Child = &ErrorCtxTupleChild{Val: "foo"} v.Nest.Children = []*ErrorCtxTupleChild{{Val: "foo"}, {Val: "bar"}} v.Nest.Map = map[string]string{"foo": "bar", "baz": "qux"} v.Nest.Nest.Val = "foo" v.Nest.Nest.Child = &ErrorCtxTupleChild{Val: "foo"} v.Nest.Nest.Children = []*ErrorCtxTupleChild{{Val: "foo"}, {Val: "bar"}} v.Nest.Nest.Map = map[string]string{"foo": "bar", "baz": "qux"} return v } type dodgifierBuf struct { *bytes.Buffer dodgifyString int strIdx int } func (o *dodgifierBuf) Write(b []byte) (n int, err error) { ilen := len(b) if msgp.NextType(b) == msgp.StrType { if o.strIdx == o.dodgifyString { // Fool msgp into thinking this value is a fixint. msgp will throw // a type error for this value. b[0] = 1 } o.strIdx++ } _, err = o.Buffer.Write(b) return ilen, err } type strCounter int func (o *strCounter) Write(b []byte) (n int, err error) { if msgp.NextType(b) == msgp.StrType { *o++ } return len(b), nil } func countStrings(bts []byte) int { r := msgp.NewReader(bytes.NewReader(bts)) strCounter := strCounter(0) for { _, err := r.CopyNext(&strCounter) if err == io.EOF { break } else if err != nil { panic(err) } } return int(strCounter) } func marshalErrorCtx(m msgp.Marshaler) []byte { bts, err := m.MarshalMsg(nil) if err != nil { panic(err) } return bts } // dodgifyMsgpString will wreck the nth string in the msgpack blob // so that it raises an error when decoded or unmarshaled. func dodgifyMsgpString(bts []byte, idx int) []byte { r := msgp.NewReader(bytes.NewReader(bts)) out := &dodgifierBuf{Buffer: &bytes.Buffer{}, dodgifyString: idx} for { _, err := r.CopyNext(out) if err == io.EOF { break } else if err != nil { panic(err) } } return out.Bytes() } func TestErrorCtxAsMapUnmarshal(t *testing.T) { bts := marshalErrorCtx(fillErrorCtxAsMap()) cnt := countStrings(bts) var as []string for i := 0; i < cnt; i++ { dodgeBts := dodgifyMsgpString(bts, i) var ec ErrorCtxAsMap _, err := (&ec).UnmarshalMsg(dodgeBts) as = append(as, err.Error()) } ok, a, b := diffstrs(as, expectedAsMap()) if !ok { t.Fatal(a, b) } } func TestErrorCtxAsMapDecode(t *testing.T) { bts := marshalErrorCtx(fillErrorCtxAsMap()) cnt := countStrings(bts) var as []string for i := 0; i < cnt; i++ { dodgeBts := dodgifyMsgpString(bts, i) r := msgp.NewReader(bytes.NewReader(dodgeBts)) var ec ErrorCtxAsMap err := (&ec).DecodeMsg(r) as = append(as, err.Error()) } ok, a, b := diffstrs(as, expectedAsMap()) if !ok { t.Fatal(a, b) } } func TestErrorCtxAsTupleUnmarshal(t *testing.T) { bts := marshalErrorCtx(fillErrorCtxAsTuple()) cnt := countStrings(bts) var as []string for i := 0; i < cnt; i++ { dodgeBts := dodgifyMsgpString(bts, i) var ec ErrorCtxAsTuple _, err := (&ec).UnmarshalMsg(dodgeBts) as = append(as, err.Error()) } ok, a, b := diffstrs(as, expectedAsTuple()) if !ok { t.Fatal(a, b) } } func TestErrorCtxAsTupleDecode(t *testing.T) { bts := marshalErrorCtx(fillErrorCtxAsTuple()) cnt := countStrings(bts) var as []string for i := 0; i < cnt; i++ { dodgeBts := dodgifyMsgpString(bts, i) r := msgp.NewReader(bytes.NewReader(dodgeBts)) var ec ErrorCtxAsTuple err := (&ec).DecodeMsg(r) as = append(as, err.Error()) } ok, a, b := diffstrs(as, expectedAsTuple()) if !ok { t.Fatal(a, b) } } func diffstrs(a, b []string) (ok bool, as, bs []string) { ma := map[string]bool{} mb := map[string]bool{} for _, x := range a { ma[x] = true } for _, x := range b { mb[x] = true } for _, x := range a { if !mb[x] { as = append(as, x) } } for _, x := range b { if !ma[x] { bs = append(bs, x) } } return len(as)+len(bs) == 0, as, bs } var errPrefix = `msgp: attempted to decode type "int" with method for "str"` func expectedAsTuple() []string { var out []string for _, s := range []string{ `Val`, `Child/Val`, `Children/0/Val`, `Children/1/Val`, `ComplexChild/Val1`, `ComplexChild/Val2`, `ComplexChild/Val3`, `ComplexChild/Val4`, `ComplexChild/Val5`, `Map`, `Map/baz`, `Map`, `Map/foo`, `Nest`, `Nest/Val`, `Nest`, `Nest/Child/Val`, `Nest`, `Nest/Children/0/Val`, `Nest/Children/1/Val`, `Nest`, `Nest/Map`, `Nest/Map/foo`, `Nest/Map`, `Nest/Map/baz`, `Nest`, `Nest/Nest`, `Nest/Nest/Val`, `Nest/Nest`, `Nest/Nest/Child/Val`, `Nest/Nest`, `Nest/Nest/Children/0/Val`, `Nest/Nest/Children/1/Val`, `Nest/Nest`, `Nest/Nest/Map`, `Nest/Nest/Map/foo`, `Nest/Nest/Map`, `Nest/Nest/Map/baz`, } { if s == "" { out = append(out, errPrefix) } else { out = append(out, errPrefix+" at "+s) } } return out } // there are a lot of extra errors in here at the struct level because we are // not discriminating between dodgy struct field map key strings and // values. dodgy struct field map keys have no field context available when // they are read. func expectedAsMap() []string { var out []string for _, s := range []string{ ``, `Val`, ``, `Child`, `Child/Val`, ``, `Children/0`, `Children/0/Val`, `Children/1`, `Children/1/Val`, `ComplexChild`, `ComplexChild/Val1`, `ComplexChild`, `ComplexChild/Val2`, `ComplexChild`, `ComplexChild/Val3`, `ComplexChild`, `ComplexChild/Val4`, `ComplexChild`, `ComplexChild/Val5`, `Map`, `Map/foo`, `Map`, `Map/baz`, ``, `Nest`, `Nest/Val`, `Nest`, `Nest/Child`, `Nest/Child/Val`, `Nest`, `Nest/Children/0`, `Nest/Children/0/Val`, `Nest/Children/1`, `Nest/Children/1/Val`, `Nest`, `Nest/Map`, `Nest/Map/foo`, `Nest/Map`, `Nest/Map/baz`, `Nest`, `Nest/Nest`, `Nest/Nest/Val`, `Nest/Nest`, `Nest/Nest/Child`, `Nest/Nest/Child/Val`, `Nest/Nest`, `Nest/Nest/Children/0`, `Nest/Nest/Children/0/Val`, `Nest/Nest/Children/1`, `Nest/Nest/Children/1/Val`, `Nest/Nest`, `Nest/Nest/Map`, `Nest/Nest/Map/baz`, `Nest/Nest/Map`, `Nest/Nest/Map/foo`, } { if s == "" { out = append(out, errPrefix) } else { out = append(out, errPrefix+" at "+s) } } return out } msgp-1.2.5/_generated/gen_test.go000066400000000000000000000121031472554463600167360ustar00rootroot00000000000000package _generated import ( "bytes" "reflect" "testing" "time" "github.com/tinylib/msgp/msgp" ) // benchmark encoding a small, "fast" type. // the point here is to see how much garbage // is generated intrinsically by the encoding/ // decoding process as opposed to the nature // of the struct. func BenchmarkFastEncode(b *testing.B) { v := &TestFast{ Lat: 40.12398, Long: -41.9082, Alt: 201.08290, Data: []byte("whaaaaargharbl"), } var buf bytes.Buffer msgp.Encode(&buf, v) en := msgp.NewWriter(msgp.Nowhere) b.SetBytes(int64(buf.Len())) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { v.EncodeMsg(en) } en.Flush() } // benchmark decoding a small, "fast" type. // the point here is to see how much garbage // is generated intrinsically by the encoding/ // decoding process as opposed to the nature // of the struct. func BenchmarkFastDecode(b *testing.B) { v := &TestFast{ Lat: 40.12398, Long: -41.9082, Alt: 201.08290, Data: []byte("whaaaaargharbl"), } var buf bytes.Buffer msgp.Encode(&buf, v) dc := msgp.NewReader(msgp.NewEndlessReader(buf.Bytes(), b)) b.SetBytes(int64(buf.Len())) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { v.DecodeMsg(dc) } } func (a *TestType) Equal(b *TestType) bool { // compare times, appended, then zero out those // fields, perform a DeepEqual, and restore them ta, tb := a.Time, b.Time if !ta.Equal(tb) { return false } aa, ab := a.Appended, b.Appended if !bytes.Equal(aa, ab) { return false } if len(a.MapStringEmpty) == 0 && len(b.MapStringEmpty) == 0 { a.MapStringEmpty = nil b.MapStringEmpty = nil } if len(a.MapStringEmpty2) == 0 && len(b.MapStringEmpty2) == 0 { a.MapStringEmpty2 = nil b.MapStringEmpty2 = nil } a.Time, b.Time = time.Time{}, time.Time{} aa, ab = nil, nil ok := reflect.DeepEqual(a, b) a.Time, b.Time = ta, tb a.Appended, b.Appended = aa, ab return ok } // This covers the following cases: // - Recursive types // - Non-builtin identifiers (and recursive types) // - time.Time // - map[string]string // - anonymous structs func Test1EncodeDecode(t *testing.T) { f := 32.00 tt := &TestType{ F: &f, Els: map[string]string{ "thing_one": "one", "thing_two": "two", }, Obj: struct { ValueA string `msg:"value_a"` ValueB []byte `msg:"value_b"` }{ ValueA: "here's the first inner value", ValueB: []byte("here's the second inner value"), }, Child: nil, Time: time.Now(), Appended: msgp.Raw([]byte{}), // 'nil' MapStringEmpty: map[string]struct{}{"Key": {}, "Key2": {}}, MapStringEmpty2: map[string]EmptyStruct{"Key3": {}, "Key4": {}}, } var buf bytes.Buffer err := msgp.Encode(&buf, tt) if err != nil { t.Fatal(err) } tnew := new(TestType) err = msgp.Decode(&buf, tnew) if err != nil { t.Error(err) } if !tt.Equal(tnew) { t.Logf("in: %#v", tt) t.Logf("out: %#v", tnew) t.Fatal("objects not equal") } tanother := new(TestType) buf.Reset() msgp.Encode(&buf, tt) var left []byte left, err = tanother.UnmarshalMsg(buf.Bytes()) if err != nil { t.Error(err) } if len(left) > 0 { t.Errorf("%d bytes left", len(left)) } if !tt.Equal(tanother) { t.Logf("in: %v", tt) t.Logf("out: %v", tanother) t.Fatal("objects not equal") } } func TestIssue168(t *testing.T) { buf := bytes.Buffer{} test := TestObj{} msgp.Encode(&buf, &TestObj{ID1: "1", ID2: "2"}) msgp.Decode(&buf, &test) if test.ID1 != "1" || test.ID2 != "2" { t.Fatalf("got back %+v", test) } } func TestIssue362(t *testing.T) { in := StructByteSlice{ ABytes: make([]byte, 0), AString: make([]string, 0), ABool: make([]bool, 0), AInt: make([]int, 0), AInt8: make([]int8, 0), AInt16: make([]int16, 0), AInt32: make([]int32, 0), AInt64: make([]int64, 0), AUint: make([]uint, 0), AUint8: make([]uint8, 0), AUint16: make([]uint16, 0), AUint32: make([]uint32, 0), AUint64: make([]uint64, 0), AFloat32: make([]float32, 0), AFloat64: make([]float64, 0), AComplex64: make([]complex64, 0), AComplex128: make([]complex128, 0), AStruct: make([]Fixed, 0), } b, err := in.MarshalMsg(nil) if err != nil { t.Fatal(err) } var dst StructByteSlice _, err = dst.UnmarshalMsg(b) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(in, dst) { t.Fatalf("mismatch %#v != %#v", in, dst) } dst2 := StructByteSlice{} dec := msgp.NewReader(bytes.NewReader(b)) err = dst2.DecodeMsg(dec) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(in, dst2) { t.Fatalf("mismatch %#v != %#v", in, dst2) } // Encode with nil zero := StructByteSlice{} b, err = zero.MarshalMsg(nil) if err != nil { t.Fatal(err) } // Decode into dst that now has values... _, err = dst.UnmarshalMsg(b) if err != nil { t.Fatal(err) } // All should be nil now. if !reflect.DeepEqual(zero, dst) { t.Fatalf("mismatch %#v != %#v", zero, dst) } dec = msgp.NewReader(bytes.NewReader(b)) err = dst2.DecodeMsg(dec) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(zero, dst2) { t.Fatalf("mismatch %#v != %#v", zero, dst2) } } msgp-1.2.5/_generated/issue102.go000066400000000000000000000012671472554463600165120ustar00rootroot00000000000000package _generated //go:generate msgp type Issue102 struct{} type Issue102deep struct { A int X struct{} Y struct{} Z int } //msgp:tuple Issue102Tuple type Issue102Tuple struct{} //msgp:tuple Issue102TupleDeep type Issue102TupleDeep struct { A int X struct{} Y struct{} Z int } type Issue102Uses struct { Nested Issue102 NestedPtr *Issue102 } //msgp:tuple Issue102TupleUsesTuple type Issue102TupleUsesTuple struct { Nested Issue102Tuple NestedPtr *Issue102Tuple } //msgp:tuple Issue102TupleUsesMap type Issue102TupleUsesMap struct { Nested Issue102 NestedPtr *Issue102 } type Issue102MapUsesTuple struct { Nested Issue102Tuple NestedPtr *Issue102Tuple } msgp-1.2.5/_generated/issue191.go000066400000000000000000000001311472554463600165070ustar00rootroot00000000000000package _generated //go:generate msgp type Issue191 struct { Foo string Bar string } msgp-1.2.5/_generated/issue191_test.go000066400000000000000000000003601472554463600175520ustar00rootroot00000000000000package _generated import ( "testing" ) // Issue #191: panic in unsafe.UnsafeString() func TestIssue191(t *testing.T) { b := []byte{0x81, 0xa0, 0xa0} var i Issue191 _, err := (&i).UnmarshalMsg(b) if err != nil { t.Error(err) } } msgp-1.2.5/_generated/issue94.go000066400000000000000000000004631472554463600164410ustar00rootroot00000000000000package _generated import ( "time" ) //go:generate msgp //msgp:shim time.Time as:string using:timetostr/strtotime type T struct { T time.Time } func timetostr(t time.Time) string { return t.Format(time.RFC3339) } func strtotime(s string) time.Time { t, _ := time.Parse(time.RFC3339, s) return t } msgp-1.2.5/_generated/issue94_test.go000066400000000000000000000010671472554463600175010ustar00rootroot00000000000000package _generated import ( "bytes" "os" "testing" ) // Issue 94: shims were not propogated recursively, // which caused shims that weren't at the top level // to be silently ignored. // // The following line will generate an error after // the code is generated if the generated code doesn't // have the right identifier in it. func TestIssue94(t *testing.T) { b, err := os.ReadFile("issue94_gen.go") if err != nil { t.Fatal(err) } const want = "timetostr" if !bytes.Contains(b, []byte(want)) { t.Errorf("generated code did not contain %q", want) } } msgp-1.2.5/_generated/newtime.go000066400000000000000000000010301472554463600165730ustar00rootroot00000000000000package _generated import "time" //go:generate msgp -v //msgp:newtime type NewTime struct { T time.Time Array []time.Time Map map[string]time.Time } func (t1 NewTime) Equal(t2 NewTime) bool { if !t1.T.Equal(t2.T) { return false } if len(t1.Array) != len(t2.Array) { return false } for i := range t1.Array { if !t1.Array[i].Equal(t2.Array[i]) { return false } } if len(t1.Map) != len(t2.Map) { return false } for k, v := range t1.Map { if !t2.Map[k].Equal(v) { return false } } return true } msgp-1.2.5/_generated/newtime_test.go000066400000000000000000000046421472554463600176460ustar00rootroot00000000000000package _generated import ( "bytes" "math/rand" "testing" "time" "github.com/tinylib/msgp/msgp" ) func TestNewTime(t *testing.T) { value := NewTime{ T: time.Now().UTC(), Array: []time.Time{time.Now().UTC(), time.Now().UTC()}, Map: map[string]time.Time{ "a": time.Now().UTC(), }, } encoded, err := value.MarshalMsg(nil) if err != nil { t.Fatal(err) } checkExtMinusOne(t, encoded) var got NewTime _, err = got.UnmarshalMsg(encoded) if err != nil { t.Fatal(err) } if !value.Equal(got) { t.Errorf("UnmarshalMsg got %v want %v", value, got) } var buf bytes.Buffer w := msgp.NewWriter(&buf) err = value.EncodeMsg(w) if err != nil { t.Fatal(err) } w.Flush() checkExtMinusOne(t, buf.Bytes()) got = NewTime{} r := msgp.NewReader(&buf) err = got.DecodeMsg(r) if err != nil { t.Fatal(err) } if !value.Equal(got) { t.Errorf("DecodeMsg got %v want %v", value, got) } } func checkExtMinusOne(t *testing.T, b []byte) { r := msgp.NewReader(bytes.NewBuffer(b)) _, err := r.ReadMapHeader() if err != nil { t.Fatal(err) } key, err := r.ReadMapKey(nil) if err != nil { t.Fatal(err) } for !bytes.Equal(key, []byte("T")) { key, err = r.ReadMapKey(nil) if err != nil { t.Fatal(err) } } n, _, err := r.ReadExtensionRaw() if err != nil { t.Fatal(err) } if n != -1 { t.Fatalf("got %v want -1", n) } t.Log("Was -1 extension") } func TestNewTimeRandom(t *testing.T) { rng := rand.New(rand.NewSource(0)) runs := int(1e6) if testing.Short() { runs = 1e4 } for i := 0; i < runs; i++ { nanos := rng.Int63n(999999999 + 1) secs := rng.Uint64() // Tweak the distribution, so we get more than average number of // length 4 and 8 timestamps. if rng.Intn(5) == 0 { secs %= uint64(time.Now().Unix()) if rng.Intn(2) == 0 { nanos = 0 } } value := NewTime{ T: time.Unix(int64(secs), nanos), } encoded, err := value.MarshalMsg(nil) if err != nil { t.Fatal(err) } var got NewTime _, err = got.UnmarshalMsg(encoded) if err != nil { t.Fatal(err) } if !value.Equal(got) { t.Fatalf("UnmarshalMsg got %v want %v", value, got) } var buf bytes.Buffer w := msgp.NewWriter(&buf) err = value.EncodeMsg(w) if err != nil { t.Fatal(err) } w.Flush() got = NewTime{} r := msgp.NewReader(&buf) err = got.DecodeMsg(r) if err != nil { t.Fatal(err) } if !value.Equal(got) { t.Fatalf("DecodeMsg got %v want %v", value, got) } } } msgp-1.2.5/_generated/omitempty.go000066400000000000000000000175121472554463600171660ustar00rootroot00000000000000package _generated import "time" //go:generate msgp type OmitEmpty0 struct { ABool bool `msg:"abool,omitempty"` AInt int `msg:"aint,omitempty"` AInt8 int8 `msg:"aint8,omitempty"` AInt16 int16 `msg:"aint16,omitempty"` AInt32 int32 `msg:"aint32,omitempty"` AInt64 int64 `msg:"aint64,omitempty"` AUint uint `msg:"auint,omitempty"` AUint8 uint8 `msg:"auint8,omitempty"` AUint16 uint16 `msg:"auint16,omitempty"` AUint32 uint32 `msg:"auint32,omitempty"` AUint64 uint64 `msg:"auint64,omitempty"` AFloat32 float32 `msg:"afloat32,omitempty"` AFloat64 float64 `msg:"afloat64,omitempty"` AComplex64 complex64 `msg:"acomplex64,omitempty"` AComplex128 complex128 `msg:"acomplex128,omitempty"` ANamedBool bool `msg:"anamedbool,omitempty"` ANamedInt int `msg:"anamedint,omitempty"` ANamedFloat64 float64 `msg:"anamedfloat64,omitempty"` AMapStrStr map[string]string `msg:"amapstrstr,omitempty"` APtrNamedStr *NamedString `msg:"aptrnamedstr,omitempty"` AString string `msg:"astring,omitempty"` ANamedString string `msg:"anamedstring,omitempty"` AByteSlice []byte `msg:"abyteslice,omitempty"` ASliceByteSlice [][]byte `msg:"aslicebyteslice,omitempty"` AMapByteSlice map[string][]byte `msg:"amapbyteslice,omitempty"` ASliceString []string `msg:"aslicestring,omitempty"` ASliceNamedString []NamedString `msg:"aslicenamedstring,omitempty"` ANamedStruct NamedStruct `msg:"anamedstruct,omitempty"` APtrNamedStruct *NamedStruct `msg:"aptrnamedstruct,omitempty"` AUnnamedStruct struct { A string `msg:"a,omitempty"` } `msg:"aunnamedstruct,omitempty"` // omitempty not supported on unnamed struct EmbeddableStruct `msg:",flatten,omitempty"` // embed flat EmbeddableStruct2 `msg:"embeddablestruct2,omitempty"` // embed non-flat AArrayInt [5]int `msg:"aarrayint,omitempty"` // not supported ATime time.Time `msg:"atime,omitempty"` } type TypeSample struct { K uint32 `msg:"k,omitempty"` V uint32 `msg:"v,omitempty"` } type TypeSamples []TypeSample type ( NamedBool bool NamedInt int NamedFloat64 float64 NamedString string ) type EmbeddableStruct struct { SomeEmbed string `msg:"someembed,omitempty"` } type EmbeddableStruct2 struct { SomeEmbed2 string `msg:"someembed2,omitempty"` } type NamedStruct struct { A string `msg:"a,omitempty"` B string `msg:"b,omitempty"` } type OmitEmptyHalfFull struct { Field00 string `msg:"field00,omitempty"` Field01 string `msg:"field01"` Field02 string `msg:"field02,omitempty"` Field03 string `msg:"field03"` } type OmitEmptyLotsOFields struct { Field00 string `msg:"field00,omitempty"` Field01 string `msg:"field01,omitempty"` Field02 string `msg:"field02,omitempty"` Field03 string `msg:"field03,omitempty"` Field04 string `msg:"field04,omitempty"` Field05 string `msg:"field05,omitempty"` Field06 string `msg:"field06,omitempty"` Field07 string `msg:"field07,omitempty"` Field08 string `msg:"field08,omitempty"` Field09 string `msg:"field09,omitempty"` Field10 string `msg:"field10,omitempty"` Field11 string `msg:"field11,omitempty"` Field12 string `msg:"field12,omitempty"` Field13 string `msg:"field13,omitempty"` Field14 string `msg:"field14,omitempty"` Field15 string `msg:"field15,omitempty"` Field16 string `msg:"field16,omitempty"` Field17 string `msg:"field17,omitempty"` Field18 string `msg:"field18,omitempty"` Field19 string `msg:"field19,omitempty"` Field20 string `msg:"field20,omitempty"` Field21 string `msg:"field21,omitempty"` Field22 string `msg:"field22,omitempty"` Field23 string `msg:"field23,omitempty"` Field24 string `msg:"field24,omitempty"` Field25 string `msg:"field25,omitempty"` Field26 string `msg:"field26,omitempty"` Field27 string `msg:"field27,omitempty"` Field28 string `msg:"field28,omitempty"` Field29 string `msg:"field29,omitempty"` Field30 string `msg:"field30,omitempty"` Field31 string `msg:"field31,omitempty"` Field32 string `msg:"field32,omitempty"` Field33 string `msg:"field33,omitempty"` Field34 string `msg:"field34,omitempty"` Field35 string `msg:"field35,omitempty"` Field36 string `msg:"field36,omitempty"` Field37 string `msg:"field37,omitempty"` Field38 string `msg:"field38,omitempty"` Field39 string `msg:"field39,omitempty"` Field40 string `msg:"field40,omitempty"` Field41 string `msg:"field41,omitempty"` Field42 string `msg:"field42,omitempty"` Field43 string `msg:"field43,omitempty"` Field44 string `msg:"field44,omitempty"` Field45 string `msg:"field45,omitempty"` Field46 string `msg:"field46,omitempty"` Field47 string `msg:"field47,omitempty"` Field48 string `msg:"field48,omitempty"` Field49 string `msg:"field49,omitempty"` Field50 string `msg:"field50,omitempty"` Field51 string `msg:"field51,omitempty"` Field52 string `msg:"field52,omitempty"` Field53 string `msg:"field53,omitempty"` Field54 string `msg:"field54,omitempty"` Field55 string `msg:"field55,omitempty"` Field56 string `msg:"field56,omitempty"` Field57 string `msg:"field57,omitempty"` Field58 string `msg:"field58,omitempty"` Field59 string `msg:"field59,omitempty"` Field60 string `msg:"field60,omitempty"` Field61 string `msg:"field61,omitempty"` Field62 string `msg:"field62,omitempty"` Field63 string `msg:"field63,omitempty"` Field64 string `msg:"field64,omitempty"` Field65 string `msg:"field65,omitempty"` Field66 string `msg:"field66,omitempty"` Field67 string `msg:"field67,omitempty"` Field68 string `msg:"field68,omitempty"` Field69 string `msg:"field69,omitempty"` } type OmitEmpty10 struct { Field00 string `msg:"field00,omitempty"` Field01 string `msg:"field01,omitempty"` Field02 string `msg:"field02,omitempty"` Field03 string `msg:"field03,omitempty"` Field04 string `msg:"field04,omitempty"` Field05 string `msg:"field05,omitempty"` Field06 string `msg:"field06,omitempty"` Field07 string `msg:"field07,omitempty"` Field08 string `msg:"field08,omitempty"` Field09 string `msg:"field09,omitempty"` } type NotOmitEmpty10 struct { Field00 string `msg:"field00"` Field01 string `msg:"field01"` Field02 string `msg:"field02"` Field03 string `msg:"field03"` Field04 string `msg:"field04"` Field05 string `msg:"field05"` Field06 string `msg:"field06"` Field07 string `msg:"field07"` Field08 string `msg:"field08"` Field09 string `msg:"field09"` } type OmitEmptyNoName struct { ABool bool `msg:",omitempty"` AInt int `msg:",omitempty"` AInt8 int8 `msg:",omitempty"` AInt16 int16 `msg:",omitempty"` AInt32 int32 `msg:",omitempty"` AInt64 int64 `msg:",omitempty"` AUint uint `msg:",omitempty"` AUint8 uint8 `msg:",omitempty"` AUint16 uint16 `msg:",omitempty"` AUint32 uint32 `msg:",omitempty"` AUint64 uint64 `msg:",omitempty"` AFloat32 float32 `msg:",omitempty"` AFloat64 float64 `msg:",omitempty"` AComplex64 complex64 `msg:",omitempty"` AComplex128 complex128 `msg:",omitempty"` ANamedBool bool `msg:",omitempty"` ANamedInt int `msg:",omitempty"` ANamedFloat64 float64 `msg:",omitempty"` AMapStrStr map[string]string `msg:",omitempty"` APtrNamedStr *NamedString `msg:",omitempty"` AString string `msg:",omitempty"` AByteSlice []byte `msg:",omitempty"` ASliceString []string `msg:",omitempty"` ASliceNamedString []NamedString `msg:",omitempty"` ANamedStruct NamedStruct `msg:",omitempty"` APtrNamedStruct *NamedStruct `msg:",omitempty"` AUnnamedStruct struct { A string `msg:",omitempty"` } `msg:",omitempty"` // omitempty not supported on unnamed struct EmbeddableStruct `msg:",flatten,omitempty"` // embed flat EmbeddableStruct2 `msg:",omitempty"` // embed non-flat AArrayInt [5]int `msg:",omitempty"` // not supported ATime time.Time `msg:",omitempty"` } msgp-1.2.5/_generated/omitempty_test.go000066400000000000000000000147131472554463600202250ustar00rootroot00000000000000package _generated import ( "bytes" "io" "reflect" "testing" "github.com/tinylib/msgp/msgp" ) func mustEncodeToJSON(o msgp.Encodable) string { var buf bytes.Buffer var err error en := msgp.NewWriter(&buf) err = o.EncodeMsg(en) if err != nil { panic(err) } en.Flush() var outbuf bytes.Buffer _, err = msgp.CopyToJSON(&outbuf, &buf) if err != nil { panic(err) } return outbuf.String() } func TestOmitEmpty0(t *testing.T) { var s string var oe0a OmitEmpty0 s = mustEncodeToJSON(&oe0a) if s != `{"aunnamedstruct":{},"aarrayint":[0,0,0,0,0]}` { t.Errorf("wrong result: %s", s) } var oe0b OmitEmpty0 oe0b.AString = "teststr" s = mustEncodeToJSON(&oe0b) if s != `{"astring":"teststr","aunnamedstruct":{},"aarrayint":[0,0,0,0,0]}` { t.Errorf("wrong result: %s", s) } // more than 15 fields filled in var oe0c OmitEmpty0 oe0c.ABool = true oe0c.AInt = 1 oe0c.AInt8 = 1 oe0c.AInt16 = 1 oe0c.AInt32 = 1 oe0c.AInt64 = 1 oe0c.AUint = 1 oe0c.AUint8 = 1 oe0c.AUint16 = 1 oe0c.AUint32 = 1 oe0c.AUint64 = 1 oe0c.AFloat32 = 1 oe0c.AFloat64 = 1 oe0c.AComplex64 = complex(1, 1) oe0c.AComplex128 = complex(1, 1) oe0c.AString = "test" oe0c.ANamedBool = true oe0c.ANamedInt = 1 oe0c.ANamedFloat64 = 1 var buf bytes.Buffer en := msgp.NewWriter(&buf) err := oe0c.EncodeMsg(en) if err != nil { t.Fatal(err) } en.Flush() de := msgp.NewReader(&buf) var oe0d OmitEmpty0 err = oe0d.DecodeMsg(de) if err != nil { t.Fatal(err) } // spot check some fields if oe0c.AFloat32 != oe0d.AFloat32 { t.Fail() } if oe0c.ANamedBool != oe0d.ANamedBool { t.Fail() } if oe0c.AInt64 != oe0d.AInt64 { t.Fail() } } func TestOmitEmptyNoNames(t *testing.T) { var s string var oe0a OmitEmptyNoName s = mustEncodeToJSON(&oe0a) if s != `{"AUnnamedStruct":{},"AArrayInt":[0,0,0,0,0]}` { t.Errorf("wrong result: %s", s) } var oe0b OmitEmptyNoName oe0b.AString = "teststr" s = mustEncodeToJSON(&oe0b) if s != `{"AString":"teststr","AUnnamedStruct":{},"AArrayInt":[0,0,0,0,0]}` { t.Errorf("wrong result: %s", s) } // more than 15 fields filled in var oe0c OmitEmptyNoName oe0c.ABool = true oe0c.AInt = 1 oe0c.AInt8 = 1 oe0c.AInt16 = 1 oe0c.AInt32 = 1 oe0c.AInt64 = 1 oe0c.AUint = 1 oe0c.AUint8 = 1 oe0c.AUint16 = 1 oe0c.AUint32 = 1 oe0c.AUint64 = 1 oe0c.AFloat32 = 1 oe0c.AFloat64 = 1 oe0c.AComplex64 = complex(1, 1) oe0c.AComplex128 = complex(1, 1) oe0c.AString = "test" oe0c.ANamedBool = true oe0c.ANamedInt = 1 oe0c.ANamedFloat64 = 1 var buf bytes.Buffer en := msgp.NewWriter(&buf) err := oe0c.EncodeMsg(en) if err != nil { t.Fatal(err) } en.Flush() de := msgp.NewReader(&buf) var oe0d OmitEmptyNoName err = oe0d.DecodeMsg(de) if err != nil { t.Fatal(err) } // spot check some fields if oe0c.AFloat32 != oe0d.AFloat32 { t.Fail() } if oe0c.ANamedBool != oe0d.ANamedBool { t.Fail() } if oe0c.AInt64 != oe0d.AInt64 { t.Fail() } } // TestOmitEmptyHalfFull tests mixed omitempty and not func TestOmitEmptyHalfFull(t *testing.T) { var s string var oeA OmitEmptyHalfFull s = mustEncodeToJSON(&oeA) if s != `{"field01":"","field03":""}` { t.Errorf("wrong result: %s", s) } var oeB OmitEmptyHalfFull oeB.Field02 = "val2" s = mustEncodeToJSON(&oeB) if s != `{"field01":"","field02":"val2","field03":""}` { t.Errorf("wrong result: %s", s) } var oeC OmitEmptyHalfFull oeC.Field03 = "val3" s = mustEncodeToJSON(&oeC) if s != `{"field01":"","field03":"val3"}` { t.Errorf("wrong result: %s", s) } } // TestOmitEmptyLotsOFields tests the case of > 64 fields (triggers the bitmask needing to be an array instead of a single value) func TestOmitEmptyLotsOFields(t *testing.T) { var s string var oeLotsA OmitEmptyLotsOFields s = mustEncodeToJSON(&oeLotsA) if s != `{}` { t.Errorf("wrong result: %s", s) } var oeLotsB OmitEmptyLotsOFields oeLotsB.Field04 = "val4" s = mustEncodeToJSON(&oeLotsB) if s != `{"field04":"val4"}` { t.Errorf("wrong result: %s", s) } var oeLotsC OmitEmptyLotsOFields oeLotsC.Field64 = "val64" s = mustEncodeToJSON(&oeLotsC) if s != `{"field64":"val64"}` { t.Errorf("wrong result: %s", s) } } func BenchmarkOmitEmpty10AllEmpty(b *testing.B) { en := msgp.NewWriter(io.Discard) var s OmitEmpty10 b.ResetTimer() for i := 0; i < b.N; i++ { err := s.EncodeMsg(en) if err != nil { b.Fatal(err) } } } func BenchmarkOmitEmpty10AllFull(b *testing.B) { en := msgp.NewWriter(io.Discard) var s OmitEmpty10 s.Field00 = "this is the value of field00" s.Field01 = "this is the value of field01" s.Field02 = "this is the value of field02" s.Field03 = "this is the value of field03" s.Field04 = "this is the value of field04" s.Field05 = "this is the value of field05" s.Field06 = "this is the value of field06" s.Field07 = "this is the value of field07" s.Field08 = "this is the value of field08" s.Field09 = "this is the value of field09" b.ResetTimer() for i := 0; i < b.N; i++ { err := s.EncodeMsg(en) if err != nil { b.Fatal(err) } } } func BenchmarkNotOmitEmpty10AllEmpty(b *testing.B) { en := msgp.NewWriter(io.Discard) var s NotOmitEmpty10 b.ResetTimer() for i := 0; i < b.N; i++ { err := s.EncodeMsg(en) if err != nil { b.Fatal(err) } } } func BenchmarkNotOmitEmpty10AllFull(b *testing.B) { en := msgp.NewWriter(io.Discard) var s NotOmitEmpty10 s.Field00 = "this is the value of field00" s.Field01 = "this is the value of field01" s.Field02 = "this is the value of field02" s.Field03 = "this is the value of field03" s.Field04 = "this is the value of field04" s.Field05 = "this is the value of field05" s.Field06 = "this is the value of field06" s.Field07 = "this is the value of field07" s.Field08 = "this is the value of field08" s.Field09 = "this is the value of field09" b.ResetTimer() for i := 0; i < b.N; i++ { err := s.EncodeMsg(en) if err != nil { b.Fatal(err) } } } func TestTypeAlias(t *testing.T) { value := TypeSamples{TypeSample{}, TypeSample{K: 1, V: 2}} encoded, err := value.MarshalMsg(nil) if err != nil { t.Fatal(err) } var got TypeSamples _, err = got.UnmarshalMsg(encoded) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(value, got) { t.Errorf("UnmarshalMsg got %v want %v", value, got) } var buf bytes.Buffer w := msgp.NewWriter(&buf) err = value.EncodeMsg(w) if err != nil { t.Fatal(err) } w.Flush() got = TypeSamples{} r := msgp.NewReader(&buf) err = got.DecodeMsg(r) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(value, got) { t.Errorf("UnmarshalMsg got %v want %v", value, got) } } msgp-1.2.5/_generated/omitzero.go000066400000000000000000000047251472554463600170110ustar00rootroot00000000000000package _generated import "time" //go:generate msgp // check some specific cases for omitzero type OmitZero0 struct { AStruct OmitZeroA `msg:"astruct,omitempty"` // leave this one omitempty BStruct OmitZeroA `msg:"bstruct,omitzero"` // and compare to this AStructPtr *OmitZeroA `msg:"astructptr,omitempty"` // a pointer case omitempty BStructPtr *OmitZeroA `msg:"bstructptr,omitzero"` // a pointer case omitzero AExt OmitZeroExt `msg:"aext,omitzero"` // external type case AExtPtr *OmitZeroExtPtr `msg:"aextptr,omitzero"` // external type pointer case // more APtrNamedStr *NamedStringOZ `msg:"aptrnamedstr,omitzero"` ANamedStruct NamedStructOZ `msg:"anamedstruct,omitzero"` APtrNamedStruct *NamedStructOZ `msg:"aptrnamedstruct,omitzero"` EmbeddableStruct `msg:",flatten,omitzero"` // embed flat EmbeddableStructOZ `msg:"embeddablestruct2,omitzero"` // embed non-flat ATime time.Time `msg:"atime,omitzero"` OmitZeroTuple OmitZeroTuple `msg:"ozt"` // the inside of a tuple should ignore both omitempty and omitzero } type OmitZeroA struct { A string `msg:"a,omitempty"` B NamedStringOZ `msg:"b,omitzero"` C NamedStringOZ `msg:"c,omitzero"` } func (o *OmitZeroA) IsZero() bool { if o == nil { return true } return *o == (OmitZeroA{}) } type NamedStructOZ struct { A string `msg:"a,omitempty"` B string `msg:"b,omitempty"` } func (ns *NamedStructOZ) IsZero() bool { if ns == nil { return true } return *ns == (NamedStructOZ{}) } type NamedStringOZ string func (ns *NamedStringOZ) IsZero() bool { if ns == nil { return true } return *ns == "" } type EmbeddableStructOZ struct { SomeEmbed string `msg:"someembed2,omitempty"` } func (es EmbeddableStructOZ) IsZero() bool { return es == (EmbeddableStructOZ{}) } type EmbeddableStructOZ2 struct { SomeEmbed2 string `msg:"someembed2,omitempty"` } func (es EmbeddableStructOZ2) IsZero() bool { return es == (EmbeddableStructOZ2{}) } //msgp:tuple OmitZeroTuple // OmitZeroTuple is flagged for tuple output, it should ignore all omitempty and omitzero functionality // since it's fundamentally incompatible. type OmitZeroTuple struct { FieldA string `msg:"fielda,omitempty"` FieldB NamedStringOZ `msg:"fieldb,omitzero"` FieldC NamedStringOZ `msg:"fieldc,omitzero"` } type OmitZero1 struct { T1 OmitZeroTuple `msg:"t1"` } msgp-1.2.5/_generated/omitzero_ext.go000066400000000000000000000047041472554463600176660ustar00rootroot00000000000000package _generated import ( "github.com/tinylib/msgp/msgp" ) // this has "external" types that will show up // as generic IDENT during code generation type OmitZeroExt struct { a int // custom type } // IsZero will return true if a is not positive func (o OmitZeroExt) IsZero() bool { return o.a <= 0 } // EncodeMsg implements msgp.Encodable func (o OmitZeroExt) EncodeMsg(en *msgp.Writer) (err error) { if o.a > 0 { return en.WriteInt(o.a) } return en.WriteNil() } // DecodeMsg implements msgp.Decodable func (o *OmitZeroExt) DecodeMsg(dc *msgp.Reader) (err error) { if dc.IsNil() { err = dc.ReadNil() if err != nil { return } o.a = 0 return } o.a, err = dc.ReadInt() return err } // MarshalMsg implements msgp.Marshaler func (o OmitZeroExt) MarshalMsg(b []byte) (ret []byte, err error) { ret = msgp.Require(b, o.Msgsize()) if o.a > 0 { return msgp.AppendInt(ret, o.a), nil } return msgp.AppendNil(ret), nil } // UnmarshalMsg implements msgp.Unmarshaler func (o *OmitZeroExt) UnmarshalMsg(bts []byte) (ret []byte, err error) { if msgp.IsNil(bts) { bts, err = msgp.ReadNilBytes(bts) return bts, err } o.a, bts, err = msgp.ReadIntBytes(bts) return bts, err } // Msgsize implements msgp.Msgsizer func (o OmitZeroExt) Msgsize() (s int) { return msgp.IntSize } type OmitZeroExtPtr struct { a int // custom type } // IsZero will return true if a is nil or not positive func (o *OmitZeroExtPtr) IsZero() bool { return o == nil || o.a <= 0 } // EncodeMsg implements msgp.Encodable func (o *OmitZeroExtPtr) EncodeMsg(en *msgp.Writer) (err error) { if o.a > 0 { return en.WriteInt(o.a) } return en.WriteNil() } // DecodeMsg implements msgp.Decodable func (o *OmitZeroExtPtr) DecodeMsg(dc *msgp.Reader) (err error) { if dc.IsNil() { err = dc.ReadNil() if err != nil { return } o.a = 0 return } o.a, err = dc.ReadInt() return err } // MarshalMsg implements msgp.Marshaler func (o *OmitZeroExtPtr) MarshalMsg(b []byte) (ret []byte, err error) { ret = msgp.Require(b, o.Msgsize()) if o.a > 0 { return msgp.AppendInt(ret, o.a), nil } return msgp.AppendNil(ret), nil } // UnmarshalMsg implements msgp.Unmarshaler func (o *OmitZeroExtPtr) UnmarshalMsg(bts []byte) (ret []byte, err error) { if msgp.IsNil(bts) { bts, err = msgp.ReadNilBytes(bts) return bts, err } o.a, bts, err = msgp.ReadIntBytes(bts) return bts, err } // Msgsize implements msgp.Msgsizer func (o *OmitZeroExtPtr) Msgsize() (s int) { return msgp.IntSize } msgp-1.2.5/_generated/omitzero_test.go000066400000000000000000000034021472554463600200370ustar00rootroot00000000000000package _generated import ( "bytes" "testing" ) func TestOmitZero(t *testing.T) { t.Run("OmitZeroExt_not_empty", func(t *testing.T) { z := OmitZero0{AExt: OmitZeroExt{a: 1}} b, err := z.MarshalMsg(nil) if err != nil { t.Fatal(err) } if !bytes.Contains(b, []byte("aext")) { t.Errorf("expected to find aext in bytes %X", b) } z = OmitZero0{} _, err = z.UnmarshalMsg(b) if err != nil { t.Fatal(err) } if z.AExt.a != 1 { t.Errorf("z.AExt.a expected 1 but got %d", z.AExt.a) } }) t.Run("OmitZeroExt_negative", func(t *testing.T) { z := OmitZero0{AExt: OmitZeroExt{a: -1}} // negative value should act as empty, via IsEmpty() call b, err := z.MarshalMsg(nil) if err != nil { t.Fatal(err) } if bytes.Contains(b, []byte("aext")) { t.Errorf("expected to not find aext in bytes %X", b) } z = OmitZero0{} _, err = z.UnmarshalMsg(b) if err != nil { t.Fatal(err) } if z.AExt.a != 0 { t.Errorf("z.AExt.a expected 0 but got %d", z.AExt.a) } }) t.Run("OmitZeroTuple", func(t *testing.T) { // make sure tuple encoding isn't affected by omitempty or omitzero z := OmitZero0{OmitZeroTuple: OmitZeroTuple{FieldA: "", FieldB: "", FieldC: "fcval"}} b, err := z.MarshalMsg(nil) if err != nil { t.Fatal(err) } // verify the exact binary encoding, that the values follow each other without field names if !bytes.Contains(b, []byte{0xA0, 0xA0, 0xA5, 'f', 'c', 'v', 'a', 'l'}) { t.Errorf("failed to find expected bytes in %X", b) } z = OmitZero0{} _, err = z.UnmarshalMsg(b) if err != nil { t.Fatal(err) } if z.OmitZeroTuple.FieldA != "" || z.OmitZeroTuple.FieldB != "" || z.OmitZeroTuple.FieldC != "fcval" { t.Errorf("z.OmitZeroTuple unexpected value: %#v", z.OmitZeroTuple) } }) } msgp-1.2.5/_generated/pointer.go000066400000000000000000000120241472554463600166100ustar00rootroot00000000000000package _generated import ( "fmt" "time" "github.com/tinylib/msgp/msgp" ) //go:generate msgp $GOFILE$ // Generate only pointer receivers: //msgp:pointer var mustNoInterf = []interface{}{ Pointer0{}, NamedBoolPointer(true), NamedIntPointer(0), NamedFloat64Pointer(0), NamedStringPointer(""), NamedMapStructPointer(nil), NamedMapStructPointer2(nil), NamedMapStringPointer(nil), NamedMapStringPointer2(nil), EmbeddableStructPointer{}, EmbeddableStruct2Pointer{}, PointerHalfFull{}, PointerNoName{}, } var mustHaveInterf = []interface{}{ &Pointer0{}, mustPtr(NamedBoolPointer(true)), mustPtr(NamedIntPointer(0)), mustPtr(NamedFloat64Pointer(0)), mustPtr(NamedStringPointer("")), mustPtr(NamedMapStructPointer(nil)), mustPtr(NamedMapStructPointer2(nil)), mustPtr(NamedMapStringPointer(nil)), mustPtr(NamedMapStringPointer2(nil)), &EmbeddableStructPointer{}, &EmbeddableStruct2Pointer{}, &PointerHalfFull{}, &PointerNoName{}, } func mustPtr[T any](v T) *T { return &v } func init() { for _, v := range mustNoInterf { if _, ok := v.(msgp.Marshaler); ok { panic(fmt.Sprintf("type %T supports interface", v)) } if _, ok := v.(msgp.Encodable); ok { panic(fmt.Sprintf("type %T supports interface", v)) } } for _, v := range mustHaveInterf { if _, ok := v.(msgp.Marshaler); !ok { panic(fmt.Sprintf("type %T does not support interface", v)) } if _, ok := v.(msgp.Encodable); !ok { panic(fmt.Sprintf("type %T does not support interface", v)) } } } type Pointer0 struct { ABool bool `msg:"abool"` AInt int `msg:"aint"` AInt8 int8 `msg:"aint8"` AInt16 int16 `msg:"aint16"` AInt32 int32 `msg:"aint32"` AInt64 int64 `msg:"aint64"` AUint uint `msg:"auint"` AUint8 uint8 `msg:"auint8"` AUint16 uint16 `msg:"auint16"` AUint32 uint32 `msg:"auint32"` AUint64 uint64 `msg:"auint64"` AFloat32 float32 `msg:"afloat32"` AFloat64 float64 `msg:"afloat64"` AComplex64 complex64 `msg:"acomplex64"` AComplex128 complex128 `msg:"acomplex128"` ANamedBool bool `msg:"anamedbool"` ANamedInt int `msg:"anamedint"` ANamedFloat64 float64 `msg:"anamedfloat64"` AMapStrStr map[string]string `msg:"amapstrstr"` APtrNamedStr *NamedString `msg:"aptrnamedstr"` AString string `msg:"astring"` ANamedString string `msg:"anamedstring"` AByteSlice []byte `msg:"abyteslice"` ASliceString []string `msg:"aslicestring"` ASliceNamedString []NamedString `msg:"aslicenamedstring"` ANamedStruct NamedStruct `msg:"anamedstruct"` APtrNamedStruct *NamedStruct `msg:"aptrnamedstruct"` AUnnamedStruct struct { A string `msg:"a"` } `msg:"aunnamedstruct"` // omitempty not supported on unnamed struct EmbeddableStruct `msg:",flatten"` // embed flat EmbeddableStruct2 `msg:"embeddablestruct2"` // embed non-flat AArrayInt [5]int `msg:"aarrayint"` // not supported ATime time.Time `msg:"atime"` } type ( NamedBoolPointer bool NamedIntPointer int NamedFloat64Pointer float64 NamedStringPointer string NamedMapStructPointer map[string]Pointer0 NamedMapStructPointer2 map[string]*Pointer0 NamedMapStringPointer map[string]NamedStringPointer NamedMapStringPointer2 map[string]*NamedStringPointer ) type EmbeddableStructPointer struct { SomeEmbed string `msg:"someembed"` } type EmbeddableStruct2Pointer struct { SomeEmbed2 string `msg:"someembed2"` } type NamedStructPointer struct { A string `msg:"a"` B string `msg:"b"` } type PointerHalfFull struct { Field00 string `msg:"field00"` Field01 string `msg:"field01"` Field02 string `msg:"field02"` Field03 string `msg:"field03"` } type PointerNoName struct { ABool bool `msg:""` AInt int `msg:""` AInt8 int8 `msg:""` AInt16 int16 `msg:""` AInt32 int32 `msg:""` AInt64 int64 `msg:""` AUint uint `msg:""` AUint8 uint8 `msg:""` AUint16 uint16 `msg:""` AUint32 uint32 `msg:""` AUint64 uint64 `msg:""` AFloat32 float32 `msg:""` AFloat64 float64 `msg:""` AComplex64 complex64 `msg:""` AComplex128 complex128 `msg:""` ANamedBool bool `msg:""` ANamedInt int `msg:""` ANamedFloat64 float64 `msg:""` AMapStrF map[string]NamedFloat64Pointer `msg:""` AMapStrStruct map[string]PointerHalfFull `msg:""` AMapStrStruct2 map[string]*PointerHalfFull `msg:""` APtrNamedStr *NamedStringPointer `msg:""` AString string `msg:""` AByteSlice []byte `msg:""` ASliceString []string `msg:""` ASliceNamedString []NamedStringPointer `msg:""` ANamedStruct NamedStructPointer `msg:""` APtrNamedStruct *NamedStructPointer `msg:""` AUnnamedStruct struct { A string `msg:""` } `msg:""` // omitempty not supported on unnamed struct EmbeddableStructPointer `msg:",flatten"` // embed flat EmbeddableStruct2Pointer `msg:""` // embed non-flat AArrayInt [5]int `msg:""` // not supported ATime time.Time `msg:""` ADur time.Duration `msg:""` } msgp-1.2.5/_generated/replace.go000066400000000000000000000034631472554463600165520ustar00rootroot00000000000000package _generated import "encoding/json" //go:generate msgp //msgp:replace Any with:any //msgp:replace MapString with:CompatibleMapString //msgp:replace MapAny with:map[string]any //msgp:replace SliceString with:[]string //msgp:replace SliceInt with:CompatibleSliceInt //msgp:replace Array8 with:CompatibleArray8 //msgp:replace Array16 with:[16]byte //msgp:replace String with:string //msgp:replace Int with:CompatibleInt //msgp:replace Uint with:uint //msgp:replace Float32 with:CompatibleFloat32 //msgp:replace Float64 with:CompatibleFloat64 //msgp:replace Time with:time.Time //msgp:replace Duration with:time.Duration //msgp:replace StructA with:CompatibleStructA //msgp:replace StructB with:CompatibleStructB //msgp:replace StructC with:CompatibleStructC //msgp:replace StructD with:CompatibleStructD //msgp:replace StructI with:CompatibleStructI //msgp:replace StructS with:CompatibleStructS type ( CompatibleMapString map[string]string CompatibleArray8 [8]byte CompatibleInt int CompatibleFloat32 float32 CompatibleFloat64 float64 CompatibleSliceInt []Int // Doesn't work // CompatibleTime time.Time CompatibleStructA struct { StructB StructB Int Int } CompatibleStructB struct { StructC StructC Any Any Array8 Array8 } CompatibleStructC struct { StructD StructD Float64 Float32 Float32 Float64 } CompatibleStructD struct { Time Time Duration Duration MapString MapString } CompatibleStructI struct { Int *Int Uint *Uint } CompatibleStructS struct { Slice SliceInt } Dummy struct { StructA StructA StructI StructI StructS StructS Array16 Array16 Uint Uint String String } ) //msgp:replace json.Number with:string type NumberJSONSampleReplace struct { Single json.Number Array []json.Number Map map[string]json.Number } msgp-1.2.5/_generated/replace_ext.go000066400000000000000000000013751472554463600174320ustar00rootroot00000000000000package _generated import "time" // external types to test replace directive type ( MapString map[string]string MapAny map[string]any SliceString []String SliceInt []Int Array8 [8]byte Array16 [16]byte Int int Uint uint String string Float32 float32 Float64 float64 Time time.Time Duration time.Duration Any any StructA struct { StructB StructB Int Int } StructB struct { StructC StructC Any Any Array8 Array8 } StructC struct { StructD StructD Float64 Float32 Float32 Float64 } StructD struct { Time Time Duration Duration MapString MapString } StructI struct { Int *Int Uint *Uint } StructS struct { Slice SliceInt } ) msgp-1.2.5/_generated/replace_test.go000066400000000000000000000163671472554463600176200ustar00rootroot00000000000000package _generated import ( "bytes" "encoding/json" "reflect" "testing" "time" "github.com/tinylib/msgp/msgp" ) func compareStructD(t *testing.T, a, b *CompatibleStructD) { t.Helper() if !time.Time(a.Time).Equal(time.Time(b.Time)) { t.Fatal("not same time") } if a.Duration != b.Duration { t.Fatal("not same duration") } if len(a.MapString) != len(b.MapString) { t.Fatal("not same map") } for k, v1 := range a.MapString { if v2, ok := b.MapString[k]; !ok || v1 != v2 { t.Fatal("not same map") } } } func compareStructC(t *testing.T, a, b *CompatibleStructC) { t.Helper() if a.Float32 != b.Float32 { t.Fatal("not same float32") } if a.Float64 != b.Float64 { t.Fatal("not same float64") } compareStructD(t, (*CompatibleStructD)(&a.StructD), (*CompatibleStructD)(&b.StructD)) } func compareStructB(t *testing.T, a, b *CompatibleStructB) { t.Helper() if a.Array8 != b.Array8 { t.Fatal("not same array") } if a.Any != b.Any { t.Fatal("not same any") } compareStructC(t, (*CompatibleStructC)(&a.StructC), (*CompatibleStructC)(&b.StructC)) } func compareStructA(t *testing.T, a, b *CompatibleStructA) { t.Helper() if a.Int != b.Int { t.Fatal("not same int") } compareStructB(t, (*CompatibleStructB)(&a.StructB), (*CompatibleStructB)(&b.StructB)) } func compareStructI(t *testing.T, a, b *CompatibleStructI) { t.Helper() if *a.Int != *b.Int { t.Fatal("not same int") } if *a.Uint != *b.Uint { t.Fatal("not same uint") } } func compareStructS(t *testing.T, a, b *CompatibleStructS) { t.Helper() if len(a.Slice) != len(b.Slice) { t.Fatal("not same slice") } for i := 0; i < len(a.Slice); i++ { if a.Slice[i] != b.Slice[i] { t.Fatal("not same slice") } } } func TestReplace_ABCD(t *testing.T) { d := CompatibleStructD{ Time: Time(time.Now()), Duration: Duration(time.Duration(1234)), MapString: map[string]string{ "foo": "bar", "hello": "word", "baz": "quux", }, } c := CompatibleStructC{ StructD: StructD(d), Float32: 1.0, Float64: 2.0, } b := CompatibleStructB{ StructC: StructC(c), Any: "sup", Array8: [8]byte{'f', 'o', 'o'}, } a := CompatibleStructA{ StructB: StructB(b), Int: 10, } t.Run("D", func(t *testing.T) { bytes, err := d.MarshalMsg(nil) if err != nil { t.Fatal(err) } ud := CompatibleStructD{} _, err = ud.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructD(t, &d, &ud) }) t.Run("C", func(t *testing.T) { bytes, err := c.MarshalMsg(nil) if err != nil { t.Fatal(err) } uc := CompatibleStructC{} _, err = uc.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructC(t, &c, &uc) }) t.Run("B", func(t *testing.T) { bytes, err := b.MarshalMsg(nil) if err != nil { t.Fatal(err) } ub := CompatibleStructB{} _, err = ub.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructB(t, &b, &ub) }) t.Run("A", func(t *testing.T) { bytes, err := a.MarshalMsg(nil) if err != nil { t.Fatal(err) } ua := CompatibleStructA{} _, err = ua.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructA(t, &a, &ua) }) } func TestReplace_I(t *testing.T) { var int0 int = -10 var uint0 uint = 12 i := CompatibleStructI{ Int: (*Int)(&int0), Uint: (*Uint)(&uint0), } bytes, err := i.MarshalMsg(nil) if err != nil { t.Fatal(err) } ui := CompatibleStructI{} _, err = ui.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructI(t, &i, &ui) } func TestReplace_S(t *testing.T) { s := CompatibleStructS{ Slice: []Int{10, 12, 14, 16}, } bytes, err := s.MarshalMsg(nil) if err != nil { t.Fatal(err) } us := CompatibleStructS{} _, err = us.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructS(t, &s, &us) } func TestReplace_Dummy(t *testing.T) { dummy := Dummy{ StructA: StructA{ StructB: StructB{ StructC: StructC{ StructD: StructD{ Time: Time(time.Now()), Duration: Duration(time.Duration(1234)), MapString: map[string]string{ "foo": "bar", "hello": "word", "baz": "quux", }, }, Float32: 1.0, Float64: 2.0, }, Any: "sup", Array8: [8]byte{'f', 'o', 'o'}, }, Int: 10, }, StructI: StructI{ Int: new(Int), Uint: new(Uint), }, StructS: StructS{ Slice: []Int{10, 12, 14, 16}, }, Uint: 10, String: "cheese", } *dummy.StructI.Int = 1234 *dummy.StructI.Uint = 555 bytes, err := dummy.MarshalMsg(nil) if err != nil { t.Fatal(err) } udummy := Dummy{} _, err = udummy.UnmarshalMsg(bytes) if err != nil { t.Fatal(err) } compareStructA(t, (*CompatibleStructA)(&dummy.StructA), (*CompatibleStructA)(&udummy.StructA)) compareStructI(t, (*CompatibleStructI)(&dummy.StructI), (*CompatibleStructI)(&udummy.StructI)) compareStructS(t, (*CompatibleStructS)(&dummy.StructS), (*CompatibleStructS)(&udummy.StructS)) if dummy.Uint != udummy.Uint { t.Fatal("not same uint") } if dummy.String != udummy.String { t.Fatal("not same string") } } func TestJSONNumberReplace(t *testing.T) { test := NumberJSONSampleReplace{ Single: "-42", Array: []json.Number{"0", "-0", "1", "-1", "0.1", "-0.1", "1234", "-1234", "12.34", "-12.34", "12E0", "12E1", "12e34", "12E-0", "12e+1", "12e-34", "-12E0", "-12E1", "-12e34", "-12E-0", "-12e+1", "-12e-34", "1.2E0", "1.2E1", "1.2e34", "1.2E-0", "1.2e+1", "1.2e-34", "-1.2E0", "-1.2E1", "-1.2e34", "-1.2E-0", "-1.2e+1", "-1.2e-34", "0E0", "0E1", "0e34", "0E-0", "0e+1", "0e-34", "-0E0", "-0E1", "-0e34", "-0E-0", "-0e+1", "-0e-34"}, Map: map[string]json.Number{ "a": json.Number("50"), }, } encoded, err := test.MarshalMsg(nil) if err != nil { t.Errorf("%v", err) } var v NumberJSONSampleReplace _, err = v.UnmarshalMsg(encoded) if err != nil { t.Errorf("%v", err) } // Symmetric since we store strings. if !reflect.DeepEqual(v, test) { t.Fatalf("want %v, got %v", test, v) } var jsBuf bytes.Buffer remain, err := msgp.UnmarshalAsJSON(&jsBuf, encoded) if err != nil { t.Errorf("%v", err) } if len(remain) != 0 { t.Errorf("remain should be empty") } // Retains number formatting. Map order is random, though. wantjs := `{"Single":"-42","Array":["0","-0","1","-1","0.1","-0.1","1234","-1234","12.34","-12.34","12E0","12E1","12e34","12E-0","12e+1","12e-34","-12E0","-12E1","-12e34","-12E-0","-12e+1","-12e-34","1.2E0","1.2E1","1.2e34","1.2E-0","1.2e+1","1.2e-34","-1.2E0","-1.2E1","-1.2e34","-1.2E-0","-1.2e+1","-1.2e-34","0E0","0E1","0e34","0E-0","0e+1","0e-34","-0E0","-0E1","-0e34","-0E-0","-0e+1","-0e-34"],"Map":{"a":"50"}}` if jsBuf.String() != wantjs { t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) } // Test encoding var buf bytes.Buffer en := msgp.NewWriter(&buf) err = test.EncodeMsg(en) if err != nil { t.Errorf("%v", err) } en.Flush() encoded = buf.Bytes() dc := msgp.NewReader(&buf) err = v.DecodeMsg(dc) if err != nil { t.Errorf("%v", err) } if !reflect.DeepEqual(v, test) { t.Fatalf("want %v, got %v", test, v) } jsBuf.Reset() remain, err = msgp.UnmarshalAsJSON(&jsBuf, encoded) if err != nil { t.Errorf("%v", err) } if len(remain) != 0 { t.Errorf("remain should be empty") } if jsBuf.String() != wantjs { t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) } } msgp-1.2.5/_generated/vet_copylocks.go000066400000000000000000000002001472554463600200050ustar00rootroot00000000000000package _generated import "sync" //go:generate msgp //go:generate go vet type Foo struct { I struct{} lock sync.Mutex } msgp-1.2.5/gen/000077500000000000000000000000001472554463600132565ustar00rootroot00000000000000msgp-1.2.5/gen/decode.go000066400000000000000000000160471472554463600150400ustar00rootroot00000000000000package gen import ( "io" "strconv" ) func decode(w io.Writer) *decodeGen { return &decodeGen{ p: printer{w: w}, hasfield: false, } } type decodeGen struct { passes p printer hasfield bool ctx *Context } func (d *decodeGen) Method() Method { return Decode } func (d *decodeGen) needsField() { if d.hasfield { return } d.p.print("\nvar field []byte; _ = field") d.hasfield = true } func (d *decodeGen) Execute(p Elem, ctx Context) error { d.ctx = &ctx p = d.applyall(p) if p == nil { return nil } d.hasfield = false if !d.p.ok() { return d.p.err } if !IsPrintable(p) { return nil } d.p.comment("DecodeMsg implements msgp.Decodable") d.p.printf("\nfunc (%s %s) DecodeMsg(dc *msgp.Reader) (err error) {", p.Varname(), methodReceiver(p)) next(d, p) d.p.nakedReturn() unsetReceiver(p) return d.p.err } func (d *decodeGen) gStruct(s *Struct) { if !d.p.ok() { return } if s.AsTuple { d.structAsTuple(s) } else { d.structAsMap(s) } } func (d *decodeGen) assignAndCheck(name string, typ string) { if !d.p.ok() { return } d.p.printf("\n%s, err = dc.Read%s()", name, typ) d.p.wrapErrCheck(d.ctx.ArgsStr()) } func (d *decodeGen) structAsTuple(s *Struct) { nfields := len(s.Fields) sz := randIdent() d.p.declare(sz, u32) d.assignAndCheck(sz, arrayHeader) d.p.arrayCheck(strconv.Itoa(nfields), sz) for i := range s.Fields { if !d.p.ok() { return } fieldElem := s.Fields[i].FieldElem anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { d.p.print("\nif dc.IsNil() {") d.p.print("\nerr = dc.ReadNil()") d.p.wrapErrCheck(d.ctx.ArgsStr()) d.p.printf("\n%s = nil\n} else {", s.Fields[i].FieldElem.Varname()) } SetIsAllowNil(fieldElem, anField) d.ctx.PushString(s.Fields[i].FieldName) next(d, fieldElem) d.ctx.Pop() if anField { d.p.printf("\n}") // close if statement } } } func (d *decodeGen) structAsMap(s *Struct) { d.needsField() sz := randIdent() d.p.declare(sz, u32) d.assignAndCheck(sz, mapHeader) oeCount := s.CountFieldTagPart("omitempty") + s.CountFieldTagPart("omitzero") if !d.ctx.clearOmitted { oeCount = 0 } bm := bmask{ bitlen: oeCount, varname: sz + "Mask", } if oeCount > 0 { // Declare mask d.p.printf("\n%s", bm.typeDecl()) d.p.printf("\n_ = %s", bm.varname) } // Index to field idx of each emitted oeEmittedIdx := []int{} d.p.printf("\nfor %s > 0 {\n%s--", sz, sz) d.assignAndCheck("field", mapKey) d.p.print("\nswitch msgp.UnsafeString(field) {") for i := range s.Fields { d.ctx.PushString(s.Fields[i].FieldName) d.p.printf("\ncase %q:", s.Fields[i].FieldTag) fieldElem := s.Fields[i].FieldElem anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { d.p.print("\nif dc.IsNil() {") d.p.print("\nerr = dc.ReadNil()") d.p.wrapErrCheck(d.ctx.ArgsStr()) d.p.printf("\n%s = nil\n} else {", fieldElem.Varname()) } SetIsAllowNil(fieldElem, anField) next(d, fieldElem) if oeCount > 0 && (s.Fields[i].HasTagPart("omitempty") || s.Fields[i].HasTagPart("omitzero")) { d.p.printf("\n%s", bm.setStmt(len(oeEmittedIdx))) oeEmittedIdx = append(oeEmittedIdx, i) } d.ctx.Pop() if !d.p.ok() { return } if anField { d.p.printf("\n}") // close if statement } } d.p.print("\ndefault:\nerr = dc.Skip()") d.p.wrapErrCheck(d.ctx.ArgsStr()) d.p.closeblock() // close switch d.p.closeblock() // close for loop if oeCount > 0 { d.p.printf("\n// Clear omitted fields.\n") if bm.bitlen > 1 { d.p.printf("if %s {\n", bm.notAllSet()) } for bitIdx, fieldIdx := range oeEmittedIdx { fieldElem := s.Fields[fieldIdx].FieldElem d.p.printf("if %s == 0 {\n", bm.readExpr(bitIdx)) fze := fieldElem.ZeroExpr() if fze != "" { d.p.printf("%s = %s\n", fieldElem.Varname(), fze) } else { d.p.printf("%s = %s{}\n", fieldElem.Varname(), fieldElem.TypeName()) } d.p.printf("}\n") } if bm.bitlen > 1 { d.p.printf("}") } } } func (d *decodeGen) gBase(b *BaseElem) { if !d.p.ok() { return } // open block for 'tmp' var tmp string if b.Convert && b.Value != IDENT { // we don't need block for 'tmp' in case of IDENT tmp = randIdent() d.p.printf("\n{ var %s %s", tmp, b.BaseType()) } vname := b.Varname() // e.g. "z.FieldOne" bname := b.BaseName() // e.g. "Float64" checkNil := vname // Name of var to check for nil // handle special cases // for object type. switch b.Value { case Bytes: if b.Convert { lowered := b.ToBase() + "(" + vname + ")" d.p.printf("\n%s, err = dc.ReadBytes(%s)", tmp, lowered) checkNil = tmp } else { d.p.printf("\n%s, err = dc.ReadBytes(%s)", vname, vname) checkNil = vname } case IDENT: if b.Convert { lowered := b.ToBase() + "(" + vname + ")" d.p.printf("\nerr = %s.DecodeMsg(dc)", lowered) } else { d.p.printf("\nerr = %s.DecodeMsg(dc)", vname) } case Ext: d.p.printf("\nerr = dc.ReadExtension(%s)", vname) default: if b.Convert { d.p.printf("\n%s, err = dc.Read%s()", tmp, bname) } else { d.p.printf("\n%s, err = dc.Read%s()", vname, bname) } } d.p.wrapErrCheck(d.ctx.ArgsStr()) if checkNil != "" && b.AllowNil() { // Ensure that 0 sized slices are allocated. d.p.printf("\nif %s == nil {\n%s = make([]byte, 0)\n}", checkNil, checkNil) } // close block for 'tmp' if b.Convert && b.Value != IDENT { if b.ShimMode == Cast { d.p.printf("\n%s = %s(%s)\n}", vname, b.FromBase(), tmp) } else { d.p.printf("\n%s, err = %s(%s)\n}", vname, b.FromBase(), tmp) d.p.wrapErrCheck(d.ctx.ArgsStr()) } } } func (d *decodeGen) gMap(m *Map) { if !d.p.ok() { return } sz := randIdent() // resize or allocate map d.p.declare(sz, u32) d.assignAndCheck(sz, mapHeader) d.p.resizeMap(sz, m) // for element in map, read string/value // pair and assign d.needsField() d.p.printf("\nfor %s > 0 {\n%s--", sz, sz) d.p.declare(m.Keyidx, "string") d.p.declare(m.Validx, m.Value.TypeName()) d.assignAndCheck(m.Keyidx, stringTyp) d.ctx.PushVar(m.Keyidx) m.Value.SetIsAllowNil(false) next(d, m.Value) d.p.mapAssign(m) d.ctx.Pop() d.p.closeblock() } func (d *decodeGen) gSlice(s *Slice) { if !d.p.ok() { return } sz := randIdent() d.p.declare(sz, u32) d.assignAndCheck(sz, arrayHeader) if s.isAllowNil { d.p.resizeSliceNoNil(sz, s) } else { d.p.resizeSlice(sz, s) } d.p.rangeBlock(d.ctx, s.Index, s.Varname(), d, s.Els) } func (d *decodeGen) gArray(a *Array) { if !d.p.ok() { return } // special case if we have [const]byte if be, ok := a.Els.(*BaseElem); ok && (be.Value == Byte || be.Value == Uint8) { d.p.printf("\nerr = dc.ReadExactBytes((%s)[:])", a.Varname()) d.p.wrapErrCheck(d.ctx.ArgsStr()) return } sz := randIdent() d.p.declare(sz, u32) d.assignAndCheck(sz, arrayHeader) d.p.arrayCheck(coerceArraySize(a.Size), sz) d.p.rangeBlock(d.ctx, a.Index, a.Varname(), d, a.Els) } func (d *decodeGen) gPtr(p *Ptr) { if !d.p.ok() { return } d.p.print("\nif dc.IsNil() {") d.p.print("\nerr = dc.ReadNil()") d.p.wrapErrCheck(d.ctx.ArgsStr()) d.p.printf("\n%s = nil\n} else {", p.Varname()) d.p.initPtr(p) next(d, p.Value) d.p.closeblock() } msgp-1.2.5/gen/elem.go000066400000000000000000000436761472554463600145470ustar00rootroot00000000000000package gen import ( "fmt" "strings" ) var ( identNext = 0 identPrefix = "za" ) func resetIdent(prefix string) { identPrefix = prefix identNext = 0 } // generate a random identifier name func randIdent() string { identNext++ return fmt.Sprintf("%s%04d", identPrefix, identNext) } // This code defines the type declaration tree. // // Consider the following: // // type Marshaler struct { // Thing1 *float64 `msg:"thing1"` // Body []byte `msg:"body"` // } // // A parser using this generator as a backend // should parse the above into: // // var val Elem = &Ptr{ // name: "z", // Value: &Struct{ // Name: "Marshaler", // Fields: []StructField{ // { // FieldTag: "thing1", // FieldElem: &Ptr{ // name: "z.Thing1", // Value: &BaseElem{ // name: "*z.Thing1", // Value: Float64, // Convert: false, // }, // }, // }, // { // FieldTag: "body", // FieldElem: &BaseElem{ // name: "z.Body", // Value: Bytes, // Convert: false, // }, // }, // }, // }, // } // Base is one of the // base types type Primitive uint8 // this is effectively the // list of currently available // ReadXxxx / WriteXxxx methods. const ( Invalid Primitive = iota Bytes String Float32 Float64 Complex64 Complex128 Uint Uint8 Uint16 Uint32 Uint64 Byte Int Int8 Int16 Int32 Int64 Bool Intf // interface{} Time // time.Time Duration // time.Duration Ext // extension JsonNumber // json.Number IDENT // IDENT means an unrecognized identifier ) // all of the recognized identities // that map to primitive types var primitives = map[string]Primitive{ "[]byte": Bytes, "string": String, "float32": Float32, "float64": Float64, "complex64": Complex64, "complex128": Complex128, "uint": Uint, "uint8": Uint8, "uint16": Uint16, "uint32": Uint32, "uint64": Uint64, "byte": Byte, "rune": Int32, "int": Int, "int8": Int8, "int16": Int16, "int32": Int32, "int64": Int64, "bool": Bool, "interface{}": Intf, "any": Intf, "time.Time": Time, "time.Duration": Duration, "msgp.Extension": Ext, "json.Number": JsonNumber, } // types built into the library // that satisfy all of the // interfaces. var builtins = map[string]struct{}{ "msgp.Raw": {}, "msgp.Number": {}, } // common data/methods for every Elem type common struct { vname, alias string ptrRcv bool } func (c *common) SetVarname(s string) { c.vname = s } func (c *common) Varname() string { return c.vname } func (c *common) Alias(typ string) { c.alias = typ } func (c *common) hidden() {} func (c *common) AllowNil() bool { return false } func (c *common) SetIsAllowNil(bool) {} func (c *common) AlwaysPtr(set *bool) bool { if c != nil && set != nil { c.ptrRcv = *set } return c.ptrRcv } func IsPrintable(e Elem) bool { if be, ok := e.(*BaseElem); ok && !be.Printable() { return false } return true } // Elem is a go type capable of being // serialized into MessagePack. It is // implemented by *Ptr, *Struct, *Array, // *Slice, *Map, and *BaseElem. type Elem interface { // SetVarname sets this nodes // variable name and recursively // sets the names of all its children. // In general, this should only be // called on the parent of the tree. SetVarname(s string) // Varname returns the variable // name of the element. Varname() string // TypeName is the canonical // go type name of the node // e.g. "string", "int", "map[string]float64" // OR the alias name, if it has been set. TypeName() string // Alias sets a type (alias) name Alias(typ string) // Copy should perform a deep copy of the object Copy() Elem // Complexity returns a measure of the // complexity of element (greater than // or equal to 1.) Complexity() int // ZeroExpr returns the expression for the correct zero/empty // value. Can be used for assignment. // Returns "" if zero/empty not supported for this Elem. ZeroExpr() string // AllowNil will return true for types that can be nil but doesn't automatically check. // This is true for slices and maps. AllowNil() bool // SetIsAllowNil will set the allownil value, if the type supports it. SetIsAllowNil(bool) // AlwaysPtr will return true if receiver should always be a pointer. AlwaysPtr(set *bool) bool // IfZeroExpr returns the expression to compare to an empty value // for this type, per the rules of the `omitempty` feature. // It is meant to be used in an if statement // and may include the simple statement form followed by // semicolon and then the expression. // Returns "" if zero/empty not supported for this Elem. // Note that this is NOT used by the `omitzero` feature. IfZeroExpr() string hidden() } // Ident returns the *BaseElem that corresponds // to the provided identity. func Ident(id string) *BaseElem { p, ok := primitives[id] if ok { return &BaseElem{Value: p} } be := &BaseElem{Value: IDENT} be.Alias(id) return be } type Array struct { common Index string // index variable name Size string // array size Els Elem // child } func (a *Array) SetVarname(s string) { a.common.SetVarname(s) ridx: a.Index = randIdent() // try to avoid using the same // index as a parent slice if strings.Contains(a.Varname(), a.Index) { goto ridx } a.Els.SetVarname(fmt.Sprintf("%s[%s]", a.Varname(), a.Index)) } func (a *Array) TypeName() string { if a.common.alias != "" { return a.common.alias } a.common.Alias(fmt.Sprintf("[%s]%s", a.Size, a.Els.TypeName())) return a.common.alias } func (a *Array) Copy() Elem { b := *a b.Els = a.Els.Copy() return &b } func (a *Array) Complexity() int { // We consider the complexity constant and leave the children to decide on their own. return 2 } // ZeroExpr returns the zero/empty expression or empty string if not supported. Unsupported for this case. func (a *Array) ZeroExpr() string { return "" } // IfZeroExpr unsupported func (a *Array) IfZeroExpr() string { return "" } // Map is a map[string]Elem type Map struct { common Keyidx string // key variable name Validx string // value variable name Value Elem // value element isAllowNil bool } func (m *Map) SetVarname(s string) { m.common.SetVarname(s) ridx: m.Keyidx = randIdent() m.Validx = randIdent() // just in case if m.Keyidx == m.Validx { goto ridx } m.Value.SetVarname(m.Validx) } func (m *Map) TypeName() string { if m.common.alias != "" { return m.common.alias } m.common.Alias("map[string]" + m.Value.TypeName()) return m.common.alias } func (m *Map) Copy() Elem { g := *m g.Value = m.Value.Copy() return &g } func (m *Map) Complexity() int { // Complexity of maps are considered constant. Children should decide on their own. return 3 } // ZeroExpr returns the zero/empty expression or empty string if not supported. Always "nil" for this case. func (m *Map) ZeroExpr() string { return "nil" } // IfZeroExpr returns the expression to compare to zero/empty. func (m *Map) IfZeroExpr() string { return m.Varname() + " == nil" } // AllowNil is true for maps. func (m *Map) AllowNil() bool { return true } // SetIsAllowNil sets whether the map is allowed to be nil. func (m *Map) SetIsAllowNil(b bool) { m.isAllowNil = b } type Slice struct { common Index string isAllowNil bool Els Elem // The type of each element } func (s *Slice) SetVarname(a string) { s.common.SetVarname(a) s.Index = randIdent() varName := s.Varname() if varName[0] == '*' { // Pointer-to-slice requires parenthesis for slicing. varName = "(" + varName + ")" } s.Els.SetVarname(fmt.Sprintf("%s[%s]", varName, s.Index)) } func (s *Slice) TypeName() string { if s.common.alias != "" { return s.common.alias } s.common.Alias("[]" + s.Els.TypeName()) return s.common.alias } func (s *Slice) Copy() Elem { z := *s z.Els = s.Els.Copy() return &z } func (s *Slice) Complexity() int { // We leave the inlining decision to the slice children. return 2 } // ZeroExpr returns the zero/empty expression or empty string if not supported. Always "nil" for this case. func (s *Slice) ZeroExpr() string { return "nil" } // IfZeroExpr returns the expression to compare to zero/empty. func (s *Slice) IfZeroExpr() string { return s.Varname() + " == nil" } // AllowNil is true for slices. func (s *Slice) AllowNil() bool { return true } // SetIsAllowNil sets whether the slice is allowed to be nil. func (s *Slice) SetIsAllowNil(b bool) { s.isAllowNil = b } // SetIsAllowNil will set whether the element is allowed to be nil. func SetIsAllowNil(e Elem, b bool) { type i interface { SetIsAllowNil(b bool) } if x, ok := e.(i); ok { x.SetIsAllowNil(b) } } type Ptr struct { common Value Elem } func (s *Ptr) SetVarname(a string) { s.common.SetVarname(a) // struct fields are dereferenced // automatically... switch x := s.Value.(type) { case *Struct: // struct fields are automatically dereferenced x.SetVarname(a) return case *BaseElem: // identities have pointer receivers if x.Value == IDENT { // replace directive sets Convert=true and Needsref=true // since BaseElem is behind a pointer we set Needsref=false if x.Convert { x.Needsref(false) } x.SetVarname(a) } else { x.SetVarname("*" + a) } return default: s.Value.SetVarname("*" + a) return } } func (s *Ptr) TypeName() string { if s.common.alias != "" { return s.common.alias } s.common.Alias("*" + s.Value.TypeName()) return s.common.alias } func (s *Ptr) Copy() Elem { v := *s v.Value = s.Value.Copy() return &v } func (s *Ptr) Complexity() int { return 1 + s.Value.Complexity() } func (s *Ptr) Needsinit() bool { if be, ok := s.Value.(*BaseElem); ok && be.needsref { return false } return true } // ZeroExpr returns the zero/empty expression or empty string if not supported. Always "nil" for this case. func (s *Ptr) ZeroExpr() string { return "nil" } // IfZeroExpr returns the expression to compare to zero/empty. func (s *Ptr) IfZeroExpr() string { return s.Varname() + " == nil" } type Struct struct { common Fields []StructField // field list AsTuple bool // write as an array instead of a map } func (s *Struct) TypeName() string { if s.common.alias != "" { return s.common.alias } str := "struct{\n" for i := range s.Fields { str += s.Fields[i].FieldName + " " + s.Fields[i].FieldElem.TypeName() + " " + s.Fields[i].RawTag + ";\n" } str += "}" s.common.Alias(str) return s.common.alias } func (s *Struct) SetVarname(a string) { s.common.SetVarname(a) writeStructFields(s.Fields, a) } func (s *Struct) Copy() Elem { g := *s g.Fields = make([]StructField, len(s.Fields)) copy(g.Fields, s.Fields) for i := range s.Fields { g.Fields[i].FieldElem = s.Fields[i].FieldElem.Copy() } return &g } func (s *Struct) Complexity() int { c := 1 for i := range s.Fields { c += s.Fields[i].FieldElem.Complexity() } return c } // ZeroExpr returns the zero/empty expression or empty string if not supported. func (s *Struct) ZeroExpr() string { if s.alias == "" { return "" // structs with no names not supported (for now) } return "(" + s.TypeName() + "{})" } // IfZeroExpr returns the expression to compare to zero/empty. func (s *Struct) IfZeroExpr() string { if s.alias == "" { return "" // structs with no names not supported (for now) } return s.Varname() + " == " + s.ZeroExpr() } // AnyHasTagPart returns true if HasTagPart(p) is true for any field. func (s *Struct) AnyHasTagPart(pname string) bool { for _, sf := range s.Fields { if sf.HasTagPart(pname) { return true } } return false } // CountFieldTagPart the count of HasTagPart(p) is true for any field. func (s *Struct) CountFieldTagPart(pname string) int { var n int for _, sf := range s.Fields { if sf.HasTagPart(pname) { n++ } } return n } type StructField struct { FieldTag string // the string inside the `msg:""` tag up to the first comma FieldTagParts []string // the string inside the `msg:""` tag split by commas RawTag string // the full struct tag FieldName string // the name of the struct field FieldElem Elem // the field type } // HasTagPart returns true if the specified tag part (option) is present. func (sf *StructField) HasTagPart(pname string) bool { if len(sf.FieldTagParts) < 2 { return false } for _, p := range sf.FieldTagParts[1:] { if p == pname { return true } } return false } type ShimMode int const ( Cast ShimMode = iota Convert ) // BaseElem is an element that // can be represented by a primitive // MessagePack type. type BaseElem struct { common ShimMode ShimMode // Method used to shim ShimToBase string // shim to base type, or empty ShimFromBase string // shim from base type, or empty Value Primitive // Type of element Convert bool // should we do an explicit conversion? mustinline bool // must inline; not printable needsref bool // needs reference for shim allowNil *bool // Override from parent. } func (s *BaseElem) Printable() bool { return !s.mustinline } func (s *BaseElem) Alias(typ string) { s.common.Alias(typ) if s.Value != IDENT { s.Convert = true } if strings.Contains(typ, ".") { s.mustinline = true } } func (s *BaseElem) AllowNil() bool { if s.allowNil == nil { return s.Value == Bytes } return *s.allowNil } // SetIsAllowNil will override allownil when tag has been parsed. func (s *BaseElem) SetIsAllowNil(b bool) { s.allowNil = &b } func (s *BaseElem) SetVarname(a string) { // extensions whose parents // are not pointers need to // be explicitly referenced if s.Value == Ext || s.needsref { if strings.HasPrefix(a, "*") { s.common.SetVarname(a[1:]) return } s.common.SetVarname("&" + a) return } s.common.SetVarname(a) } // TypeName returns the syntactically correct Go // type name for the base element. func (s *BaseElem) TypeName() string { if s.common.alias != "" { return s.common.alias } s.common.Alias(s.BaseType()) return s.common.alias } // ToBase, used if Convert==true, is used as tmp = {{ToBase}}({{Varname}}) func (s *BaseElem) ToBase() string { if s.ShimToBase != "" { return s.ShimToBase } return s.BaseType() } // FromBase, used if Convert==true, is used as {{Varname}} = {{FromBase}}(tmp) func (s *BaseElem) FromBase() string { if s.ShimFromBase != "" { return s.ShimFromBase } return s.TypeName() } // BaseName returns the string form of the // base type (e.g. Float64, Ident, etc) func (s *BaseElem) BaseName() string { // time.Time and time.Duration are special cases; // we strip the package prefix if s.Value == Time { return "Time" } if s.Value == Duration { return "Duration" } if s.Value == JsonNumber { return "JSONNumber" } return s.Value.String() } func (s *BaseElem) BaseType() string { switch s.Value { case IDENT: return s.TypeName() // exceptions to the naming/capitalization // rule: case Intf: return "interface{}" case Bytes: return "[]byte" case Time: return "time.Time" case Duration: return "time.Duration" case JsonNumber: return "json.Number" case Ext: return "msgp.Extension" // everything else is base.String() with // the first letter as lowercase default: return strings.ToLower(s.BaseName()) } } func (s *BaseElem) Needsref(b bool) { s.needsref = b } func (s *BaseElem) Copy() Elem { g := *s return &g } func (s *BaseElem) Complexity() int { if s.Convert && !s.mustinline { return 2 } // we need to return 1 if !printable(), // in order to make sure that stuff gets // inlined appropriately return 1 } // Resolved returns whether or not // the type of the element is // a primitive or a builtin provided // by the package. func (s *BaseElem) Resolved() bool { if s.Value == IDENT { _, ok := builtins[s.TypeName()] return ok } return true } // ZeroExpr returns the zero/empty expression or empty string if not supported. func (s *BaseElem) ZeroExpr() string { switch s.Value { case Bytes: return "nil" case String: return "\"\"" case Complex64, Complex128: return "complex(0,0)" case Float32, Float64, Uint, Uint8, Uint16, Uint32, Uint64, Byte, Int, Int8, Int16, Int32, Int64, Duration: return "0" case Bool: return "false" case Time: return "(time.Time{})" case JsonNumber: return `""` } return "" } // IfZeroExpr returns the expression to compare to zero/empty. func (s *BaseElem) IfZeroExpr() string { z := s.ZeroExpr() if z == "" { return "" } return s.Varname() + " == " + z } func (k Primitive) String() string { switch k { case String: return "String" case Bytes: return "Bytes" case Float32: return "Float32" case Float64: return "Float64" case Complex64: return "Complex64" case Complex128: return "Complex128" case Uint: return "Uint" case Uint8: return "Uint8" case Uint16: return "Uint16" case Uint32: return "Uint32" case Uint64: return "Uint64" case Byte: return "Byte" case Int: return "Int" case Int8: return "Int8" case Int16: return "Int16" case Int32: return "Int32" case Int64: return "Int64" case Bool: return "Bool" case Intf: return "Intf" case Time: return "time.Time" case Duration: return "time.Duration" case Ext: return "Extension" case JsonNumber: return "json.Number" case IDENT: return "Ident" default: return "INVALID" } } // writeStructFields is a trampoline for writeBase for // all of the fields in a struct func writeStructFields(s []StructField, name string) { for i := range s { s[i].FieldElem.SetVarname(fmt.Sprintf("%s.%s", name, s[i].FieldName)) } } // coerceArraySize ensures we can compare constant array lengths. // // msgpack array headers are 32 bit unsigned, which is reflected in the // ArrayHeader implementation in this library using uint32. On the Go side, we // can declare array lengths as any constant integer width, which breaks when // attempting a direct comparison to an array header's uint32. func coerceArraySize(asz string) string { return fmt.Sprintf("uint32(%s)", asz) } msgp-1.2.5/gen/encode.go000066400000000000000000000156451472554463600150550ustar00rootroot00000000000000package gen import ( "fmt" "io" "github.com/tinylib/msgp/msgp" ) func encode(w io.Writer) *encodeGen { return &encodeGen{ p: printer{w: w}, } } type encodeGen struct { passes p printer fuse []byte ctx *Context } func (e *encodeGen) Method() Method { return Encode } func (e *encodeGen) Apply(dirs []string) error { return nil } func (e *encodeGen) writeAndCheck(typ string, argfmt string, arg interface{}) { if e.ctx.compFloats && typ == "Float64" { typ = "Float" } if e.ctx.newTime && typ == "Time" { typ = "TimeExt" } e.p.printf("\nerr = en.Write%s(%s)", typ, fmt.Sprintf(argfmt, arg)) e.p.wrapErrCheck(e.ctx.ArgsStr()) } func (e *encodeGen) fuseHook() { if len(e.fuse) > 0 { e.appendraw(e.fuse) e.fuse = e.fuse[:0] } } func (e *encodeGen) Fuse(b []byte) { if len(e.fuse) > 0 { e.fuse = append(e.fuse, b...) } else { e.fuse = b } } func (e *encodeGen) Execute(p Elem, ctx Context) error { e.ctx = &ctx if !e.p.ok() { return e.p.err } p = e.applyall(p) if p == nil { return nil } if !IsPrintable(p) { return nil } e.p.comment("EncodeMsg implements msgp.Encodable") rcv := imutMethodReceiver(p) ogVar := p.Varname() if p.AlwaysPtr(nil) { rcv = methodReceiver(p) } e.p.printf("\nfunc (%s %s) EncodeMsg(en *msgp.Writer) (err error) {", ogVar, rcv) next(e, p) if p.AlwaysPtr(nil) { p.SetVarname(ogVar) } e.p.nakedReturn() return e.p.err } func (e *encodeGen) gStruct(s *Struct) { if !e.p.ok() { return } if s.AsTuple { e.tuple(s) } else { e.structmap(s) } } func (e *encodeGen) tuple(s *Struct) { nfields := len(s.Fields) data := msgp.AppendArrayHeader(nil, uint32(nfields)) e.p.printf("\n// array header, size %d", nfields) e.Fuse(data) if len(s.Fields) == 0 { e.fuseHook() } for i := range s.Fields { if !e.p.ok() { return } fieldElem := s.Fields[i].FieldElem anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { e.p.printf("\nif %s { // allownil: if nil", fieldElem.IfZeroExpr()) e.p.printf("\nerr = en.WriteNil(); if err != nil { return; }") e.p.printf("\n} else {") } SetIsAllowNil(fieldElem, anField) e.ctx.PushString(s.Fields[i].FieldName) next(e, s.Fields[i].FieldElem) e.ctx.Pop() if anField { e.p.print("\n}") // close if statement } } } func (e *encodeGen) appendraw(bts []byte) { e.p.print("\nerr = en.Append(") for i, b := range bts { if i != 0 { e.p.print(", ") } e.p.printf("0x%x", b) } e.p.print(")\nif err != nil { return }") } func (e *encodeGen) structmap(s *Struct) { oeIdentPrefix := randIdent() var data []byte nfields := len(s.Fields) bm := bmask{ bitlen: nfields, varname: oeIdentPrefix + "Mask", } omitempty := s.AnyHasTagPart("omitempty") omitzero := s.AnyHasTagPart("omitzero") var closeZero bool var fieldNVar string if omitempty || omitzero { fieldNVar = oeIdentPrefix + "Len" e.p.printf("\n// check for omitted fields") e.p.printf("\n%s := uint32(%d)", fieldNVar, nfields) e.p.printf("\n%s", bm.typeDecl()) e.p.printf("\n_ = %s", bm.varname) for i, sf := range s.Fields { if !e.p.ok() { return } if ize := sf.FieldElem.IfZeroExpr(); ize != "" && sf.HasTagPart("omitempty") { e.p.printf("\nif %s {", ize) e.p.printf("\n%s--", fieldNVar) e.p.printf("\n%s", bm.setStmt(i)) e.p.printf("\n}") } else if sf.HasTagPart("omitzero") { e.p.printf("\nif %s.IsZero() {", sf.FieldElem.Varname()) e.p.printf("\n%s--", fieldNVar) e.p.printf("\n%s", bm.setStmt(i)) e.p.printf("\n}") } } e.p.printf("\n// variable map header, size %s", fieldNVar) e.p.varWriteMapHeader("en", fieldNVar, nfields) e.p.print("\nif err != nil { return }") if !e.p.ok() { return } // Skip block, if no fields are set. if nfields > 1 { e.p.printf("\n\n// skip if no fields are to be emitted") e.p.printf("\nif %s != 0 {", fieldNVar) closeZero = true } } else { // non-omit version data = msgp.AppendMapHeader(nil, uint32(nfields)) e.p.printf("\n// map header, size %d", nfields) e.Fuse(data) if len(s.Fields) == 0 { e.p.printf("\n_ = %s", s.vname) e.fuseHook() } } for i := range s.Fields { if !e.p.ok() { return } // if field is omitempty or omitzero, wrap with if statement based on the emptymask oeField := (omitempty || omitzero) && ((s.Fields[i].HasTagPart("omitempty") && s.Fields[i].FieldElem.IfZeroExpr() != "") || s.Fields[i].HasTagPart("omitzero")) if oeField { e.p.printf("\nif %s == 0 { // if not omitted", bm.readExpr(i)) } data = msgp.AppendString(nil, s.Fields[i].FieldTag) e.p.printf("\n// write %q", s.Fields[i].FieldTag) e.Fuse(data) e.fuseHook() fieldElem := s.Fields[i].FieldElem anField := !oeField && s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { e.p.printf("\nif %s { // allownil: if nil", s.Fields[i].FieldElem.IfZeroExpr()) e.p.printf("\nerr = en.WriteNil(); if err != nil { return; }") e.p.printf("\n} else {") } SetIsAllowNil(fieldElem, anField) e.ctx.PushString(s.Fields[i].FieldName) next(e, s.Fields[i].FieldElem) e.ctx.Pop() if oeField || anField { e.p.print("\n}") // close if statement } } if closeZero { e.p.printf("\n}") // close if statement } } func (e *encodeGen) gMap(m *Map) { if !e.p.ok() { return } e.fuseHook() vname := m.Varname() e.writeAndCheck(mapHeader, lenAsUint32, vname) e.p.printf("\nfor %s, %s := range %s {", m.Keyidx, m.Validx, vname) e.writeAndCheck(stringTyp, literalFmt, m.Keyidx) e.ctx.PushVar(m.Keyidx) m.Value.SetIsAllowNil(false) next(e, m.Value) e.ctx.Pop() e.p.closeblock() } func (e *encodeGen) gPtr(s *Ptr) { if !e.p.ok() { return } e.fuseHook() e.p.printf("\nif %s == nil { err = en.WriteNil(); if err != nil { return; } } else {", s.Varname()) next(e, s.Value) e.p.closeblock() } func (e *encodeGen) gSlice(s *Slice) { if !e.p.ok() { return } e.fuseHook() e.writeAndCheck(arrayHeader, lenAsUint32, s.Varname()) e.p.rangeBlock(e.ctx, s.Index, s.Varname(), e, s.Els) } func (e *encodeGen) gArray(a *Array) { if !e.p.ok() { return } e.fuseHook() // shortcut for [const]byte if be, ok := a.Els.(*BaseElem); ok && (be.Value == Byte || be.Value == Uint8) { e.p.printf("\nerr = en.WriteBytes((%s)[:])", a.Varname()) e.p.wrapErrCheck(e.ctx.ArgsStr()) return } e.writeAndCheck(arrayHeader, literalFmt, coerceArraySize(a.Size)) e.p.rangeBlock(e.ctx, a.Index, a.Varname(), e, a.Els) } func (e *encodeGen) gBase(b *BaseElem) { if !e.p.ok() { return } e.fuseHook() vname := b.Varname() if b.Convert { if b.ShimMode == Cast { vname = tobaseConvert(b) } else { vname = randIdent() e.p.printf("\nvar %s %s", vname, b.BaseType()) e.p.printf("\n%s, err = %s", vname, tobaseConvert(b)) e.p.wrapErrCheck(e.ctx.ArgsStr()) } } if b.Value == IDENT { // unknown identity e.p.printf("\nerr = %s.EncodeMsg(en)", vname) e.p.wrapErrCheck(e.ctx.ArgsStr()) } else { // typical case e.writeAndCheck(b.BaseName(), literalFmt, vname) } } msgp-1.2.5/gen/marshal.go000066400000000000000000000157371472554463600152510ustar00rootroot00000000000000package gen import ( "fmt" "io" "github.com/tinylib/msgp/msgp" ) func marshal(w io.Writer) *marshalGen { return &marshalGen{ p: printer{w: w}, } } type marshalGen struct { passes p printer fuse []byte ctx *Context } func (m *marshalGen) Method() Method { return Marshal } func (m *marshalGen) Apply(dirs []string) error { return nil } func (m *marshalGen) Execute(p Elem, ctx Context) error { m.ctx = &ctx if !m.p.ok() { return m.p.err } p = m.applyall(p) if p == nil { return nil } if !IsPrintable(p) { return nil } m.p.comment("MarshalMsg implements msgp.Marshaler") // save the vname before // calling methodReceiver so // that z.Msgsize() is printed correctly c := p.Varname() rcv := imutMethodReceiver(p) ogVar := p.Varname() if p.AlwaysPtr(nil) { rcv = methodReceiver(p) } m.p.printf("\nfunc (%s %s) MarshalMsg(b []byte) (o []byte, err error) {", ogVar, rcv) m.p.printf("\no = msgp.Require(b, %s.Msgsize())", c) next(m, p) if p.AlwaysPtr(nil) { p.SetVarname(ogVar) } m.p.nakedReturn() return m.p.err } func (m *marshalGen) rawAppend(typ string, argfmt string, arg interface{}) { if m.ctx.compFloats && typ == "Float64" { typ = "Float" } if m.ctx.newTime && typ == "Time" { typ = "TimeExt" } m.p.printf("\no = msgp.Append%s(o, %s)", typ, fmt.Sprintf(argfmt, arg)) } func (m *marshalGen) fuseHook() { if len(m.fuse) > 0 { m.rawbytes(m.fuse) m.fuse = m.fuse[:0] } } func (m *marshalGen) Fuse(b []byte) { if len(m.fuse) == 0 { m.fuse = b } else { m.fuse = append(m.fuse, b...) } } func (m *marshalGen) gStruct(s *Struct) { if !m.p.ok() { return } if s.AsTuple { m.tuple(s) } else { m.mapstruct(s) } } func (m *marshalGen) tuple(s *Struct) { data := make([]byte, 0, 5) data = msgp.AppendArrayHeader(data, uint32(len(s.Fields))) m.p.printf("\n// array header, size %d", len(s.Fields)) m.Fuse(data) if len(s.Fields) == 0 { m.fuseHook() } for i := range s.Fields { if !m.p.ok() { return } fieldElem := s.Fields[i].FieldElem anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { m.p.printf("\nif %s { // allownil: if nil", fieldElem.IfZeroExpr()) m.p.printf("\no = msgp.AppendNil(o)") m.p.printf("\n} else {") } m.ctx.PushString(s.Fields[i].FieldName) SetIsAllowNil(fieldElem, anField) next(m, fieldElem) m.ctx.Pop() if anField { m.p.printf("\n}") // close if statement } } } func (m *marshalGen) mapstruct(s *Struct) { oeIdentPrefix := randIdent() var data []byte nfields := len(s.Fields) bm := bmask{ bitlen: nfields, varname: oeIdentPrefix + "Mask", } omitempty := s.AnyHasTagPart("omitempty") omitzero := s.AnyHasTagPart("omitzero") var closeZero bool var fieldNVar string if omitempty || omitzero { fieldNVar = oeIdentPrefix + "Len" m.p.printf("\n// check for omitted fields") m.p.printf("\n%s := uint32(%d)", fieldNVar, nfields) m.p.printf("\n%s", bm.typeDecl()) m.p.printf("\n_ = %s", bm.varname) for i, sf := range s.Fields { if !m.p.ok() { return } if ize := sf.FieldElem.IfZeroExpr(); ize != "" && sf.HasTagPart("omitempty") { m.p.printf("\nif %s {", ize) m.p.printf("\n%s--", fieldNVar) m.p.printf("\n%s", bm.setStmt(i)) m.p.printf("\n}") } else if sf.HasTagPart("omitzero") { m.p.printf("\nif %s.IsZero() {", sf.FieldElem.Varname()) m.p.printf("\n%s--", fieldNVar) m.p.printf("\n%s", bm.setStmt(i)) m.p.printf("\n}") } } m.p.printf("\n// variable map header, size %s", fieldNVar) m.p.varAppendMapHeader("o", fieldNVar, nfields) if !m.p.ok() { return } // Skip block, if no fields are set. if nfields > 1 { m.p.printf("\n\n// skip if no fields are to be emitted") m.p.printf("\nif %s != 0 {", fieldNVar) closeZero = true } } else { // non-omitempty version data = make([]byte, 0, 64) data = msgp.AppendMapHeader(data, uint32(len(s.Fields))) m.p.printf("\n// map header, size %d", len(s.Fields)) m.Fuse(data) if len(s.Fields) == 0 { m.p.printf("\n_ = %s", s.vname) m.fuseHook() } } for i := range s.Fields { if !m.p.ok() { return } // if field is omitempty or omitzero, wrap with if statement based on the emptymask oeField := (omitempty || omitzero) && ((s.Fields[i].HasTagPart("omitempty") && s.Fields[i].FieldElem.IfZeroExpr() != "") || s.Fields[i].HasTagPart("omitzero")) if oeField { m.p.printf("\nif %s == 0 { // if not omitted", bm.readExpr(i)) } data = msgp.AppendString(nil, s.Fields[i].FieldTag) m.p.printf("\n// string %q", s.Fields[i].FieldTag) m.Fuse(data) m.fuseHook() fieldElem := s.Fields[i].FieldElem anField := !oeField && s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { m.p.printf("\nif %s { // allownil: if nil", fieldElem.IfZeroExpr()) m.p.printf("\no = msgp.AppendNil(o)") m.p.printf("\n} else {") } m.ctx.PushString(s.Fields[i].FieldName) SetIsAllowNil(fieldElem, anField) next(m, fieldElem) m.ctx.Pop() if oeField || anField { m.p.printf("\n}") // close if statement } } if closeZero { m.p.printf("\n}") // close if statement } } // append raw data func (m *marshalGen) rawbytes(bts []byte) { m.p.print("\no = append(o, ") for _, b := range bts { m.p.printf("0x%x,", b) } m.p.print(")") } func (m *marshalGen) gMap(s *Map) { if !m.p.ok() { return } m.fuseHook() vname := s.Varname() m.rawAppend(mapHeader, lenAsUint32, vname) m.p.printf("\nfor %s, %s := range %s {", s.Keyidx, s.Validx, vname) m.rawAppend(stringTyp, literalFmt, s.Keyidx) m.ctx.PushVar(s.Keyidx) s.Value.SetIsAllowNil(false) next(m, s.Value) m.ctx.Pop() m.p.closeblock() } func (m *marshalGen) gSlice(s *Slice) { if !m.p.ok() { return } m.fuseHook() vname := s.Varname() m.rawAppend(arrayHeader, lenAsUint32, vname) m.p.rangeBlock(m.ctx, s.Index, vname, m, s.Els) } func (m *marshalGen) gArray(a *Array) { if !m.p.ok() { return } m.fuseHook() if be, ok := a.Els.(*BaseElem); ok && be.Value == Byte { m.rawAppend("Bytes", "(%s)[:]", a.Varname()) return } m.rawAppend(arrayHeader, literalFmt, coerceArraySize(a.Size)) m.p.rangeBlock(m.ctx, a.Index, a.Varname(), m, a.Els) } func (m *marshalGen) gPtr(p *Ptr) { if !m.p.ok() { return } m.fuseHook() m.p.printf("\nif %s == nil {\no = msgp.AppendNil(o)\n} else {", p.Varname()) next(m, p.Value) m.p.closeblock() } func (m *marshalGen) gBase(b *BaseElem) { if !m.p.ok() { return } m.fuseHook() vname := b.Varname() if b.Convert { if b.ShimMode == Cast { vname = tobaseConvert(b) } else { vname = randIdent() m.p.printf("\nvar %s %s", vname, b.BaseType()) m.p.printf("\n%s, err = %s", vname, tobaseConvert(b)) m.p.wrapErrCheck(m.ctx.ArgsStr()) } } var echeck bool switch b.Value { case IDENT: echeck = true m.p.printf("\no, err = %s.MarshalMsg(o)", vname) case Intf, Ext, JsonNumber: echeck = true m.p.printf("\no, err = msgp.Append%s(o, %s)", b.BaseName(), vname) default: m.rawAppend(b.BaseName(), literalFmt, vname) } if echeck { m.p.wrapErrCheck(m.ctx.ArgsStr()) } } msgp-1.2.5/gen/size.go000066400000000000000000000140011472554463600145530ustar00rootroot00000000000000package gen import ( "fmt" "io" "strconv" "github.com/tinylib/msgp/msgp" ) type sizeState uint8 const ( // need to write "s = ..." assign sizeState = iota // need to write "s += ..." add // can just append "+ ..." expr ) func sizes(w io.Writer) *sizeGen { return &sizeGen{ p: printer{w: w}, state: assign, } } type sizeGen struct { passes p printer state sizeState ctx *Context } func (s *sizeGen) Method() Method { return Size } func (s *sizeGen) Apply(dirs []string) error { return nil } func builtinSize(typ string) string { return "msgp." + typ + "Size" } // this lets us chain together addition // operations where possible func (s *sizeGen) addConstant(sz string) { if !s.p.ok() { return } switch s.state { case assign: s.p.print("\ns = " + sz) s.state = expr return case add: s.p.print("\ns += " + sz) s.state = expr return case expr: s.p.print(" + " + sz) return } panic("unknown size state") } func (s *sizeGen) Execute(p Elem, ctx Context) error { s.ctx = &ctx if !s.p.ok() { return s.p.err } p = s.applyall(p) if p == nil { return nil } if !IsPrintable(p) { return nil } s.ctx.PushString(p.TypeName()) s.p.comment("Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message") rcv := imutMethodReceiver(p) ogVar := p.Varname() if p.AlwaysPtr(nil) { rcv = methodReceiver(p) } s.p.printf("\nfunc (%s %s) Msgsize() (s int) {", ogVar, rcv) s.state = assign next(s, p) if p.AlwaysPtr(nil) { p.SetVarname(ogVar) } s.p.nakedReturn() return s.p.err } func (s *sizeGen) gStruct(st *Struct) { if !s.p.ok() { return } nfields := uint32(len(st.Fields)) if st.AsTuple { data := msgp.AppendArrayHeader(nil, nfields) s.addConstant(strconv.Itoa(len(data))) for i := range st.Fields { if !s.p.ok() { return } next(s, st.Fields[i].FieldElem) } } else { data := msgp.AppendMapHeader(nil, nfields) s.addConstant(strconv.Itoa(len(data))) for i := range st.Fields { data = data[:0] data = msgp.AppendString(data, st.Fields[i].FieldTag) s.addConstant(strconv.Itoa(len(data))) next(s, st.Fields[i].FieldElem) } } } func (s *sizeGen) gPtr(p *Ptr) { s.state = add // inner must use add s.p.printf("\nif %s == nil {\ns += msgp.NilSize\n} else {", p.Varname()) next(s, p.Value) s.state = add // closing block; reset to add s.p.closeblock() } func (s *sizeGen) gSlice(sl *Slice) { if !s.p.ok() { return } s.addConstant(builtinSize(arrayHeader)) // if the slice's element is a fixed size // (e.g. float64, [32]int, etc.), then // print the length times the element size directly if str, ok := fixedsizeExpr(sl.Els); ok { s.addConstant(fmt.Sprintf("(%s * (%s))", lenExpr(sl), str)) return } // add inside the range block, and immediately after s.state = add s.p.rangeBlock(s.ctx, sl.Index, sl.Varname(), s, sl.Els) s.state = add } func (s *sizeGen) gArray(a *Array) { if !s.p.ok() { return } s.addConstant(builtinSize(arrayHeader)) // if the array's children are a fixed // size, we can compile an expression // that always represents the array's wire size if str, ok := fixedsizeExpr(a); ok { s.addConstant(str) return } s.state = add s.p.rangeBlock(s.ctx, a.Index, a.Varname(), s, a.Els) s.state = add } func (s *sizeGen) gMap(m *Map) { s.addConstant(builtinSize(mapHeader)) vn := m.Varname() s.p.printf("\nif %s != nil {", vn) s.p.printf("\nfor %s, %s := range %s {", m.Keyidx, m.Validx, vn) s.p.printf("\n_ = %s", m.Validx) // we may not use the value s.p.printf("\ns += msgp.StringPrefixSize + len(%s)", m.Keyidx) s.state = expr s.ctx.PushVar(m.Keyidx) next(s, m.Value) s.ctx.Pop() s.p.closeblock() s.p.closeblock() s.state = add } func (s *sizeGen) gBase(b *BaseElem) { if !s.p.ok() { return } if b.Convert && b.ShimMode == Convert { s.state = add vname := randIdent() s.p.printf("\nvar %s %s", vname, b.BaseType()) // ensure we don't get "unused variable" warnings from outer slice iterations s.p.printf("\n_ = %s", b.Varname()) s.p.printf("\ns += %s", basesizeExpr(b.Value, vname, b.BaseName())) s.state = expr } else { vname := b.Varname() if b.Convert { vname = tobaseConvert(b) } s.addConstant(basesizeExpr(b.Value, vname, b.BaseName())) } } // returns "len(slice)" func lenExpr(sl *Slice) string { return "len(" + sl.Varname() + ")" } // is a given primitive always the same (max) // size on the wire? func fixedSize(p Primitive) bool { switch p { case Intf, Ext, IDENT, Bytes, String: return false default: return true } } // strip reference from string func stripRef(s string) string { if s[0] == '&' { return s[1:] } return s } // return a fixed-size expression, if possible. // only possible for *BaseElem and *Array. // returns (expr, ok) func fixedsizeExpr(e Elem) (string, bool) { switch e := e.(type) { case *Array: if str, ok := fixedsizeExpr(e.Els); ok { return fmt.Sprintf("(%s * (%s))", e.Size, str), true } case *BaseElem: if fixedSize(e.Value) { return builtinSize(e.BaseName()), true } case *Struct: var str string for _, f := range e.Fields { if fs, ok := fixedsizeExpr(f.FieldElem); ok { if str == "" { str = fs } else { str += "+" + fs } } else { return "", false } } var hdrlen int mhdr := msgp.AppendMapHeader(nil, uint32(len(e.Fields))) hdrlen += len(mhdr) var strbody []byte for _, f := range e.Fields { strbody = msgp.AppendString(strbody[:0], f.FieldTag) hdrlen += len(strbody) } return fmt.Sprintf("%d + %s", hdrlen, str), true } return "", false } // print size expression of a variable name func basesizeExpr(value Primitive, vname, basename string) string { switch value { case Ext: return "msgp.ExtensionPrefixSize + " + stripRef(vname) + ".Len()" case Intf: return "msgp.GuessSize(" + vname + ")" case IDENT: return vname + ".Msgsize()" case Bytes: return "msgp.BytesPrefixSize + len(" + vname + ")" case String: return "msgp.StringPrefixSize + len(" + vname + ")" default: return builtinSize(basename) } } msgp-1.2.5/gen/spec.go000066400000000000000000000301531472554463600145410ustar00rootroot00000000000000package gen import ( "bytes" "fmt" "io" "strings" ) const ( lenAsUint32 = "uint32(len(%s))" literalFmt = "%s" intFmt = "%d" quotedFmt = `"%s"` mapHeader = "MapHeader" arrayHeader = "ArrayHeader" mapKey = "MapKeyPtr" stringTyp = "String" u32 = "uint32" ) // Method is a bitfield representing something that the // generator knows how to print. type Method uint8 // are the bits in 'f' set in 'm'? func (m Method) isset(f Method) bool { return (m&f == f) } // String implements fmt.Stringer func (m Method) String() string { switch m { case 0, invalidmeth: return "" case Decode: return "decode" case Encode: return "encode" case Marshal: return "marshal" case Unmarshal: return "unmarshal" case Size: return "size" case Test: return "test" default: // return e.g. "decode+encode+test" modes := [...]Method{Decode, Encode, Marshal, Unmarshal, Size, Test} any := false nm := "" for _, mm := range modes { if m.isset(mm) { if any { nm += "+" + mm.String() } else { nm += mm.String() any = true } } } return nm } } const ( Decode Method = 1 << iota // msgp.Decodable Encode // msgp.Encodable Marshal // msgp.Marshaler Unmarshal // msgp.Unmarshaler Size // msgp.Sizer Test // generate tests invalidmeth // this isn't a method encodetest = Encode | Decode | Test // tests for Encodable and Decodable marshaltest = Marshal | Unmarshal | Test // tests for Marshaler and Unmarshaler ) type Printer struct { gens []generator CompactFloats bool ClearOmitted bool NewTime bool } func NewPrinter(m Method, out io.Writer, tests io.Writer) *Printer { if m.isset(Test) && tests == nil { panic("cannot print tests with 'nil' tests argument!") } gens := make([]generator, 0, 7) if m.isset(Decode) { gens = append(gens, decode(out)) } if m.isset(Encode) { gens = append(gens, encode(out)) } if m.isset(Marshal) { gens = append(gens, marshal(out)) } if m.isset(Unmarshal) { gens = append(gens, unmarshal(out)) } if m.isset(Size) { gens = append(gens, sizes(out)) } if m.isset(marshaltest) { gens = append(gens, mtest(tests)) } if m.isset(encodetest) { gens = append(gens, etest(tests)) } if len(gens) == 0 { panic("NewPrinter called with invalid method flags") } return &Printer{gens: gens} } // TransformPass is a pass that transforms individual // elements. (Note that if the returned is different from // the argument, it should not point to the same objects.) type TransformPass func(Elem) Elem // IgnoreTypename is a pass that just ignores // types of a given name. func IgnoreTypename(name string) TransformPass { return func(e Elem) Elem { if e.TypeName() == name { return nil } return e } } // ApplyDirective applies a directive to a named pass // and all of its dependents. func (p *Printer) ApplyDirective(pass Method, t TransformPass) { for _, g := range p.gens { if g.Method().isset(pass) { g.Add(t) } } } // Print prints an Elem. func (p *Printer) Print(e Elem) error { e.SetIsAllowNil(false) for _, g := range p.gens { // Elem.SetVarname() is called before the Print() step in parse.FileSet.PrintTo(). // Elem.SetVarname() generates identifiers as it walks the Elem. This can cause // collisions between idents created during SetVarname and idents created during Print, // hence the separate prefixes. resetIdent("zb") err := g.Execute(e, Context{ compFloats: p.CompactFloats, clearOmitted: p.ClearOmitted, newTime: p.NewTime, }) resetIdent("za") if err != nil { return err } } return nil } type contextItem interface { Arg() string } type contextString string func (c contextString) Arg() string { return fmt.Sprintf("%q", c) } type contextVar string func (c contextVar) Arg() string { return string(c) } type Context struct { path []contextItem compFloats bool clearOmitted bool newTime bool } func (c *Context) PushString(s string) { c.path = append(c.path, contextString(s)) } func (c *Context) PushVar(s string) { c.path = append(c.path, contextVar(s)) } func (c *Context) Pop() { c.path = c.path[:len(c.path)-1] } func (c *Context) ArgsStr() string { var out string for idx, p := range c.path { if idx > 0 { out += ", " } out += p.Arg() } return out } // generator is the interface through // which code is generated. type generator interface { Method() Method Add(p TransformPass) Execute(Elem, Context) error // execute writes the method for the provided object. } type passes []TransformPass func (p *passes) Add(t TransformPass) { *p = append(*p, t) } func (p *passes) applyall(e Elem) Elem { for _, t := range *p { e = t(e) if e == nil { return nil } } return e } type traversal interface { gMap(*Map) gSlice(*Slice) gArray(*Array) gPtr(*Ptr) gBase(*BaseElem) gStruct(*Struct) } // type-switch dispatch to the correct // method given the type of 'e' func next(t traversal, e Elem) { switch e := e.(type) { case *Map: t.gMap(e) case *Struct: t.gStruct(e) case *Slice: t.gSlice(e) case *Array: t.gArray(e) case *Ptr: t.gPtr(e) case *BaseElem: t.gBase(e) default: panic("bad element type") } } // possibly-immutable method receiver func imutMethodReceiver(p Elem) string { switch e := p.(type) { case *Struct: // TODO(HACK): actually do real math here. if len(e.Fields) <= 3 { for i := range e.Fields { if be, ok := e.Fields[i].FieldElem.(*BaseElem); !ok || (be.Value == IDENT || be.Value == Bytes) { goto nope } } return p.TypeName() } nope: return "*" + p.TypeName() // gets dereferenced automatically case *Array: return "*" + p.TypeName() // everything else can be // by-value. default: return p.TypeName() } } // if necessary, wraps a type // so that its method receiver // is of the write type. func methodReceiver(p Elem) string { switch p.(type) { // structs and arrays are // dereferenced automatically, // so no need to alter varname case *Struct, *Array: return "*" + p.TypeName() // set variable name to // *varname default: p.SetVarname("(*" + p.Varname() + ")") return "*" + p.TypeName() } } func unsetReceiver(p Elem) { switch p.(type) { case *Struct, *Array: default: p.SetVarname("z") } } // shared utility for generators type printer struct { w io.Writer err error } // writes "var {{name}} {{typ}};" func (p *printer) declare(name string, typ string) { p.printf("\nvar %s %s", name, typ) } // does: // // if m == nil { // m = make(type, size) // } else if len(m) > 0 { // // for key := range m { delete(m, key) } // } func (p *printer) resizeMap(size string, m *Map) { vn := m.Varname() if !p.ok() { return } p.printf("\nif %s == nil {", vn) p.printf("\n%s = make(%s, %s)", vn, m.TypeName(), size) p.printf("\n} else if len(%s) > 0 {", vn) p.clearMap(vn) p.closeblock() } // assign key to value based on varnames func (p *printer) mapAssign(m *Map) { if !p.ok() { return } p.printf("\n%s[%s] = %s", m.Varname(), m.Keyidx, m.Validx) } // clear map keys func (p *printer) clearMap(name string) { p.printf("\nfor key := range %[1]s { delete(%[1]s, key) }", name) } func (p *printer) wrapErrCheck(ctx string) { p.print("\nif err != nil {") p.printf("\nerr = msgp.WrapError(err, %s)", ctx) p.printf("\nreturn") p.print("\n}") } func (p *printer) resizeSlice(size string, s *Slice) { p.printf("\nif cap(%[1]s) >= int(%[2]s) { %[1]s = (%[1]s)[:%[2]s] } else { %[1]s = make(%[3]s, %[2]s) }", s.Varname(), size, s.TypeName()) } // resizeSliceNoNil will resize a slice and will not allow nil slices. func (p *printer) resizeSliceNoNil(size string, s *Slice) { p.printf("\nif %[1]s != nil && cap(%[1]s) >= int(%[2]s) {", s.Varname(), size) p.printf("\n%[1]s = (%[1]s)[:%[2]s]", s.Varname(), size) p.printf("\n} else { %[1]s = make(%[3]s, %[2]s) }", s.Varname(), size, s.TypeName()) } func (p *printer) arrayCheck(want string, got string) { p.printf("\nif %[1]s != %[2]s { err = msgp.ArrayError{Wanted: %[2]s, Got: %[1]s}; return }", got, want) } func (p *printer) closeblock() { p.print("\n}") } // does: // // for idx := range iter { // {{generate inner}} // } func (p *printer) rangeBlock(ctx *Context, idx string, iter string, t traversal, inner Elem) { ctx.PushVar(idx) // Tags on slices do not extend to the elements, so we always disable allownil on elements. // If we want this to happen in the future, it should be a unique tag. inner.SetIsAllowNil(false) p.printf("\n for %s := range %s {", idx, iter) next(t, inner) p.closeblock() ctx.Pop() } func (p *printer) nakedReturn() { if p.ok() { p.print("\nreturn\n}\n") } } func (p *printer) comment(s string) { p.print("\n// " + s) } func (p *printer) printf(format string, args ...interface{}) { if p.err == nil { _, p.err = fmt.Fprintf(p.w, format, args...) } } func (p *printer) print(format string) { if p.err == nil { _, p.err = io.WriteString(p.w, format) } } func (p *printer) initPtr(pt *Ptr) { if pt.Needsinit() { vname := pt.Varname() p.printf("\nif %s == nil { %s = new(%s); }", vname, vname, pt.Value.TypeName()) } } func (p *printer) ok() bool { return p.err == nil } func tobaseConvert(b *BaseElem) string { return b.ToBase() + "(" + b.Varname() + ")" } func (p *printer) varWriteMapHeader(receiver string, sizeVarname string, maxSize int) { if maxSize <= 15 { p.printf("\nerr = %s.Append(0x80 | uint8(%s))", receiver, sizeVarname) } else { p.printf("\nerr = %s.WriteMapHeader(%s)", receiver, sizeVarname) } } func (p *printer) varAppendMapHeader(sliceVarname string, sizeVarname string, maxSize int) { if maxSize <= 15 { p.printf("\n%s = append(%s, 0x80 | uint8(%s))", sliceVarname, sliceVarname, sizeVarname) } else { p.printf("\n%s = msgp.AppendMapHeader(%s, %s)", sliceVarname, sliceVarname, sizeVarname) } } // bmask is a bitmask of a the specified number of bits type bmask struct { bitlen int varname string } // typeDecl returns the variable declaration as a var statement func (b *bmask) typeDecl() string { return fmt.Sprintf("var %s %s /* %d bits */", b.varname, b.typeName(), b.bitlen) } // typeName returns the type, e.g. "uint8" or "[2]uint64" func (b *bmask) typeName() string { if b.bitlen <= 8 { return "uint8" } if b.bitlen <= 16 { return "uint16" } if b.bitlen <= 32 { return "uint32" } if b.bitlen <= 64 { return "uint64" } return fmt.Sprintf("[%d]uint64", (b.bitlen+64-1)/64) } // readExpr returns the expression to read from a position in the bitmask. // Compare ==0 for false or !=0 for true. func (b *bmask) readExpr(bitoffset int) string { if bitoffset < 0 || bitoffset >= b.bitlen { panic(fmt.Errorf("bitoffset %d out of range for bitlen %d", bitoffset, b.bitlen)) } var buf bytes.Buffer buf.Grow(len(b.varname) + 16) buf.WriteByte('(') buf.WriteString(b.varname) if b.bitlen > 64 { fmt.Fprintf(&buf, "[%d]", (bitoffset / 64)) } buf.WriteByte('&') fmt.Fprintf(&buf, "0x%X", (uint64(1) << (uint64(bitoffset) % 64))) buf.WriteByte(')') return buf.String() } // setStmt returns the statement to set the specified bit in the bitmask. func (b *bmask) setStmt(bitoffset int) string { var buf bytes.Buffer buf.Grow(len(b.varname) + 16) buf.WriteString(b.varname) if b.bitlen > 64 { fmt.Fprintf(&buf, "[%d]", (bitoffset / 64)) } fmt.Fprintf(&buf, " |= 0x%X", (uint64(1) << (uint64(bitoffset) % 64))) return buf.String() } // notAllSet returns a check against all fields having been set in set. func (b *bmask) notAllSet() string { var buf bytes.Buffer buf.Grow(len(b.varname) + 16) buf.WriteString(b.varname) if b.bitlen > 64 { var bytes []string remain := b.bitlen for remain >= 8 { bytes = append(bytes, "0xff") } if remain > 0 { bytes = append(bytes, fmt.Sprintf("0x%X", remain)) } fmt.Fprintf(&buf, " != [%d]byte{%s}\n", (b.bitlen+63)/64, strings.Join(bytes, ",")) } fmt.Fprintf(&buf, " != 0x%x", uint64(1< 0 { t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) } left, err = msgp.Skip(bts) if err != nil { t.Fatal(err) } if len(left) > 0 { t.Errorf("%d bytes left over after Skip(): %q", len(left), left) } } func BenchmarkMarshalMsg{{.TypeName}}(b *testing.B) { v := {{.TypeName}}{} b.ReportAllocs() b.ResetTimer() for i:=0; i m { t.Log("WARNING: TestEncodeDecode{{.TypeName}} Msgsize() is inaccurate") } vn := {{.TypeName}}{} err := msgp.Decode(&buf, &vn) if err != nil { t.Error(err) } buf.Reset() msgp.Encode(&buf, &v) err = msgp.NewReader(&buf).Skip() if err != nil { t.Error(err) } } func BenchmarkEncode{{.TypeName}}(b *testing.B) { v := {{.TypeName}}{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) en := msgp.NewWriter(msgp.Nowhere) b.ReportAllocs() b.ResetTimer() for i:=0; i 0 { // Declare mask u.p.printf("\n%s", bm.typeDecl()) u.p.printf("\n_ = %s", bm.varname) } // Index to field idx of each emitted oeEmittedIdx := []int{} u.p.printf("\nfor %s > 0 {", sz) u.p.printf("\n%s--; field, bts, err = msgp.ReadMapKeyZC(bts)", sz) u.p.wrapErrCheck(u.ctx.ArgsStr()) u.p.print("\nswitch msgp.UnsafeString(field) {") for i := range s.Fields { if !u.p.ok() { return } u.p.printf("\ncase %q:", s.Fields[i].FieldTag) u.ctx.PushString(s.Fields[i].FieldName) fieldElem := s.Fields[i].FieldElem anField := s.Fields[i].HasTagPart("allownil") && fieldElem.AllowNil() if anField { u.p.printf("\nif msgp.IsNil(bts) {\nbts = bts[1:]\n%s = nil\n} else {", fieldElem.Varname()) } SetIsAllowNil(fieldElem, anField) next(u, fieldElem) u.ctx.Pop() if oeCount > 0 && (s.Fields[i].HasTagPart("omitempty") || s.Fields[i].HasTagPart("omitzero")) { u.p.printf("\n%s", bm.setStmt(len(oeEmittedIdx))) oeEmittedIdx = append(oeEmittedIdx, i) } if anField { u.p.printf("\n}") } } u.p.print("\ndefault:\nbts, err = msgp.Skip(bts)") u.p.wrapErrCheck(u.ctx.ArgsStr()) u.p.print("\n}\n}") // close switch and for loop if oeCount > 0 { u.p.printf("\n// Clear omitted fields.\n") if bm.bitlen > 1 { u.p.printf("if %s {\n", bm.notAllSet()) } for bitIdx, fieldIdx := range oeEmittedIdx { fieldElem := s.Fields[fieldIdx].FieldElem u.p.printf("if %s == 0 {\n", bm.readExpr(bitIdx)) fze := fieldElem.ZeroExpr() if fze != "" { u.p.printf("%s = %s\n", fieldElem.Varname(), fze) } else { u.p.printf("%s = %s{}\n", fieldElem.Varname(), fieldElem.TypeName()) } u.p.printf("}\n") } if bm.bitlen > 1 { u.p.printf("}") } } } func (u *unmarshalGen) gBase(b *BaseElem) { if !u.p.ok() { return } refname := b.Varname() // assigned to lowered := b.Varname() // passed as argument // begin 'tmp' block if b.Convert && b.Value != IDENT { // we don't need block for 'tmp' in case of IDENT refname = randIdent() lowered = b.ToBase() + "(" + lowered + ")" u.p.printf("\n{\nvar %s %s", refname, b.BaseType()) } switch b.Value { case Bytes: u.p.printf("\n%s, bts, err = msgp.ReadBytesBytes(bts, %s)", refname, lowered) case Ext: u.p.printf("\nbts, err = msgp.ReadExtensionBytes(bts, %s)", lowered) case IDENT: if b.Convert { lowered = b.ToBase() + "(" + lowered + ")" } u.p.printf("\nbts, err = %s.UnmarshalMsg(bts)", lowered) default: u.p.printf("\n%s, bts, err = msgp.Read%sBytes(bts)", refname, b.BaseName()) } u.p.wrapErrCheck(u.ctx.ArgsStr()) if b.Value == Bytes && b.AllowNil() { // Ensure that 0 sized slices are allocated. u.p.printf("\nif %s == nil {\n%s = make([]byte, 0)\n}", refname, refname) } // close 'tmp' block if b.Convert && b.Value != IDENT { if b.ShimMode == Cast { u.p.printf("\n%s = %s(%s)\n", b.Varname(), b.FromBase(), refname) } else { u.p.printf("\n%s, err = %s(%s)\n", b.Varname(), b.FromBase(), refname) u.p.wrapErrCheck(u.ctx.ArgsStr()) } u.p.printf("}") } } func (u *unmarshalGen) gArray(a *Array) { if !u.p.ok() { return } // special case for [const]byte objects // see decode.go for symmetry if be, ok := a.Els.(*BaseElem); ok && be.Value == Byte { u.p.printf("\nbts, err = msgp.ReadExactBytes(bts, (%s)[:])", a.Varname()) u.p.wrapErrCheck(u.ctx.ArgsStr()) return } sz := randIdent() u.p.declare(sz, u32) u.assignAndCheck(sz, arrayHeader) u.p.arrayCheck(coerceArraySize(a.Size), sz) u.p.rangeBlock(u.ctx, a.Index, a.Varname(), u, a.Els) } func (u *unmarshalGen) gSlice(s *Slice) { if !u.p.ok() { return } sz := randIdent() u.p.declare(sz, u32) u.assignAndCheck(sz, arrayHeader) if s.isAllowNil { u.p.resizeSliceNoNil(sz, s) } else { u.p.resizeSlice(sz, s) } u.p.rangeBlock(u.ctx, s.Index, s.Varname(), u, s.Els) } func (u *unmarshalGen) gMap(m *Map) { if !u.p.ok() { return } sz := randIdent() u.p.declare(sz, u32) u.assignAndCheck(sz, mapHeader) // allocate or clear map u.p.resizeMap(sz, m) // We likely need a field. // Add now to not be inside for scope. u.needsField() // loop and get key,value u.p.printf("\nfor %s > 0 {", sz) u.p.printf("\nvar %s string; var %s %s; %s--", m.Keyidx, m.Validx, m.Value.TypeName(), sz) u.assignAndCheck(m.Keyidx, stringTyp) u.ctx.PushVar(m.Keyidx) m.Value.SetIsAllowNil(false) next(u, m.Value) u.ctx.Pop() u.p.mapAssign(m) u.p.closeblock() } func (u *unmarshalGen) gPtr(p *Ptr) { u.p.printf("\nif msgp.IsNil(bts) { bts, err = msgp.ReadNilBytes(bts); if err != nil { return }; %s = nil; } else { ", p.Varname()) u.p.initPtr(p) next(u, p.Value) u.p.closeblock() } msgp-1.2.5/go.mod000066400000000000000000000002761472554463600136200ustar00rootroot00000000000000module github.com/tinylib/msgp go 1.20 require ( github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c golang.org/x/tools v0.22.0 ) require golang.org/x/mod v0.18.0 // indirect msgp-1.2.5/go.sum000066400000000000000000000011421472554463600136360ustar00rootroot00000000000000github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= msgp-1.2.5/issue185_test.go000066400000000000000000000152551472554463600154710ustar00rootroot00000000000000package main import ( "fmt" "go/ast" "go/parser" "go/token" "os" "path/filepath" "reflect" "sort" "testing" "text/template" "github.com/tinylib/msgp/gen" ) // When stuff's going wrong, you'll be glad this is here! const debugTemp = false // Ensure that consistent identifiers are generated on a per-method basis by msgp. // // Also ensure that no duplicate identifiers appear in a method. // // structs are currently processed alphabetically by msgp. this test relies on // that property. func TestIssue185Idents(t *testing.T) { identCases := []struct { tpl *template.Template expectedChanged []string }{ {tpl: issue185IdentsTpl, expectedChanged: []string{"Test1"}}, {tpl: issue185ComplexIdentsTpl, expectedChanged: []string{"Test2"}}, } methods := []string{"DecodeMsg", "EncodeMsg", "Msgsize", "MarshalMsg", "UnmarshalMsg"} for idx, identCase := range identCases { // generate the code, extract the generated variable names, mapped to function name var tplData issue185TplData varsBefore, err := loadVars(t, identCase.tpl, tplData) if err != nil { t.Fatalf("%d: could not extract before vars: %v", idx, err) } // regenerate the code with extra field(s), extract the generated variable // names, mapped to function name tplData.Extra = true varsAfter, err := loadVars(t, identCase.tpl, tplData) if err != nil { t.Fatalf("%d: could not extract after vars: %v", idx, err) } // ensure that all declared variable names inside each of the methods we // expect to change have actually changed for _, stct := range identCase.expectedChanged { for _, method := range methods { fn := fmt.Sprintf("%s.%s", stct, method) bv, av := varsBefore.Value(fn), varsAfter.Value(fn) if len(bv) > 0 && len(av) > 0 && reflect.DeepEqual(bv, av) { t.Fatalf("%d vars identical! expected vars to change for %s", idx, fn) } delete(varsBefore, fn) delete(varsAfter, fn) } } // all of the remaining keys should not have changed for bmethod, bvars := range varsBefore { avars := varsAfter.Value(bmethod) if !reflect.DeepEqual(bvars, avars) { t.Fatalf("%d: vars changed! expected vars identical for %s", idx, bmethod) } delete(varsBefore, bmethod) delete(varsAfter, bmethod) } if len(varsBefore) > 0 || len(varsAfter) > 0 { t.Fatalf("%d: unexpected methods remaining", idx) } } } type issue185TplData struct { Extra bool } func TestIssue185Overlap(t *testing.T) { overlapCases := []struct { tpl *template.Template data issue185TplData }{ {tpl: issue185IdentsTpl, data: issue185TplData{Extra: false}}, {tpl: issue185IdentsTpl, data: issue185TplData{Extra: true}}, {tpl: issue185ComplexIdentsTpl, data: issue185TplData{Extra: false}}, {tpl: issue185ComplexIdentsTpl, data: issue185TplData{Extra: true}}, } for idx, o := range overlapCases { // regenerate the code with extra field(s), extract the generated variable // names, mapped to function name mvars, err := loadVars(t, o.tpl, o.data) if err != nil { t.Fatalf("%d: could not extract after vars: %v", idx, err) } identCnt := 0 for fn, vars := range mvars { sort.Strings(vars) // Loose sanity check to make sure the tests expectations aren't broken. // If the prefix ever changes, this needs to change. for _, v := range vars { if v[0] == 'z' { identCnt++ } } for i := 0; i < len(vars)-1; i++ { if vars[i] == vars[i+1] { t.Fatalf("%d: duplicate var %s in function %s", idx, vars[i], fn) } } } // one last sanity check: if there aren't any vars that start with 'z', // this test's expectations are unsatisfiable. if identCnt == 0 { t.Fatalf("%d: no generated identifiers found", idx) } } } func loadVars(t *testing.T, tpl *template.Template, tplData interface{}) (vars extractedVars, err error) { tempDir := t.TempDir() if !debugTemp { defer os.RemoveAll(tempDir) } else { fmt.Println(tempDir) } tfile := filepath.Join(tempDir, "msg.go") genFile := newFilename(tfile, "") if err = goGenerateTpl(tempDir, tfile, tpl, tplData); err != nil { err = fmt.Errorf("could not generate code: %v", err) return } vars, err = extractVars(genFile) if err != nil { err = fmt.Errorf("could not extract after vars: %v", err) return } return } type varVisitor struct { vars []string fset *token.FileSet } func (v *varVisitor) Visit(node ast.Node) (w ast.Visitor) { gen, ok := node.(*ast.GenDecl) if !ok { return v } for _, spec := range gen.Specs { if vspec, ok := spec.(*ast.ValueSpec); ok { for _, n := range vspec.Names { v.vars = append(v.vars, n.Name) } } } return v } type extractedVars map[string][]string func (e extractedVars) Value(key string) []string { if v, ok := e[key]; ok { return v } panic(fmt.Errorf("unknown key %s", key)) } func extractVars(file string) (extractedVars, error) { fset := token.NewFileSet() f, err := parser.ParseFile(fset, file, nil, 0) if err != nil { return nil, err } vars := make(map[string][]string) for _, d := range f.Decls { switch d := d.(type) { case *ast.FuncDecl: sn := "" switch rt := d.Recv.List[0].Type.(type) { case *ast.Ident: sn = rt.Name case *ast.StarExpr: sn = rt.X.(*ast.Ident).Name default: panic("unknown receiver type") } key := fmt.Sprintf("%s.%s", sn, d.Name.Name) vis := &varVisitor{fset: fset} ast.Walk(vis, d.Body) vars[key] = vis.vars } } return vars, nil } func goGenerateTpl(cwd, tfile string, tpl *template.Template, tplData interface{}) error { outf, err := os.OpenFile(tfile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600) if err != nil { return err } defer outf.Close() if err := tpl.Execute(outf, tplData); err != nil { return err } mode := gen.Encode | gen.Decode | gen.Size | gen.Marshal | gen.Unmarshal return Run(tfile, mode, false) } var issue185IdentsTpl = template.Must(template.New("").Parse(` package issue185 //go:generate msgp type Test1 struct { Foo string Bar string {{ if .Extra }}Baz []string{{ end }} Qux string } type Test2 struct { Foo string Bar string Baz string } `)) var issue185ComplexIdentsTpl = template.Must(template.New("").Parse(` package issue185 //go:generate msgp type Test1 struct { Foo string Bar string Baz string } type Test2 struct { Foo string Bar string Baz []string Qux map[string]string Yep map[string]map[string]string Quack struct { Quack struct { Quack struct { {{ if .Extra }}Extra []string{{ end }} Quack string } } } Nup struct { Foo string Bar string Baz []string Qux map[string]string Yep map[string]map[string]string } Ding struct { Dong struct { Dung struct { Thing string } } } } type Test3 struct { Foo string Bar string Baz string } `)) msgp-1.2.5/main.go000066400000000000000000000066711472554463600137720ustar00rootroot00000000000000// msgp is a code generation tool for // creating methods to serialize and de-serialize // Go data structures to and from MessagePack. // // This package is targeted at the `go generate` tool. // To use it, include the following directive in a // go source file with types requiring source generation: // // //go:generate msgp // // The go generate tool should set the proper environment variables for // the generator to execute without any command-line flags. However, the // following options are supported, if you need them: // // -o = output file name (default is {input}_gen.go) // -file = input file name (or directory; default is $GOFILE, which is set by the `go generate` command) // -io = satisfy the `msgp.Decodable` and `msgp.Encodable` interfaces (default is true) // -marshal = satisfy the `msgp.Marshaler` and `msgp.Unmarshaler` interfaces (default is true) // -tests = generate tests and benchmarks (default is true) // // For more information, please read README.md, and the wiki at github.com/tinylib/msgp package main import ( "flag" "fmt" "os" "path/filepath" "strings" "github.com/tinylib/msgp/gen" "github.com/tinylib/msgp/parse" "github.com/tinylib/msgp/printer" ) var ( out = flag.String("o", "", "output file") file = flag.String("file", "", "input file") encode = flag.Bool("io", true, "create Encode and Decode methods") marshal = flag.Bool("marshal", true, "create Marshal and Unmarshal methods") tests = flag.Bool("tests", true, "create tests and benchmarks") unexported = flag.Bool("unexported", false, "also process unexported types") verbose = flag.Bool("v", false, "verbose diagnostics") ) func diagf(f string, args ...interface{}) { if !*verbose { return } if f[len(f)-1] != '\n' { f += "\n" } fmt.Fprintf(os.Stderr, f, args...) } func exitln(res string) { fmt.Fprintln(os.Stderr, res) os.Exit(1) } func main() { flag.Parse() if *verbose { printer.Logf = diagf parse.Logf = diagf } // GOFILE is set by go generate if *file == "" { *file = os.Getenv("GOFILE") if *file == "" { exitln("No file to parse.") } } var mode gen.Method if *encode { mode |= (gen.Encode | gen.Decode | gen.Size) } if *marshal { mode |= (gen.Marshal | gen.Unmarshal | gen.Size) } if *tests { mode |= gen.Test } if mode&^gen.Test == 0 { exitln("No methods to generate; -io=false && -marshal=false") } if err := Run(*file, mode, *unexported); err != nil { exitln(err.Error()) } } // Run writes all methods using the associated file or path, e.g. // // err := msgp.Run("path/to/myfile.go", gen.Size|gen.Marshal|gen.Unmarshal|gen.Test, false) func Run(gofile string, mode gen.Method, unexported bool) error { if mode&^gen.Test == 0 { return nil } diagf("Input: \"%s\"\n", gofile) fs, err := parse.File(gofile, unexported) if err != nil { return err } if len(fs.Identities) == 0 { diagf("No types requiring code generation were found!") } return printer.PrintFile(newFilename(gofile, fs.Package), fs, mode) } // picks a new file name based on input flags and input filename(s). func newFilename(old string, pkg string) string { if *out != "" { if pre := strings.TrimPrefix(*out, old); len(pre) > 0 && !strings.HasSuffix(*out, ".go") { return filepath.Join(old, *out) } return *out } if fi, err := os.Stat(old); err == nil && fi.IsDir() { old = filepath.Join(old, pkg) } // new file name is old file name + _gen.go return strings.TrimSuffix(old, ".go") + "_gen.go" } msgp-1.2.5/msgp/000077500000000000000000000000001472554463600134535ustar00rootroot00000000000000msgp-1.2.5/msgp/advise_linux.go000066400000000000000000000007241472554463600164770ustar00rootroot00000000000000//go:build linux && !appengine && !tinygo // +build linux,!appengine,!tinygo package msgp import ( "os" "syscall" ) func adviseRead(mem []byte) { syscall.Madvise(mem, syscall.MADV_SEQUENTIAL|syscall.MADV_WILLNEED) } func adviseWrite(mem []byte) { syscall.Madvise(mem, syscall.MADV_SEQUENTIAL) } func fallocate(f *os.File, sz int64) error { err := syscall.Fallocate(int(f.Fd()), 0, 0, sz) if err == syscall.ENOTSUP { return f.Truncate(sz) } return err } msgp-1.2.5/msgp/advise_other.go000066400000000000000000000004521472554463600164570ustar00rootroot00000000000000//go:build (!linux && !tinygo && !windows) || appengine // +build !linux,!tinygo,!windows appengine package msgp import ( "os" ) // TODO: darwin, BSD support func adviseRead(mem []byte) {} func adviseWrite(mem []byte) {} func fallocate(f *os.File, sz int64) error { return f.Truncate(sz) } msgp-1.2.5/msgp/circular.go000066400000000000000000000015641472554463600156140ustar00rootroot00000000000000package msgp type timer interface { StartTimer() StopTimer() } // EndlessReader is an io.Reader // that loops over the same data // endlessly. It is used for benchmarking. type EndlessReader struct { tb timer data []byte offset int } // NewEndlessReader returns a new endless reader func NewEndlessReader(b []byte, tb timer) *EndlessReader { // Double until we reach 4K. for len(b) < 4<<10 { b = append(b, b...) } return &EndlessReader{tb: tb, data: b, offset: 0} } // Read implements io.Reader. In practice, it // always returns (len(p), nil), although it // fills the supplied slice while the benchmark // timer is stopped. func (c *EndlessReader) Read(p []byte) (int, error) { var n int l := len(p) m := len(c.data) nn := copy(p[n:], c.data[c.offset:]) n += nn for n < l { n += copy(p[n:], c.data[:]) } c.offset = (c.offset + l) % m return n, nil } msgp-1.2.5/msgp/defs.go000066400000000000000000000064311472554463600147270ustar00rootroot00000000000000// This package is the support library for the msgp code generator (http://github.com/tinylib/msgp). // // This package defines the utilites used by the msgp code generator for encoding and decoding MessagePack // from []byte and io.Reader/io.Writer types. Much of this package is devoted to helping the msgp code // generator implement the Marshaler/Unmarshaler and Encodable/Decodable interfaces. // // This package defines four "families" of functions: // - AppendXxxx() appends an object to a []byte in MessagePack encoding. // - ReadXxxxBytes() reads an object from a []byte and returns the remaining bytes. // - (*Writer).WriteXxxx() writes an object to the buffered *Writer type. // - (*Reader).ReadXxxx() reads an object from a buffered *Reader type. // // Once a type has satisfied the `Encodable` and `Decodable` interfaces, // it can be written and read from arbitrary `io.Writer`s and `io.Reader`s using // // msgp.Encode(io.Writer, msgp.Encodable) // // and // // msgp.Decode(io.Reader, msgp.Decodable) // // There are also methods for converting MessagePack to JSON without // an explicit de-serialization step. // // For additional tips, tricks, and gotchas, please visit // the wiki at http://github.com/tinylib/msgp package msgp const ( last4 = 0x0f first4 = 0xf0 last5 = 0x1f first3 = 0xe0 last7 = 0x7f // recursionLimit is the limit of recursive calls. // This limits the call depth of dynamic code, like Skip and interface conversions. recursionLimit = 100000 ) func isfixint(b byte) bool { return b>>7 == 0 } func isnfixint(b byte) bool { return b&first3 == mnfixint } func isfixmap(b byte) bool { return b&first4 == mfixmap } func isfixarray(b byte) bool { return b&first4 == mfixarray } func isfixstr(b byte) bool { return b&first3 == mfixstr } func wfixint(u uint8) byte { return u & last7 } func rfixint(b byte) uint8 { return b } func wnfixint(i int8) byte { return byte(i) | mnfixint } func rnfixint(b byte) int8 { return int8(b) } func rfixmap(b byte) uint8 { return b & last4 } func wfixmap(u uint8) byte { return mfixmap | (u & last4) } func rfixstr(b byte) uint8 { return b & last5 } func wfixstr(u uint8) byte { return (u & last5) | mfixstr } func rfixarray(b byte) uint8 { return (b & last4) } func wfixarray(u uint8) byte { return (u & last4) | mfixarray } // These are all the byte // prefixes defined by the // msgpack standard const ( // 0XXXXXXX mfixint uint8 = 0x00 // 111XXXXX mnfixint uint8 = 0xe0 // 1000XXXX mfixmap uint8 = 0x80 // 1001XXXX mfixarray uint8 = 0x90 // 101XXXXX mfixstr uint8 = 0xa0 mnil uint8 = 0xc0 mfalse uint8 = 0xc2 mtrue uint8 = 0xc3 mbin8 uint8 = 0xc4 mbin16 uint8 = 0xc5 mbin32 uint8 = 0xc6 mext8 uint8 = 0xc7 mext16 uint8 = 0xc8 mext32 uint8 = 0xc9 mfloat32 uint8 = 0xca mfloat64 uint8 = 0xcb muint8 uint8 = 0xcc muint16 uint8 = 0xcd muint32 uint8 = 0xce muint64 uint8 = 0xcf mint8 uint8 = 0xd0 mint16 uint8 = 0xd1 mint32 uint8 = 0xd2 mint64 uint8 = 0xd3 mfixext1 uint8 = 0xd4 mfixext2 uint8 = 0xd5 mfixext4 uint8 = 0xd6 mfixext8 uint8 = 0xd7 mfixext16 uint8 = 0xd8 mstr8 uint8 = 0xd9 mstr16 uint8 = 0xda mstr32 uint8 = 0xdb marray16 uint8 = 0xdc marray32 uint8 = 0xdd mmap16 uint8 = 0xde mmap32 uint8 = 0xdf ) msgp-1.2.5/msgp/defs_test.go000066400000000000000000000003461472554463600157650ustar00rootroot00000000000000package msgp_test //go:generate msgp -o=defgen_test.go -tests=false type Blobs []Blob type Blob struct { Name string `msg:"name"` Float float64 `msg:"float"` Bytes []byte `msg:"bytes"` Amount int64 `msg:"amount"` } msgp-1.2.5/msgp/edit.go000066400000000000000000000130111472554463600147230ustar00rootroot00000000000000package msgp import ( "math" ) // Locate returns a []byte pointing to the field // in a messagepack map with the provided key. (The returned []byte // points to a sub-slice of 'raw'; Locate does no allocations.) If the // key doesn't exist in the map, a zero-length []byte will be returned. func Locate(key string, raw []byte) []byte { s, n := locate(raw, key) return raw[s:n] } // Replace takes a key ("key") in a messagepack map ("raw") // and replaces its value with the one provided and returns // the new []byte. The returned []byte may point to the same // memory as "raw". Replace makes no effort to evaluate the validity // of the contents of 'val'. It may use up to the full capacity of 'raw.' // Replace returns 'nil' if the field doesn't exist or if the object in 'raw' // is not a map. func Replace(key string, raw []byte, val []byte) []byte { start, end := locate(raw, key) if start == end { return nil } return replace(raw, start, end, val, true) } // CopyReplace works similarly to Replace except that the returned // byte slice does not point to the same memory as 'raw'. CopyReplace // returns 'nil' if the field doesn't exist or 'raw' isn't a map. func CopyReplace(key string, raw []byte, val []byte) []byte { start, end := locate(raw, key) if start == end { return nil } return replace(raw, start, end, val, false) } // Remove removes a key-value pair from 'raw'. It returns // 'raw' unchanged if the key didn't exist. func Remove(key string, raw []byte) []byte { start, end := locateKV(raw, key) if start == end { return raw } raw = raw[:start+copy(raw[start:], raw[end:])] return resizeMap(raw, -1) } // HasKey returns whether the map in 'raw' has // a field with key 'key' func HasKey(key string, raw []byte) bool { sz, bts, err := ReadMapHeaderBytes(raw) if err != nil { return false } var field []byte for i := uint32(0); i < sz; i++ { field, bts, err = ReadStringZC(bts) if err != nil { return false } if UnsafeString(field) == key { return true } } return false } func replace(raw []byte, start int, end int, val []byte, inplace bool) []byte { ll := end - start // length of segment to replace lv := len(val) if inplace { extra := lv - ll // fastest case: we're doing // a 1:1 replacement if extra == 0 { copy(raw[start:], val) return raw } else if extra < 0 { // 'val' smaller than replaced value // copy in place and shift back x := copy(raw[start:], val) y := copy(raw[start+x:], raw[end:]) return raw[:start+x+y] } else if extra < cap(raw)-len(raw) { // 'val' less than (cap-len) extra bytes // copy in place and shift forward raw = raw[0 : len(raw)+extra] // shift end forward copy(raw[end+extra:], raw[end:]) copy(raw[start:], val) return raw } } // we have to allocate new space out := make([]byte, len(raw)+len(val)-ll) x := copy(out, raw[:start]) y := copy(out[x:], val) copy(out[x+y:], raw[end:]) return out } // locate does a naive O(n) search for the map key; returns start, end // (returns 0,0 on error) func locate(raw []byte, key string) (start int, end int) { var ( sz uint32 bts []byte field []byte err error ) sz, bts, err = ReadMapHeaderBytes(raw) if err != nil { return } // loop and locate field for i := uint32(0); i < sz; i++ { field, bts, err = ReadStringZC(bts) if err != nil { return 0, 0 } if UnsafeString(field) == key { // start location l := len(raw) start = l - len(bts) bts, err = Skip(bts) if err != nil { return 0, 0 } end = l - len(bts) return } bts, err = Skip(bts) if err != nil { return 0, 0 } } return 0, 0 } // locate key AND value func locateKV(raw []byte, key string) (start int, end int) { var ( sz uint32 bts []byte field []byte err error ) sz, bts, err = ReadMapHeaderBytes(raw) if err != nil { return 0, 0 } for i := uint32(0); i < sz; i++ { tmp := len(bts) field, bts, err = ReadStringZC(bts) if err != nil { return 0, 0 } if UnsafeString(field) == key { start = len(raw) - tmp bts, err = Skip(bts) if err != nil { return 0, 0 } end = len(raw) - len(bts) return } bts, err = Skip(bts) if err != nil { return 0, 0 } } return 0, 0 } // delta is delta on map size func resizeMap(raw []byte, delta int64) []byte { var sz int64 switch raw[0] { case mmap16: sz = int64(big.Uint16(raw[1:])) if sz+delta <= math.MaxUint16 { big.PutUint16(raw[1:], uint16(sz+delta)) return raw } if cap(raw)-len(raw) >= 2 { raw = raw[0 : len(raw)+2] copy(raw[5:], raw[3:]) raw[0] = mmap32 big.PutUint32(raw[1:], uint32(sz+delta)) return raw } n := make([]byte, 0, len(raw)+5) n = AppendMapHeader(n, uint32(sz+delta)) return append(n, raw[3:]...) case mmap32: sz = int64(big.Uint32(raw[1:])) big.PutUint32(raw[1:], uint32(sz+delta)) return raw default: sz = int64(rfixmap(raw[0])) if sz+delta < 16 { raw[0] = wfixmap(uint8(sz + delta)) return raw } else if sz+delta <= math.MaxUint16 { if cap(raw)-len(raw) >= 2 { raw = raw[0 : len(raw)+2] copy(raw[3:], raw[1:]) raw[0] = mmap16 big.PutUint16(raw[1:], uint16(sz+delta)) return raw } n := make([]byte, 0, len(raw)+5) n = AppendMapHeader(n, uint32(sz+delta)) return append(n, raw[1:]...) } if cap(raw)-len(raw) >= 4 { raw = raw[0 : len(raw)+4] copy(raw[5:], raw[1:]) raw[0] = mmap32 big.PutUint32(raw[1:], uint32(sz+delta)) return raw } n := make([]byte, 0, len(raw)+5) n = AppendMapHeader(n, uint32(sz+delta)) return append(n, raw[1:]...) } } msgp-1.2.5/msgp/edit_test.go000066400000000000000000000106111472554463600157650ustar00rootroot00000000000000package msgp import ( "bytes" "reflect" "testing" ) func TestRemove(t *testing.T) { var buf bytes.Buffer w := NewWriter(&buf) w.WriteMapHeader(3) w.WriteString("first") w.WriteFloat64(-3.1) w.WriteString("second") w.WriteString("DELETE ME!!!") w.WriteString("third") w.WriteBytes([]byte("blah")) w.Flush() raw := Remove("second", buf.Bytes()) m, _, err := ReadMapStrIntfBytes(raw, nil) if err != nil { t.Fatal(err) } if len(m) != 2 { t.Errorf("expected %d fields; found %d", 2, len(m)) } if _, ok := m["first"]; !ok { t.Errorf("field %q not found", "first") } if _, ok := m["third"]; !ok { t.Errorf("field %q not found", "third") } if _, ok := m["second"]; ok { t.Errorf("field %q (deleted field) still present", "second") } } func TestLocate(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(2) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(2.0) en.Flush() field := Locate("thing_one", buf.Bytes()) if len(field) == 0 { t.Fatal("field not found") } if !HasKey("thing_one", buf.Bytes()) { t.Fatal("field not found") } var zbuf bytes.Buffer w := NewWriter(&zbuf) w.WriteString("value_one") w.Flush() if !bytes.Equal(zbuf.Bytes(), field) { t.Errorf("got %q; wanted %q", field, zbuf.Bytes()) } zbuf.Reset() w.WriteFloat64(2.0) w.Flush() field = Locate("thing_two", buf.Bytes()) if len(field) == 0 { t.Fatal("field not found") } if !bytes.Equal(zbuf.Bytes(), field) { t.Errorf("got %q; wanted %q", field, zbuf.Bytes()) } field = Locate("nope", buf.Bytes()) if len(field) != 0 { t.Fatalf("wanted a zero-length returned slice") } } func TestReplace(t *testing.T) { // there are 4 cases that need coverage: // - new value is smaller than old value // - new value is the same size as the old value // - new value is larger than old, but fits within cap(b) // - new value is larger than old, and doesn't fit within cap(b) var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(3) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(2.0) en.WriteString("some_bytes") en.WriteBytes([]byte("here are some bytes")) en.Flush() // same-size replacement var fbuf bytes.Buffer w := NewWriter(&fbuf) w.WriteFloat64(4.0) w.Flush() // replace 2.0 with 4.0 in field two raw := Replace("thing_two", buf.Bytes(), fbuf.Bytes()) if len(raw) == 0 { t.Fatal("field not found") } var err error m := make(map[string]interface{}) m, _, err = ReadMapStrIntfBytes(raw, m) if err != nil { t.Logf("%q", raw) t.Fatal(err) } if !reflect.DeepEqual(m["thing_two"], 4.0) { t.Errorf("wanted %v; got %v", 4.0, m["thing_two"]) } // smaller-size replacement // replace 2.0 with []byte("hi!") fbuf.Reset() w.WriteBytes([]byte("hi!")) w.Flush() raw = Replace("thing_two", raw, fbuf.Bytes()) if len(raw) == 0 { t.Fatal("field not found") } m, _, err = ReadMapStrIntfBytes(raw, m) if err != nil { t.Logf("%q", raw) t.Fatal(err) } if !reflect.DeepEqual(m["thing_two"], []byte("hi!")) { t.Errorf("wanted %v; got %v", []byte("hi!"), m["thing_two"]) } // larger-size replacement fbuf.Reset() w.WriteBytes([]byte("some even larger bytes than before")) w.Flush() raw = Replace("some_bytes", raw, fbuf.Bytes()) if len(raw) == 0 { t.Logf("%q", raw) t.Fatal(err) } m, _, err = ReadMapStrIntfBytes(raw, m) if err != nil { t.Logf("%q", raw) t.Fatal(err) } if !reflect.DeepEqual(m["some_bytes"], []byte("some even larger bytes than before")) { t.Errorf("wanted %v; got %v", []byte("hello there!"), m["some_bytes"]) } // identical in-place replacement field := Locate("some_bytes", raw) newraw := CopyReplace("some_bytes", raw, field) if !bytes.Equal(newraw, raw) { t.Logf("in: %q", raw) t.Logf("out: %q", newraw) t.Error("bytes not equal after copyreplace") } } func BenchmarkLocate(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(3) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(2.0) en.WriteString("thing_three") en.WriteBytes([]byte("hello!")) en.Flush() raw := buf.Bytes() // bytes/s will be the number of bytes traversed per unit of time field := Locate("thing_three", raw) b.SetBytes(int64(len(raw) - len(field))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { Locate("thing_three", raw) } } msgp-1.2.5/msgp/elsize.go000066400000000000000000000075671472554463600153140ustar00rootroot00000000000000package msgp func calcBytespec(v byte) bytespec { // single byte values switch v { case mnil: return bytespec{size: 1, extra: constsize, typ: NilType} case mfalse: return bytespec{size: 1, extra: constsize, typ: BoolType} case mtrue: return bytespec{size: 1, extra: constsize, typ: BoolType} case mbin8: return bytespec{size: 2, extra: extra8, typ: BinType} case mbin16: return bytespec{size: 3, extra: extra16, typ: BinType} case mbin32: return bytespec{size: 5, extra: extra32, typ: BinType} case mext8: return bytespec{size: 3, extra: extra8, typ: ExtensionType} case mext16: return bytespec{size: 4, extra: extra16, typ: ExtensionType} case mext32: return bytespec{size: 6, extra: extra32, typ: ExtensionType} case mfloat32: return bytespec{size: 5, extra: constsize, typ: Float32Type} case mfloat64: return bytespec{size: 9, extra: constsize, typ: Float64Type} case muint8: return bytespec{size: 2, extra: constsize, typ: UintType} case muint16: return bytespec{size: 3, extra: constsize, typ: UintType} case muint32: return bytespec{size: 5, extra: constsize, typ: UintType} case muint64: return bytespec{size: 9, extra: constsize, typ: UintType} case mint8: return bytespec{size: 2, extra: constsize, typ: IntType} case mint16: return bytespec{size: 3, extra: constsize, typ: IntType} case mint32: return bytespec{size: 5, extra: constsize, typ: IntType} case mint64: return bytespec{size: 9, extra: constsize, typ: IntType} case mfixext1: return bytespec{size: 3, extra: constsize, typ: ExtensionType} case mfixext2: return bytespec{size: 4, extra: constsize, typ: ExtensionType} case mfixext4: return bytespec{size: 6, extra: constsize, typ: ExtensionType} case mfixext8: return bytespec{size: 10, extra: constsize, typ: ExtensionType} case mfixext16: return bytespec{size: 18, extra: constsize, typ: ExtensionType} case mstr8: return bytespec{size: 2, extra: extra8, typ: StrType} case mstr16: return bytespec{size: 3, extra: extra16, typ: StrType} case mstr32: return bytespec{size: 5, extra: extra32, typ: StrType} case marray16: return bytespec{size: 3, extra: array16v, typ: ArrayType} case marray32: return bytespec{size: 5, extra: array32v, typ: ArrayType} case mmap16: return bytespec{size: 3, extra: map16v, typ: MapType} case mmap32: return bytespec{size: 5, extra: map32v, typ: MapType} } switch { // fixint case v >= mfixint && v < 0x80: return bytespec{size: 1, extra: constsize, typ: IntType} // fixstr gets constsize, since the prefix yields the size case v >= mfixstr && v < 0xc0: return bytespec{size: 1 + rfixstr(v), extra: constsize, typ: StrType} // fixmap case v >= mfixmap && v < 0x90: return bytespec{size: 1, extra: varmode(2 * rfixmap(v)), typ: MapType} // fixarray case v >= mfixarray && v < 0xa0: return bytespec{size: 1, extra: varmode(rfixarray(v)), typ: ArrayType} // nfixint case v >= mnfixint && uint16(v) < 0x100: return bytespec{size: 1, extra: constsize, typ: IntType} } // 0xC1 is unused per the spec and falls through to here, // everything else is covered above return bytespec{} } func getType(v byte) Type { return getBytespec(v).typ } // a valid bytespsec has // non-zero 'size' and // non-zero 'typ' type bytespec struct { size uint8 // prefix size information extra varmode // extra size information typ Type // type _ byte // makes bytespec 4 bytes (yes, this matters) } // size mode // if positive, # elements for composites type varmode int8 const ( constsize varmode = 0 // constant size (size bytes + uint8(varmode) objects) extra8 varmode = -1 // has uint8(p[1]) extra bytes extra16 varmode = -2 // has be16(p[1:]) extra bytes extra32 varmode = -3 // has be32(p[1:]) extra bytes map16v varmode = -4 // use map16 map32v varmode = -5 // use map32 array16v varmode = -6 // use array16 array32v varmode = -7 // use array32 ) msgp-1.2.5/msgp/elsize_default.go000066400000000000000000000006321472554463600170020ustar00rootroot00000000000000//go:build !tinygo // +build !tinygo package msgp // size of every object on the wire, // plus type information. gives us // constant-time type information // for traversing composite objects. var sizes [256]bytespec func init() { for i := 0; i < 256; i++ { sizes[i] = calcBytespec(byte(i)) } } // getBytespec gets inlined to a simple array index func getBytespec(v byte) bytespec { return sizes[v] } msgp-1.2.5/msgp/elsize_test.go000066400000000000000000000062711472554463600163420ustar00rootroot00000000000000package msgp import "testing" func TestBytespec(t *testing.T) { // verify that bytespec refactor for TinyGo behaves the same as the old code // previous sizes array setup verbatim: // size of every object on the wire, // plus type information. gives us // constant-time type information // for traversing composite objects. // sizes := [256]bytespec{ mnil: {size: 1, extra: constsize, typ: NilType}, mfalse: {size: 1, extra: constsize, typ: BoolType}, mtrue: {size: 1, extra: constsize, typ: BoolType}, mbin8: {size: 2, extra: extra8, typ: BinType}, mbin16: {size: 3, extra: extra16, typ: BinType}, mbin32: {size: 5, extra: extra32, typ: BinType}, mext8: {size: 3, extra: extra8, typ: ExtensionType}, mext16: {size: 4, extra: extra16, typ: ExtensionType}, mext32: {size: 6, extra: extra32, typ: ExtensionType}, mfloat32: {size: 5, extra: constsize, typ: Float32Type}, mfloat64: {size: 9, extra: constsize, typ: Float64Type}, muint8: {size: 2, extra: constsize, typ: UintType}, muint16: {size: 3, extra: constsize, typ: UintType}, muint32: {size: 5, extra: constsize, typ: UintType}, muint64: {size: 9, extra: constsize, typ: UintType}, mint8: {size: 2, extra: constsize, typ: IntType}, mint16: {size: 3, extra: constsize, typ: IntType}, mint32: {size: 5, extra: constsize, typ: IntType}, mint64: {size: 9, extra: constsize, typ: IntType}, mfixext1: {size: 3, extra: constsize, typ: ExtensionType}, mfixext2: {size: 4, extra: constsize, typ: ExtensionType}, mfixext4: {size: 6, extra: constsize, typ: ExtensionType}, mfixext8: {size: 10, extra: constsize, typ: ExtensionType}, mfixext16: {size: 18, extra: constsize, typ: ExtensionType}, mstr8: {size: 2, extra: extra8, typ: StrType}, mstr16: {size: 3, extra: extra16, typ: StrType}, mstr32: {size: 5, extra: extra32, typ: StrType}, marray16: {size: 3, extra: array16v, typ: ArrayType}, marray32: {size: 5, extra: array32v, typ: ArrayType}, mmap16: {size: 3, extra: map16v, typ: MapType}, mmap32: {size: 5, extra: map32v, typ: MapType}, } // set up fixed fields // fixint for i := mfixint; i < 0x80; i++ { sizes[i] = bytespec{size: 1, extra: constsize, typ: IntType} } // nfixint for i := uint16(mnfixint); i < 0x100; i++ { sizes[uint8(i)] = bytespec{size: 1, extra: constsize, typ: IntType} } // fixstr gets constsize, // since the prefix yields the size for i := mfixstr; i < 0xc0; i++ { sizes[i] = bytespec{size: 1 + rfixstr(i), extra: constsize, typ: StrType} } // fixmap for i := mfixmap; i < 0x90; i++ { sizes[i] = bytespec{size: 1, extra: varmode(2 * rfixmap(i)), typ: MapType} } // fixarray for i := mfixarray; i < 0xa0; i++ { sizes[i] = bytespec{size: 1, extra: varmode(rfixarray(i)), typ: ArrayType} } // compare all values to calcBytespec for i := 0; i < 256; i++ { sizeb := sizes[byte(i)] cb := calcBytespec(byte(i)) if sizeb != cb { t.Errorf("at index 0x%x calculated sizes array %#v does not match calcBytespec %#v", i, sizeb, cb) } // all except the unused byte C1 should have non-zero size if i != 0xC1 && sizeb.size == 0 { t.Errorf("unexpected zero size for index 0x%x", i) } } } msgp-1.2.5/msgp/elsize_tinygo.go000066400000000000000000000004631472554463600166710ustar00rootroot00000000000000//go:build tinygo // +build tinygo package msgp // for tinygo, getBytespec just calls calcBytespec // a simple/slow function with a switch statement - // doesn't require any heap alloc, moves the space // requirements into code instad of ram func getBytespec(v byte) bytespec { return calcBytespec(v) } msgp-1.2.5/msgp/errors.go000066400000000000000000000242421472554463600153220ustar00rootroot00000000000000package msgp import ( "reflect" "strconv" ) const resumableDefault = false var ( // ErrShortBytes is returned when the // slice being decoded is too short to // contain the contents of the message ErrShortBytes error = errShort{} // ErrRecursion is returned when the maximum recursion limit is reached for an operation. // This should only realistically be seen on adversarial data trying to exhaust the stack. ErrRecursion error = errRecursion{} // this error is only returned // if we reach code that should // be unreachable fatal error = errFatal{} ) // Error is the interface satisfied // by all of the errors that originate // from this package. type Error interface { error // Resumable returns whether // or not the error means that // the stream of data is malformed // and the information is unrecoverable. Resumable() bool } // contextError allows msgp Error instances to be enhanced with additional // context about their origin. type contextError interface { Error // withContext must not modify the error instance - it must clone and // return a new error with the context added. withContext(ctx string) error } // Cause returns the underlying cause of an error that has been wrapped // with additional context. func Cause(e error) error { out := e if e, ok := e.(errWrapped); ok && e.cause != nil { out = e.cause } return out } // Resumable returns whether or not the error means that the stream of data is // malformed and the information is unrecoverable. func Resumable(e error) bool { if e, ok := e.(Error); ok { return e.Resumable() } return resumableDefault } // WrapError wraps an error with additional context that allows the part of the // serialized type that caused the problem to be identified. Underlying errors // can be retrieved using Cause() // // The input error is not modified - a new error should be returned. // // ErrShortBytes is not wrapped with any context due to backward compatibility // issues with the public API. func WrapError(err error, ctx ...interface{}) error { switch e := err.(type) { case errShort: return e case contextError: return e.withContext(ctxString(ctx)) default: return errWrapped{cause: err, ctx: ctxString(ctx)} } } func addCtx(ctx, add string) string { if ctx != "" { return add + "/" + ctx } else { return add } } // errWrapped allows arbitrary errors passed to WrapError to be enhanced with // context and unwrapped with Cause() type errWrapped struct { cause error ctx string } func (e errWrapped) Error() string { if e.ctx != "" { return e.cause.Error() + " at " + e.ctx } else { return e.cause.Error() } } func (e errWrapped) Resumable() bool { if e, ok := e.cause.(Error); ok { return e.Resumable() } return resumableDefault } // Unwrap returns the cause. func (e errWrapped) Unwrap() error { return e.cause } type errShort struct{} func (e errShort) Error() string { return "msgp: too few bytes left to read object" } func (e errShort) Resumable() bool { return false } type errFatal struct { ctx string } func (f errFatal) Error() string { out := "msgp: fatal decoding error (unreachable code)" if f.ctx != "" { out += " at " + f.ctx } return out } func (f errFatal) Resumable() bool { return false } func (f errFatal) withContext(ctx string) error { f.ctx = addCtx(f.ctx, ctx); return f } type errRecursion struct{} func (e errRecursion) Error() string { return "msgp: recursion limit reached" } func (e errRecursion) Resumable() bool { return false } // ArrayError is an error returned // when decoding a fix-sized array // of the wrong size type ArrayError struct { Wanted uint32 Got uint32 ctx string } // Error implements the error interface func (a ArrayError) Error() string { out := "msgp: wanted array of size " + strconv.Itoa(int(a.Wanted)) + "; got " + strconv.Itoa(int(a.Got)) if a.ctx != "" { out += " at " + a.ctx } return out } // Resumable is always 'true' for ArrayErrors func (a ArrayError) Resumable() bool { return true } func (a ArrayError) withContext(ctx string) error { a.ctx = addCtx(a.ctx, ctx); return a } // IntOverflow is returned when a call // would downcast an integer to a type // with too few bits to hold its value. type IntOverflow struct { Value int64 // the value of the integer FailedBitsize int // the bit size that the int64 could not fit into ctx string } // Error implements the error interface func (i IntOverflow) Error() string { str := "msgp: " + strconv.FormatInt(i.Value, 10) + " overflows int" + strconv.Itoa(i.FailedBitsize) if i.ctx != "" { str += " at " + i.ctx } return str } // Resumable is always 'true' for overflows func (i IntOverflow) Resumable() bool { return true } func (i IntOverflow) withContext(ctx string) error { i.ctx = addCtx(i.ctx, ctx); return i } // UintOverflow is returned when a call // would downcast an unsigned integer to a type // with too few bits to hold its value type UintOverflow struct { Value uint64 // value of the uint FailedBitsize int // the bit size that couldn't fit the value ctx string } // Error implements the error interface func (u UintOverflow) Error() string { str := "msgp: " + strconv.FormatUint(u.Value, 10) + " overflows uint" + strconv.Itoa(u.FailedBitsize) if u.ctx != "" { str += " at " + u.ctx } return str } // Resumable is always 'true' for overflows func (u UintOverflow) Resumable() bool { return true } func (u UintOverflow) withContext(ctx string) error { u.ctx = addCtx(u.ctx, ctx); return u } // InvalidTimestamp is returned when an invalid timestamp is encountered type InvalidTimestamp struct { Nanos int64 // value of the nano, if invalid FieldLength int // Unexpected field length. ctx string } // Error implements the error interface func (u InvalidTimestamp) Error() (str string) { if u.Nanos > 0 { str = "msgp: timestamp nanosecond field value " + strconv.FormatInt(u.Nanos, 10) + " exceeds maximum allows of 999999999" } else if u.FieldLength >= 0 { str = "msgp: invalid timestamp field length " + strconv.FormatInt(int64(u.FieldLength), 10) + " - must be 4, 8 or 12" } if u.ctx != "" { str += " at " + u.ctx } return str } // Resumable is always 'true' for overflows func (u InvalidTimestamp) Resumable() bool { return true } func (u InvalidTimestamp) withContext(ctx string) error { u.ctx = addCtx(u.ctx, ctx); return u } // UintBelowZero is returned when a call // would cast a signed integer below zero // to an unsigned integer. type UintBelowZero struct { Value int64 // value of the incoming int ctx string } // Error implements the error interface func (u UintBelowZero) Error() string { str := "msgp: attempted to cast int " + strconv.FormatInt(u.Value, 10) + " to unsigned" if u.ctx != "" { str += " at " + u.ctx } return str } // Resumable is always 'true' for overflows func (u UintBelowZero) Resumable() bool { return true } func (u UintBelowZero) withContext(ctx string) error { u.ctx = ctx return u } // A TypeError is returned when a particular // decoding method is unsuitable for decoding // a particular MessagePack value. type TypeError struct { Method Type // Type expected by method Encoded Type // Type actually encoded ctx string } // Error implements the error interface func (t TypeError) Error() string { out := "msgp: attempted to decode type " + quoteStr(t.Encoded.String()) + " with method for " + quoteStr(t.Method.String()) if t.ctx != "" { out += " at " + t.ctx } return out } // Resumable returns 'true' for TypeErrors func (t TypeError) Resumable() bool { return true } func (t TypeError) withContext(ctx string) error { t.ctx = addCtx(t.ctx, ctx); return t } // returns either InvalidPrefixError or // TypeError depending on whether or not // the prefix is recognized func badPrefix(want Type, lead byte) error { t := getType(lead) if t == InvalidType { return InvalidPrefixError(lead) } return TypeError{Method: want, Encoded: t} } // InvalidPrefixError is returned when a bad encoding // uses a prefix that is not recognized in the MessagePack standard. // This kind of error is unrecoverable. type InvalidPrefixError byte // Error implements the error interface func (i InvalidPrefixError) Error() string { return "msgp: unrecognized type prefix 0x" + strconv.FormatInt(int64(i), 16) } // Resumable returns 'false' for InvalidPrefixErrors func (i InvalidPrefixError) Resumable() bool { return false } // ErrUnsupportedType is returned // when a bad argument is supplied // to a function that takes `interface{}`. type ErrUnsupportedType struct { T reflect.Type ctx string } // Error implements error func (e *ErrUnsupportedType) Error() string { out := "msgp: type " + quoteStr(e.T.String()) + " not supported" if e.ctx != "" { out += " at " + e.ctx } return out } // Resumable returns 'true' for ErrUnsupportedType func (e *ErrUnsupportedType) Resumable() bool { return true } func (e *ErrUnsupportedType) withContext(ctx string) error { o := *e o.ctx = addCtx(o.ctx, ctx) return &o } // simpleQuoteStr is a simplified version of strconv.Quote for TinyGo, // which takes up a lot less code space by escaping all non-ASCII // (UTF-8) bytes with \x. Saves about 4k of code size // (unicode tables, needed for IsPrint(), are big). // It lives in errors.go just so we can test it in errors_test.go func simpleQuoteStr(s string) string { const ( lowerhex = "0123456789abcdef" ) sb := make([]byte, 0, len(s)+2) sb = append(sb, `"`...) l: // loop through string bytes (not UTF-8 characters) for i := 0; i < len(s); i++ { b := s[i] // specific escape chars switch b { case '\\': sb = append(sb, `\\`...) case '"': sb = append(sb, `\"`...) case '\a': sb = append(sb, `\a`...) case '\b': sb = append(sb, `\b`...) case '\f': sb = append(sb, `\f`...) case '\n': sb = append(sb, `\n`...) case '\r': sb = append(sb, `\r`...) case '\t': sb = append(sb, `\t`...) case '\v': sb = append(sb, `\v`...) default: // no escaping needed (printable ASCII) if b >= 0x20 && b <= 0x7E { sb = append(sb, b) continue l } // anything else is \x sb = append(sb, `\x`...) sb = append(sb, lowerhex[byte(b)>>4]) sb = append(sb, lowerhex[byte(b)&0xF]) continue l } } sb = append(sb, `"`...) return string(sb) } msgp-1.2.5/msgp/errors_default.go000066400000000000000000000005751472554463600170310ustar00rootroot00000000000000//go:build !tinygo // +build !tinygo package msgp import ( "fmt" "strconv" ) // ctxString converts the incoming interface{} slice into a single string. func ctxString(ctx []interface{}) string { out := "" for idx, cv := range ctx { if idx > 0 { out += "/" } out += fmt.Sprintf("%v", cv) } return out } func quoteStr(s string) string { return strconv.Quote(s) } msgp-1.2.5/msgp/errors_test.go000066400000000000000000000060371472554463600163630ustar00rootroot00000000000000package msgp import ( "errors" "fmt" "io" "strings" "testing" ) func TestWrapVanillaErrorWithNoAdditionalContext(t *testing.T) { err := errors.New("test") w := WrapError(err) if w == err { t.Fatal() } if w.Error() != err.Error() { t.Fatal() } if w.(errWrapped).Resumable() { t.Fatal() } } func TestWrapVanillaErrorWithAdditionalContext(t *testing.T) { err := errors.New("test") w := WrapError(err, "foo", "bar") if w == err { t.Fatal() } if w.Error() == err.Error() { t.Fatal() } if w.(Error).Resumable() { t.Fatal() } if !strings.HasPrefix(w.Error(), err.Error()) { t.Fatal() } rest := w.Error()[len(err.Error()):] if rest != " at foo/bar" { t.Fatal() } } func TestWrapResumableError(t *testing.T) { err := ArrayError{} w := WrapError(err) if !w.(Error).Resumable() { t.Fatal() } } func TestWrapMultiple(t *testing.T) { err := &TypeError{} w := WrapError(WrapError(err, "b"), "a") expected := `msgp: attempted to decode type "" with method for "" at a/b` if expected != w.Error() { t.Fatal() } } func TestCause(t *testing.T) { for idx, err := range []error{ errors.New("test"), ArrayError{}, &ErrUnsupportedType{}, } { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { cerr := WrapError(err, "test") if cerr == err { t.Fatal() } if Cause(err) != err { t.Fatal() } }) } } func TestCauseShortByte(t *testing.T) { err := ErrShortBytes cerr := WrapError(err, "test") if cerr != err { t.Fatal() } if Cause(err) != err { t.Fatal() } } func TestUnwrap(t *testing.T) { // check errors that get wrapped for idx, err := range []error{ errors.New("test"), io.EOF, } { t.Run(fmt.Sprintf("wrapped_%d", idx), func(t *testing.T) { cerr := WrapError(err, "test") if cerr == err { t.Fatal() } uwerr := errors.Unwrap(cerr) if uwerr != err { t.Fatal() } if !errors.Is(cerr, err) { t.Fatal() } }) } // check errors where only context is applied for idx, err := range []error{ ArrayError{}, &ErrUnsupportedType{}, } { t.Run(fmt.Sprintf("ctx_only_%d", idx), func(t *testing.T) { cerr := WrapError(err, "test") if cerr == err { t.Fatal() } if errors.Unwrap(cerr) != nil { t.Fatal() } }) } } func TestSimpleQuoteStr(t *testing.T) { // check some cases for simpleQuoteStr type tcase struct { in string out string } tcaseList := []tcase{ { in: ``, out: `""`, }, { in: `abc`, out: `"abc"`, }, { in: `"`, out: `"\""`, }, { in: `'`, out: `"'"`, }, { in: `on🔥!`, out: `"on\xf0\x9f\x94\xa5!"`, }, { in: "line\r\nbr", out: `"line\r\nbr"`, }, { in: "\x00", out: `"\x00"`, }, { // invalid UTF-8 should make no difference but check it regardless in: "not\x80valid", out: `"not\x80valid"`, }, } for i, tc := range tcaseList { tc := tc t.Run(fmt.Sprint(i), func(t *testing.T) { out := simpleQuoteStr(tc.in) if out != tc.out { t.Errorf("input %q; expected: %s; but got: %s", tc.in, tc.out, out) } }) } } msgp-1.2.5/msgp/errors_tinygo.go000066400000000000000000000012121472554463600167030ustar00rootroot00000000000000//go:build tinygo // +build tinygo package msgp import ( "reflect" ) // ctxString converts the incoming interface{} slice into a single string, // without using fmt under tinygo func ctxString(ctx []interface{}) string { out := "" for idx, cv := range ctx { if idx > 0 { out += "/" } out += ifToStr(cv) } return out } type stringer interface { String() string } func ifToStr(i interface{}) string { switch v := i.(type) { case stringer: return v.String() case error: return v.Error() case string: return v default: return reflect.ValueOf(i).String() } } func quoteStr(s string) string { return simpleQuoteStr(s) } msgp-1.2.5/msgp/extension.go000066400000000000000000000310211472554463600160130ustar00rootroot00000000000000package msgp import ( "errors" "math" "strconv" ) const ( // Complex64Extension is the extension number used for complex64 Complex64Extension = 3 // Complex128Extension is the extension number used for complex128 Complex128Extension = 4 // TimeExtension is the extension number used for time.Time TimeExtension = 5 // MsgTimeExtension is the extension number for timestamps as defined in // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type MsgTimeExtension = -1 ) // msgTimeExtension is a painful workaround to avoid "constant -1 overflows byte". var msgTimeExtension = int8(MsgTimeExtension) // our extensions live here var extensionReg = make(map[int8]func() Extension) // RegisterExtension registers extensions so that they // can be initialized and returned by methods that // decode `interface{}` values. This should only // be called during initialization. f() should return // a newly-initialized zero value of the extension. Keep in // mind that extensions 3, 4, and 5 are reserved for // complex64, complex128, and time.Time, respectively, // and that MessagePack reserves extension types from -127 to -1. // // For example, if you wanted to register a user-defined struct: // // msgp.RegisterExtension(10, func() msgp.Extension { &MyExtension{} }) // // RegisterExtension will panic if you call it multiple times // with the same 'typ' argument, or if you use a reserved // type (3, 4, or 5). func RegisterExtension(typ int8, f func() Extension) { switch typ { case Complex64Extension, Complex128Extension, TimeExtension: panic(errors.New("msgp: forbidden extension type: " + strconv.Itoa(int(typ)))) } if _, ok := extensionReg[typ]; ok { panic(errors.New("msgp: RegisterExtension() called with typ " + strconv.Itoa(int(typ)) + " more than once")) } extensionReg[typ] = f } // ExtensionTypeError is an error type returned // when there is a mis-match between an extension type // and the type encoded on the wire type ExtensionTypeError struct { Got int8 Want int8 } // Error implements the error interface func (e ExtensionTypeError) Error() string { return "msgp: error decoding extension: wanted type " + strconv.Itoa(int(e.Want)) + "; got type " + strconv.Itoa(int(e.Got)) } // Resumable returns 'true' for ExtensionTypeErrors func (e ExtensionTypeError) Resumable() bool { return true } func errExt(got int8, wanted int8) error { return ExtensionTypeError{Got: got, Want: wanted} } // Extension is the interface fulfilled // by types that want to define their // own binary encoding. type Extension interface { // ExtensionType should return // a int8 that identifies the concrete // type of the extension. (Types <0 are // officially reserved by the MessagePack // specifications.) ExtensionType() int8 // Len should return the length // of the data to be encoded Len() int // MarshalBinaryTo should copy // the data into the supplied slice, // assuming that the slice has length Len() MarshalBinaryTo([]byte) error UnmarshalBinary([]byte) error } // RawExtension implements the Extension interface type RawExtension struct { Data []byte Type int8 } // ExtensionType implements Extension.ExtensionType, and returns r.Type func (r *RawExtension) ExtensionType() int8 { return r.Type } // Len implements Extension.Len, and returns len(r.Data) func (r *RawExtension) Len() int { return len(r.Data) } // MarshalBinaryTo implements Extension.MarshalBinaryTo, // and returns a copy of r.Data func (r *RawExtension) MarshalBinaryTo(d []byte) error { copy(d, r.Data) return nil } // UnmarshalBinary implements Extension.UnmarshalBinary, // and sets r.Data to the contents of the provided slice func (r *RawExtension) UnmarshalBinary(b []byte) error { if cap(r.Data) >= len(b) { r.Data = r.Data[0:len(b)] } else { r.Data = make([]byte, len(b)) } copy(r.Data, b) return nil } func (mw *Writer) writeExtensionHeader(length int, extType int8) error { switch length { case 0: o, err := mw.require(3) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = 0 mw.buf[o+2] = byte(extType) case 1: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext1 mw.buf[o+1] = byte(extType) case 2: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext2 mw.buf[o+1] = byte(extType) case 4: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext4 mw.buf[o+1] = byte(extType) case 8: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext8 mw.buf[o+1] = byte(extType) case 16: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext16 mw.buf[o+1] = byte(extType) default: switch { case length < math.MaxUint8: o, err := mw.require(3) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = byte(uint8(length)) mw.buf[o+2] = byte(extType) case length < math.MaxUint16: o, err := mw.require(4) if err != nil { return err } mw.buf[o] = mext16 big.PutUint16(mw.buf[o+1:], uint16(length)) mw.buf[o+3] = byte(extType) default: o, err := mw.require(6) if err != nil { return err } mw.buf[o] = mext32 big.PutUint32(mw.buf[o+1:], uint32(length)) mw.buf[o+5] = byte(extType) } } return nil } // WriteExtension writes an extension type to the writer func (mw *Writer) WriteExtension(e Extension) error { length := e.Len() err := mw.writeExtensionHeader(length, e.ExtensionType()) if err != nil { return err } // we can only write directly to the // buffer if we're sure that it // fits the object if length <= mw.bufsize() { o, err := mw.require(length) if err != nil { return err } return e.MarshalBinaryTo(mw.buf[o:]) } // here we create a new buffer // just large enough for the body // and save it as the write buffer err = mw.flush() if err != nil { return err } buf := make([]byte, length) err = e.MarshalBinaryTo(buf) if err != nil { return err } mw.buf = buf mw.wloc = length return nil } // WriteExtensionRaw writes an extension type to the writer func (mw *Writer) WriteExtensionRaw(extType int8, payload []byte) error { if err := mw.writeExtensionHeader(len(payload), extType); err != nil { return err } // instead of using mw.Write(), we'll copy the data through the internal // buffer, otherwise the payload would be moved to the heap // (meaning we can use stack-allocated buffers with zero allocations) for len(payload) > 0 { chunkSize := mw.avail() if chunkSize == 0 { if err := mw.flush(); err != nil { return err } chunkSize = mw.avail() } if chunkSize > len(payload) { chunkSize = len(payload) } mw.wloc += copy(mw.buf[mw.wloc:], payload[:chunkSize]) payload = payload[chunkSize:] } return nil } // peek at the extension type, assuming the next // kind to be read is Extension func (m *Reader) peekExtensionType() (int8, error) { _, _, extType, err := m.peekExtensionHeader() return extType, err } // peekExtension peeks at the extension encoding type // (must guarantee at least 1 byte in 'b') func peekExtension(b []byte) (int8, error) { spec := getBytespec(b[0]) size := spec.size if spec.typ != ExtensionType { return 0, badPrefix(ExtensionType, b[0]) } if len(b) < int(size) { return 0, ErrShortBytes } // for fixed extensions, // the type information is in // the second byte if spec.extra == constsize { return int8(b[1]), nil } // otherwise, it's in the last // part of the prefix return int8(b[size-1]), nil } func (m *Reader) peekExtensionHeader() (offset int, length int, extType int8, err error) { var p []byte p, err = m.R.Peek(2) if err != nil { return } offset = 2 lead := p[0] switch lead { case mfixext1: extType = int8(p[1]) length = 1 return case mfixext2: extType = int8(p[1]) length = 2 return case mfixext4: extType = int8(p[1]) length = 4 return case mfixext8: extType = int8(p[1]) length = 8 return case mfixext16: extType = int8(p[1]) length = 16 return case mext8: p, err = m.R.Peek(3) if err != nil { return } offset = 3 extType = int8(p[2]) length = int(uint8(p[1])) case mext16: p, err = m.R.Peek(4) if err != nil { return } offset = 4 extType = int8(p[3]) length = int(big.Uint16(p[1:])) case mext32: p, err = m.R.Peek(6) if err != nil { return } offset = 6 extType = int8(p[5]) length = int(big.Uint32(p[1:])) default: err = badPrefix(ExtensionType, lead) return } return } // ReadExtension reads the next object from the reader // as an extension. ReadExtension will fail if the next // object in the stream is not an extension, or if // e.Type() is not the same as the wire type. func (m *Reader) ReadExtension(e Extension) error { offset, length, extType, err := m.peekExtensionHeader() if err != nil { return err } if expectedType := e.ExtensionType(); extType != expectedType { return errExt(extType, expectedType) } p, err := m.R.Peek(offset + length) if err != nil { return err } err = e.UnmarshalBinary(p[offset:]) if err == nil { // consume the peeked bytes _, err = m.R.Skip(offset + length) } return err } // ReadExtensionRaw reads the next object from the reader // as an extension. The returned slice is only // valid until the next *Reader method call. func (m *Reader) ReadExtensionRaw() (int8, []byte, error) { offset, length, extType, err := m.peekExtensionHeader() if err != nil { return 0, nil, err } payload, err := m.R.Next(offset + length) if err != nil { return 0, nil, err } return extType, payload[offset:], nil } // AppendExtension appends a MessagePack extension to the provided slice func AppendExtension(b []byte, e Extension) ([]byte, error) { l := e.Len() var o []byte var n int switch l { case 0: o, n = ensure(b, 3) o[n] = mext8 o[n+1] = 0 o[n+2] = byte(e.ExtensionType()) return o[:n+3], nil case 1: o, n = ensure(b, 3) o[n] = mfixext1 o[n+1] = byte(e.ExtensionType()) n += 2 case 2: o, n = ensure(b, 4) o[n] = mfixext2 o[n+1] = byte(e.ExtensionType()) n += 2 case 4: o, n = ensure(b, 6) o[n] = mfixext4 o[n+1] = byte(e.ExtensionType()) n += 2 case 8: o, n = ensure(b, 10) o[n] = mfixext8 o[n+1] = byte(e.ExtensionType()) n += 2 case 16: o, n = ensure(b, 18) o[n] = mfixext16 o[n+1] = byte(e.ExtensionType()) n += 2 default: switch { case l < math.MaxUint8: o, n = ensure(b, l+3) o[n] = mext8 o[n+1] = byte(uint8(l)) o[n+2] = byte(e.ExtensionType()) n += 3 case l < math.MaxUint16: o, n = ensure(b, l+4) o[n] = mext16 big.PutUint16(o[n+1:], uint16(l)) o[n+3] = byte(e.ExtensionType()) n += 4 default: o, n = ensure(b, l+6) o[n] = mext32 big.PutUint32(o[n+1:], uint32(l)) o[n+5] = byte(e.ExtensionType()) n += 6 } } return o, e.MarshalBinaryTo(o[n:]) } // ReadExtensionBytes reads an extension from 'b' into 'e' // and returns any remaining bytes. // Possible errors: // - ErrShortBytes ('b' not long enough) // - ExtensionTypeError{} (wire type not the same as e.Type()) // - TypeError{} (next object not an extension) // - InvalidPrefixError // - An umarshal error returned from e.UnmarshalBinary func ReadExtensionBytes(b []byte, e Extension) ([]byte, error) { typ, remain, data, err := readExt(b) if err != nil { return b, err } if typ != e.ExtensionType() { return b, errExt(typ, e.ExtensionType()) } return remain, e.UnmarshalBinary(data) } // readExt will read the extension type, and return remaining bytes, // as well as the data of the extension. func readExt(b []byte) (typ int8, remain []byte, data []byte, err error) { l := len(b) if l < 3 { return 0, b, nil, ErrShortBytes } lead := b[0] var ( sz int // size of 'data' off int // offset of 'data' ) switch lead { case mfixext1: typ = int8(b[1]) sz = 1 off = 2 case mfixext2: typ = int8(b[1]) sz = 2 off = 2 case mfixext4: typ = int8(b[1]) sz = 4 off = 2 case mfixext8: typ = int8(b[1]) sz = 8 off = 2 case mfixext16: typ = int8(b[1]) sz = 16 off = 2 case mext8: sz = int(uint8(b[1])) typ = int8(b[2]) off = 3 if sz == 0 { return typ, b[3:], b[3:3], nil } case mext16: if l < 4 { return 0, b, nil, ErrShortBytes } sz = int(big.Uint16(b[1:])) typ = int8(b[3]) off = 4 case mext32: if l < 6 { return 0, b, nil, ErrShortBytes } sz = int(big.Uint32(b[1:])) typ = int8(b[5]) off = 6 default: return 0, b, nil, badPrefix(ExtensionType, lead) } // the data of the extension starts // at 'off' and is 'sz' bytes long tot := off + sz if len(b[off:]) < sz { return 0, b, nil, ErrShortBytes } return typ, b[tot:], b[off:tot:tot], nil } msgp-1.2.5/msgp/extension_test.go000066400000000000000000000111711472554463600170560ustar00rootroot00000000000000package msgp import ( "bytes" "math/rand" "testing" ) var extSizes = [...]int{0, 1, 2, 4, 8, 16, int(tint8), int(tuint16), int(tuint32)} func randomExt() RawExtension { e := RawExtension{} e.Type = int8(rand.Int()) e.Data = RandBytes(extSizes[rand.Intn(len(extSizes))]) return e } func TestReadWriteExtension(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) dc := NewReader(&buf) t.Run("interface", func(t *testing.T) { for i := 0; i < 25; i++ { buf.Reset() e := randomExt() en.WriteExtension(&e) en.Flush() err := dc.ReadExtension(&e) if err != nil { t.Errorf("error with extension (length %d): %s", len(buf.Bytes()), err) } } }) t.Run("raw", func(t *testing.T) { for i := 0; i < 25; i++ { buf.Reset() e := randomExt() en.WriteExtensionRaw(e.Type, e.Data) en.Flush() typ, payload, err := dc.ReadExtensionRaw() if err != nil { t.Errorf("error with extension (length %d): %s", len(buf.Bytes()), err) } if typ != e.Type || !bytes.Equal(payload, e.Data) { t.Errorf("extension mismatch: %d %x != %d %x", typ, payload, e.Type, e.Data) } } }) } func TestReadWriteLargeExtensionRaw(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) dc := NewReader(&buf) largeExt := RawExtension{Type: int8(rand.Int()), Data: RandBytes(int(tuint32))} err := en.WriteExtensionRaw(largeExt.Type, largeExt.Data) if err != nil { t.Errorf("error with large extension write: %s", err) } // write a nil as a marker err = en.WriteNil() if err != nil { t.Errorf("error with large extension write: %s", err) } en.Flush() typ, payload, err := dc.ReadExtensionRaw() if err != nil { t.Errorf("error with large extension read: %s", err) } if typ != largeExt.Type || !bytes.Equal(payload, largeExt.Data) { t.Errorf("large extension mismatch: %d %x != %d %x", typ, payload, largeExt.Type, largeExt.Data) } err = dc.ReadNil() if err != nil { t.Errorf("error with large extension read: %s", err) } } func TestExtensionRawStackBuffer(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) dc := NewReader(&buf) const bufSize = 128 e := &RawExtension{Type: int8(rand.Int()), Data: RandBytes(bufSize)} allocs := testing.AllocsPerRun(100, func() { buf.Reset() var staticBuf [bufSize]byte slc := e.Data[:rand.Intn(bufSize)] copy(staticBuf[:], slc) err := en.WriteExtensionRaw(e.Type, staticBuf[:len(slc)]) if err != nil { t.Errorf("error writing extension: %s", err) } en.Flush() typ, payload, err := dc.ReadExtensionRaw() if err != nil { t.Errorf("error reading extension: %s", err) } if typ != e.Type || !bytes.Equal(payload, slc) { t.Errorf("extension mismatch: %d %x != %d %x", typ, payload, e.Type, slc) } }) if allocs != 0 { t.Errorf("using stack allocated buffer with WriteExtensionRaw caused %f allocations", allocs) } } func TestReadWriteExtensionBytes(t *testing.T) { var bts []byte for i := 0; i < 24; i++ { e := randomExt() bts, _ = AppendExtension(bts[0:0], &e) _, err := ReadExtensionBytes(bts, &e) if err != nil { t.Errorf("error with extension (length %d): %s", len(bts), err) } } } func TestAppendAndWriteCompatibility(t *testing.T) { var bts []byte var buf bytes.Buffer en := NewWriter(&buf) for i := 0; i < 24; i++ { buf.Reset() e := randomExt() bts, _ = AppendExtension(bts[0:0], &e) en.WriteExtension(&e) en.Flush() if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("the outputs are different:\n\t%x\n\t%x", buf.Bytes(), bts) } _, err := ReadExtensionBytes(bts, &e) if err != nil { t.Errorf("error with extension (length %d): %s", len(bts), err) } } } func BenchmarkExtensionReadWrite(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) dc := NewReader(&buf) b.Run("interface", func(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { buf.Reset() e := randomExt() err := en.WriteExtension(&e) if err != nil { b.Errorf("error writing extension: %s", err) } en.Flush() err = dc.ReadExtension(&e) if err != nil { b.Errorf("error reading extension: %s", err) } } }) b.Run("raw", func(b *testing.B) { // this should have zero allocations b.ReportAllocs() for i := 0; i < b.N; i++ { buf.Reset() e := randomExt() err := en.WriteExtensionRaw(e.Type, e.Data) if err != nil { b.Errorf("error writing extension: %s", err) } en.Flush() typ, payload, err := dc.ReadExtensionRaw() if err != nil { b.Errorf("error reading extension: %s", err) } if typ != e.Type || !bytes.Equal(payload, e.Data) { b.Errorf("extension mismatch: %d %x != %d %x", typ, payload, e.Type, e.Data) } buf.Reset() } }) } msgp-1.2.5/msgp/file.go000066400000000000000000000047371472554463600147340ustar00rootroot00000000000000//go:build (linux || darwin || dragonfly || freebsd || illumos || netbsd || openbsd) && !appengine && !tinygo // +build linux darwin dragonfly freebsd illumos netbsd openbsd // +build !appengine // +build !tinygo package msgp import ( "os" "syscall" ) // ReadFile reads a file into 'dst' using // a read-only memory mapping. Consequently, // the file must be mmap-able, and the // Unmarshaler should never write to // the source memory. (Methods generated // by the msgp tool obey that constraint, but // user-defined implementations may not.) // // Reading and writing through file mappings // is only efficient for large files; small // files are best read and written using // the ordinary streaming interfaces. func ReadFile(dst Unmarshaler, file *os.File) error { stat, err := file.Stat() if err != nil { return err } data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { return err } adviseRead(data) _, err = dst.UnmarshalMsg(data) uerr := syscall.Munmap(data) if err == nil { err = uerr } return err } // MarshalSizer is the combination // of the Marshaler and Sizer // interfaces. type MarshalSizer interface { Marshaler Sizer } // WriteFile writes a file from 'src' using // memory mapping. It overwrites the entire // contents of the previous file. // The mapping size is calculated // using the `Msgsize()` method // of 'src', so it must produce a result // equal to or greater than the actual encoded // size of the object. Otherwise, // a fault (SIGBUS) will occur. // // Reading and writing through file mappings // is only efficient for large files; small // files are best read and written using // the ordinary streaming interfaces. // // NOTE: The performance of this call // is highly OS- and filesystem-dependent. // Users should take care to test that this // performs as expected in a production environment. // (Linux users should run a kernel and filesystem // that support fallocate(2) for the best results.) func WriteFile(src MarshalSizer, file *os.File) error { sz := src.Msgsize() err := fallocate(file, int64(sz)) if err != nil { return err } data, err := syscall.Mmap(int(file.Fd()), 0, sz, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) if err != nil { return err } adviseWrite(data) chunk := data[:0] chunk, err = src.MarshalMsg(chunk) if err != nil { return err } uerr := syscall.Munmap(data) if uerr != nil { return uerr } return file.Truncate(int64(len(chunk))) } msgp-1.2.5/msgp/file_port.go000066400000000000000000000014301472554463600157630ustar00rootroot00000000000000//go:build windows || appengine || tinygo // +build windows appengine tinygo package msgp import ( "io" "os" ) // MarshalSizer is the combination // of the Marshaler and Sizer // interfaces. type MarshalSizer interface { Marshaler Sizer } func ReadFile(dst Unmarshaler, file *os.File) error { if u, ok := dst.(Decodable); ok { return u.DecodeMsg(NewReader(file)) } data, err := io.ReadAll(file) if err != nil { return err } _, err = dst.UnmarshalMsg(data) return err } func WriteFile(src MarshalSizer, file *os.File) error { if e, ok := src.(Encodable); ok { w := NewWriter(file) err := e.EncodeMsg(w) if err == nil { err = w.Flush() } return err } raw, err := src.MarshalMsg(nil) if err != nil { return err } _, err = file.Write(raw) return err } msgp-1.2.5/msgp/file_test.go000066400000000000000000000044221472554463600157620ustar00rootroot00000000000000//go:build linux || darwin || dragonfly || freebsd || illumos || netbsd || openbsd // +build linux darwin dragonfly freebsd illumos netbsd openbsd package msgp_test import ( "bytes" "crypto/rand" "io" prand "math/rand" "os" "testing" "github.com/tinylib/msgp/msgp" ) type rawBytes []byte func (r rawBytes) MarshalMsg(b []byte) ([]byte, error) { return msgp.AppendBytes(b, []byte(r)), nil } func (r rawBytes) Msgsize() int { return msgp.BytesPrefixSize + len(r) } func (r *rawBytes) UnmarshalMsg(b []byte) ([]byte, error) { tmp, out, err := msgp.ReadBytesBytes(b, (*(*[]byte)(r))[:0]) *r = rawBytes(tmp) return out, err } func TestReadWriteFile(t *testing.T) { t.Parallel() f, err := os.Create("tmpfile") if err != nil { t.Fatal(err) } defer func() { f.Close() os.Remove("tmpfile") }() data := make([]byte, 1024*1024) rand.Read(data) err = msgp.WriteFile(rawBytes(data), f) if err != nil { t.Fatal(err) } var out rawBytes f.Seek(0, io.SeekStart) err = msgp.ReadFile(&out, f) if err != nil { t.Fatal(err) } if !bytes.Equal([]byte(out), []byte(data)) { t.Fatal("Input and output not equal.") } } var ( blobstrings = []string{"", "a string", "a longer string here!"} blobfloats = []float64{0.0, -1.0, 1.0, 3.1415926535} blobints = []int64{0, 1, -1, 80000, 1 << 30} blobbytes = [][]byte{{}, []byte("hello"), []byte(`{"is_json":true,"is_compact":"unable to determine"}`)} ) func BenchmarkWriteReadFile(b *testing.B) { // let's not run out of disk space... if b.N > 10000000 { b.N = 10000000 //nolint:staticcheck // ignoring "SA3001: should not assign to b.N (staticcheck)" as this should not usually happen. } fname := "bench-tmpfile" f, err := os.Create(fname) if err != nil { b.Fatal(err) } defer func(f *os.File, name string) { f.Close() os.Remove(name) }(f, fname) data := make(Blobs, b.N) for i := range data { data[i].Name = blobstrings[prand.Intn(len(blobstrings))] data[i].Float = blobfloats[prand.Intn(len(blobfloats))] data[i].Amount = blobints[prand.Intn(len(blobints))] data[i].Bytes = blobbytes[prand.Intn(len(blobbytes))] } b.SetBytes(int64(data.Msgsize() / b.N)) b.ResetTimer() err = msgp.WriteFile(data, f) if err != nil { b.Fatal(err) } err = msgp.ReadFile(&data, f) if err != nil { b.Fatal(err) } } msgp-1.2.5/msgp/floatbench_test.go000066400000000000000000000007521472554463600171520ustar00rootroot00000000000000package msgp import ( "testing" ) func BenchmarkReadWriteFloat32(b *testing.B) { var f float32 = 3.9081 bts := AppendFloat32([]byte{}, f) b.ResetTimer() for i := 0; i < b.N; i++ { bts = AppendFloat32(bts[0:0], f) f, bts, _ = ReadFloat32Bytes(bts) } } func BenchmarkReadWriteFloat64(b *testing.B) { var f float64 = 3.9081 bts := AppendFloat64([]byte{}, f) b.ResetTimer() for i := 0; i < b.N; i++ { bts = AppendFloat64(bts[0:0], f) f, bts, _ = ReadFloat64Bytes(bts) } } msgp-1.2.5/msgp/integers.go000066400000000000000000000077051472554463600156330ustar00rootroot00000000000000package msgp import "encoding/binary" /* ---------------------------------- integer encoding utilities (inline-able) TODO(tinylib): there are faster, albeit non-portable solutions to the code below. implement byteswap? ---------------------------------- */ func putMint64(b []byte, i int64) { _ = b[8] // bounds check elimination b[0] = mint64 b[1] = byte(i >> 56) b[2] = byte(i >> 48) b[3] = byte(i >> 40) b[4] = byte(i >> 32) b[5] = byte(i >> 24) b[6] = byte(i >> 16) b[7] = byte(i >> 8) b[8] = byte(i) } func getMint64(b []byte) int64 { _ = b[8] // bounds check elimination return (int64(b[1]) << 56) | (int64(b[2]) << 48) | (int64(b[3]) << 40) | (int64(b[4]) << 32) | (int64(b[5]) << 24) | (int64(b[6]) << 16) | (int64(b[7]) << 8) | (int64(b[8])) } func putMint32(b []byte, i int32) { _ = b[4] // bounds check elimination b[0] = mint32 b[1] = byte(i >> 24) b[2] = byte(i >> 16) b[3] = byte(i >> 8) b[4] = byte(i) } func getMint32(b []byte) int32 { _ = b[4] // bounds check elimination return (int32(b[1]) << 24) | (int32(b[2]) << 16) | (int32(b[3]) << 8) | (int32(b[4])) } func putMint16(b []byte, i int16) { _ = b[2] // bounds check elimination b[0] = mint16 b[1] = byte(i >> 8) b[2] = byte(i) } func getMint16(b []byte) (i int16) { _ = b[2] // bounds check elimination return (int16(b[1]) << 8) | int16(b[2]) } func putMint8(b []byte, i int8) { _ = b[1] // bounds check elimination b[0] = mint8 b[1] = byte(i) } func getMint8(b []byte) (i int8) { return int8(b[1]) } func putMuint64(b []byte, u uint64) { _ = b[8] // bounds check elimination b[0] = muint64 b[1] = byte(u >> 56) b[2] = byte(u >> 48) b[3] = byte(u >> 40) b[4] = byte(u >> 32) b[5] = byte(u >> 24) b[6] = byte(u >> 16) b[7] = byte(u >> 8) b[8] = byte(u) } func getMuint64(b []byte) uint64 { _ = b[8] // bounds check elimination return (uint64(b[1]) << 56) | (uint64(b[2]) << 48) | (uint64(b[3]) << 40) | (uint64(b[4]) << 32) | (uint64(b[5]) << 24) | (uint64(b[6]) << 16) | (uint64(b[7]) << 8) | (uint64(b[8])) } func putMuint32(b []byte, u uint32) { _ = b[4] // bounds check elimination b[0] = muint32 b[1] = byte(u >> 24) b[2] = byte(u >> 16) b[3] = byte(u >> 8) b[4] = byte(u) } func getMuint32(b []byte) uint32 { _ = b[4] // bounds check elimination return (uint32(b[1]) << 24) | (uint32(b[2]) << 16) | (uint32(b[3]) << 8) | (uint32(b[4])) } func putMuint16(b []byte, u uint16) { _ = b[2] // bounds check elimination b[0] = muint16 b[1] = byte(u >> 8) b[2] = byte(u) } func getMuint16(b []byte) uint16 { _ = b[2] // bounds check elimination return (uint16(b[1]) << 8) | uint16(b[2]) } func putMuint8(b []byte, u uint8) { _ = b[1] // bounds check elimination b[0] = muint8 b[1] = byte(u) } func getMuint8(b []byte) uint8 { return uint8(b[1]) } func getUnix(b []byte) (sec int64, nsec int32) { sec = int64(binary.BigEndian.Uint64(b)) nsec = int32(binary.BigEndian.Uint32(b[8:])) return } func putUnix(b []byte, sec int64, nsec int32) { binary.BigEndian.PutUint64(b, uint64(sec)) binary.BigEndian.PutUint32(b[8:], uint32(nsec)) } /* ----------------------------- prefix utilities ----------------------------- */ // write prefix and uint8 func prefixu8(b []byte, pre byte, sz uint8) { _ = b[1] // bounds check elimination b[0] = pre b[1] = byte(sz) } // write prefix and big-endian uint16 func prefixu16(b []byte, pre byte, sz uint16) { _ = b[2] // bounds check elimination b[0] = pre b[1] = byte(sz >> 8) b[2] = byte(sz) } // write prefix and big-endian uint32 func prefixu32(b []byte, pre byte, sz uint32) { _ = b[4] // bounds check elimination b[0] = pre b[1] = byte(sz >> 24) b[2] = byte(sz >> 16) b[3] = byte(sz >> 8) b[4] = byte(sz) } func prefixu64(b []byte, pre byte, sz uint64) { _ = b[8] // bounds check elimination b[0] = pre b[1] = byte(sz >> 56) b[2] = byte(sz >> 48) b[3] = byte(sz >> 40) b[4] = byte(sz >> 32) b[5] = byte(sz >> 24) b[6] = byte(sz >> 16) b[7] = byte(sz >> 8) b[8] = byte(sz) } msgp-1.2.5/msgp/integers_test.go000066400000000000000000000056101472554463600166630ustar00rootroot00000000000000package msgp import ( "encoding/binary" "testing" ) func BenchmarkIntegers(b *testing.B) { bytes64 := make([]byte, 9) bytes32 := make([]byte, 5) bytes16 := make([]byte, 3) b.Run("Int64", func(b *testing.B) { b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putMint64(bytes64, -1234567890123456789) } }) b.Run("Get", func(b *testing.B) { putMint64(bytes64, -1234567890123456789) for i := 0; i < b.N; i++ { getMint64(bytes64) } }) }) b.Run("Int32", func(b *testing.B) { b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putMint32(bytes32, -123456789) } }) b.Run("Get", func(b *testing.B) { putMint32(bytes32, -123456789) for i := 0; i < b.N; i++ { getMint32(bytes32) } }) }) b.Run("Int16", func(b *testing.B) { b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putMint16(bytes16, -12345) } }) b.Run("Get", func(b *testing.B) { putMint16(bytes16, -12345) for i := 0; i < b.N; i++ { getMint16(bytes16) } }) }) b.Run("Uint64", func(b *testing.B) { b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putMuint64(bytes64, 1234567890123456789) } }) b.Run("Get", func(b *testing.B) { putMuint64(bytes64, 1234567890123456789) for i := 0; i < b.N; i++ { getMuint64(bytes64) } }) }) b.Run("Uint32", func(b *testing.B) { b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putMuint32(bytes32, 123456789) } }) b.Run("Get", func(b *testing.B) { putMuint32(bytes32, 123456789) for i := 0; i < b.N; i++ { getMuint32(bytes32) } }) }) b.Run("Uint16", func(b *testing.B) { b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putMuint16(bytes16, 12345) } }) b.Run("Get", func(b *testing.B) { putMuint16(bytes16, 12345) for i := 0; i < b.N; i++ { getMuint16(bytes16) } }) }) } func BenchmarkIntegersUnix(b *testing.B) { bytes := make([]byte, 12) var sec int64 = 1609459200 var nsec int32 = 123456789 b.Run("Get", func(b *testing.B) { binary.BigEndian.PutUint64(bytes, uint64(sec)) binary.BigEndian.PutUint32(bytes[8:], uint32(nsec)) for i := 0; i < b.N; i++ { getUnix(bytes) } }) b.Run("Put", func(b *testing.B) { for i := 0; i < b.N; i++ { putUnix(bytes, sec, nsec) } }) } func BenchmarkIntegersPrefix(b *testing.B) { bytesU16 := make([]byte, 3) bytesU32 := make([]byte, 5) bytesU64 := make([]byte, 9) b.Run("u16", func(b *testing.B) { var pre byte = 0x01 var sz uint16 = 12345 for i := 0; i < b.N; i++ { prefixu16(bytesU16, pre, sz) } }) b.Run("u32", func(b *testing.B) { var pre byte = 0x02 var sz uint32 = 123456789 for i := 0; i < b.N; i++ { prefixu32(bytesU32, pre, sz) } }) b.Run("u64", func(b *testing.B) { var pre byte = 0x03 var sz uint64 = 1234567890123456789 for i := 0; i < b.N; i++ { prefixu64(bytesU64, pre, sz) } }) } msgp-1.2.5/msgp/json.go000066400000000000000000000241451472554463600147610ustar00rootroot00000000000000package msgp import ( "bufio" "encoding/base64" "encoding/json" "io" "strconv" "unicode/utf8" ) var ( null = []byte("null") hex = []byte("0123456789abcdef") ) var defuns [_maxtype]func(jsWriter, *Reader) (int, error) // note: there is an initialization loop if // this isn't set up during init() func init() { // since none of these functions are inline-able, // there is not much of a penalty to the indirect // call. however, this is best expressed as a jump-table... defuns = [_maxtype]func(jsWriter, *Reader) (int, error){ StrType: rwString, BinType: rwBytes, MapType: rwMap, ArrayType: rwArray, Float64Type: rwFloat64, Float32Type: rwFloat32, BoolType: rwBool, IntType: rwInt, UintType: rwUint, NilType: rwNil, ExtensionType: rwExtension, Complex64Type: rwExtension, Complex128Type: rwExtension, TimeType: rwTime, } } // this is the interface // used to write json type jsWriter interface { io.Writer io.ByteWriter WriteString(string) (int, error) } // CopyToJSON reads MessagePack from 'src' and copies it // as JSON to 'dst' until EOF. func CopyToJSON(dst io.Writer, src io.Reader) (n int64, err error) { r := NewReader(src) n, err = r.WriteToJSON(dst) freeR(r) return } // WriteToJSON translates MessagePack from 'r' and writes it as // JSON to 'w' until the underlying reader returns io.EOF. It returns // the number of bytes written, and an error if it stopped before EOF. func (r *Reader) WriteToJSON(w io.Writer) (n int64, err error) { var j jsWriter var bf *bufio.Writer if jsw, ok := w.(jsWriter); ok { j = jsw } else { bf = bufio.NewWriter(w) j = bf } var nn int for err == nil { nn, err = rwNext(j, r) n += int64(nn) } if err != io.EOF { if bf != nil { bf.Flush() } return } err = nil if bf != nil { err = bf.Flush() } return } func rwNext(w jsWriter, src *Reader) (int, error) { t, err := src.NextType() if err != nil { return 0, err } return defuns[t](w, src) } func rwMap(dst jsWriter, src *Reader) (n int, err error) { var comma bool var sz uint32 var field []byte sz, err = src.ReadMapHeader() if err != nil { return } if sz == 0 { return dst.WriteString("{}") } // This is potentially a recursive call. if done, err := src.recursiveCall(); err != nil { return 0, err } else { defer done() } err = dst.WriteByte('{') if err != nil { return } n++ var nn int for i := uint32(0); i < sz; i++ { if comma { err = dst.WriteByte(',') if err != nil { return } n++ } field, err = src.ReadMapKeyPtr() if err != nil { return } nn, err = rwquoted(dst, field) n += nn if err != nil { return } err = dst.WriteByte(':') if err != nil { return } n++ nn, err = rwNext(dst, src) n += nn if err != nil { return } if !comma { comma = true } } err = dst.WriteByte('}') if err != nil { return } n++ return } func rwArray(dst jsWriter, src *Reader) (n int, err error) { err = dst.WriteByte('[') if err != nil { return } // This is potentially a recursive call. if done, err := src.recursiveCall(); err != nil { return 0, err } else { defer done() } var sz uint32 var nn int sz, err = src.ReadArrayHeader() if err != nil { return } comma := false for i := uint32(0); i < sz; i++ { if comma { err = dst.WriteByte(',') if err != nil { return } n++ } nn, err = rwNext(dst, src) n += nn if err != nil { return } comma = true } err = dst.WriteByte(']') if err != nil { return } n++ return } func rwNil(dst jsWriter, src *Reader) (int, error) { err := src.ReadNil() if err != nil { return 0, err } return dst.Write(null) } func rwFloat32(dst jsWriter, src *Reader) (int, error) { f, err := src.ReadFloat32() if err != nil { return 0, err } src.scratch = strconv.AppendFloat(src.scratch[:0], float64(f), 'f', -1, 32) return dst.Write(src.scratch) } func rwFloat64(dst jsWriter, src *Reader) (int, error) { f, err := src.ReadFloat64() if err != nil { return 0, err } src.scratch = strconv.AppendFloat(src.scratch[:0], f, 'f', -1, 64) return dst.Write(src.scratch) } func rwInt(dst jsWriter, src *Reader) (int, error) { i, err := src.ReadInt64() if err != nil { return 0, err } src.scratch = strconv.AppendInt(src.scratch[:0], i, 10) return dst.Write(src.scratch) } func rwUint(dst jsWriter, src *Reader) (int, error) { u, err := src.ReadUint64() if err != nil { return 0, err } src.scratch = strconv.AppendUint(src.scratch[:0], u, 10) return dst.Write(src.scratch) } func rwBool(dst jsWriter, src *Reader) (int, error) { b, err := src.ReadBool() if err != nil { return 0, err } if b { return dst.WriteString("true") } return dst.WriteString("false") } func rwTime(dst jsWriter, src *Reader) (int, error) { t, err := src.ReadTime() if err != nil { return 0, err } bts, err := t.MarshalJSON() if err != nil { return 0, err } return dst.Write(bts) } func rwExtension(dst jsWriter, src *Reader) (n int, err error) { et, err := src.peekExtensionType() if err != nil { return 0, err } // registered extensions can override // the JSON encoding if j, ok := extensionReg[et]; ok { var bts []byte e := j() err = src.ReadExtension(e) if err != nil { return } bts, err = json.Marshal(e) if err != nil { return } return dst.Write(bts) } e := RawExtension{} e.Type = et err = src.ReadExtension(&e) if err != nil { return } var nn int err = dst.WriteByte('{') if err != nil { return } n++ nn, err = dst.WriteString(`"type":`) n += nn if err != nil { return } src.scratch = strconv.AppendInt(src.scratch[0:0], int64(e.Type), 10) nn, err = dst.Write(src.scratch) n += nn if err != nil { return } nn, err = dst.WriteString(`,"data":"`) n += nn if err != nil { return } enc := base64.NewEncoder(base64.StdEncoding, dst) nn, err = enc.Write(e.Data) n += nn if err != nil { return } err = enc.Close() if err != nil { return } nn, err = dst.WriteString(`"}`) n += nn return } func rwString(dst jsWriter, src *Reader) (n int, err error) { lead, err := src.R.PeekByte() if err != nil { return } var read int var p []byte if isfixstr(lead) { read = int(rfixstr(lead)) src.R.Skip(1) goto write } switch lead { case mstr8: p, err = src.R.Next(2) if err != nil { return } read = int(uint8(p[1])) case mstr16: p, err = src.R.Next(3) if err != nil { return } read = int(big.Uint16(p[1:])) case mstr32: p, err = src.R.Next(5) if err != nil { return } read = int(big.Uint32(p[1:])) default: err = badPrefix(StrType, lead) return } write: p, err = src.R.Next(read) if err != nil { return } n, err = rwquoted(dst, p) return } func rwBytes(dst jsWriter, src *Reader) (n int, err error) { var nn int err = dst.WriteByte('"') if err != nil { return } n++ src.scratch, err = src.ReadBytes(src.scratch[:0]) if err != nil { return } enc := base64.NewEncoder(base64.StdEncoding, dst) nn, err = enc.Write(src.scratch) n += nn if err != nil { return } err = enc.Close() if err != nil { return } err = dst.WriteByte('"') if err != nil { return } n++ return } // Below (c) The Go Authors, 2009-2014 // Subject to the BSD-style license found at http://golang.org // // see: encoding/json/encode.go:(*encodeState).stringbytes() func rwquoted(dst jsWriter, s []byte) (n int, err error) { var nn int err = dst.WriteByte('"') if err != nil { return } n++ start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { i++ continue } if start < i { nn, err = dst.Write(s[start:i]) n += nn if err != nil { return } } switch b { case '\\', '"': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte(b) if err != nil { return } n++ case '\n': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte('n') if err != nil { return } n++ case '\r': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte('r') if err != nil { return } n++ case '\t': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte('t') if err != nil { return } n++ default: // This encodes bytes < 0x20 except for \t, \n and \r. // It also escapes <, >, and & // because they can lead to security holes when // user-controlled strings are rendered into JSON // and served to some browsers. nn, err = dst.WriteString(`\u00`) n += nn if err != nil { return } err = dst.WriteByte(hex[b>>4]) if err != nil { return } n++ err = dst.WriteByte(hex[b&0xF]) if err != nil { return } n++ } i++ start = i continue } c, size := utf8.DecodeRune(s[i:]) if c == utf8.RuneError && size == 1 { if start < i { nn, err = dst.Write(s[start:i]) n += nn if err != nil { return } } nn, err = dst.WriteString(`\ufffd`) n += nn if err != nil { return } i += size start = i continue } // U+2028 is LINE SEPARATOR. // U+2029 is PARAGRAPH SEPARATOR. // They are both technically valid characters in JSON strings, // but don't work in JSONP, which has to be evaluated as JavaScript, // and can lead to security holes there. It is valid JSON to // escape them, so we do so unconditionally. // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. if c == '\u2028' || c == '\u2029' { if start < i { nn, err = dst.Write(s[start:i]) n += nn if err != nil { return } } nn, err = dst.WriteString(`\u202`) n += nn if err != nil { return } err = dst.WriteByte(hex[c&0xF]) if err != nil { return } n++ i += size start = i continue } i += size } if start < len(s) { nn, err = dst.Write(s[start:]) n += nn if err != nil { return } } err = dst.WriteByte('"') if err != nil { return } n++ return } msgp-1.2.5/msgp/json_bytes.go000066400000000000000000000203131472554463600161600ustar00rootroot00000000000000package msgp import ( "bufio" "encoding/base64" "encoding/json" "io" "strconv" "time" ) var unfuns [_maxtype]func(jsWriter, []byte, []byte, int) ([]byte, []byte, error) func init() { // NOTE(pmh): this is best expressed as a jump table, // but gc doesn't do that yet. revisit post-go1.5. unfuns = [_maxtype]func(jsWriter, []byte, []byte, int) ([]byte, []byte, error){ StrType: rwStringBytes, BinType: rwBytesBytes, MapType: rwMapBytes, ArrayType: rwArrayBytes, Float64Type: rwFloat64Bytes, Float32Type: rwFloat32Bytes, BoolType: rwBoolBytes, IntType: rwIntBytes, UintType: rwUintBytes, NilType: rwNullBytes, ExtensionType: rwExtensionBytes, Complex64Type: rwExtensionBytes, Complex128Type: rwExtensionBytes, TimeType: rwTimeBytes, } } // UnmarshalAsJSON takes raw messagepack and writes // it as JSON to 'w'. If an error is returned, the // bytes not translated will also be returned. If // no errors are encountered, the length of the returned // slice will be zero. func UnmarshalAsJSON(w io.Writer, msg []byte) ([]byte, error) { var ( scratch []byte cast bool dst jsWriter err error ) if jsw, ok := w.(jsWriter); ok { dst = jsw cast = true } else { dst = bufio.NewWriterSize(w, 512) } for len(msg) > 0 && err == nil { msg, scratch, err = writeNext(dst, msg, scratch, 0) } if !cast && err == nil { err = dst.(*bufio.Writer).Flush() } return msg, err } func writeNext(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { if len(msg) < 1 { return msg, scratch, ErrShortBytes } t := getType(msg[0]) if t == InvalidType { return msg, scratch, InvalidPrefixError(msg[0]) } if t == ExtensionType { et, err := peekExtension(msg) if err != nil { return nil, scratch, err } if et == TimeExtension || et == MsgTimeExtension { t = TimeType } } return unfuns[t](w, msg, scratch, depth) } func rwArrayBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { if depth >= recursionLimit { return msg, scratch, ErrRecursion } sz, msg, err := ReadArrayHeaderBytes(msg) if err != nil { return msg, scratch, err } err = w.WriteByte('[') if err != nil { return msg, scratch, err } for i := uint32(0); i < sz; i++ { if i != 0 { err = w.WriteByte(',') if err != nil { return msg, scratch, err } } msg, scratch, err = writeNext(w, msg, scratch, depth+1) if err != nil { return msg, scratch, err } } err = w.WriteByte(']') return msg, scratch, err } func rwMapBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { if depth >= recursionLimit { return msg, scratch, ErrRecursion } sz, msg, err := ReadMapHeaderBytes(msg) if err != nil { return msg, scratch, err } err = w.WriteByte('{') if err != nil { return msg, scratch, err } for i := uint32(0); i < sz; i++ { if i != 0 { err = w.WriteByte(',') if err != nil { return msg, scratch, err } } msg, scratch, err = rwMapKeyBytes(w, msg, scratch, depth) if err != nil { return msg, scratch, err } err = w.WriteByte(':') if err != nil { return msg, scratch, err } msg, scratch, err = writeNext(w, msg, scratch, depth+1) if err != nil { return msg, scratch, err } } err = w.WriteByte('}') return msg, scratch, err } func rwMapKeyBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { msg, scratch, err := rwStringBytes(w, msg, scratch, depth) if err != nil { if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { return rwBytesBytes(w, msg, scratch, depth) } } return msg, scratch, err } func rwStringBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { str, msg, err := ReadStringZC(msg) if err != nil { return msg, scratch, err } _, err = rwquoted(w, str) return msg, scratch, err } func rwBytesBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { bts, msg, err := ReadBytesZC(msg) if err != nil { return msg, scratch, err } l := base64.StdEncoding.EncodedLen(len(bts)) if cap(scratch) >= l { scratch = scratch[0:l] } else { scratch = make([]byte, l) } base64.StdEncoding.Encode(scratch, bts) err = w.WriteByte('"') if err != nil { return msg, scratch, err } _, err = w.Write(scratch) if err != nil { return msg, scratch, err } err = w.WriteByte('"') return msg, scratch, err } func rwNullBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { msg, err := ReadNilBytes(msg) if err != nil { return msg, scratch, err } _, err = w.Write(null) return msg, scratch, err } func rwBoolBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { b, msg, err := ReadBoolBytes(msg) if err != nil { return msg, scratch, err } if b { _, err = w.WriteString("true") return msg, scratch, err } _, err = w.WriteString("false") return msg, scratch, err } func rwIntBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { i, msg, err := ReadInt64Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendInt(scratch[0:0], i, 10) _, err = w.Write(scratch) return msg, scratch, err } func rwUintBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { u, msg, err := ReadUint64Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendUint(scratch[0:0], u, 10) _, err = w.Write(scratch) return msg, scratch, err } func rwFloat32Bytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { var f float32 var err error f, msg, err = ReadFloat32Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendFloat(scratch[:0], float64(f), 'f', -1, 32) _, err = w.Write(scratch) return msg, scratch, err } func rwFloat64Bytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { var f float64 var err error f, msg, err = ReadFloat64Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendFloat(scratch[:0], f, 'f', -1, 64) _, err = w.Write(scratch) return msg, scratch, err } func rwTimeBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { var t time.Time var err error t, msg, err = ReadTimeBytes(msg) if err != nil { return msg, scratch, err } bts, err := t.MarshalJSON() if err != nil { return msg, scratch, err } _, err = w.Write(bts) return msg, scratch, err } func rwExtensionBytes(w jsWriter, msg []byte, scratch []byte, depth int) ([]byte, []byte, error) { var err error var et int8 et, err = peekExtension(msg) if err != nil { return msg, scratch, err } // if it's time.Time if et == TimeExtension || et == MsgTimeExtension { var tm time.Time tm, msg, err = ReadTimeBytes(msg) if err != nil { return msg, scratch, err } bts, err := tm.MarshalJSON() if err != nil { return msg, scratch, err } _, err = w.Write(bts) return msg, scratch, err } // if the extension is registered, // use its canonical JSON form if f, ok := extensionReg[et]; ok { e := f() msg, err = ReadExtensionBytes(msg, e) if err != nil { return msg, scratch, err } bts, err := json.Marshal(e) if err != nil { return msg, scratch, err } _, err = w.Write(bts) return msg, scratch, err } // otherwise, write `{"type": , "data": ""}` r := RawExtension{} r.Type = et msg, err = ReadExtensionBytes(msg, &r) if err != nil { return msg, scratch, err } scratch, err = writeExt(w, r, scratch) return msg, scratch, err } func writeExt(w jsWriter, r RawExtension, scratch []byte) ([]byte, error) { _, err := w.WriteString(`{"type":`) if err != nil { return scratch, err } scratch = strconv.AppendInt(scratch[0:0], int64(r.Type), 10) _, err = w.Write(scratch) if err != nil { return scratch, err } _, err = w.WriteString(`,"data":"`) if err != nil { return scratch, err } l := base64.StdEncoding.EncodedLen(len(r.Data)) if cap(scratch) >= l { scratch = scratch[0:l] } else { scratch = make([]byte, l) } base64.StdEncoding.Encode(scratch, r.Data) _, err = w.Write(scratch) if err != nil { return scratch, err } _, err = w.WriteString(`"}`) return scratch, err } msgp-1.2.5/msgp/json_bytes_test.go000066400000000000000000000045341472554463600172260ustar00rootroot00000000000000package msgp import ( "bytes" "encoding/json" "testing" "time" ) func TestUnmarshalJSON(t *testing.T) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(5) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_map") enc.WriteMapHeader(2) // INNER enc.WriteString("cmplx") enc.WriteComplex64(complex(1.0, 1.0)) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("an extension") enc.WriteExtension(&RawExtension{Type: 1, Data: []byte("blaaahhh")}) enc.WriteString("some bytes") enc.WriteBytes([]byte("here are some bytes")) enc.WriteString("now") enc.WriteTime(time.Now()) enc.Flush() var js bytes.Buffer _, err := UnmarshalAsJSON(&js, buf.Bytes()) if err != nil { t.Logf("%s", js.Bytes()) t.Fatal(err) } mp := make(map[string]interface{}) err = json.Unmarshal(js.Bytes(), &mp) if err != nil { t.Log(js.String()) t.Fatalf("Error unmarshaling: %s", err) } if len(mp) != 5 { t.Errorf("map length should be %d, not %d", 5, len(mp)) } so, ok := mp["thing_1"] if !ok || so != "a string object" { t.Errorf("expected %q; got %q", "a string object", so) } if _, ok := mp["now"]; !ok { t.Error(`"now" field doesn't exist`) } c, ok := mp["a_map"] if !ok { t.Error(`"a_map" field doesn't exist`) } else { if m, ok := c.(map[string]interface{}); ok { if _, ok := m["cmplx"]; !ok { t.Error(`"a_map.cmplx" doesn't exist`) } } else { t.Error(`can't type-assert "c" to map[string]interface{}`) } } t.Logf("JSON: %s", js.Bytes()) } func BenchmarkUnmarshalAsJSON(b *testing.B) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(4) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_first_map") enc.WriteMapHeader(2) enc.WriteString("float_a") enc.WriteFloat32(1.0) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("an array") enc.WriteArrayHeader(2) enc.WriteBool(true) enc.WriteUint(2089) enc.WriteString("a_second_map") enc.WriteMapStrStr(map[string]string{ "internal_one": "blah", "internal_two": "blahhh...", }) enc.Flush() var js bytes.Buffer bts := buf.Bytes() _, err := UnmarshalAsJSON(&js, bts) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(js.Bytes()))) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { js.Reset() UnmarshalAsJSON(&js, bts) } } msgp-1.2.5/msgp/json_test.go000066400000000000000000000075301472554463600160170ustar00rootroot00000000000000package msgp import ( "bytes" "encoding/json" "reflect" "testing" "unicode/utf8" ) func TestCopyJSON(t *testing.T) { var buf bytes.Buffer enc := NewWriter(&buf) const mapLength = 6 enc.WriteMapHeader(mapLength) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_map") enc.WriteMapHeader(2) enc.WriteString("float_a") enc.WriteFloat32(1.0) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("some bytes") enc.WriteBytes([]byte("here are some bytes")) enc.WriteString("a bool") enc.WriteBool(true) enc.WriteString("a map") enc.WriteMapStrStr(map[string]string{ "internal_one": "blah", "internal_two": "blahhh...", }) enc.WriteString("float64") const encodedFloat64 = 1672209023 enc.WriteFloat64(encodedFloat64) enc.Flush() var js bytes.Buffer _, err := CopyToJSON(&js, &buf) if err != nil { t.Fatal(err) } mp := make(map[string]interface{}) err = json.Unmarshal(js.Bytes(), &mp) if err != nil { t.Log(js.String()) t.Fatalf("Error unmarshaling: %s", err) } if len(mp) != mapLength { t.Errorf("map length should be %d, not %d", mapLength, len(mp)) } so, ok := mp["thing_1"] if !ok || so != "a string object" { t.Errorf("expected %q; got %q", "a string object", so) } in, ok := mp["a map"] if !ok { t.Error("no key 'a map'") } if inm, ok := in.(map[string]interface{}); !ok { t.Error("inner map not type-assertable to map[string]interface{}") } else { inm1, ok := inm["internal_one"] if !ok || !reflect.DeepEqual(inm1, "blah") { t.Errorf("inner map field %q should be %q, not %q", "internal_one", "blah", inm1) } } if actual := mp["float64"]; float64(encodedFloat64) != actual.(float64) { t.Errorf("expected %G, got %G", float64(encodedFloat64), actual) } } // Encoder should generate valid utf-8 even if passed bad input func TestCopyJSONNegativeUTF8(t *testing.T) { // Single string with non-compliant utf-8 byte stringWithBadUTF8 := []byte{ 0xa1, 0xe0, } src := bytes.NewBuffer(stringWithBadUTF8) var js bytes.Buffer _, err := CopyToJSON(&js, src) if err != nil { t.Fatal(err) } // Even though we provided bad input, should have escaped the naughty character if !utf8.Valid(js.Bytes()) { t.Errorf("Expected JSON to be valid utf-8 even when provided bad input") } // Expect a bad character string expected := `"\ufffd"` if js.String() != expected { t.Errorf("Expected: '%s', got: '%s'", expected, js.String()) } } func BenchmarkCopyToJSON(b *testing.B) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(4) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_first_map") enc.WriteMapHeader(2) enc.WriteString("float_a") enc.WriteFloat32(1.0) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("an array") enc.WriteArrayHeader(2) enc.WriteBool(true) enc.WriteUint(2089) enc.WriteString("a_second_map") enc.WriteMapStrStr(map[string]string{ "internal_one": "blah", "internal_two": "blahhh...", }) enc.Flush() var js bytes.Buffer bts := buf.Bytes() _, err := CopyToJSON(&js, &buf) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(js.Bytes()))) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { js.Reset() CopyToJSON(&js, bytes.NewReader(bts)) } } func BenchmarkStdlibJSON(b *testing.B) { obj := map[string]interface{}{ "thing_1": "a string object", "a_first_map": map[string]interface{}{ "float_a": float32(1.0), "float_b": -100, }, "an array": []interface{}{ "part_A", "part_B", }, "a_second_map": map[string]interface{}{ "internal_one": "blah", "internal_two": "blahhh...", }, } var js bytes.Buffer err := json.NewEncoder(&js).Encode(&obj) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(js.Bytes()))) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { js.Reset() json.NewEncoder(&js).Encode(&obj) } } msgp-1.2.5/msgp/number.go000066400000000000000000000127121472554463600152750ustar00rootroot00000000000000package msgp import ( "math" "strconv" ) // The portable parts of the Number implementation // Number can be // an int64, uint64, float32, // or float64 internally. // It can decode itself // from any of the native // messagepack number types. // The zero-value of Number // is Int(0). Using the equality // operator with Number compares // both the type and the value // of the number. type Number struct { // internally, this // is just a tagged union. // the raw bits of the number // are stored the same way regardless. bits uint64 typ Type } // AsInt sets the number to an int64. func (n *Number) AsInt(i int64) { // we always store int(0) // as {0, InvalidType} in // order to preserve // the behavior of the == operator if i == 0 { n.typ = InvalidType n.bits = 0 return } n.typ = IntType n.bits = uint64(i) } // AsUint sets the number to a uint64. func (n *Number) AsUint(u uint64) { n.typ = UintType n.bits = u } // AsFloat32 sets the value of the number // to a float32. func (n *Number) AsFloat32(f float32) { n.typ = Float32Type n.bits = uint64(math.Float32bits(f)) } // AsFloat64 sets the value of the // number to a float64. func (n *Number) AsFloat64(f float64) { n.typ = Float64Type n.bits = math.Float64bits(f) } // Int casts the number as an int64, and // returns whether or not that was the // underlying type. func (n *Number) Int() (int64, bool) { return int64(n.bits), n.typ == IntType || n.typ == InvalidType } // Uint casts the number as a uint64, and returns // whether or not that was the underlying type. func (n *Number) Uint() (uint64, bool) { return n.bits, n.typ == UintType } // Float casts the number to a float64, and // returns whether or not that was the underlying // type (either a float64 or a float32). func (n *Number) Float() (float64, bool) { switch n.typ { case Float32Type: return float64(math.Float32frombits(uint32(n.bits))), true case Float64Type: return math.Float64frombits(n.bits), true default: return 0.0, false } } // Type will return one of: // Float64Type, Float32Type, UintType, or IntType. func (n *Number) Type() Type { if n.typ == InvalidType { return IntType } return n.typ } // DecodeMsg implements msgp.Decodable func (n *Number) DecodeMsg(r *Reader) error { typ, err := r.NextType() if err != nil { return err } switch typ { case Float32Type: f, err := r.ReadFloat32() if err != nil { return err } n.AsFloat32(f) return nil case Float64Type: f, err := r.ReadFloat64() if err != nil { return err } n.AsFloat64(f) return nil case IntType: i, err := r.ReadInt64() if err != nil { return err } n.AsInt(i) return nil case UintType: u, err := r.ReadUint64() if err != nil { return err } n.AsUint(u) return nil default: return TypeError{Encoded: typ, Method: IntType} } } // UnmarshalMsg implements msgp.Unmarshaler func (n *Number) UnmarshalMsg(b []byte) ([]byte, error) { typ := NextType(b) switch typ { case IntType: i, o, err := ReadInt64Bytes(b) if err != nil { return b, err } n.AsInt(i) return o, nil case UintType: u, o, err := ReadUint64Bytes(b) if err != nil { return b, err } n.AsUint(u) return o, nil case Float64Type: f, o, err := ReadFloat64Bytes(b) if err != nil { return b, err } n.AsFloat64(f) return o, nil case Float32Type: f, o, err := ReadFloat32Bytes(b) if err != nil { return b, err } n.AsFloat32(f) return o, nil default: return b, TypeError{Method: IntType, Encoded: typ} } } // MarshalMsg implements msgp.Marshaler func (n *Number) MarshalMsg(b []byte) ([]byte, error) { switch n.typ { case IntType: return AppendInt64(b, int64(n.bits)), nil case UintType: return AppendUint64(b, uint64(n.bits)), nil case Float64Type: return AppendFloat64(b, math.Float64frombits(n.bits)), nil case Float32Type: return AppendFloat32(b, math.Float32frombits(uint32(n.bits))), nil default: return AppendInt64(b, 0), nil } } // EncodeMsg implements msgp.Encodable func (n *Number) EncodeMsg(w *Writer) error { switch n.typ { case IntType: return w.WriteInt64(int64(n.bits)) case UintType: return w.WriteUint64(n.bits) case Float64Type: return w.WriteFloat64(math.Float64frombits(n.bits)) case Float32Type: return w.WriteFloat32(math.Float32frombits(uint32(n.bits))) default: return w.WriteInt64(0) } } // Msgsize implements msgp.Sizer func (n *Number) Msgsize() int { switch n.typ { case Float32Type: return Float32Size case Float64Type: return Float64Size case IntType: return Int64Size case UintType: return Uint64Size default: return 1 // fixint(0) } } // MarshalJSON implements json.Marshaler func (n *Number) MarshalJSON() ([]byte, error) { t := n.Type() if t == InvalidType { return []byte{'0'}, nil } out := make([]byte, 0, 32) switch t { case Float32Type, Float64Type: f, _ := n.Float() return strconv.AppendFloat(out, f, 'f', -1, 64), nil case IntType: i, _ := n.Int() return strconv.AppendInt(out, i, 10), nil case UintType: u, _ := n.Uint() return strconv.AppendUint(out, u, 10), nil default: panic("(*Number).typ is invalid") } } // String implements fmt.Stringer func (n *Number) String() string { switch n.typ { case InvalidType: return "0" case Float32Type, Float64Type: f, _ := n.Float() return strconv.FormatFloat(f, 'f', -1, 64) case IntType: i, _ := n.Int() return strconv.FormatInt(i, 10) case UintType: u, _ := n.Uint() return strconv.FormatUint(u, 10) default: panic("(*Number).typ is invalid") } } msgp-1.2.5/msgp/number_test.go000066400000000000000000000036201472554463600163320ustar00rootroot00000000000000package msgp import ( "bytes" "testing" ) func TestNumber(t *testing.T) { n := Number{} if n.Type() != IntType { t.Errorf("expected zero-value type to be %s; got %s", IntType, n.Type()) } if n.String() != "0" { t.Errorf("expected Number{}.String() to be \"0\" but got %q", n.String()) } n.AsInt(248) i, ok := n.Int() if !ok || i != 248 || n.Type() != IntType || n.String() != "248" { t.Errorf("%d in; %d out!", 248, i) } n.AsFloat64(3.141) f, ok := n.Float() if !ok || f != 3.141 || n.Type() != Float64Type || n.String() != "3.141" { t.Errorf("%f in; %f out!", 3.141, f) } n.AsUint(40000) u, ok := n.Uint() if !ok || u != 40000 || n.Type() != UintType || n.String() != "40000" { t.Errorf("%d in; %d out!", 40000, u) } nums := []interface{}{ float64(3.14159), int64(-29081), uint64(90821983), float32(3.141), } var dat []byte var buf bytes.Buffer wr := NewWriter(&buf) for _, n := range nums { dat, _ = AppendIntf(dat, n) wr.WriteIntf(n) } wr.Flush() mout := make([]Number, len(nums)) dout := make([]Number, len(nums)) rd := NewReader(&buf) unm := dat for i := range nums { var err error unm, err = mout[i].UnmarshalMsg(unm) if err != nil { t.Fatal("unmarshal error:", err) } err = dout[i].DecodeMsg(rd) if err != nil { t.Fatal("decode error:", err) } if mout[i] != dout[i] { t.Errorf("for %#v, got %#v from unmarshal and %#v from decode", nums[i], mout[i], dout[i]) } } buf.Reset() var odat []byte for i := range nums { var err error odat, err = mout[i].MarshalMsg(odat) if err != nil { t.Fatal("marshal error:", err) } err = dout[i].EncodeMsg(wr) if err != nil { t.Fatal("encode error", err) } } wr.Flush() if !bytes.Equal(dat, odat) { t.Errorf("marshal: expected output %#v; got %#v", dat, odat) } if !bytes.Equal(dat, buf.Bytes()) { t.Errorf("encode: expected output %#v; got %#v", dat, buf.Bytes()) } } msgp-1.2.5/msgp/purego.go000066400000000000000000000004421472554463600153030ustar00rootroot00000000000000//go:build (purego && !unsafe) || appengine // +build purego,!unsafe appengine package msgp // let's just assume appengine // uses 64-bit hardware... const smallint = false func UnsafeString(b []byte) string { return string(b) } func UnsafeBytes(s string) []byte { return []byte(s) } msgp-1.2.5/msgp/raw_test.go000066400000000000000000000041551472554463600156370ustar00rootroot00000000000000package msgp import ( "bytes" "testing" "time" ) // all standard interfaces type allifaces interface { Encodable Decodable Marshaler Unmarshaler Sizer } func TestRaw(t *testing.T) { bts := make([]byte, 0, 512) bts = AppendMapHeader(bts, 3) bts = AppendString(bts, "key_one") bts = AppendFloat64(bts, -1.0) bts = AppendString(bts, "key_two") bts = AppendString(bts, "value_two") bts = AppendString(bts, "key_three") bts = AppendTime(bts, time.Now()) var r Raw // verify that Raw satisfies // the interfaces we want it to var _ allifaces = &r // READ TESTS extra, err := r.UnmarshalMsg(bts) if err != nil { t.Fatal("error from UnmarshalMsg:", err) } if len(extra) != 0 { t.Errorf("expected 0 bytes left; found %d", len(extra)) } if !bytes.Equal([]byte(r), bts) { t.Fatal("value of raw and input slice are not equal after UnmarshalMsg") } r = r[:0] var buf bytes.Buffer buf.Write(bts) rd := NewReader(&buf) err = r.DecodeMsg(rd) if err != nil { t.Fatal("error from DecodeMsg:", err) } if !bytes.Equal([]byte(r), bts) { t.Fatal("value of raw and input slice are not equal after DecodeMsg") } // WRITE TESTS buf.Reset() wr := NewWriter(&buf) err = r.EncodeMsg(wr) if err != nil { t.Fatal("error from EncodeMsg:", err) } wr.Flush() if !bytes.Equal(buf.Bytes(), bts) { t.Fatal("value of buf.Bytes() and input slice are not equal after EncodeMsg") } var outsl []byte outsl, err = r.MarshalMsg(outsl) if err != nil { t.Fatal("error from MarshalMsg:", err) } if !bytes.Equal(outsl, bts) { t.Fatal("value of output and input of MarshalMsg are not equal.") } } func TestNullRaw(t *testing.T) { // Marshal/Unmarshal var x, y Raw if bts, err := x.MarshalMsg(nil); err != nil { t.Fatal(err) } else if _, err = y.UnmarshalMsg(bts); err != nil { t.Fatal(err) } if !bytes.Equal(x, y) { t.Fatal("compare") } // Encode/Decode var buf bytes.Buffer wr := NewWriter(&buf) if err := x.EncodeMsg(wr); err != nil { t.Fatal(err) } wr.Flush() rd := NewReader(&buf) if err := y.DecodeMsg(rd); err != nil { t.Fatal(err) } if !bytes.Equal(x, y) { t.Fatal("compare") } } msgp-1.2.5/msgp/read.go000066400000000000000000000713161472554463600147250ustar00rootroot00000000000000package msgp import ( "encoding/binary" "encoding/json" "io" "math" "strconv" "sync" "time" "github.com/philhofer/fwd" ) // where we keep old *Readers var readerPool = sync.Pool{New: func() interface{} { return &Reader{} }} // Type is a MessagePack wire type, // including this package's built-in // extension types. type Type byte // MessagePack Types // // The zero value of Type // is InvalidType. const ( InvalidType Type = iota // MessagePack built-in types StrType BinType MapType ArrayType Float64Type Float32Type BoolType IntType UintType NilType DurationType ExtensionType // pseudo-types provided // by extensions Complex64Type Complex128Type TimeType NumberType _maxtype ) // String implements fmt.Stringer func (t Type) String() string { switch t { case StrType: return "str" case BinType: return "bin" case MapType: return "map" case ArrayType: return "array" case Float64Type: return "float64" case Float32Type: return "float32" case BoolType: return "bool" case UintType: return "uint" case IntType: return "int" case ExtensionType: return "ext" case NilType: return "nil" case NumberType: return "number" default: return "" } } func freeR(m *Reader) { readerPool.Put(m) } // Unmarshaler is the interface fulfilled // by objects that know how to unmarshal // themselves from MessagePack. // UnmarshalMsg unmarshals the object // from binary, returing any leftover // bytes and any errors encountered. type Unmarshaler interface { UnmarshalMsg([]byte) ([]byte, error) } // Decodable is the interface fulfilled // by objects that know how to read // themselves from a *Reader. type Decodable interface { DecodeMsg(*Reader) error } // Decode decodes 'd' from 'r'. func Decode(r io.Reader, d Decodable) error { rd := NewReader(r) err := d.DecodeMsg(rd) freeR(rd) return err } // NewReader returns a *Reader that // reads from the provided reader. The // reader will be buffered. func NewReader(r io.Reader) *Reader { p := readerPool.Get().(*Reader) if p.R == nil { p.R = fwd.NewReader(r) } else { p.R.Reset(r) } return p } // NewReaderSize returns a *Reader with a buffer of the given size. // (This is vastly preferable to passing the decoder a reader that is already buffered.) func NewReaderSize(r io.Reader, sz int) *Reader { return &Reader{R: fwd.NewReaderSize(r, sz)} } // NewReaderBuf returns a *Reader with a provided buffer. func NewReaderBuf(r io.Reader, buf []byte) *Reader { return &Reader{R: fwd.NewReaderBuf(r, buf)} } // Reader wraps an io.Reader and provides // methods to read MessagePack-encoded values // from it. Readers are buffered. type Reader struct { // R is the buffered reader // that the Reader uses // to decode MessagePack. // The Reader itself // is stateless; all the // buffering is done // within R. R *fwd.Reader scratch []byte recursionDepth int } // Read implements `io.Reader` func (m *Reader) Read(p []byte) (int, error) { return m.R.Read(p) } // CopyNext reads the next object from m without decoding it and writes it to w. // It avoids unnecessary copies internally. func (m *Reader) CopyNext(w io.Writer) (int64, error) { sz, o, err := getNextSize(m.R) if err != nil { return 0, err } var n int64 // Opportunistic optimization: if we can fit the whole thing in the m.R // buffer, then just get a pointer to that, and pass it to w.Write, // avoiding an allocation. if int(sz) <= m.R.BufferSize() { var nn int var buf []byte buf, err = m.R.Next(int(sz)) if err != nil { if err == io.ErrUnexpectedEOF { err = ErrShortBytes } return 0, err } nn, err = w.Write(buf) n += int64(nn) } else { // Fall back to io.CopyN. // May avoid allocating if w is a ReaderFrom (e.g. bytes.Buffer) n, err = io.CopyN(w, m.R, int64(sz)) if err == io.ErrUnexpectedEOF { err = ErrShortBytes } } if err != nil { return n, err } else if n < int64(sz) { return n, io.ErrShortWrite } if done, err := m.recursiveCall(); err != nil { return n, err } else { defer done() } // for maps and slices, read elements for x := uintptr(0); x < o; x++ { var n2 int64 n2, err = m.CopyNext(w) if err != nil { return n, err } n += n2 } return n, nil } // recursiveCall will increment the recursion depth and return an error if it is exceeded. // If a nil error is returned, done must be called to decrement the counter. func (m *Reader) recursiveCall() (done func(), err error) { if m.recursionDepth >= recursionLimit { return func() {}, ErrRecursion } m.recursionDepth++ return func() { m.recursionDepth-- }, nil } // ReadFull implements `io.ReadFull` func (m *Reader) ReadFull(p []byte) (int, error) { return m.R.ReadFull(p) } // Reset resets the underlying reader. func (m *Reader) Reset(r io.Reader) { m.R.Reset(r) } // Buffered returns the number of bytes currently in the read buffer. func (m *Reader) Buffered() int { return m.R.Buffered() } // BufferSize returns the capacity of the read buffer. func (m *Reader) BufferSize() int { return m.R.BufferSize() } // NextType returns the next object type to be decoded. func (m *Reader) NextType() (Type, error) { next, err := m.R.PeekByte() if err != nil { return InvalidType, err } t := getType(next) if t == InvalidType { return t, InvalidPrefixError(next) } if t == ExtensionType { v, err := m.peekExtensionType() if err != nil { return InvalidType, err } switch v { case Complex64Extension: return Complex64Type, nil case Complex128Extension: return Complex128Type, nil case TimeExtension, MsgTimeExtension: return TimeType, nil } } return t, nil } // IsNil returns whether or not // the next byte is a null messagepack byte func (m *Reader) IsNil() bool { p, err := m.R.PeekByte() return err == nil && p == mnil } // getNextSize returns the size of the next object on the wire. // returns (obj size, obj elements, error) // only maps and arrays have non-zero obj elements // for maps and arrays, obj size does not include elements // // use uintptr b/c it's guaranteed to be large enough // to hold whatever we can fit in memory. func getNextSize(r *fwd.Reader) (uintptr, uintptr, error) { lead, err := r.PeekByte() if err != nil { return 0, 0, err } spec := getBytespec(lead) size, mode := spec.size, spec.extra if size == 0 { return 0, 0, InvalidPrefixError(lead) } if mode >= 0 { return uintptr(size), uintptr(mode), nil } b, err := r.Peek(int(size)) if err != nil { return 0, 0, err } switch mode { case extra8: return uintptr(size) + uintptr(b[1]), 0, nil case extra16: return uintptr(size) + uintptr(big.Uint16(b[1:])), 0, nil case extra32: return uintptr(size) + uintptr(big.Uint32(b[1:])), 0, nil case map16v: return uintptr(size), 2 * uintptr(big.Uint16(b[1:])), nil case map32v: return uintptr(size), 2 * uintptr(big.Uint32(b[1:])), nil case array16v: return uintptr(size), uintptr(big.Uint16(b[1:])), nil case array32v: return uintptr(size), uintptr(big.Uint32(b[1:])), nil default: return 0, 0, fatal } } // Skip skips over the next object, regardless of // its type. If it is an array or map, the whole array // or map will be skipped. func (m *Reader) Skip() error { var ( v uintptr // bytes o uintptr // objects err error p []byte ) // we can use the faster // method if we have enough // buffered data if m.R.Buffered() >= 5 { p, err = m.R.Peek(5) if err != nil { return err } v, o, err = getSize(p) if err != nil { return err } } else { v, o, err = getNextSize(m.R) if err != nil { return err } } // 'v' is always non-zero // if err == nil _, err = m.R.Skip(int(v)) if err != nil { return err } // for maps and slices, skip elements with recursive call if done, err := m.recursiveCall(); err != nil { return err } else { defer done() } for x := uintptr(0); x < o; x++ { err = m.Skip() if err != nil { return err } } return nil } // ReadMapHeader reads the next object // as a map header and returns the size // of the map and the number of bytes written. // It will return a TypeError{} if the next // object is not a map. func (m *Reader) ReadMapHeader() (sz uint32, err error) { var p []byte var lead byte lead, err = m.R.PeekByte() if err != nil { return } if isfixmap(lead) { sz = uint32(rfixmap(lead)) _, err = m.R.Skip(1) return } switch lead { case mmap16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case mmap32: p, err = m.R.Next(5) if err != nil { return } sz = big.Uint32(p[1:]) return default: err = badPrefix(MapType, lead) return } } // ReadMapKey reads either a 'str' or 'bin' field from // the reader and returns the value as a []byte. It uses // scratch for storage if it is large enough. func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) { out, err := m.ReadStringAsBytes(scratch) if err != nil { if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { return m.ReadBytes(scratch) } return nil, err } return out, nil } // ReadMapKeyPtr returns a []byte pointing to the contents // of a valid map key. The key cannot be empty, and it // must be shorter than the total buffer size of the // *Reader. Additionally, the returned slice is only // valid until the next *Reader method call. Users // should exercise extreme care when using this // method; writing into the returned slice may // corrupt future reads. func (m *Reader) ReadMapKeyPtr() ([]byte, error) { lead, err := m.R.PeekByte() if err != nil { return nil, err } var read int var p []byte if isfixstr(lead) { read = int(rfixstr(lead)) m.R.Skip(1) goto fill } switch lead { case mstr8, mbin8: p, err = m.R.Next(2) if err != nil { return nil, err } read = int(p[1]) case mstr16, mbin16: p, err = m.R.Next(3) if err != nil { return nil, err } read = int(big.Uint16(p[1:])) case mstr32, mbin32: p, err = m.R.Next(5) if err != nil { return nil, err } read = int(big.Uint32(p[1:])) default: return nil, badPrefix(StrType, lead) } fill: if read == 0 { return nil, ErrShortBytes } return m.R.Next(read) } // ReadArrayHeader reads the next object as an // array header and returns the size of the array // and the number of bytes read. func (m *Reader) ReadArrayHeader() (sz uint32, err error) { lead, err := m.R.PeekByte() if err != nil { return } if isfixarray(lead) { sz = uint32(rfixarray(lead)) _, err = m.R.Skip(1) return } var p []byte switch lead { case marray16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case marray32: p, err = m.R.Next(5) if err != nil { return } sz = big.Uint32(p[1:]) return default: err = badPrefix(ArrayType, lead) return } } // ReadNil reads a 'nil' MessagePack byte from the reader func (m *Reader) ReadNil() error { p, err := m.R.PeekByte() if err != nil { return err } if p != mnil { return badPrefix(NilType, p) } _, err = m.R.Skip(1) return err } // ReadFloat64 reads a float64 from the reader. // (If the value on the wire is encoded as a float32, // it will be up-cast to a float64.) func (m *Reader) ReadFloat64() (f float64, err error) { var p []byte p, err = m.R.Peek(9) if err != nil { // we'll allow a coversion from float32 to float64, // since we don't lose any precision if err == io.EOF && len(p) > 0 && p[0] == mfloat32 { ef, err := m.ReadFloat32() return float64(ef), err } return } if p[0] != mfloat64 { // see above if p[0] == mfloat32 { ef, err := m.ReadFloat32() return float64(ef), err } err = badPrefix(Float64Type, p[0]) return } f = math.Float64frombits(getMuint64(p)) _, err = m.R.Skip(9) return } // ReadFloat32 reads a float32 from the reader func (m *Reader) ReadFloat32() (f float32, err error) { var p []byte p, err = m.R.Peek(5) if err != nil { return } if p[0] != mfloat32 { err = badPrefix(Float32Type, p[0]) return } f = math.Float32frombits(getMuint32(p)) _, err = m.R.Skip(5) return } // ReadBool reads a bool from the reader func (m *Reader) ReadBool() (b bool, err error) { var p byte p, err = m.R.PeekByte() if err != nil { return } switch p { case mtrue: b = true case mfalse: default: err = badPrefix(BoolType, p) return } _, err = m.R.Skip(1) return } // ReadDuration reads a time.Duration from the reader func (m *Reader) ReadDuration() (d time.Duration, err error) { i, err := m.ReadInt64() return time.Duration(i), err } // ReadInt64 reads an int64 from the reader func (m *Reader) ReadInt64() (i int64, err error) { var p []byte lead, err := m.R.PeekByte() if err != nil { return } if isfixint(lead) { i = int64(rfixint(lead)) _, err = m.R.Skip(1) return } else if isnfixint(lead) { i = int64(rnfixint(lead)) _, err = m.R.Skip(1) return } switch lead { case mint8: p, err = m.R.Next(2) if err != nil { return } i = int64(getMint8(p)) return case muint8: p, err = m.R.Next(2) if err != nil { return } i = int64(getMuint8(p)) return case mint16: p, err = m.R.Next(3) if err != nil { return } i = int64(getMint16(p)) return case muint16: p, err = m.R.Next(3) if err != nil { return } i = int64(getMuint16(p)) return case mint32: p, err = m.R.Next(5) if err != nil { return } i = int64(getMint32(p)) return case muint32: p, err = m.R.Next(5) if err != nil { return } i = int64(getMuint32(p)) return case mint64: p, err = m.R.Next(9) if err != nil { return } i = getMint64(p) return case muint64: p, err = m.R.Next(9) if err != nil { return } u := getMuint64(p) if u > math.MaxInt64 { err = UintOverflow{Value: u, FailedBitsize: 64} return } i = int64(u) return default: err = badPrefix(IntType, lead) return } } // ReadInt32 reads an int32 from the reader func (m *Reader) ReadInt32() (i int32, err error) { var in int64 in, err = m.ReadInt64() if in > math.MaxInt32 || in < math.MinInt32 { err = IntOverflow{Value: in, FailedBitsize: 32} return } i = int32(in) return } // ReadInt16 reads an int16 from the reader func (m *Reader) ReadInt16() (i int16, err error) { var in int64 in, err = m.ReadInt64() if in > math.MaxInt16 || in < math.MinInt16 { err = IntOverflow{Value: in, FailedBitsize: 16} return } i = int16(in) return } // ReadInt8 reads an int8 from the reader func (m *Reader) ReadInt8() (i int8, err error) { var in int64 in, err = m.ReadInt64() if in > math.MaxInt8 || in < math.MinInt8 { err = IntOverflow{Value: in, FailedBitsize: 8} return } i = int8(in) return } // ReadInt reads an int from the reader func (m *Reader) ReadInt() (i int, err error) { if smallint { var in int32 in, err = m.ReadInt32() i = int(in) return } var in int64 in, err = m.ReadInt64() i = int(in) return } // ReadUint64 reads a uint64 from the reader func (m *Reader) ReadUint64() (u uint64, err error) { var p []byte lead, err := m.R.PeekByte() if err != nil { return } if isfixint(lead) { u = uint64(rfixint(lead)) _, err = m.R.Skip(1) return } switch lead { case mint8: p, err = m.R.Next(2) if err != nil { return } v := int64(getMint8(p)) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) return case muint8: p, err = m.R.Next(2) if err != nil { return } u = uint64(getMuint8(p)) return case mint16: p, err = m.R.Next(3) if err != nil { return } v := int64(getMint16(p)) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) return case muint16: p, err = m.R.Next(3) if err != nil { return } u = uint64(getMuint16(p)) return case mint32: p, err = m.R.Next(5) if err != nil { return } v := int64(getMint32(p)) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) return case muint32: p, err = m.R.Next(5) if err != nil { return } u = uint64(getMuint32(p)) return case mint64: p, err = m.R.Next(9) if err != nil { return } v := int64(getMint64(p)) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) return case muint64: p, err = m.R.Next(9) if err != nil { return } u = getMuint64(p) return default: if isnfixint(lead) { err = UintBelowZero{Value: int64(rnfixint(lead))} } else { err = badPrefix(UintType, lead) } return } } // ReadUint32 reads a uint32 from the reader func (m *Reader) ReadUint32() (u uint32, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint32 { err = UintOverflow{Value: in, FailedBitsize: 32} return } u = uint32(in) return } // ReadUint16 reads a uint16 from the reader func (m *Reader) ReadUint16() (u uint16, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint16 { err = UintOverflow{Value: in, FailedBitsize: 16} return } u = uint16(in) return } // ReadUint8 reads a uint8 from the reader func (m *Reader) ReadUint8() (u uint8, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint8 { err = UintOverflow{Value: in, FailedBitsize: 8} return } u = uint8(in) return } // ReadUint reads a uint from the reader func (m *Reader) ReadUint() (u uint, err error) { if smallint { var un uint32 un, err = m.ReadUint32() u = uint(un) return } var un uint64 un, err = m.ReadUint64() u = uint(un) return } // ReadByte is analogous to ReadUint8. // // NOTE: this is *not* an implementation // of io.ByteReader. func (m *Reader) ReadByte() (b byte, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint8 { err = UintOverflow{Value: in, FailedBitsize: 8} return } b = byte(in) return } // ReadBytes reads a MessagePack 'bin' object // from the reader and returns its value. It may // use 'scratch' for storage if it is non-nil. func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) { var p []byte var lead byte p, err = m.R.Peek(2) if err != nil { return } lead = p[0] var read int64 switch lead { case mbin8: read = int64(p[1]) m.R.Skip(2) case mbin16: p, err = m.R.Next(3) if err != nil { return } read = int64(big.Uint16(p[1:])) case mbin32: p, err = m.R.Next(5) if err != nil { return } read = int64(big.Uint32(p[1:])) default: err = badPrefix(BinType, lead) return } if int64(cap(scratch)) < read { b = make([]byte, read) } else { b = scratch[0:read] } _, err = m.R.ReadFull(b) return } // ReadBytesHeader reads the size header // of a MessagePack 'bin' object. The user // is responsible for dealing with the next // 'sz' bytes from the reader in an application-specific // way. func (m *Reader) ReadBytesHeader() (sz uint32, err error) { var p []byte lead, err := m.R.PeekByte() if err != nil { return } switch lead { case mbin8: p, err = m.R.Next(2) if err != nil { return } sz = uint32(p[1]) return case mbin16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case mbin32: p, err = m.R.Next(5) if err != nil { return } sz = uint32(big.Uint32(p[1:])) return default: err = badPrefix(BinType, p[0]) return } } // ReadExactBytes reads a MessagePack 'bin'-encoded // object off of the wire into the provided slice. An // ArrayError will be returned if the object is not // exactly the length of the input slice. func (m *Reader) ReadExactBytes(into []byte) error { p, err := m.R.Peek(2) if err != nil { return err } lead := p[0] var read int64 // bytes to read var skip int // prefix size to skip switch lead { case mbin8: read = int64(p[1]) skip = 2 case mbin16: p, err = m.R.Peek(3) if err != nil { return err } read = int64(big.Uint16(p[1:])) skip = 3 case mbin32: p, err = m.R.Peek(5) if err != nil { return err } read = int64(big.Uint32(p[1:])) skip = 5 default: return badPrefix(BinType, lead) } if read != int64(len(into)) { return ArrayError{Wanted: uint32(len(into)), Got: uint32(read)} } m.R.Skip(skip) _, err = m.R.ReadFull(into) return err } // ReadStringAsBytes reads a MessagePack 'str' (utf-8) string // and returns its value as bytes. It may use 'scratch' for storage // if it is non-nil. func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) { var p []byte lead, err := m.R.PeekByte() if err != nil { return } var read int64 if isfixstr(lead) { read = int64(rfixstr(lead)) m.R.Skip(1) goto fill } switch lead { case mstr8: p, err = m.R.Next(2) if err != nil { return } read = int64(uint8(p[1])) case mstr16: p, err = m.R.Next(3) if err != nil { return } read = int64(big.Uint16(p[1:])) case mstr32: p, err = m.R.Next(5) if err != nil { return } read = int64(big.Uint32(p[1:])) default: err = badPrefix(StrType, lead) return } fill: if int64(cap(scratch)) < read { b = make([]byte, read) } else { b = scratch[0:read] } _, err = m.R.ReadFull(b) return } // ReadStringHeader reads a string header // off of the wire. The user is then responsible // for dealing with the next 'sz' bytes from // the reader in an application-specific manner. func (m *Reader) ReadStringHeader() (sz uint32, err error) { lead, err := m.R.PeekByte() if err != nil { return } if isfixstr(lead) { sz = uint32(rfixstr(lead)) m.R.Skip(1) return } var p []byte switch lead { case mstr8: p, err = m.R.Next(2) if err != nil { return } sz = uint32(p[1]) return case mstr16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case mstr32: p, err = m.R.Next(5) if err != nil { return } sz = big.Uint32(p[1:]) return default: err = badPrefix(StrType, lead) return } } // ReadString reads a utf-8 string from the reader func (m *Reader) ReadString() (s string, err error) { var read int64 lead, err := m.R.PeekByte() if err != nil { return } var p []byte if isfixstr(lead) { read = int64(rfixstr(lead)) m.R.Skip(1) goto fill } switch lead { case mstr8: p, err = m.R.Next(2) if err != nil { return } read = int64(uint8(p[1])) case mstr16: p, err = m.R.Next(3) if err != nil { return } read = int64(big.Uint16(p[1:])) case mstr32: p, err = m.R.Next(5) if err != nil { return } read = int64(big.Uint32(p[1:])) default: err = badPrefix(StrType, lead) return } fill: if read == 0 { s, err = "", nil return } // reading into the memory // that will become the string // itself has vastly superior // worst-case performance, because // the reader buffer doesn't have // to be large enough to hold the string. // the idea here is to make it more // difficult for someone malicious // to cause the system to run out of // memory by sending very large strings. // // NOTE: this works because the argument // passed to (*fwd.Reader).ReadFull escapes // to the heap; its argument may, in turn, // be passed to the underlying reader, and // thus escape analysis *must* conclude that // 'out' escapes. out := make([]byte, read) _, err = m.R.ReadFull(out) if err != nil { return } s = UnsafeString(out) return } // ReadComplex64 reads a complex64 from the reader func (m *Reader) ReadComplex64() (f complex64, err error) { var p []byte p, err = m.R.Peek(10) if err != nil { return } if p[0] != mfixext8 { err = badPrefix(Complex64Type, p[0]) return } if int8(p[1]) != Complex64Extension { err = errExt(int8(p[1]), Complex64Extension) return } f = complex(math.Float32frombits(big.Uint32(p[2:])), math.Float32frombits(big.Uint32(p[6:]))) _, err = m.R.Skip(10) return } // ReadComplex128 reads a complex128 from the reader func (m *Reader) ReadComplex128() (f complex128, err error) { var p []byte p, err = m.R.Peek(18) if err != nil { return } if p[0] != mfixext16 { err = badPrefix(Complex128Type, p[0]) return } if int8(p[1]) != Complex128Extension { err = errExt(int8(p[1]), Complex128Extension) return } f = complex(math.Float64frombits(big.Uint64(p[2:])), math.Float64frombits(big.Uint64(p[10:]))) _, err = m.R.Skip(18) return } // ReadMapStrIntf reads a MessagePack map into a map[string]interface{}. // (You must pass a non-nil map into the function.) func (m *Reader) ReadMapStrIntf(mp map[string]interface{}) (err error) { var sz uint32 sz, err = m.ReadMapHeader() if err != nil { return } for key := range mp { delete(mp, key) } for i := uint32(0); i < sz; i++ { var key string var val interface{} key, err = m.ReadString() if err != nil { return } val, err = m.ReadIntf() if err != nil { return } mp[key] = val } return } // ReadTime reads a time.Time object from the reader. // The returned time's location will be set to time.Local. func (m *Reader) ReadTime() (t time.Time, err error) { offset, length, extType, err := m.peekExtensionHeader() if err != nil { return t, err } switch extType { case TimeExtension: var p []byte p, err = m.R.Peek(15) if err != nil { return } if p[0] != mext8 || p[1] != 12 { err = badPrefix(TimeType, p[0]) return } if int8(p[2]) != TimeExtension { err = errExt(int8(p[2]), TimeExtension) return } sec, nsec := getUnix(p[3:]) t = time.Unix(sec, int64(nsec)).Local() _, err = m.R.Skip(15) return case MsgTimeExtension: switch length { case 4, 8, 12: var tmp [12]byte _, err = m.R.Skip(offset) if err != nil { return } var n int n, err = m.R.Read(tmp[:length]) if err != nil { return } if n != length { err = ErrShortBytes return } b := tmp[:length] switch length { case 4: t = time.Unix(int64(binary.BigEndian.Uint32(b)), 0).Local() case 8: v := binary.BigEndian.Uint64(b) nanos := int64(v >> 34) if nanos > 999999999 { // In timestamp 64 and timestamp 96 formats, nanoseconds must not be larger than 999999999. err = InvalidTimestamp{Nanos: nanos} return } t = time.Unix(int64(v&(1<<34-1)), nanos).Local() case 12: nanos := int64(binary.BigEndian.Uint32(b)) if nanos > 999999999 { // In timestamp 64 and timestamp 96 formats, nanoseconds must not be larger than 999999999. err = InvalidTimestamp{Nanos: nanos} return } ux := int64(binary.BigEndian.Uint64(b[4:])) t = time.Unix(ux, nanos).Local() } default: err = InvalidTimestamp{FieldLength: length} } default: err = errExt(extType, TimeExtension) } return } // ReadJSONNumber reads an integer or a float value and return as json.Number func (m *Reader) ReadJSONNumber() (n json.Number, err error) { t, err := m.NextType() if err != nil { return } switch t { case IntType: v, err := m.ReadInt64() if err == nil { return json.Number(strconv.FormatInt(v, 10)), nil } return "", err case UintType: v, err := m.ReadUint64() if err == nil { return json.Number(strconv.FormatUint(v, 10)), nil } return "", err case Float32Type, Float64Type: v, err := m.ReadFloat64() if err == nil { return json.Number(strconv.FormatFloat(v, 'f', -1, 64)), nil } return "", err } return "", TypeError{Method: NumberType, Encoded: t} } // ReadIntf reads out the next object as a raw interface{}/any. // Arrays are decoded as []interface{}, and maps are decoded // as map[string]interface{}. Integers are decoded as int64 // and unsigned integers are decoded as uint64. func (m *Reader) ReadIntf() (i interface{}, err error) { var t Type t, err = m.NextType() if err != nil { return } switch t { case BoolType: i, err = m.ReadBool() return case IntType: i, err = m.ReadInt64() return case UintType: i, err = m.ReadUint64() return case BinType: i, err = m.ReadBytes(nil) return case StrType: i, err = m.ReadString() return case Complex64Type: i, err = m.ReadComplex64() return case Complex128Type: i, err = m.ReadComplex128() return case TimeType: i, err = m.ReadTime() return case DurationType: i, err = m.ReadDuration() return case ExtensionType: var t int8 t, err = m.peekExtensionType() if err != nil { return } f, ok := extensionReg[t] if ok { e := f() err = m.ReadExtension(e) i = e return } var e RawExtension e.Type = t err = m.ReadExtension(&e) i = &e return case MapType: // This can call back here, so treat as recursive call. if done, err := m.recursiveCall(); err != nil { return nil, err } else { defer done() } mp := make(map[string]interface{}) err = m.ReadMapStrIntf(mp) i = mp return case NilType: err = m.ReadNil() i = nil return case Float32Type: i, err = m.ReadFloat32() return case Float64Type: i, err = m.ReadFloat64() return case ArrayType: var sz uint32 sz, err = m.ReadArrayHeader() if err != nil { return } if done, err := m.recursiveCall(); err != nil { return nil, err } else { defer done() } out := make([]interface{}, int(sz)) for j := range out { out[j], err = m.ReadIntf() if err != nil { return } } i = out return default: return nil, fatal // unreachable } } msgp-1.2.5/msgp/read_bytes.go000066400000000000000000000662521472554463600161360ustar00rootroot00000000000000package msgp import ( "bytes" "encoding/binary" "encoding/json" "math" "strconv" "time" ) var big = binary.BigEndian // NextType returns the type of the next // object in the slice. If the length // of the input is zero, it returns // [InvalidType]. func NextType(b []byte) Type { if len(b) == 0 { return InvalidType } spec := getBytespec(b[0]) t := spec.typ if t == ExtensionType && len(b) > int(spec.size) { var tp int8 if spec.extra == constsize { tp = int8(b[1]) } else { tp = int8(b[spec.size-1]) } switch tp { case TimeExtension, MsgTimeExtension: return TimeType case Complex128Extension: return Complex128Type case Complex64Extension: return Complex64Type default: return ExtensionType } } return t } // IsNil returns true if len(b)>0 and // the leading byte is a 'nil' MessagePack // byte; false otherwise func IsNil(b []byte) bool { if len(b) != 0 && b[0] == mnil { return true } return false } // Raw is raw MessagePack. // Raw allows you to read and write // data without interpreting its contents. type Raw []byte // MarshalMsg implements [Marshaler]. // It appends the raw contents of 'raw' // to the provided byte slice. If 'raw' // is 0 bytes, 'nil' will be appended instead. func (r Raw) MarshalMsg(b []byte) ([]byte, error) { i := len(r) if i == 0 { return AppendNil(b), nil } o, l := ensure(b, i) copy(o[l:], []byte(r)) return o, nil } // UnmarshalMsg implements [Unmarshaler]. // It sets the contents of *Raw to be the next // object in the provided byte slice. func (r *Raw) UnmarshalMsg(b []byte) ([]byte, error) { l := len(b) out, err := Skip(b) if err != nil { return b, err } rlen := l - len(out) if IsNil(b[:rlen]) { rlen = 0 } if cap(*r) < rlen { *r = make(Raw, rlen) } else { *r = (*r)[0:rlen] } copy(*r, b[:rlen]) return out, nil } // EncodeMsg implements [Encodable]. // It writes the raw bytes to the writer. // If r is empty, it writes 'nil' instead. func (r Raw) EncodeMsg(w *Writer) error { if len(r) == 0 { return w.WriteNil() } _, err := w.Write([]byte(r)) return err } // DecodeMsg implements [Decodable]. // It sets the value of *Raw to be the // next object on the wire. func (r *Raw) DecodeMsg(f *Reader) error { *r = (*r)[:0] err := appendNext(f, (*[]byte)(r)) if IsNil(*r) { *r = (*r)[:0] } return err } // Msgsize implements [Sizer]. func (r Raw) Msgsize() int { l := len(r) if l == 0 { return 1 // for 'nil' } return l } func appendNext(f *Reader, d *[]byte) error { amt, o, err := getNextSize(f.R) if err != nil { return err } var i int *d, i = ensure(*d, int(amt)) _, err = f.R.ReadFull((*d)[i:]) if err != nil { return err } for o > 0 { err = appendNext(f, d) if err != nil { return err } o-- } return nil } // MarshalJSON implements [json.Marshaler]. func (r *Raw) MarshalJSON() ([]byte, error) { var buf bytes.Buffer _, err := UnmarshalAsJSON(&buf, []byte(*r)) return buf.Bytes(), err } // ReadMapHeaderBytes reads a map header size // from 'b' and returns the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a map) func ReadMapHeaderBytes(b []byte) (sz uint32, o []byte, err error) { l := len(b) if l < 1 { err = ErrShortBytes return } lead := b[0] b = b[1:] if isfixmap(lead) { sz = uint32(rfixmap(lead)) o = b return } switch lead { case mmap16: if len(b) < 2 { err = ErrShortBytes return } sz = uint32(big.Uint16(b)) o = b[2:] return case mmap32: if len(b) < 4 { err = ErrShortBytes return } sz = big.Uint32(b) o = b[4:] return default: err = badPrefix(MapType, lead) return } } // ReadMapKeyZC attempts to read a map key // from 'b' and returns the key bytes and the remaining bytes // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a str or bin) func ReadMapKeyZC(b []byte) ([]byte, []byte, error) { o, x, err := ReadStringZC(b) if err != nil { if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { return ReadBytesZC(b) } return nil, b, err } return o, x, nil } // ReadArrayHeaderBytes attempts to read // the array header size off of 'b' and return // the size and remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not an array) func ReadArrayHeaderBytes(b []byte) (sz uint32, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes } lead := b[0] b = b[1:] if isfixarray(lead) { sz = uint32(rfixarray(lead)) o = b return } switch lead { case marray16: if len(b) < 2 { err = ErrShortBytes return } sz = uint32(big.Uint16(b)) o = b[2:] return case marray32: if len(b) < 4 { err = ErrShortBytes return } sz = big.Uint32(b) o = b[4:] return default: err = badPrefix(ArrayType, lead) return } } // ReadBytesHeader reads the 'bin' header size // off of 'b' and returns the size and remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a bin object) func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes } switch b[0] { case mbin8: if len(b) < 2 { err = ErrShortBytes return } sz = uint32(b[1]) o = b[2:] return case mbin16: if len(b) < 3 { err = ErrShortBytes return } sz = uint32(big.Uint16(b[1:])) o = b[3:] return case mbin32: if len(b) < 5 { err = ErrShortBytes return } sz = big.Uint32(b[1:]) o = b[5:] return default: err = badPrefix(BinType, b[0]) return } } // ReadNilBytes tries to read a "nil" byte // off of 'b' and return the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a 'nil') // - [InvalidPrefixError] func ReadNilBytes(b []byte) ([]byte, error) { if len(b) < 1 { return nil, ErrShortBytes } if b[0] != mnil { return b, badPrefix(NilType, b[0]) } return b[1:], nil } // ReadFloat64Bytes tries to read a float64 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a float64) func ReadFloat64Bytes(b []byte) (f float64, o []byte, err error) { if len(b) < 9 { if len(b) >= 5 && b[0] == mfloat32 { var tf float32 tf, o, err = ReadFloat32Bytes(b) f = float64(tf) return } err = ErrShortBytes return } if b[0] != mfloat64 { if b[0] == mfloat32 { var tf float32 tf, o, err = ReadFloat32Bytes(b) f = float64(tf) return } err = badPrefix(Float64Type, b[0]) return } f = math.Float64frombits(getMuint64(b)) o = b[9:] return } // ReadFloat32Bytes tries to read a float64 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a float32) func ReadFloat32Bytes(b []byte) (f float32, o []byte, err error) { if len(b) < 5 { err = ErrShortBytes return } if b[0] != mfloat32 { err = TypeError{Method: Float32Type, Encoded: getType(b[0])} return } f = math.Float32frombits(getMuint32(b)) o = b[5:] return } // ReadBoolBytes tries to read a float64 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a bool) func ReadBoolBytes(b []byte) (bool, []byte, error) { if len(b) < 1 { return false, b, ErrShortBytes } switch b[0] { case mtrue: return true, b[1:], nil case mfalse: return false, b[1:], nil default: return false, b, badPrefix(BoolType, b[0]) } } // ReadDurationBytes tries to read a time.Duration // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - TypeError (not a int) func ReadDurationBytes(b []byte) (d time.Duration, o []byte, err error) { i, o, err := ReadInt64Bytes(b) return time.Duration(i), o, err } // ReadInt64Bytes tries to read an int64 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a int) func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes } lead := b[0] b = b[1:] if isfixint(lead) { i = int64(rfixint(lead)) o = b return } if isnfixint(lead) { i = int64(rnfixint(lead)) o = b return } switch lead { case mint8: if len(b) < 1 { err = ErrShortBytes return } i = int64(int8(b[0])) o = b[1:] return case muint8: if len(b) < 1 { err = ErrShortBytes return } i = int64(b[0]) o = b[1:] return case mint16: if len(b) < 2 { err = ErrShortBytes return } i = int64(int16(big.Uint16(b))) o = b[2:] return case muint16: if len(b) < 2 { err = ErrShortBytes return } i = int64(big.Uint16(b)) o = b[2:] return case mint32: if len(b) < 4 { err = ErrShortBytes return } i = int64(int32(big.Uint32(b))) o = b[4:] return case muint32: if len(b) < 4 { err = ErrShortBytes return } i = int64(big.Uint32(b)) o = b[4:] return case mint64: if len(b) < 8 { err = ErrShortBytes return } i = int64(big.Uint64(b)) o = b[8:] return case muint64: if len(b) < 8 { err = ErrShortBytes return } u := big.Uint64(b) if u > math.MaxInt64 { err = UintOverflow{Value: u, FailedBitsize: 64} return } i = int64(u) o = b[8:] return default: err = badPrefix(IntType, lead) return } } // ReadInt32Bytes tries to read an int32 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a int) // - [IntOverflow] (value doesn't fit in int32) func ReadInt32Bytes(b []byte) (int32, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt32 || i < math.MinInt32 { return 0, o, IntOverflow{Value: i, FailedBitsize: 32} } return int32(i), o, err } // ReadInt16Bytes tries to read an int16 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a int) // - [IntOverflow] (value doesn't fit in int16) func ReadInt16Bytes(b []byte) (int16, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt16 || i < math.MinInt16 { return 0, o, IntOverflow{Value: i, FailedBitsize: 16} } return int16(i), o, err } // ReadInt8Bytes tries to read an int16 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a int) // - [IntOverflow] (value doesn't fit in int8) func ReadInt8Bytes(b []byte) (int8, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt8 || i < math.MinInt8 { return 0, o, IntOverflow{Value: i, FailedBitsize: 8} } return int8(i), o, err } // ReadIntBytes tries to read an int // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a int) // - [IntOverflow] (value doesn't fit in int; 32-bit platforms only) func ReadIntBytes(b []byte) (int, []byte, error) { if smallint { i, b, err := ReadInt32Bytes(b) return int(i), b, err } i, b, err := ReadInt64Bytes(b) return int(i), b, err } // ReadUint64Bytes tries to read a uint64 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a uint) func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes } lead := b[0] b = b[1:] if isfixint(lead) { u = uint64(rfixint(lead)) o = b return } switch lead { case mint8: if len(b) < 1 { err = ErrShortBytes return } v := int64(int8(b[0])) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) o = b[1:] return case muint8: if len(b) < 1 { err = ErrShortBytes return } u = uint64(b[0]) o = b[1:] return case mint16: if len(b) < 2 { err = ErrShortBytes return } v := int64((int16(b[0]) << 8) | int16(b[1])) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) o = b[2:] return case muint16: if len(b) < 2 { err = ErrShortBytes return } u = uint64(big.Uint16(b)) o = b[2:] return case mint32: if len(b) < 4 { err = ErrShortBytes return } v := int64(int32(big.Uint32(b))) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) o = b[4:] return case muint32: if len(b) < 4 { err = ErrShortBytes return } u = uint64(big.Uint32(b)) o = b[4:] return case mint64: if len(b) < 8 { err = ErrShortBytes return } v := int64(big.Uint64(b)) if v < 0 { err = UintBelowZero{Value: v} return } u = uint64(v) o = b[8:] return case muint64: if len(b) < 8 { err = ErrShortBytes return } u = big.Uint64(b) o = b[8:] return default: if isnfixint(lead) { err = UintBelowZero{Value: int64(rnfixint(lead))} } else { err = badPrefix(UintType, lead) } return } } // ReadUint32Bytes tries to read a uint32 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a uint) // - [UintOverflow] (value too large for uint32) func ReadUint32Bytes(b []byte) (uint32, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint32 { return 0, nil, UintOverflow{Value: v, FailedBitsize: 32} } return uint32(v), o, err } // ReadUint16Bytes tries to read a uint16 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a uint) // - [UintOverflow] (value too large for uint16) func ReadUint16Bytes(b []byte) (uint16, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint16 { return 0, nil, UintOverflow{Value: v, FailedBitsize: 16} } return uint16(v), o, err } // ReadUint8Bytes tries to read a uint8 // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a uint) // - [UintOverflow] (value too large for uint8) func ReadUint8Bytes(b []byte) (uint8, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint8 { return 0, nil, UintOverflow{Value: v, FailedBitsize: 8} } return uint8(v), o, err } // ReadUintBytes tries to read a uint // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a uint) // - [UintOverflow] (value too large for uint; 32-bit platforms only) func ReadUintBytes(b []byte) (uint, []byte, error) { if smallint { u, b, err := ReadUint32Bytes(b) return uint(u), b, err } u, b, err := ReadUint64Bytes(b) return uint(u), b, err } // ReadByteBytes is analogous to ReadUint8Bytes func ReadByteBytes(b []byte) (byte, []byte, error) { return ReadUint8Bytes(b) } // ReadBytesBytes reads a 'bin' object // from 'b' and returns its vaue and // the remaining bytes in 'b'. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - [TypeError] (not a 'bin' object) func ReadBytesBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { return readBytesBytes(b, scratch, false) } func readBytesBytes(b []byte, scratch []byte, zc bool) (v []byte, o []byte, err error) { l := len(b) if l < 1 { return nil, nil, ErrShortBytes } lead := b[0] b = b[1:] var read int switch lead { case mbin8: if len(b) < 1 { err = ErrShortBytes return } read = int(b[0]) b = b[1:] case mbin16: if len(b) < 2 { err = ErrShortBytes return } read = int(big.Uint16(b)) b = b[2:] case mbin32: if len(b) < 4 { err = ErrShortBytes return } read = int(big.Uint32(b)) b = b[4:] default: err = badPrefix(BinType, lead) return } if len(b) < read { err = ErrShortBytes return } // zero-copy if zc { v = b[0:read] o = b[read:] return } if cap(scratch) >= read { v = scratch[0:read] } else { v = make([]byte, read) } o = b[copy(v, b):] return } // ReadBytesZC extracts the messagepack-encoded // binary field without copying. The returned []byte // points to the same memory as the input slice. // // Possible errors: // // - [ErrShortBytes] (b not long enough) // - [TypeError] (object not 'bin') func ReadBytesZC(b []byte) (v []byte, o []byte, err error) { return readBytesBytes(b, nil, true) } func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { if len(b) < 1 { err = ErrShortBytes return } lead := b[0] var read uint32 b = b[1:] switch lead { case mbin8: if len(b) < 1 { err = ErrShortBytes return } read = uint32(b[0]) b = b[1:] case mbin16: if len(b) < 2 { err = ErrShortBytes return } read = uint32(big.Uint16(b)) b = b[2:] case mbin32: if len(b) < 4 { err = ErrShortBytes return } read = big.Uint32(b) b = b[4:] default: err = badPrefix(BinType, lead) return } if read != uint32(len(into)) { err = ArrayError{Wanted: uint32(len(into)), Got: read} return } o = b[copy(into, b):] return } // ReadStringZC reads a messagepack string field // without copying. The returned []byte points // to the same memory as the input slice. // // Possible errors: // // - [ErrShortBytes] (b not long enough) // - [TypeError] (object not 'str') func ReadStringZC(b []byte) (v []byte, o []byte, err error) { if len(b) < 1 { return nil, nil, ErrShortBytes } lead := b[0] var read int b = b[1:] if isfixstr(lead) { read = int(rfixstr(lead)) } else { switch lead { case mstr8: if len(b) < 1 { err = ErrShortBytes return } read = int(b[0]) b = b[1:] case mstr16: if len(b) < 2 { err = ErrShortBytes return } read = int(big.Uint16(b)) b = b[2:] case mstr32: if len(b) < 4 { err = ErrShortBytes return } read = int(big.Uint32(b)) b = b[4:] default: err = TypeError{Method: StrType, Encoded: getType(lead)} return } } if len(b) < read { err = ErrShortBytes return } v = b[0:read] o = b[read:] return } // ReadStringBytes reads a 'str' object // from 'b' and returns its value and the // remaining bytes in 'b'. // // Possible errors: // // - [ErrShortBytes] (b not long enough) // - [TypeError] (not 'str' type) // - [InvalidPrefixError] func ReadStringBytes(b []byte) (string, []byte, error) { v, o, err := ReadStringZC(b) return string(v), o, err } // ReadStringAsBytes reads a 'str' object // into a slice of bytes. 'v' is the value of // the 'str' object, which may reside in memory // pointed to by 'scratch.' 'o' is the remaining bytes // in 'b'. // // Possible errors: // // - [ErrShortBytes] (b not long enough) // - [TypeError] (not 'str' type) // - [InvalidPrefixError] (unknown type marker) func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { var tmp []byte tmp, o, err = ReadStringZC(b) v = append(scratch[:0], tmp...) return } // ReadComplex128Bytes reads a complex128 // extension object from 'b' and returns the // remaining bytes. // // Possible errors: // // - [ErrShortBytes] (not enough bytes in 'b') // - [TypeError] (object not a complex128) // - [InvalidPrefixError] // - [ExtensionTypeError] (object an extension of the correct size, but not a complex128) func ReadComplex128Bytes(b []byte) (c complex128, o []byte, err error) { if len(b) < 18 { err = ErrShortBytes return } if b[0] != mfixext16 { err = badPrefix(Complex128Type, b[0]) return } if int8(b[1]) != Complex128Extension { err = errExt(int8(b[1]), Complex128Extension) return } c = complex(math.Float64frombits(big.Uint64(b[2:])), math.Float64frombits(big.Uint64(b[10:]))) o = b[18:] return } // ReadComplex64Bytes reads a complex64 // extension object from 'b' and returns the // remaining bytes. // // Possible errors: // // - [ErrShortBytes] (not enough bytes in 'b') // - [TypeError] (object not a complex64) // - [ExtensionTypeError] (object an extension of the correct size, but not a complex64) func ReadComplex64Bytes(b []byte) (c complex64, o []byte, err error) { if len(b) < 10 { err = ErrShortBytes return } if b[0] != mfixext8 { err = badPrefix(Complex64Type, b[0]) return } if b[1] != Complex64Extension { err = errExt(int8(b[1]), Complex64Extension) return } c = complex(math.Float32frombits(big.Uint32(b[2:])), math.Float32frombits(big.Uint32(b[6:]))) o = b[10:] return } // ReadTimeBytes reads a time.Time // extension object from 'b' and returns the // remaining bytes. // Both the official and the format in this package will be read. // // Possible errors: // // - [ErrShortBytes] (not enough bytes in 'b') // - [TypeError] (object not a time extension 5 or -1) // - [ExtensionTypeError] (object an extension of the correct size, but not a time.Time) func ReadTimeBytes(b []byte) (t time.Time, o []byte, err error) { if len(b) < 6 { err = ErrShortBytes return } typ, o, b, err := readExt(b) if err != nil { return } switch typ { case TimeExtension: if len(b) != 12 { err = ErrShortBytes return } sec, nsec := getUnix(b) t = time.Unix(sec, int64(nsec)).Local() return case MsgTimeExtension: switch len(b) { case 4: t = time.Unix(int64(binary.BigEndian.Uint32(b)), 0).Local() return case 8: v := binary.BigEndian.Uint64(b) nanos := int64(v >> 34) if nanos > 999999999 { // In timestamp 64 and timestamp 96 formats, nanoseconds must not be larger than 999999999. err = InvalidTimestamp{Nanos: nanos} return } t = time.Unix(int64(v&(1<<34-1)), nanos).Local() return case 12: nanos := int64(binary.BigEndian.Uint32(b)) if nanos > 999999999 { // In timestamp 64 and timestamp 96 formats, nanoseconds must not be larger than 999999999. err = InvalidTimestamp{Nanos: nanos} return } ux := int64(binary.BigEndian.Uint64(b[4:])) t = time.Unix(ux, nanos).Local() return default: err = InvalidTimestamp{FieldLength: len(b)} return } default: err = errExt(int8(b[2]), TimeExtension) return } } // ReadMapStrIntfBytes reads a map[string]interface{} // out of 'b' and returns the map and remaining bytes. // If 'old' is non-nil, the values will be read into that map. func ReadMapStrIntfBytes(b []byte, old map[string]interface{}) (v map[string]interface{}, o []byte, err error) { return readMapStrIntfBytesDepth(b, old, 0) } func readMapStrIntfBytesDepth(b []byte, old map[string]interface{}, depth int) (v map[string]interface{}, o []byte, err error) { if depth >= recursionLimit { err = ErrRecursion return } var sz uint32 o = b sz, o, err = ReadMapHeaderBytes(o) if err != nil { return } if old != nil { for key := range old { delete(old, key) } v = old } else { v = make(map[string]interface{}, int(sz)) } for z := uint32(0); z < sz; z++ { if len(o) < 1 { err = ErrShortBytes return } var key []byte key, o, err = ReadMapKeyZC(o) if err != nil { return } var val interface{} val, o, err = readIntfBytesDepth(o, depth) if err != nil { return } v[string(key)] = val } return } // ReadIntfBytes attempts to read // the next object out of 'b' as a raw interface{} and // return the remaining bytes. func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) { return readIntfBytesDepth(b, 0) } func readIntfBytesDepth(b []byte, depth int) (i interface{}, o []byte, err error) { if depth >= recursionLimit { err = ErrRecursion return } if len(b) < 1 { err = ErrShortBytes return } k := NextType(b) switch k { case MapType: i, o, err = readMapStrIntfBytesDepth(b, nil, depth+1) return case ArrayType: var sz uint32 sz, o, err = ReadArrayHeaderBytes(b) if err != nil { return } j := make([]interface{}, int(sz)) i = j for d := range j { j[d], o, err = readIntfBytesDepth(o, depth+1) if err != nil { return } } return case Float32Type: i, o, err = ReadFloat32Bytes(b) return case Float64Type: i, o, err = ReadFloat64Bytes(b) return case IntType: i, o, err = ReadInt64Bytes(b) return case UintType: i, o, err = ReadUint64Bytes(b) return case BoolType: i, o, err = ReadBoolBytes(b) return case TimeType: i, o, err = ReadTimeBytes(b) return case Complex64Type: i, o, err = ReadComplex64Bytes(b) return case Complex128Type: i, o, err = ReadComplex128Bytes(b) return case ExtensionType: var t int8 t, err = peekExtension(b) if err != nil { return } // use a user-defined extension, // if it's been registered f, ok := extensionReg[t] if ok { e := f() o, err = ReadExtensionBytes(b, e) i = e return } // last resort is a raw extension e := RawExtension{} e.Type = int8(t) o, err = ReadExtensionBytes(b, &e) i = &e return case NilType: o, err = ReadNilBytes(b) return case BinType: i, o, err = ReadBytesBytes(b, nil) return case StrType: i, o, err = ReadStringBytes(b) return default: err = InvalidPrefixError(b[0]) return } } // Skip skips the next object in 'b' and // returns the remaining bytes. If the object // is a map or array, all of its elements // will be skipped. // // Possible errors: // // - [ErrShortBytes] (not enough bytes in b) // - [InvalidPrefixError] (bad encoding) // - [ErrRecursion] (too deeply nested data) func Skip(b []byte) ([]byte, error) { return skipDepth(b, 0) } func skipDepth(b []byte, depth int) ([]byte, error) { if depth >= recursionLimit { return b, ErrRecursion } sz, asz, err := getSize(b) if err != nil { return b, err } if uintptr(len(b)) < sz { return b, ErrShortBytes } b = b[sz:] for asz > 0 { b, err = skipDepth(b, depth+1) if err != nil { return b, err } asz-- } return b, nil } // returns (skip N bytes, skip M objects, error) func getSize(b []byte) (uintptr, uintptr, error) { l := len(b) if l == 0 { return 0, 0, ErrShortBytes } lead := b[0] spec := getBytespec(lead) // get type information size, mode := spec.size, spec.extra if size == 0 { return 0, 0, InvalidPrefixError(lead) } if mode >= 0 { // fixed composites return uintptr(size), uintptr(mode), nil } if l < int(size) { return 0, 0, ErrShortBytes } switch mode { case extra8: return uintptr(size) + uintptr(b[1]), 0, nil case extra16: return uintptr(size) + uintptr(big.Uint16(b[1:])), 0, nil case extra32: return uintptr(size) + uintptr(big.Uint32(b[1:])), 0, nil case map16v: return uintptr(size), 2 * uintptr(big.Uint16(b[1:])), nil case map32v: return uintptr(size), 2 * uintptr(big.Uint32(b[1:])), nil case array16v: return uintptr(size), uintptr(big.Uint16(b[1:])), nil case array32v: return uintptr(size), uintptr(big.Uint32(b[1:])), nil default: return 0, 0, fatal } } // ReadJSONNumberBytes tries to read a number // from 'b' and return the value and the remaining bytes. // // Possible errors: // // - [ErrShortBytes] (too few bytes) // - TypeError (not a number (int/float)) func ReadJSONNumberBytes(b []byte) (number json.Number, o []byte, err error) { if len(b) < 1 { return "", nil, ErrShortBytes } if i, o, err := ReadInt64Bytes(b); err == nil { return json.Number(strconv.FormatInt(i, 10)), o, nil } f, o, err := ReadFloat64Bytes(b) if err == nil { return json.Number(strconv.FormatFloat(f, 'f', -1, 64)), o, nil } return "", nil, TypeError{Method: NumberType, Encoded: getType(b[0])} } msgp-1.2.5/msgp/read_bytes_test.go000066400000000000000000000367421472554463600171760ustar00rootroot00000000000000package msgp import ( "bytes" "fmt" "log" "math" "reflect" "testing" "time" ) func TestReadMapHeaderBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []uint32{0, 1, 5, 49082} for i, v := range tests { buf.Reset() en.WriteMapHeader(v) en.Flush() out, left, err := ReadMapHeaderBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func BenchmarkReadMapHeaderBytes(b *testing.B) { sizes := []uint32{1, 100, tuint16, tuint32} buf := make([]byte, 0, 5*len(sizes)) for _, sz := range sizes { buf = AppendMapHeader(buf, sz) } b.SetBytes(int64(len(buf) / len(sizes))) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadMapHeaderBytes(buf) if len(buf) == 0 { buf = o } } } func TestReadArrayHeaderBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []uint32{0, 1, 5, 49082} for i, v := range tests { buf.Reset() en.WriteArrayHeader(v) en.Flush() out, left, err := ReadArrayHeaderBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func BenchmarkReadArrayHeaderBytes(b *testing.B) { sizes := []uint32{1, 100, tuint16, tuint32} buf := make([]byte, 0, 5*len(sizes)) for _, sz := range sizes { buf = AppendArrayHeader(buf, sz) } b.SetBytes(int64(len(buf) / len(sizes))) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadArrayHeaderBytes(buf) if len(buf) == 0 { buf = o } } } func TestReadBytesHeader(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []uint32{0, 1, 5, 49082, 1 << 16, math.MaxUint32} for i, v := range tests { buf.Reset() en.WriteBytesHeader(v) en.Flush() out, left, err := ReadBytesHeader(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func BenchmarkTestReadBytesHeader(b *testing.B) { sizes := []uint32{1, 100, tuint16, tuint32} buf := make([]byte, 0, 5*len(sizes)) for _, sz := range sizes { buf = AppendBytesHeader(buf, sz) } b.SetBytes(int64(len(buf) / len(sizes))) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadBytesHeader(buf) if len(buf) == 0 { buf = o } } } func TestReadNilBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteNil() en.Flush() left, err := ReadNilBytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } } func BenchmarkReadNilByte(b *testing.B) { buf := []byte{mnil} b.SetBytes(1) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadNilBytes(buf) } } func TestReadFloat64Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteFloat64(3.14159) en.Flush() out, left, err := ReadFloat64Bytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != 3.14159 { t.Errorf("%f in; %f out", 3.14159, out) } } func BenchmarkReadFloat64Bytes(b *testing.B) { f := float64(3.14159) buf := make([]byte, 0, 9) buf = AppendFloat64(buf, f) b.SetBytes(int64(len(buf))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadFloat64Bytes(buf) } } func TestReadFloat32Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteFloat32(3.1) en.Flush() out, left, err := ReadFloat32Bytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != 3.1 { t.Errorf("%f in; %f out", 3.1, out) } } func BenchmarkReadFloat32Bytes(b *testing.B) { f := float32(3.14159) buf := make([]byte, 0, 5) buf = AppendFloat32(buf, f) b.SetBytes(int64(len(buf))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadFloat32Bytes(buf) } } func TestReadBoolBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []bool{true, false} for i, v := range tests { buf.Reset() en.WriteBool(v) en.Flush() out, left, err := ReadBoolBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%t in; %t out", v, out) } } } func BenchmarkReadBoolBytes(b *testing.B) { buf := []byte{mtrue, mfalse, mtrue, mfalse} b.SetBytes(1) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadBoolBytes(buf) if len(buf) == 0 { buf = o } } } func TestReadInt64Bytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) ints := []int64{ -100000, -5000, -5, 0, 8, 240, int64(tuint16), int64(tuint32), int64(tuint64), -5, -30, 0, 1, 127, 300, 40921, 34908219, } uints := []uint64{0, 8, 240, uint64(tuint16), uint64(tuint32), uint64(tuint64)} all := make([]interface{}, 0, len(ints)+len(uints)) for _, v := range ints { all = append(all, v) } for _, v := range uints { all = append(all, v) } for i, num := range all { buf.Reset() var err error var in int64 switch num := num.(type) { case int64: err = wr.WriteInt64(num) in = num case uint64: err = wr.WriteUint64(num) in = int64(num) default: panic(num) } if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, left, err := ReadInt64Bytes(buf.Bytes()) if out != in { t.Errorf("Test case %d: put %d in and got %d out", i, num, in) } if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } } } func TestReadUint64Bytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) vs := []interface{}{ int64(0), int64(8), int64(240), int64(tuint16), int64(tuint32), int64(tuint64), uint64(0), uint64(8), uint64(240), uint64(tuint16), uint64(tuint32), uint64(tuint64), uint64(math.MaxUint64), } for i, num := range vs { buf.Reset() var err error var in uint64 switch num := num.(type) { case int64: err = wr.WriteInt64(num) in = uint64(num) case uint64: err = wr.WriteUint64(num) in = (num) default: panic(num) } if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, left, err := ReadUint64Bytes(buf.Bytes()) if out != in { t.Errorf("Test case %d: put %d in and got %d out", i, num, in) } if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } } } func TestReadIntBytesOverflows(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) i8, i16, i32, i64, u8, u16, u32, u64 := 1, 2, 3, 4, 5, 6, 7, 8 overflowErr := func(err error, failBits int) bool { bits := 0 switch err := err.(type) { case IntOverflow: bits = err.FailedBitsize case UintOverflow: bits = err.FailedBitsize } if bits == failBits { return true } log.Println("bits mismatch", bits, failBits) return false } belowZeroErr := func(err error, failBits int) bool { switch err.(type) { case UintBelowZero: return true } return false } vs := []struct { v interface{} rdBits int failBits int errCheck func(err error, failBits int) bool }{ {uint64(math.MaxInt64), i32, 32, overflowErr}, {uint64(math.MaxInt64), i16, 16, overflowErr}, {uint64(math.MaxInt64), i8, 8, overflowErr}, {uint64(math.MaxUint64), i64, 64, overflowErr}, {uint64(math.MaxUint64), i32, 64, overflowErr}, {uint64(math.MaxUint64), i16, 64, overflowErr}, {uint64(math.MaxUint64), i8, 64, overflowErr}, {uint64(math.MaxUint32), i32, 32, overflowErr}, {uint64(math.MaxUint32), i16, 16, overflowErr}, {uint64(math.MaxUint32), i8, 8, overflowErr}, {int64(math.MinInt64), u64, 64, belowZeroErr}, {int64(math.MinInt64), u32, 64, belowZeroErr}, {int64(math.MinInt64), u16, 64, belowZeroErr}, {int64(math.MinInt64), u8, 64, belowZeroErr}, {int64(math.MinInt32), u64, 64, belowZeroErr}, {int64(math.MinInt32), u32, 32, belowZeroErr}, {int64(math.MinInt32), u16, 16, belowZeroErr}, {int64(math.MinInt32), u8, 8, belowZeroErr}, {int64(math.MinInt16), u64, 64, belowZeroErr}, {int64(math.MinInt16), u32, 32, belowZeroErr}, {int64(math.MinInt16), u16, 16, belowZeroErr}, {int64(math.MinInt16), u8, 8, belowZeroErr}, {int64(math.MinInt8), u64, 64, belowZeroErr}, {int64(math.MinInt8), u32, 32, belowZeroErr}, {int64(math.MinInt8), u16, 16, belowZeroErr}, {int64(math.MinInt8), u8, 8, belowZeroErr}, {-1, u64, 64, belowZeroErr}, {-1, u32, 32, belowZeroErr}, {-1, u16, 16, belowZeroErr}, {-1, u8, 8, belowZeroErr}, } for i, v := range vs { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { buf.Reset() switch num := v.v.(type) { case int: wr.WriteInt64(int64(num)) case int64: wr.WriteInt64(num) case uint64: wr.WriteUint64(num) default: panic(num) } wr.Flush() var err error switch v.rdBits { case i64: _, _, err = ReadInt64Bytes(buf.Bytes()) case i32: _, _, err = ReadInt32Bytes(buf.Bytes()) case i16: _, _, err = ReadInt16Bytes(buf.Bytes()) case i8: _, _, err = ReadInt8Bytes(buf.Bytes()) case u64: _, _, err = ReadUint64Bytes(buf.Bytes()) case u32: _, _, err = ReadUint32Bytes(buf.Bytes()) case u16: _, _, err = ReadUint16Bytes(buf.Bytes()) case u8: _, _, err = ReadUint8Bytes(buf.Bytes()) } if !v.errCheck(err, v.failBits) { t.Fatal(err, v.rdBits, v.failBits) } }) } } func TestReadBytesBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := [][]byte{{}, []byte("some bytes"), []byte("some more bytes")} var scratch []byte for i, v := range tests { buf.Reset() en.WriteBytes(v) en.Flush() out, left, err := ReadBytesBytes(buf.Bytes(), scratch) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !bytes.Equal(out, v) { t.Errorf("%q in; %q out", v, out) } } } func TestReadZCBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := [][]byte{{}, []byte("some bytes"), []byte("some more bytes")} for i, v := range tests { buf.Reset() en.WriteBytes(v) en.Flush() out, left, err := ReadBytesZC(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !bytes.Equal(out, v) { t.Errorf("%q in; %q out", v, out) } } } func TestReadZCString(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []string{"", "hello", "here's another string......"} for i, v := range tests { buf.Reset() en.WriteString(v) en.Flush() out, left, err := ReadStringZC(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if string(out) != v { t.Errorf("%q in; %q out", v, out) } } } func TestReadStringBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []string{"", "hello", "here's another string......"} for i, v := range tests { buf.Reset() en.WriteString(v) en.Flush() out, left, err := ReadStringBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%q in; %q out", v, out) } } } func TestReadComplex128Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []complex128{complex(0, 0), complex(12.8, 32.0)} for i, v := range tests { buf.Reset() en.WriteComplex128(v) en.Flush() out, left, err := ReadComplex128Bytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%f in; %f out", v, out) } } } // both the 'str' and 'bin' types are acceptable map keys func TestReadMapKey(t *testing.T) { args := []string{ "a", "ab", "qwertyuiop", } var buf bytes.Buffer en := NewWriter(&buf) for i := range args { en.WriteString(args[i]) en.WriteBytes([]byte(args[i])) } en.Flush() b := buf.Bytes() for i := range args { var s0, s1 []byte var err error s0, b, err = ReadMapKeyZC(b) if err != nil { t.Fatalf("couldn't read string as map key: %q", err) } s1, b, err = ReadMapKeyZC(b) if err != nil { t.Fatalf("couldn't read bytes as map key: %q", err) } if !bytes.Equal(s0, s1) { t.Fatalf("%q != %q", s0, s1) } if string(s0) != args[i] { t.Fatalf("%q != %q", s0, args[i]) } } } func TestReadComplex64Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []complex64{complex(0, 0), complex(12.8, 32.0)} for i, v := range tests { buf.Reset() en.WriteComplex64(v) en.Flush() out, left, err := ReadComplex64Bytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%f in; %f out", v, out) } } } func TestReadTimeBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) now := time.Now() en.WriteTime(now) en.Flush() out, left, err := ReadTimeBytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !now.Equal(out) { t.Errorf("%s in; %s out", now, out) } } func BenchmarkReadTimeBytes(b *testing.B) { data := AppendTime(nil, time.Now()) b.SetBytes(15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadTimeBytes(data) } } func TestReadIntfBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := make([]interface{}, 0, 10) tests = append(tests, float64(3.5)) tests = append(tests, int64(-49082)) tests = append(tests, uint64(34908)) tests = append(tests, string("hello!")) tests = append(tests, []byte("blah.")) tests = append(tests, map[string]interface{}{ "key_one": 3.5, "key_two": "hi.", }) for i, v := range tests { buf.Reset() if err := en.WriteIntf(v); err != nil { t.Fatal(err) } en.Flush() out, left, err := ReadIntfBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !reflect.DeepEqual(v, out) { t.Errorf("ReadIntf(): %v in; %v out", v, out) } } } func BenchmarkSkipBytes(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(6) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(3.14159) en.WriteString("some_bytes") en.WriteBytes([]byte("nkl4321rqw908vxzpojnlk2314rqew098-s09123rdscasd")) en.WriteString("the_time") en.WriteTime(time.Now()) en.WriteString("what?") en.WriteBool(true) en.WriteString("ext") en.WriteExtension(&RawExtension{Type: 55, Data: []byte("raw data!!!")}) en.Flush() bts := buf.Bytes() b.SetBytes(int64(len(bts))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := Skip(bts) if err != nil { b.Fatal(err) } } } msgp-1.2.5/msgp/read_test.go000066400000000000000000000534341472554463600157650ustar00rootroot00000000000000package msgp import ( "bytes" "errors" "fmt" "io" "math" "math/rand" "reflect" "testing" "time" ) func TestSanity(t *testing.T) { if !isfixint(0) { t.Fatal("WUT.") } } func TestReadIntf(t *testing.T) { // NOTE: if you include cases // with, say, int32s, the test // will fail, b/c integers are // always read out as int64, and // unsigned integers as uint64 testCases := []interface{}{ float64(128.032), float32(9082.092), int64(-40), uint64(9082981), time.Now(), 48*time.Hour + 3*time.Minute + 2*time.Second + 3*time.Nanosecond, "hello!", []byte("hello!"), map[string]interface{}{ "thing-1": "thing-1-value", "thing-2": int64(800), "thing-3": []byte("some inner bytes..."), "thing-4": false, }, } var buf bytes.Buffer var v interface{} dec := NewReader(&buf) enc := NewWriter(&buf) for i, ts := range testCases { buf.Reset() err := enc.WriteIntf(ts) if err != nil { t.Errorf("Test case %d: %s", i, err) continue } err = enc.Flush() if err != nil { t.Fatal(err) } v, err = dec.ReadIntf() if err != nil { t.Errorf("Test case: %d: %s", i, err) } /* for time, use time.Equal instead of reflect.DeepEqual */ if tm, ok := v.(time.Time); ok { if !tm.Equal(v.(time.Time)) { t.Errorf("%v != %v", ts, v) } } else if intd, ok := ts.(time.Duration); ok { /* for time.Duration, cast before comparing */ outtd := time.Duration(v.(int64)) if intd != outtd { t.Errorf("%v in; %v out", intd, outtd) } } else if !reflect.DeepEqual(v, ts) { t.Errorf("%v in; %v out", ts, v) } } } func TestReadIntfRecursion(t *testing.T) { var buf bytes.Buffer dec := NewReader(&buf) enc := NewWriter(&buf) // Test array recursion... for i := 0; i < recursionLimit*2; i++ { enc.WriteArrayHeader(1) } enc.Flush() b := buf.Bytes() _, err := dec.ReadIntf() if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, _, err = ReadIntfBytes(b) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } // Test JSON dec.Reset(bytes.NewReader(b)) _, err = dec.WriteToJSON(io.Discard) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, err = UnmarshalAsJSON(io.Discard, b) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } _, err = CopyToJSON(io.Discard, bytes.NewReader(b)) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } // Test map recursion... buf.Reset() for i := 0; i < recursionLimit*2; i++ { enc.WriteMapHeader(1) // Write a key... enc.WriteString("a") } enc.Flush() b = buf.Bytes() dec.Reset(bytes.NewReader(b)) _, err = dec.ReadIntf() if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, _, err = ReadIntfBytes(b) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } // Test ReadMapStrInt using same input dec.Reset(bytes.NewReader(b)) err = dec.ReadMapStrIntf(map[string]interface{}{}) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, _, err = ReadMapStrIntfBytes(b, map[string]interface{}{}) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } // Test CopyNext dec.Reset(bytes.NewReader(b)) _, err = dec.CopyNext(io.Discard) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } // Test JSON dec.Reset(bytes.NewReader(b)) _, err = dec.WriteToJSON(io.Discard) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, err = UnmarshalAsJSON(io.Discard, b) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } _, err = CopyToJSON(io.Discard, bytes.NewReader(b)) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } } func TestSkipRecursion(t *testing.T) { var buf bytes.Buffer dec := NewReader(&buf) enc := NewWriter(&buf) // Test array recursion... for i := 0; i < recursionLimit*2; i++ { enc.WriteArrayHeader(1) } enc.Flush() b := buf.Bytes() err := dec.Skip() if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, err = Skip(b) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } buf.Reset() // Test map recursion... for i := 0; i < recursionLimit*2; i++ { enc.WriteMapHeader(1) // Write a key... enc.WriteString("a") } enc.Flush() b = buf.Bytes() err = dec.Skip() if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Reader error: %v", err) } _, err = Skip(b) if !errors.Is(err, ErrRecursion) { t.Errorf("unexpected Bytes error: %v", err) } } func TestReadMapHeader(t *testing.T) { tests := []struct { Sz uint32 }{ {0}, {1}, {tuint16}, {tuint32}, } var buf bytes.Buffer var sz uint32 var err error wr := NewWriter(&buf) rd := NewReader(&buf) for i, test := range tests { buf.Reset() err = wr.WriteMapHeader(test.Sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } sz, err = rd.ReadMapHeader() if err != nil { t.Errorf("Test case %d: got error %s", i, err) } if sz != test.Sz { t.Errorf("Test case %d: wrote size %d; got size %d", i, test.Sz, sz) } } } func BenchmarkReadMapHeader(b *testing.B) { sizes := []uint32{0, 1, tuint16, tuint32} data := make([]byte, 0, len(sizes)*5) for _, d := range sizes { data = AppendMapHeader(data, d) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(sizes))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { rd.ReadMapHeader() } } func TestReadArrayHeader(t *testing.T) { tests := []struct { Sz uint32 }{ {0}, {1}, {tuint16}, {tuint32}, } var buf bytes.Buffer var sz uint32 var err error wr := NewWriter(&buf) rd := NewReader(&buf) for i, test := range tests { buf.Reset() err = wr.WriteArrayHeader(test.Sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } sz, err = rd.ReadArrayHeader() if err != nil { t.Errorf("Test case %d: got error %s", i, err) } if sz != test.Sz { t.Errorf("Test case %d: wrote size %d; got size %d", i, test.Sz, sz) } } } func BenchmarkReadArrayHeader(b *testing.B) { sizes := []uint32{0, 1, tuint16, tuint32} data := make([]byte, 0, len(sizes)*5) for _, d := range sizes { data = AppendArrayHeader(data, d) } rd := NewReader(NewEndlessReader(data, b)) b.ReportAllocs() b.SetBytes(int64(len(data) / len(sizes))) b.ResetTimer() for i := 0; i < b.N; i++ { rd.ReadArrayHeader() } } func TestReadNil(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) wr.WriteNil() wr.Flush() err := rd.ReadNil() if err != nil { t.Fatal(err) } } func BenchmarkReadNil(b *testing.B) { data := AppendNil(nil) rd := NewReader(NewEndlessReader(data, b)) b.ReportAllocs() b.SetBytes(1) b.ResetTimer() for i := 0; i < b.N; i++ { err := rd.ReadNil() if err != nil { b.Fatal(err) } } } func TestReadFloat64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 100; i++ { buf.Reset() flt := (rand.Float64() - 0.5) * math.MaxFloat64 err := wr.WriteFloat64(flt) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadFloat64() if err != nil { t.Errorf("Error reading %f: %s", flt, err) continue } if out != flt { t.Errorf("Put in %f but got out %f", flt, out) } } } func BenchmarkReadFloat64(b *testing.B) { fs := []float64{rand.Float64(), rand.Float64(), rand.Float64(), rand.Float64()} data := make([]byte, 0, 9*len(fs)) for _, f := range fs { data = AppendFloat64(data, f) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadFloat64() if err != nil { b.Fatal(err) } } } func TestReadFloat32(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 10000; i++ { buf.Reset() flt := (rand.Float32() - 0.5) * math.MaxFloat32 err := wr.WriteFloat32(flt) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadFloat32() if err != nil { t.Errorf("Error reading %f: %s", flt, err) continue } if out != flt { t.Errorf("Put in %f but got out %f", flt, out) } } } func BenchmarkReadFloat32(b *testing.B) { fs := []float32{rand.Float32(), rand.Float32(), rand.Float32(), rand.Float32()} data := make([]byte, 0, 5*len(fs)) for _, f := range fs { data = AppendFloat32(data, f) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(5) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadFloat32() if err != nil { b.Fatal(err) } } } func TestReadInt64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) ints := []int64{-100000, -5000, -5, 0, 8, 240, int64(tuint16), int64(tuint32), int64(tuint64)} uints := []uint64{0, 8, 240, uint64(tuint16), uint64(tuint32), uint64(tuint64)} all := make([]interface{}, 0, len(ints)+len(uints)) for _, v := range ints { all = append(all, v) } for _, v := range uints { all = append(all, v) } for i, num := range all { buf.Reset() var err error var in int64 switch num := num.(type) { case int64: err = wr.WriteInt64(num) in = num case uint64: err = wr.WriteUint64(num) in = int64(num) default: panic(num) } if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadInt64() if err != nil { t.Fatal(err) } if out != in { t.Errorf("Test case %d: put %d in and got %d out", i, num, in) } } } func TestReadIntOverflows(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) i8, i16, i32, i64, u8, u16, u32, u64 := 1, 2, 3, 4, 5, 6, 7, 8 overflowErr := func(err error, failBits int) bool { bits := 0 switch err := err.(type) { case IntOverflow: bits = err.FailedBitsize case UintOverflow: bits = err.FailedBitsize } return bits == failBits } belowZeroErr := func(err error, failBits int) bool { switch err.(type) { case UintBelowZero: return true } return false } vs := []struct { v interface{} rdBits int failBits int errCheck func(err error, failBits int) bool }{ {uint64(math.MaxInt64), i32, 32, overflowErr}, {uint64(math.MaxInt64), i16, 16, overflowErr}, {uint64(math.MaxInt64), i8, 8, overflowErr}, {uint64(math.MaxUint64), i64, 64, overflowErr}, {uint64(math.MaxUint64), i32, 64, overflowErr}, {uint64(math.MaxUint64), i16, 64, overflowErr}, {uint64(math.MaxUint64), i8, 64, overflowErr}, {uint64(math.MaxUint32), i32, 32, overflowErr}, {uint64(math.MaxUint32), i16, 16, overflowErr}, {uint64(math.MaxUint32), i8, 8, overflowErr}, {int64(math.MinInt64), u64, 64, belowZeroErr}, {int64(math.MinInt64), u32, 64, belowZeroErr}, {int64(math.MinInt64), u16, 64, belowZeroErr}, {int64(math.MinInt64), u8, 64, belowZeroErr}, {int64(math.MinInt32), u64, 64, belowZeroErr}, {int64(math.MinInt32), u32, 32, belowZeroErr}, {int64(math.MinInt32), u16, 16, belowZeroErr}, {int64(math.MinInt32), u8, 8, belowZeroErr}, {int64(math.MinInt16), u64, 64, belowZeroErr}, {int64(math.MinInt16), u32, 32, belowZeroErr}, {int64(math.MinInt16), u16, 16, belowZeroErr}, {int64(math.MinInt16), u8, 8, belowZeroErr}, {int64(math.MinInt8), u64, 64, belowZeroErr}, {int64(math.MinInt8), u32, 32, belowZeroErr}, {int64(math.MinInt8), u16, 16, belowZeroErr}, {int64(math.MinInt8), u8, 8, belowZeroErr}, {-1, u64, 64, belowZeroErr}, {-1, u32, 32, belowZeroErr}, {-1, u16, 16, belowZeroErr}, {-1, u8, 8, belowZeroErr}, } for i, v := range vs { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { switch num := v.v.(type) { case int: wr.WriteInt64(int64(num)) case int64: wr.WriteInt64(num) case uint64: wr.WriteUint64(num) default: panic(num) } wr.Flush() var err error switch v.rdBits { case i64: _, err = rd.ReadInt64() case i32: _, err = rd.ReadInt32() case i16: _, err = rd.ReadInt16() case i8: _, err = rd.ReadInt8() case u64: _, err = rd.ReadUint64() case u32: _, err = rd.ReadUint32() case u16: _, err = rd.ReadUint16() case u8: _, err = rd.ReadUint8() } if !v.errCheck(err, v.failBits) { t.Fatal(err) } }) } } func BenchmarkReadInt64(b *testing.B) { is := []int64{0, 1, 65000, rand.Int63()} data := make([]byte, 0, 9*len(is)) for _, n := range is { data = AppendInt64(data, n) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(is))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadInt64() if err != nil { b.Fatal(err) } } } func BenchmarkReadUintWithInt64(b *testing.B) { us := []uint64{0, 1, 10000, uint64(rand.Uint32() * 4)} data := make([]byte, 0, 9*len(us)) for _, n := range us { data = AppendUint64(data, n) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(us))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadInt64() if err != nil { b.Fatal(err) } } } func TestReadUint64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) ints := []uint64{0, 8, 240, uint64(tuint16), uint64(tuint32), uint64(tuint64)} for i, num := range ints { buf.Reset() err := wr.WriteUint64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadUint64() if err != nil { t.Fatal(err) } if out != num { t.Errorf("Test case %d: put %d in and got %d out", i, num, out) } } } func BenchmarkReadUint64(b *testing.B) { us := []uint64{0, 1, 10000, uint64(rand.Uint32() * 4)} data := make([]byte, 0, 9*len(us)) for _, n := range us { data = AppendUint64(data, n) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(us))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadUint64() if err != nil { b.Fatal(err) } } } func BenchmarkReadIntWithUint64(b *testing.B) { is := []int64{0, 1, 65000, rand.Int63()} data := make([]byte, 0, 9*len(is)) for _, n := range is { data = AppendInt64(data, n) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(is))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadUint64() if err != nil { b.Fatal(err) } } } func TestReadBytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) sizes := []int{0, 1, 225, int(tuint32)} var scratch []byte for i, size := range sizes { buf.Reset() bts := RandBytes(size) err := wr.WriteBytes(bts) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadBytes(scratch) if err != nil { t.Errorf("test case %d: %s", i, err) continue } if !bytes.Equal(bts, out) { t.Errorf("test case %d: Bytes not equal.", i) } } } func benchBytes(size uint32, b *testing.B) { data := make([]byte, 0, size+5) data = AppendBytes(data, RandBytes(int(size))) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() var scratch []byte var err error for i := 0; i < b.N; i++ { scratch, err = rd.ReadBytes(scratch) if err != nil { b.Fatal(err) } } } func BenchmarkRead16Bytes(b *testing.B) { benchBytes(16, b) } func BenchmarkRead256Bytes(b *testing.B) { benchBytes(256, b) } // This particular case creates // an object larger than the default // read buffer size, so it's a decent // indicator of worst-case performance. func BenchmarkRead2048Bytes(b *testing.B) { benchBytes(2048, b) } func TestReadString(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) sizes := []int{0, 1, 225, int(math.MaxUint16 + 5)} for i, size := range sizes { buf.Reset() in := string(RandBytes(size)) err := wr.WriteString(in) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadString() if err != nil { t.Errorf("test case %d: %s", i, err) } if out != in { t.Errorf("test case %d: strings not equal.", i) t.Errorf("string (len = %d) in; string (len = %d) out", size, len(out)) } } } func benchString(size uint32, b *testing.B) { str := string(RandBytes(int(size))) data := make([]byte, 0, len(str)+5) data = AppendString(data, str) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadString() if err != nil { b.Fatal(err) } } } func benchStringAsBytes(size uint32, b *testing.B) { str := string(RandBytes(int(size))) data := make([]byte, 0, len(str)+5) data = AppendString(data, str) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() var scratch []byte var err error for i := 0; i < b.N; i++ { scratch, err = rd.ReadStringAsBytes(scratch) if err != nil { b.Fatal(err) } } } func BenchmarkRead16StringAsBytes(b *testing.B) { benchStringAsBytes(16, b) } func BenchmarkRead256StringAsBytes(b *testing.B) { benchStringAsBytes(256, b) } func BenchmarkRead16String(b *testing.B) { benchString(16, b) } func BenchmarkRead256String(b *testing.B) { benchString(256, b) } func TestReadComplex64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 100; i++ { buf.Reset() f := complex(rand.Float32()*math.MaxFloat32, rand.Float32()*math.MaxFloat32) wr.WriteComplex64(f) err := wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadComplex64() if err != nil { t.Error(err) continue } if out != f { t.Errorf("Wrote %f; read %f", f, out) } } } func BenchmarkReadComplex64(b *testing.B) { f := complex(rand.Float32(), rand.Float32()) data := AppendComplex64(nil, f) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadComplex64() if err != nil { b.Fatal(err) } } } func TestReadComplex128(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 10; i++ { buf.Reset() f := complex(rand.Float64()*math.MaxFloat64, rand.Float64()*math.MaxFloat64) wr.WriteComplex128(f) err := wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadComplex128() if err != nil { t.Error(err) continue } if out != f { t.Errorf("Wrote %f; read %f", f, out) } } } func BenchmarkReadComplex128(b *testing.B) { f := complex(rand.Float64(), rand.Float64()) data := AppendComplex128(nil, f) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadComplex128() if err != nil { b.Fatal(err) } } } func TestTime(t *testing.T) { var buf bytes.Buffer now := time.Now() en := NewWriter(&buf) dc := NewReader(&buf) err := en.WriteTime(now) if err != nil { t.Fatal(err) } err = en.Flush() if err != nil { t.Fatal(err) } out, err := dc.ReadTime() if err != nil { t.Fatal(err) } // check for equivalence if !now.Equal(out) { t.Fatalf("%s in; %s out", now, out) } } func BenchmarkReadTime(b *testing.B) { t := time.Now() data := AppendTime(nil, t) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadTime() if err != nil { b.Fatal(err) } } } func TestSkip(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) wr.WriteMapHeader(4) wr.WriteString("key_1") wr.WriteBytes([]byte("value_1")) wr.WriteString("key_2") wr.WriteFloat64(2.0) wr.WriteString("key_3") wr.WriteComplex128(3.0i) wr.WriteString("key_4") wr.WriteInt64(49080432189) wr.Flush() // this should skip the whole map err := rd.Skip() if err != nil { t.Fatal(err) } tp, err := rd.NextType() if err != io.EOF { t.Errorf("expected %q; got %q", io.EOF, err) t.Errorf("returned type %q", tp) } } func BenchmarkSkip(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(6) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(3.14159) en.WriteString("some_bytes") en.WriteBytes([]byte("nkl4321rqw908vxzpojnlk2314rqew098-s09123rdscasd")) en.WriteString("the_time") en.WriteTime(time.Now()) en.WriteString("what?") en.WriteBool(true) en.WriteString("ext") en.WriteExtension(&RawExtension{Type: 55, Data: []byte("raw data!!!")}) en.Flush() bts := buf.Bytes() b.SetBytes(int64(len(bts))) b.ReportAllocs() b.ResetTimer() rd := NewReader(NewEndlessReader(bts, b)) for i := 0; i < b.N; i++ { err := rd.Skip() if err != nil { b.Fatal(err) } } } func TestCopyNext(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(6) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(3.14159) en.WriteString("some_bytes") en.WriteBytes([]byte("nkl4321rqw908vxzpojnlk2314rqew098-s09123rdscasd")) en.WriteString("the_time") en.WriteTime(time.Now()) en.WriteString("what?") en.WriteBool(true) en.WriteString("ext") en.WriteExtension(&RawExtension{Type: 55, Data: []byte("raw data!!!")}) en.Flush() // Read from a copy of the original buf. de := NewReader(bytes.NewReader(buf.Bytes())) w := new(bytes.Buffer) n, err := de.CopyNext(w) if err != nil { t.Fatal(err) } if n != int64(buf.Len()) { t.Fatalf("CopyNext returned the wrong value (%d != %d)", n, buf.Len()) } if !bytes.Equal(buf.Bytes(), w.Bytes()) { t.Fatalf("not equal! %v, %v", buf.Bytes(), w.Bytes()) } } msgp-1.2.5/msgp/size.go000066400000000000000000000014731472554463600147610ustar00rootroot00000000000000package msgp // The sizes provided // are the worst-case // encoded sizes for // each type. For variable- // length types ([]byte, string), // the total encoded size is // the prefix size plus the // length of the object. const ( Int64Size = 9 IntSize = Int64Size UintSize = Int64Size Int8Size = 2 Int16Size = 3 Int32Size = 5 Uint8Size = 2 ByteSize = Uint8Size Uint16Size = 3 Uint32Size = 5 Uint64Size = Int64Size Float64Size = 9 Float32Size = 5 Complex64Size = 10 Complex128Size = 18 DurationSize = Int64Size TimeSize = 15 BoolSize = 1 NilSize = 1 JSONNumberSize = Int64Size // Same as Float64Size MapHeaderSize = 5 ArrayHeaderSize = 5 BytesPrefixSize = 5 StringPrefixSize = 5 ExtensionPrefixSize = 6 ) msgp-1.2.5/msgp/unsafe.go000066400000000000000000000016241472554463600152660ustar00rootroot00000000000000//go:build (!purego && !appengine) || (!appengine && purego && unsafe) // +build !purego,!appengine !appengine,purego,unsafe package msgp import ( "unsafe" ) // NOTE: // all of the definition in this file // should be repeated in appengine.go, // but without using unsafe const ( // spec says int and uint are always // the same size, but that int/uint // size may not be machine word size smallint = unsafe.Sizeof(int(0)) == 4 ) // UnsafeString returns the byte slice as a volatile string // THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. // THIS IS EVIL CODE. // YOU HAVE BEEN WARNED. func UnsafeString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } // UnsafeBytes returns the string as a byte slice // // Deprecated: // Since this code is no longer used by the code generator, // UnsafeBytes(s) is precisely equivalent to []byte(s) func UnsafeBytes(s string) []byte { return []byte(s) } msgp-1.2.5/msgp/write.go000066400000000000000000000503431472554463600151410ustar00rootroot00000000000000package msgp import ( "encoding/binary" "encoding/json" "errors" "io" "math" "reflect" "sync" "time" ) const ( // min buffer size for the writer minWriterSize = 18 ) // Sizer is an interface implemented // by types that can estimate their // size when MessagePack encoded. // This interface is optional, but // encoding/marshaling implementations // may use this as a way to pre-allocate // memory for serialization. type Sizer interface { Msgsize() int } var ( // Nowhere is an io.Writer to nowhere Nowhere io.Writer = nwhere{} btsType = reflect.TypeOf(([]byte)(nil)) writerPool = sync.Pool{ New: func() interface{} { return &Writer{buf: make([]byte, 2048)} }, } ) func popWriter(w io.Writer) *Writer { wr := writerPool.Get().(*Writer) wr.Reset(w) return wr } func pushWriter(wr *Writer) { wr.w = nil wr.wloc = 0 writerPool.Put(wr) } // freeW frees a writer for use // by other processes. It is not necessary // to call freeW on a writer. However, maintaining // a reference to a *Writer after calling freeW on // it will cause undefined behavior. func freeW(w *Writer) { pushWriter(w) } // Require ensures that cap(old)-len(old) >= extra. func Require(old []byte, extra int) []byte { l := len(old) c := cap(old) r := l + extra if c >= r { return old } else if l == 0 { return make([]byte, 0, extra) } // the new size is the greater // of double the old capacity // and the sum of the old length // and the number of new bytes // necessary. c <<= 1 if c < r { c = r } n := make([]byte, l, c) copy(n, old) return n } // nowhere writer type nwhere struct{} func (n nwhere) Write(p []byte) (int, error) { return len(p), nil } // Marshaler is the interface implemented // by types that know how to marshal themselves // as MessagePack. MarshalMsg appends the marshalled // form of the object to the provided // byte slice, returning the extended // slice and any errors encountered. type Marshaler interface { MarshalMsg([]byte) ([]byte, error) } // Encodable is the interface implemented // by types that know how to write themselves // as MessagePack using a *msgp.Writer. type Encodable interface { EncodeMsg(*Writer) error } // Writer is a buffered writer // that can be used to write // MessagePack objects to an io.Writer. // You must call *Writer.Flush() in order // to flush all of the buffered data // to the underlying writer. type Writer struct { w io.Writer buf []byte wloc int } // NewWriter returns a new *Writer. func NewWriter(w io.Writer) *Writer { if wr, ok := w.(*Writer); ok { return wr } return popWriter(w) } // NewWriterSize returns a writer with a custom buffer size. func NewWriterSize(w io.Writer, sz int) *Writer { // we must be able to require() 'minWriterSize' // contiguous bytes, so that is the // practical minimum buffer size if sz < minWriterSize { sz = minWriterSize } buf := make([]byte, sz) return NewWriterBuf(w, buf) } // NewWriterBuf returns a writer with a provided buffer. // 'buf' is not used when the capacity is smaller than 18, // custom buffer is allocated instead. func NewWriterBuf(w io.Writer, buf []byte) *Writer { if cap(buf) < minWriterSize { buf = make([]byte, minWriterSize) } buf = buf[:cap(buf)] return &Writer{ w: w, buf: buf, } } // Encode encodes an Encodable to an io.Writer. func Encode(w io.Writer, e Encodable) error { wr := NewWriter(w) err := e.EncodeMsg(wr) if err == nil { err = wr.Flush() } freeW(wr) return err } func (mw *Writer) flush() error { if mw.wloc == 0 { return nil } n, err := mw.w.Write(mw.buf[:mw.wloc]) if err != nil { if n > 0 { mw.wloc = copy(mw.buf, mw.buf[n:mw.wloc]) } return err } mw.wloc = 0 return nil } // Flush flushes all of the buffered // data to the underlying writer. func (mw *Writer) Flush() error { return mw.flush() } // Buffered returns the number bytes in the write buffer func (mw *Writer) Buffered() int { return len(mw.buf) - mw.wloc } func (mw *Writer) avail() int { return len(mw.buf) - mw.wloc } func (mw *Writer) bufsize() int { return len(mw.buf) } // NOTE: this should only be called with // a number that is guaranteed to be less than // len(mw.buf). typically, it is called with a constant. // // NOTE: this is a hot code path func (mw *Writer) require(n int) (int, error) { c := len(mw.buf) wl := mw.wloc if c-wl < n { if err := mw.flush(); err != nil { return 0, err } wl = mw.wloc } mw.wloc += n return wl, nil } func (mw *Writer) Append(b ...byte) error { if mw.avail() < len(b) { err := mw.flush() if err != nil { return err } } mw.wloc += copy(mw.buf[mw.wloc:], b) return nil } // push one byte onto the buffer // // NOTE: this is a hot code path func (mw *Writer) push(b byte) error { if mw.wloc == len(mw.buf) { if err := mw.flush(); err != nil { return err } } mw.buf[mw.wloc] = b mw.wloc++ return nil } func (mw *Writer) prefix8(b byte, u uint8) error { const need = 2 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu8(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } func (mw *Writer) prefix16(b byte, u uint16) error { const need = 3 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu16(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } func (mw *Writer) prefix32(b byte, u uint32) error { const need = 5 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu32(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } func (mw *Writer) prefix64(b byte, u uint64) error { const need = 9 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu64(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } // Write implements io.Writer, and writes // data directly to the buffer. func (mw *Writer) Write(p []byte) (int, error) { l := len(p) if mw.avail() < l { if err := mw.flush(); err != nil { return 0, err } if l > len(mw.buf) { return mw.w.Write(p) } } mw.wloc += copy(mw.buf[mw.wloc:], p) return l, nil } // implements io.WriteString func (mw *Writer) writeString(s string) error { l := len(s) if mw.avail() < l { if err := mw.flush(); err != nil { return err } if l > len(mw.buf) { _, err := io.WriteString(mw.w, s) return err } } mw.wloc += copy(mw.buf[mw.wloc:], s) return nil } // Reset changes the underlying writer used by the Writer func (mw *Writer) Reset(w io.Writer) { mw.buf = mw.buf[:cap(mw.buf)] mw.w = w mw.wloc = 0 } // WriteMapHeader writes a map header of the given // size to the writer func (mw *Writer) WriteMapHeader(sz uint32) error { switch { case sz <= 15: return mw.push(wfixmap(uint8(sz))) case sz <= math.MaxUint16: return mw.prefix16(mmap16, uint16(sz)) default: return mw.prefix32(mmap32, sz) } } // WriteArrayHeader writes an array header of the // given size to the writer func (mw *Writer) WriteArrayHeader(sz uint32) error { switch { case sz <= 15: return mw.push(wfixarray(uint8(sz))) case sz <= math.MaxUint16: return mw.prefix16(marray16, uint16(sz)) default: return mw.prefix32(marray32, sz) } } // WriteNil writes a nil byte to the buffer func (mw *Writer) WriteNil() error { return mw.push(mnil) } // WriteFloat writes a float to the writer as either float64 // or float32 when it represents the exact same value func (mw *Writer) WriteFloat(f float64) error { f32 := float32(f) if float64(f32) == f { return mw.prefix32(mfloat32, math.Float32bits(f32)) } return mw.prefix64(mfloat64, math.Float64bits(f)) } // WriteFloat64 writes a float64 to the writer func (mw *Writer) WriteFloat64(f float64) error { return mw.prefix64(mfloat64, math.Float64bits(f)) } // WriteFloat32 writes a float32 to the writer func (mw *Writer) WriteFloat32(f float32) error { return mw.prefix32(mfloat32, math.Float32bits(f)) } // WriteDuration writes a time.Duration to the writer func (mw *Writer) WriteDuration(d time.Duration) error { return mw.WriteInt64(int64(d)) } // WriteInt64 writes an int64 to the writer func (mw *Writer) WriteInt64(i int64) error { if i >= 0 { switch { case i <= math.MaxInt8: return mw.push(wfixint(uint8(i))) case i <= math.MaxInt16: return mw.prefix16(mint16, uint16(i)) case i <= math.MaxInt32: return mw.prefix32(mint32, uint32(i)) default: return mw.prefix64(mint64, uint64(i)) } } switch { case i >= -32: return mw.push(wnfixint(int8(i))) case i >= math.MinInt8: return mw.prefix8(mint8, uint8(i)) case i >= math.MinInt16: return mw.prefix16(mint16, uint16(i)) case i >= math.MinInt32: return mw.prefix32(mint32, uint32(i)) default: return mw.prefix64(mint64, uint64(i)) } } // WriteInt8 writes an int8 to the writer func (mw *Writer) WriteInt8(i int8) error { return mw.WriteInt64(int64(i)) } // WriteInt16 writes an int16 to the writer func (mw *Writer) WriteInt16(i int16) error { return mw.WriteInt64(int64(i)) } // WriteInt32 writes an int32 to the writer func (mw *Writer) WriteInt32(i int32) error { return mw.WriteInt64(int64(i)) } // WriteInt writes an int to the writer func (mw *Writer) WriteInt(i int) error { return mw.WriteInt64(int64(i)) } // WriteUint64 writes a uint64 to the writer func (mw *Writer) WriteUint64(u uint64) error { switch { case u <= (1<<7)-1: return mw.push(wfixint(uint8(u))) case u <= math.MaxUint8: return mw.prefix8(muint8, uint8(u)) case u <= math.MaxUint16: return mw.prefix16(muint16, uint16(u)) case u <= math.MaxUint32: return mw.prefix32(muint32, uint32(u)) default: return mw.prefix64(muint64, u) } } // WriteByte is analogous to WriteUint8 func (mw *Writer) WriteByte(u byte) error { return mw.WriteUint8(uint8(u)) } // WriteUint8 writes a uint8 to the writer func (mw *Writer) WriteUint8(u uint8) error { return mw.WriteUint64(uint64(u)) } // WriteUint16 writes a uint16 to the writer func (mw *Writer) WriteUint16(u uint16) error { return mw.WriteUint64(uint64(u)) } // WriteUint32 writes a uint32 to the writer func (mw *Writer) WriteUint32(u uint32) error { return mw.WriteUint64(uint64(u)) } // WriteUint writes a uint to the writer func (mw *Writer) WriteUint(u uint) error { return mw.WriteUint64(uint64(u)) } // WriteBytes writes binary as 'bin' to the writer func (mw *Writer) WriteBytes(b []byte) error { sz := uint32(len(b)) var err error switch { case sz <= math.MaxUint8: err = mw.prefix8(mbin8, uint8(sz)) case sz <= math.MaxUint16: err = mw.prefix16(mbin16, uint16(sz)) default: err = mw.prefix32(mbin32, sz) } if err != nil { return err } _, err = mw.Write(b) return err } // WriteBytesHeader writes just the size header // of a MessagePack 'bin' object. The user is responsible // for then writing 'sz' more bytes into the stream. func (mw *Writer) WriteBytesHeader(sz uint32) error { switch { case sz <= math.MaxUint8: return mw.prefix8(mbin8, uint8(sz)) case sz <= math.MaxUint16: return mw.prefix16(mbin16, uint16(sz)) default: return mw.prefix32(mbin32, sz) } } // WriteBool writes a bool to the writer func (mw *Writer) WriteBool(b bool) error { if b { return mw.push(mtrue) } return mw.push(mfalse) } // WriteString writes a messagepack string to the writer. // (This is NOT an implementation of io.StringWriter) func (mw *Writer) WriteString(s string) error { sz := uint32(len(s)) var err error switch { case sz <= 31: err = mw.push(wfixstr(uint8(sz))) case sz <= math.MaxUint8: err = mw.prefix8(mstr8, uint8(sz)) case sz <= math.MaxUint16: err = mw.prefix16(mstr16, uint16(sz)) default: err = mw.prefix32(mstr32, sz) } if err != nil { return err } return mw.writeString(s) } // WriteStringHeader writes just the string size // header of a MessagePack 'str' object. The user // is responsible for writing 'sz' more valid UTF-8 // bytes to the stream. func (mw *Writer) WriteStringHeader(sz uint32) error { switch { case sz <= 31: return mw.push(wfixstr(uint8(sz))) case sz <= math.MaxUint8: return mw.prefix8(mstr8, uint8(sz)) case sz <= math.MaxUint16: return mw.prefix16(mstr16, uint16(sz)) default: return mw.prefix32(mstr32, sz) } } // WriteStringFromBytes writes a 'str' object // from a []byte. func (mw *Writer) WriteStringFromBytes(str []byte) error { sz := uint32(len(str)) var err error switch { case sz <= 31: err = mw.push(wfixstr(uint8(sz))) case sz <= math.MaxUint8: err = mw.prefix8(mstr8, uint8(sz)) case sz <= math.MaxUint16: err = mw.prefix16(mstr16, uint16(sz)) default: err = mw.prefix32(mstr32, sz) } if err != nil { return err } _, err = mw.Write(str) return err } // WriteComplex64 writes a complex64 to the writer func (mw *Writer) WriteComplex64(f complex64) error { o, err := mw.require(10) if err != nil { return err } mw.buf[o] = mfixext8 mw.buf[o+1] = Complex64Extension big.PutUint32(mw.buf[o+2:], math.Float32bits(real(f))) big.PutUint32(mw.buf[o+6:], math.Float32bits(imag(f))) return nil } // WriteComplex128 writes a complex128 to the writer func (mw *Writer) WriteComplex128(f complex128) error { o, err := mw.require(18) if err != nil { return err } mw.buf[o] = mfixext16 mw.buf[o+1] = Complex128Extension big.PutUint64(mw.buf[o+2:], math.Float64bits(real(f))) big.PutUint64(mw.buf[o+10:], math.Float64bits(imag(f))) return nil } // WriteMapStrStr writes a map[string]string to the writer func (mw *Writer) WriteMapStrStr(mp map[string]string) (err error) { err = mw.WriteMapHeader(uint32(len(mp))) if err != nil { return } for key, val := range mp { err = mw.WriteString(key) if err != nil { return } err = mw.WriteString(val) if err != nil { return } } return nil } // WriteMapStrIntf writes a map[string]interface to the writer func (mw *Writer) WriteMapStrIntf(mp map[string]interface{}) (err error) { err = mw.WriteMapHeader(uint32(len(mp))) if err != nil { return } for key, val := range mp { err = mw.WriteString(key) if err != nil { return } err = mw.WriteIntf(val) if err != nil { return } } return } // WriteTime writes a time.Time object to the wire. // // Time is encoded as Unix time, which means that // location (time zone) data is removed from the object. // The encoded object itself is 12 bytes: 8 bytes for // a big-endian 64-bit integer denoting seconds // elapsed since "zero" Unix time, followed by 4 bytes // for a big-endian 32-bit signed integer denoting // the nanosecond offset of the time. This encoding // is intended to ease portability across languages. // (Note that this is *not* the standard time.Time // binary encoding, because its implementation relies // heavily on the internal representation used by the // time package.) func (mw *Writer) WriteTime(t time.Time) error { t = t.UTC() o, err := mw.require(15) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = 12 mw.buf[o+2] = TimeExtension putUnix(mw.buf[o+3:], t.Unix(), int32(t.Nanosecond())) return nil } // WriteTimeExt will write t using the official msgpack extension spec. // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type func (mw *Writer) WriteTimeExt(t time.Time) error { // Time rounded towards zero. secPrec := t.Truncate(time.Second) remain := t.Sub(secPrec).Nanoseconds() asSecs := secPrec.Unix() switch { case remain == 0 && asSecs > 0 && asSecs <= math.MaxUint32: // 4 bytes o, err := mw.require(6) if err != nil { return err } mw.buf[o] = mfixext4 mw.buf[o+1] = byte(msgTimeExtension) binary.BigEndian.PutUint32(mw.buf[o+2:], uint32(asSecs)) return nil case asSecs < 0 || asSecs >= (1<<34): // 12 bytes o, err := mw.require(12 + 3) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = 12 mw.buf[o+2] = byte(msgTimeExtension) binary.BigEndian.PutUint32(mw.buf[o+3:], uint32(remain)) binary.BigEndian.PutUint64(mw.buf[o+3+4:], uint64(asSecs)) default: // 8 bytes o, err := mw.require(10) if err != nil { return err } mw.buf[o] = mfixext8 mw.buf[o+1] = byte(msgTimeExtension) binary.BigEndian.PutUint64(mw.buf[o+2:], uint64(asSecs)|(uint64(remain)<<34)) } return nil } // WriteJSONNumber writes the json.Number to the stream as either integer or float. func (mw *Writer) WriteJSONNumber(n json.Number) error { if n == "" { // The zero value outputs the 0 integer. return mw.push(0) } ii, err := n.Int64() if err == nil { return mw.WriteInt64(ii) } ff, err := n.Float64() if err == nil { return mw.WriteFloat(ff) } return err } // WriteIntf writes the concrete type of 'v'. // WriteIntf will error if 'v' is not one of the following: // - A bool, float, string, []byte, int, uint, or complex // - A map of supported types (with string keys) // - An array or slice of supported types // - A pointer to a supported type // - A type that satisfies the msgp.Encodable interface // - A type that satisfies the msgp.Extension interface func (mw *Writer) WriteIntf(v interface{}) error { if v == nil { return mw.WriteNil() } switch v := v.(type) { // preferred interfaces case Encodable: return v.EncodeMsg(mw) case Extension: return mw.WriteExtension(v) // concrete types case bool: return mw.WriteBool(v) case float32: return mw.WriteFloat32(v) case float64: return mw.WriteFloat64(v) case complex64: return mw.WriteComplex64(v) case complex128: return mw.WriteComplex128(v) case uint8: return mw.WriteUint8(v) case uint16: return mw.WriteUint16(v) case uint32: return mw.WriteUint32(v) case uint64: return mw.WriteUint64(v) case uint: return mw.WriteUint(v) case int8: return mw.WriteInt8(v) case int16: return mw.WriteInt16(v) case int32: return mw.WriteInt32(v) case int64: return mw.WriteInt64(v) case int: return mw.WriteInt(v) case string: return mw.WriteString(v) case []byte: return mw.WriteBytes(v) case map[string]string: return mw.WriteMapStrStr(v) case map[string]interface{}: return mw.WriteMapStrIntf(v) case time.Time: return mw.WriteTime(v) case time.Duration: return mw.WriteDuration(v) case json.Number: return mw.WriteJSONNumber(v) } val := reflect.ValueOf(v) if !isSupported(val.Kind()) || !val.IsValid() { return errors.New("msgp: type " + val.String() + " not supported") } switch val.Kind() { case reflect.Ptr: if val.IsNil() { return mw.WriteNil() } return mw.WriteIntf(val.Elem().Interface()) case reflect.Slice: return mw.writeSlice(val) case reflect.Map: return mw.writeMap(val) } return &ErrUnsupportedType{T: val.Type()} } func (mw *Writer) writeMap(v reflect.Value) (err error) { if v.Type().Key().Kind() != reflect.String { return errors.New("msgp: map keys must be strings") } ks := v.MapKeys() err = mw.WriteMapHeader(uint32(len(ks))) if err != nil { return } for _, key := range ks { val := v.MapIndex(key) err = mw.WriteString(key.String()) if err != nil { return } err = mw.WriteIntf(val.Interface()) if err != nil { return } } return } func (mw *Writer) writeSlice(v reflect.Value) (err error) { // is []byte if v.Type().ConvertibleTo(btsType) { return mw.WriteBytes(v.Bytes()) } sz := uint32(v.Len()) err = mw.WriteArrayHeader(sz) if err != nil { return } for i := uint32(0); i < sz; i++ { err = mw.WriteIntf(v.Index(int(i)).Interface()) if err != nil { return } } return } // is the reflect.Kind encodable? func isSupported(k reflect.Kind) bool { switch k { case reflect.Func, reflect.Chan, reflect.Invalid, reflect.UnsafePointer: return false default: return true } } // GuessSize guesses the size of the underlying // value of 'i'. If the underlying value is not // a simple builtin (or []byte), GuessSize defaults // to 512. func GuessSize(i interface{}) int { if i == nil { return NilSize } switch i := i.(type) { case Sizer: return i.Msgsize() case Extension: return ExtensionPrefixSize + i.Len() case float64: return Float64Size case float32: return Float32Size case uint8, uint16, uint32, uint64, uint: return UintSize case int8, int16, int32, int64, int: return IntSize case []byte: return BytesPrefixSize + len(i) case string: return StringPrefixSize + len(i) case complex64: return Complex64Size case complex128: return Complex128Size case bool: return BoolSize case map[string]interface{}: s := MapHeaderSize for key, val := range i { s += StringPrefixSize + len(key) + GuessSize(val) } return s case map[string]string: s := MapHeaderSize for key, val := range i { s += 2*StringPrefixSize + len(key) + len(val) } return s default: return 512 } } msgp-1.2.5/msgp/write_bytes.go000066400000000000000000000307121472554463600163450ustar00rootroot00000000000000package msgp import ( "encoding/binary" "encoding/json" "errors" "math" "reflect" "time" ) // ensure 'sz' extra bytes in 'b' btw len(b) and cap(b) func ensure(b []byte, sz int) ([]byte, int) { l := len(b) c := cap(b) if c-l < sz { o := make([]byte, (2*c)+sz) // exponential growth n := copy(o, b) return o[:n+sz], n } return b[:l+sz], l } // AppendMapHeader appends a map header with the // given size to the slice func AppendMapHeader(b []byte, sz uint32) []byte { switch { case sz <= 15: return append(b, wfixmap(uint8(sz))) case sz <= math.MaxUint16: o, n := ensure(b, 3) prefixu16(o[n:], mmap16, uint16(sz)) return o default: o, n := ensure(b, 5) prefixu32(o[n:], mmap32, sz) return o } } // AppendArrayHeader appends an array header with // the given size to the slice func AppendArrayHeader(b []byte, sz uint32) []byte { switch { case sz <= 15: return append(b, wfixarray(uint8(sz))) case sz <= math.MaxUint16: o, n := ensure(b, 3) prefixu16(o[n:], marray16, uint16(sz)) return o default: o, n := ensure(b, 5) prefixu32(o[n:], marray32, sz) return o } } // AppendNil appends a 'nil' byte to the slice func AppendNil(b []byte) []byte { return append(b, mnil) } // AppendFloat appends a float to the slice as either float64 // or float32 when it represents the exact same value func AppendFloat(b []byte, f float64) []byte { f32 := float32(f) if float64(f32) == f { return AppendFloat32(b, f32) } return AppendFloat64(b, f) } // AppendFloat64 appends a float64 to the slice func AppendFloat64(b []byte, f float64) []byte { o, n := ensure(b, Float64Size) prefixu64(o[n:], mfloat64, math.Float64bits(f)) return o } // AppendFloat32 appends a float32 to the slice func AppendFloat32(b []byte, f float32) []byte { o, n := ensure(b, Float32Size) prefixu32(o[n:], mfloat32, math.Float32bits(f)) return o } // AppendDuration appends a time.Duration to the slice func AppendDuration(b []byte, d time.Duration) []byte { return AppendInt64(b, int64(d)) } // AppendInt64 appends an int64 to the slice func AppendInt64(b []byte, i int64) []byte { if i >= 0 { switch { case i <= math.MaxInt8: return append(b, wfixint(uint8(i))) case i <= math.MaxInt16: o, n := ensure(b, 3) putMint16(o[n:], int16(i)) return o case i <= math.MaxInt32: o, n := ensure(b, 5) putMint32(o[n:], int32(i)) return o default: o, n := ensure(b, 9) putMint64(o[n:], i) return o } } switch { case i >= -32: return append(b, wnfixint(int8(i))) case i >= math.MinInt8: o, n := ensure(b, 2) putMint8(o[n:], int8(i)) return o case i >= math.MinInt16: o, n := ensure(b, 3) putMint16(o[n:], int16(i)) return o case i >= math.MinInt32: o, n := ensure(b, 5) putMint32(o[n:], int32(i)) return o default: o, n := ensure(b, 9) putMint64(o[n:], i) return o } } // AppendInt appends an int to the slice func AppendInt(b []byte, i int) []byte { return AppendInt64(b, int64(i)) } // AppendInt8 appends an int8 to the slice func AppendInt8(b []byte, i int8) []byte { return AppendInt64(b, int64(i)) } // AppendInt16 appends an int16 to the slice func AppendInt16(b []byte, i int16) []byte { return AppendInt64(b, int64(i)) } // AppendInt32 appends an int32 to the slice func AppendInt32(b []byte, i int32) []byte { return AppendInt64(b, int64(i)) } // AppendUint64 appends a uint64 to the slice func AppendUint64(b []byte, u uint64) []byte { switch { case u <= (1<<7)-1: return append(b, wfixint(uint8(u))) case u <= math.MaxUint8: o, n := ensure(b, 2) putMuint8(o[n:], uint8(u)) return o case u <= math.MaxUint16: o, n := ensure(b, 3) putMuint16(o[n:], uint16(u)) return o case u <= math.MaxUint32: o, n := ensure(b, 5) putMuint32(o[n:], uint32(u)) return o default: o, n := ensure(b, 9) putMuint64(o[n:], u) return o } } // AppendUint appends a uint to the slice func AppendUint(b []byte, u uint) []byte { return AppendUint64(b, uint64(u)) } // AppendUint8 appends a uint8 to the slice func AppendUint8(b []byte, u uint8) []byte { return AppendUint64(b, uint64(u)) } // AppendByte is analogous to AppendUint8 func AppendByte(b []byte, u byte) []byte { return AppendUint8(b, uint8(u)) } // AppendUint16 appends a uint16 to the slice func AppendUint16(b []byte, u uint16) []byte { return AppendUint64(b, uint64(u)) } // AppendUint32 appends a uint32 to the slice func AppendUint32(b []byte, u uint32) []byte { return AppendUint64(b, uint64(u)) } // AppendBytes appends bytes to the slice as MessagePack 'bin' data func AppendBytes(b []byte, bts []byte) []byte { sz := len(bts) var o []byte var n int switch { case sz <= math.MaxUint8: o, n = ensure(b, 2+sz) prefixu8(o[n:], mbin8, uint8(sz)) n += 2 case sz <= math.MaxUint16: o, n = ensure(b, 3+sz) prefixu16(o[n:], mbin16, uint16(sz)) n += 3 default: o, n = ensure(b, 5+sz) prefixu32(o[n:], mbin32, uint32(sz)) n += 5 } return o[:n+copy(o[n:], bts)] } // AppendBytesHeader appends an 'bin' header with // the given size to the slice. func AppendBytesHeader(b []byte, sz uint32) []byte { var o []byte var n int switch { case sz <= math.MaxUint8: o, n = ensure(b, 2) prefixu8(o[n:], mbin8, uint8(sz)) return o case sz <= math.MaxUint16: o, n = ensure(b, 3) prefixu16(o[n:], mbin16, uint16(sz)) return o } o, n = ensure(b, 5) prefixu32(o[n:], mbin32, sz) return o } // AppendBool appends a bool to the slice func AppendBool(b []byte, t bool) []byte { if t { return append(b, mtrue) } return append(b, mfalse) } // AppendString appends a string as a MessagePack 'str' to the slice func AppendString(b []byte, s string) []byte { sz := len(s) var n int var o []byte switch { case sz <= 31: o, n = ensure(b, 1+sz) o[n] = wfixstr(uint8(sz)) n++ case sz <= math.MaxUint8: o, n = ensure(b, 2+sz) prefixu8(o[n:], mstr8, uint8(sz)) n += 2 case sz <= math.MaxUint16: o, n = ensure(b, 3+sz) prefixu16(o[n:], mstr16, uint16(sz)) n += 3 default: o, n = ensure(b, 5+sz) prefixu32(o[n:], mstr32, uint32(sz)) n += 5 } return o[:n+copy(o[n:], s)] } // AppendStringFromBytes appends a []byte // as a MessagePack 'str' to the slice 'b.' func AppendStringFromBytes(b []byte, str []byte) []byte { sz := len(str) var n int var o []byte switch { case sz <= 31: o, n = ensure(b, 1+sz) o[n] = wfixstr(uint8(sz)) n++ case sz <= math.MaxUint8: o, n = ensure(b, 2+sz) prefixu8(o[n:], mstr8, uint8(sz)) n += 2 case sz <= math.MaxUint16: o, n = ensure(b, 3+sz) prefixu16(o[n:], mstr16, uint16(sz)) n += 3 default: o, n = ensure(b, 5+sz) prefixu32(o[n:], mstr32, uint32(sz)) n += 5 } return o[:n+copy(o[n:], str)] } // AppendComplex64 appends a complex64 to the slice as a MessagePack extension func AppendComplex64(b []byte, c complex64) []byte { o, n := ensure(b, Complex64Size) o[n] = mfixext8 o[n+1] = Complex64Extension big.PutUint32(o[n+2:], math.Float32bits(real(c))) big.PutUint32(o[n+6:], math.Float32bits(imag(c))) return o } // AppendComplex128 appends a complex128 to the slice as a MessagePack extension func AppendComplex128(b []byte, c complex128) []byte { o, n := ensure(b, Complex128Size) o[n] = mfixext16 o[n+1] = Complex128Extension big.PutUint64(o[n+2:], math.Float64bits(real(c))) big.PutUint64(o[n+10:], math.Float64bits(imag(c))) return o } // AppendTime appends a time.Time to the slice as a MessagePack extension func AppendTime(b []byte, t time.Time) []byte { o, n := ensure(b, TimeSize) t = t.UTC() o[n] = mext8 o[n+1] = 12 o[n+2] = TimeExtension putUnix(o[n+3:], t.Unix(), int32(t.Nanosecond())) return o } // AppendTimeExt will write t using the official msgpack extension spec. // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type func AppendTimeExt(b []byte, t time.Time) []byte { // Time rounded towards zero. secPrec := t.Truncate(time.Second) remain := t.Sub(secPrec).Nanoseconds() asSecs := secPrec.Unix() switch { case remain == 0 && asSecs > 0 && asSecs <= math.MaxUint32: // 4 bytes o, n := ensure(b, 2+4) o[n+0] = mfixext4 o[n+1] = byte(msgTimeExtension) binary.BigEndian.PutUint32(o[n+2:], uint32(asSecs)) return o case asSecs < 0 || asSecs >= (1<<34): // 12 bytes o, n := ensure(b, 3+12) o[n+0] = mext8 o[n+1] = 12 o[n+2] = byte(msgTimeExtension) binary.BigEndian.PutUint32(o[n+3:], uint32(remain)) binary.BigEndian.PutUint64(o[n+3+4:], uint64(asSecs)) return o default: // 8 bytes o, n := ensure(b, 2+8) o[n+0] = mfixext8 o[n+1] = byte(msgTimeExtension) binary.BigEndian.PutUint64(o[n+2:], uint64(asSecs)|(uint64(remain)<<34)) return o } } // AppendMapStrStr appends a map[string]string to the slice // as a MessagePack map with 'str'-type keys and values func AppendMapStrStr(b []byte, m map[string]string) []byte { sz := uint32(len(m)) b = AppendMapHeader(b, sz) for key, val := range m { b = AppendString(b, key) b = AppendString(b, val) } return b } // AppendMapStrIntf appends a map[string]interface{} to the slice // as a MessagePack map with 'str'-type keys. func AppendMapStrIntf(b []byte, m map[string]interface{}) ([]byte, error) { sz := uint32(len(m)) b = AppendMapHeader(b, sz) var err error for key, val := range m { b = AppendString(b, key) b, err = AppendIntf(b, val) if err != nil { return b, err } } return b, nil } // AppendIntf appends the concrete type of 'i' to the // provided []byte. 'i' must be one of the following: // - 'nil' // - A bool, float, string, []byte, int, uint, or complex // - A map[string]T where T is another supported type // - A []T, where T is another supported type // - A *T, where T is another supported type // - A type that satisfies the msgp.Marshaler interface // - A type that satisfies the msgp.Extension interface func AppendIntf(b []byte, i interface{}) ([]byte, error) { if i == nil { return AppendNil(b), nil } // all the concrete types // for which we have methods switch i := i.(type) { case Marshaler: return i.MarshalMsg(b) case Extension: return AppendExtension(b, i) case bool: return AppendBool(b, i), nil case float32: return AppendFloat32(b, i), nil case float64: return AppendFloat64(b, i), nil case complex64: return AppendComplex64(b, i), nil case complex128: return AppendComplex128(b, i), nil case string: return AppendString(b, i), nil case []byte: return AppendBytes(b, i), nil case int8: return AppendInt8(b, i), nil case int16: return AppendInt16(b, i), nil case int32: return AppendInt32(b, i), nil case int64: return AppendInt64(b, i), nil case int: return AppendInt64(b, int64(i)), nil case uint: return AppendUint64(b, uint64(i)), nil case uint8: return AppendUint8(b, i), nil case uint16: return AppendUint16(b, i), nil case uint32: return AppendUint32(b, i), nil case uint64: return AppendUint64(b, i), nil case time.Time: return AppendTime(b, i), nil case time.Duration: return AppendDuration(b, i), nil case map[string]interface{}: return AppendMapStrIntf(b, i) case map[string]string: return AppendMapStrStr(b, i), nil case json.Number: return AppendJSONNumber(b, i) case []interface{}: b = AppendArrayHeader(b, uint32(len(i))) var err error for _, k := range i { b, err = AppendIntf(b, k) if err != nil { return b, err } } return b, nil } var err error v := reflect.ValueOf(i) switch v.Kind() { case reflect.Map: if v.Type().Key().Kind() != reflect.String { return b, errors.New("msgp: map keys must be strings") } ks := v.MapKeys() b = AppendMapHeader(b, uint32(len(ks))) for _, key := range ks { val := v.MapIndex(key) b = AppendString(b, key.String()) b, err = AppendIntf(b, val.Interface()) if err != nil { return nil, err } } return b, nil case reflect.Array, reflect.Slice: l := v.Len() b = AppendArrayHeader(b, uint32(l)) for i := 0; i < l; i++ { b, err = AppendIntf(b, v.Index(i).Interface()) if err != nil { return b, err } } return b, nil case reflect.Ptr: if v.IsNil() { return AppendNil(b), err } b, err = AppendIntf(b, v.Elem().Interface()) return b, err default: return b, &ErrUnsupportedType{T: v.Type()} } } // AppendJSONNumber appends a json.Number to the slice. // An error will be returned if the json.Number returns error as both integer and float. func AppendJSONNumber(b []byte, n json.Number) ([]byte, error) { if n == "" { // The zero value outputs the 0 integer. return append(b, 0), nil } ii, err := n.Int64() if err == nil { return AppendInt64(b, ii), nil } ff, err := n.Float64() if err == nil { return AppendFloat(b, ff), nil } return b, err } msgp-1.2.5/msgp/write_bytes_test.go000066400000000000000000000314521472554463600174060ustar00rootroot00000000000000package msgp import ( "bytes" "math" "math/rand" "reflect" "strings" "testing" "time" ) func TestIssue116(t *testing.T) { data := AppendInt64(nil, math.MinInt64) i, _, err := ReadInt64Bytes(data) if err != nil { t.Fatal(err) } if i != math.MinInt64 { t.Errorf("put %d in and got %d out", int64(math.MinInt64), i) } var buf bytes.Buffer w := NewWriter(&buf) w.WriteInt64(math.MinInt64) w.Flush() i, err = NewReader(&buf).ReadInt64() if err != nil { t.Fatal(err) } if i != math.MinInt64 { t.Errorf("put %d in and got %d out", int64(math.MinInt64), i) } } func TestAppendMapHeader(t *testing.T) { szs := []uint32{0, 1, uint32(tint8), uint32(tint16), tuint32} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range szs { buf.Reset() en.WriteMapHeader(sz) en.Flush() bts = AppendMapHeader(bts[0:0], sz) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for size %d, encoder wrote %q and append wrote %q", sz, buf.Bytes(), bts) } } } func BenchmarkAppendMapHeader(b *testing.B) { buf := make([]byte, 0, 9) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { AppendMapHeader(buf[:0], 0) AppendMapHeader(buf[:0], uint32(tint8)) AppendMapHeader(buf[:0], tuint16) AppendMapHeader(buf[:0], tuint32) } } func TestAppendArrayHeader(t *testing.T) { szs := []uint32{0, 1, uint32(tint8), uint32(tint16), tuint32} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range szs { buf.Reset() en.WriteArrayHeader(sz) en.Flush() bts = AppendArrayHeader(bts[0:0], sz) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for size %d, encoder wrote %q and append wrote %q", sz, buf.Bytes(), bts) } } } func BenchmarkAppendArrayHeader(b *testing.B) { buf := make([]byte, 0, 9) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { AppendArrayHeader(buf[:0], 0) AppendArrayHeader(buf[:0], uint32(tint8)) AppendArrayHeader(buf[:0], tuint16) AppendArrayHeader(buf[:0], tuint32) } } func TestAppendBytesHeader(t *testing.T) { szs := []uint32{0, 1, uint32(tint8), uint32(tint16), tuint32, math.MaxUint32} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range szs { buf.Reset() en.WriteBytesHeader(sz) en.Flush() bts = AppendBytesHeader(bts[0:0], sz) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for size %d, encoder wrote %q and append wrote %q", sz, buf.Bytes(), bts) } } } func BenchmarkAppendBytesHeader(b *testing.B) { buf := make([]byte, 0, 9) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { AppendBytesHeader(buf[:0], 0) AppendBytesHeader(buf[:0], uint32(tint8)) AppendBytesHeader(buf[:0], tuint16) AppendBytesHeader(buf[:0], tuint32) } } func TestAppendNil(t *testing.T) { var bts []byte bts = AppendNil(bts[0:0]) if bts[0] != mnil { t.Fatal("bts[0] is not 'nil'") } } func TestAppendFloat(t *testing.T) { rng := rand.New(rand.NewSource(0)) const n = 1e7 src := make([]float64, n) for i := range src { // ~50% full float64, 50% converted from float32. if rng.Uint32()&1 == 1 { src[i] = rng.NormFloat64() } else { src[i] = float64(math.MaxFloat32 * (0.5 - rng.Float32())) } } var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, f := range src { en.WriteFloat(f) bts = AppendFloat(bts, f) } en.Flush() if buf.Len() != len(bts) { t.Errorf("encoder wrote %d; append wrote %d bytes", buf.Len(), len(bts)) } t.Logf("%f bytes/value", float64(buf.Len())/n) a, b := bts, buf.Bytes() for i := range a { if a[i] != b[i] { t.Errorf("mismatch at byte %d, %d != %d", i, a[i], b[i]) break } } for i, want := range src { var got float64 var err error got, a, err = ReadFloat64Bytes(a) if err != nil { t.Fatal(err) } if want != got { t.Errorf("value #%d: want %v; got %v", i, want, got) } } } func BenchmarkAppendFloat(b *testing.B) { rng := rand.New(rand.NewSource(0)) const n = 1 << 16 src := make([]float64, n) for i := range src { // ~50% full float64, 50% converted from float32. if rng.Uint32()&1 == 1 { src[i] = rng.NormFloat64() } else { src[i] = float64(math.MaxFloat32 * (0.5 - rng.Float32())) } } buf := make([]byte, 0, 9) b.SetBytes(8) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendFloat(buf, src[i&(n-1)]) } } func TestAppendFloat64(t *testing.T) { f := float64(3.14159) var buf bytes.Buffer en := NewWriter(&buf) var bts []byte en.WriteFloat64(f) en.Flush() bts = AppendFloat64(bts[0:0], f) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for float %f, encoder wrote %q; append wrote %q", f, buf.Bytes(), bts) } } func BenchmarkAppendFloat64(b *testing.B) { f := float64(3.14159) buf := make([]byte, 0, 9) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendFloat64(buf[0:0], f) } } func TestAppendFloat32(t *testing.T) { f := float32(3.14159) var buf bytes.Buffer en := NewWriter(&buf) var bts []byte en.WriteFloat32(f) en.Flush() bts = AppendFloat32(bts[0:0], f) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for float %f, encoder wrote %q; append wrote %q", f, buf.Bytes(), bts) } } func BenchmarkAppendFloat32(b *testing.B) { f := float32(3.14159) buf := make([]byte, 0, 5) b.SetBytes(5) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendFloat32(buf[0:0], f) } } func TestAppendInt64(t *testing.T) { is := []int64{0, 1, -5, -50, int64(tint16), int64(tint32), int64(tint64)} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, i := range is { buf.Reset() en.WriteInt64(i) en.Flush() bts = AppendInt64(bts[0:0], i) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for int64 %d, encoder wrote %q; append wrote %q", i, buf.Bytes(), bts) } } } func BenchmarkAppendInt64(b *testing.B) { is := []int64{0, 1, -5, -50, int64(tint16), int64(tint32), int64(tint64)} l := len(is) buf := make([]byte, 0, 9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendInt64(buf[0:0], is[i%l]) } } func TestAppendUint64(t *testing.T) { us := []uint64{0, 1, uint64(tuint16), uint64(tuint32), tuint64} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, u := range us { buf.Reset() en.WriteUint64(u) en.Flush() bts = AppendUint64(bts[0:0], u) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for uint64 %d, encoder wrote %q; append wrote %q", u, buf.Bytes(), bts) } } } func BenchmarkAppendUint64(b *testing.B) { us := []uint64{0, 1, 15, uint64(tuint16), uint64(tuint32), tuint64} buf := make([]byte, 0, 9) b.ReportAllocs() b.ResetTimer() l := len(us) for i := 0; i < b.N; i++ { AppendUint64(buf[0:0], us[i%l]) } } func TestAppendBytes(t *testing.T) { sizes := []int{0, 1, 225, int(tuint32)} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range sizes { buf.Reset() b := RandBytes(sz) en.WriteBytes(b) en.Flush() bts = AppendBytes(b[0:0], b) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for bytes of length %d, encoder wrote %d bytes and append wrote %d bytes", sz, buf.Len(), len(bts)) } } } func benchappendBytes(size uint32, b *testing.B) { bts := RandBytes(int(size)) buf := make([]byte, 0, len(bts)+5) b.SetBytes(int64(len(bts) + 5)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendBytes(buf[0:0], bts) } } func BenchmarkAppend16Bytes(b *testing.B) { benchappendBytes(16, b) } func BenchmarkAppend256Bytes(b *testing.B) { benchappendBytes(256, b) } func BenchmarkAppend2048Bytes(b *testing.B) { benchappendBytes(2048, b) } func TestAppendString(t *testing.T) { sizes := []int{0, 1, 225, int(tuint32)} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range sizes { buf.Reset() s := string(RandBytes(sz)) en.WriteString(s) en.Flush() bts = AppendString(bts[0:0], s) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for string of length %d, encoder wrote %d bytes and append wrote %d bytes", sz, buf.Len(), len(bts)) t.Errorf("WriteString prefix: %x", buf.Bytes()[0:5]) t.Errorf("Appendstring prefix: %x", bts[0:5]) } } } func benchappendString(size uint32, b *testing.B) { str := string(RandBytes(int(size))) buf := make([]byte, 0, len(str)+5) b.SetBytes(int64(len(str) + 5)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendString(buf[0:0], str) } } func BenchmarkAppend16String(b *testing.B) { benchappendString(16, b) } func BenchmarkAppend256String(b *testing.B) { benchappendString(256, b) } func BenchmarkAppend2048String(b *testing.B) { benchappendString(2048, b) } func TestAppendBool(t *testing.T) { vs := []bool{true, false} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, v := range vs { buf.Reset() en.WriteBool(v) en.Flush() bts = AppendBool(bts[0:0], v) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for %t, encoder wrote %q and append wrote %q", v, buf.Bytes(), bts) } } } func BenchmarkAppendBool(b *testing.B) { vs := []bool{true, false} buf := make([]byte, 0, 1) b.SetBytes(1) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendBool(buf[0:0], vs[i%2]) } } func BenchmarkAppendTime(b *testing.B) { t := time.Now() b.SetBytes(15) buf := make([]byte, 0, 15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendTime(buf[0:0], t) } } func BenchmarkAppendTimeExt(b *testing.B) { t := time.Now() buf := make([]byte, 0, 15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendTimeExt(buf[0:0], t) } } // TestEncodeDecode does a back-and-forth test of encoding and decoding and compare the value with a given output. func TestEncodeDecode(t *testing.T) { for _, tc := range []struct { name string input interface{} output interface{} encodeError string }{ { name: "nil", input: nil, }, { name: "bool", input: true, }, { name: "int", input: int64(42), }, { name: "float", input: 3.14159, }, { name: "string", input: "hello", }, { name: "bytes", input: []byte("hello"), }, { name: "array-empty", input: []interface{}{}, }, { name: "array", input: []interface{}{int64(1), int64(2), int64(3)}, }, { name: "map-empty", input: map[string]interface{}{}, }, { name: "map", input: map[string]interface{}{"a": int64(1), "b": int64(2)}, }, { name: "map-interface", input: map[string]interface{}{"a": int64(1), "b": "2"}, }, { name: "map-string", input: map[string]string{"a": "1", "b": "2"}, output: map[string]interface{}{"a": "1", "b": "2"}, }, { name: "map-array", input: map[string][]int64{"a": {1, 2}, "b": {3}}, output: map[string]interface{}{"a": []interface{}{int64(1), int64(2)}, "b": []interface{}{int64(3)}}, }, { name: "map-map", input: map[string]map[string]int64{"a": {"a": 1, "b": 2}, "b": {"c": 3}}, output: map[string]interface{}{"a": map[string]interface{}{"a": int64(1), "b": int64(2)}, "b": map[string]interface{}{"c": int64(3)}}, }, { name: "array-map", input: []interface{}{map[string]interface{}{"a": int64(1), "b": "2"}, map[string]int64{"c": 3}}, output: []interface{}{map[string]interface{}{"a": int64(1), "b": "2"}, map[string]interface{}{"c": int64(3)}}, }, { name: "array-array", input: []interface{}{[]int64{1, 2}, []interface{}{int64(3)}}, output: []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3)}}, }, { name: "array-array-map", input: []interface{}{[]interface{}{int64(1), int64(2)}, map[string]interface{}{"c": int64(3)}}, }, { name: "map-array-map", input: map[string]interface{}{"a": []interface{}{int64(1), int64(2)}, "b": map[string]interface{}{"c": int64(3)}}, }, { name: "map-invalid-keys", input: map[interface{}]interface{}{int64(1): int64(2)}, encodeError: "msgp: map keys must be strings", }, { name: "map-nested-invalid-keys", input: map[string]interface{}{"a": map[int64]string{1: "2"}}, encodeError: "msgp: map keys must be strings", }, { name: "invalid-type", input: struct{}{}, encodeError: "msgp: type \"struct {}\" not supported", }, } { t.Run(tc.name, func(t *testing.T) { // If no output is given, use the input as output if tc.output == nil { tc.output = tc.input } buf, err := AppendIntf(nil, tc.input) if tc.encodeError != "" { if err == nil || !strings.Contains(err.Error(), tc.encodeError) { t.Fatalf("expected encode error '%s' but got '%s'", tc.encodeError, err) } return } if tc.encodeError == "" && err != nil { t.Fatalf("expected no encode error but got '%s'", err.Error()) } out, _, _ := ReadIntfBytes(buf) if err != nil { t.Fatalf("expected no decode error but got '%s'", err.Error()) } if !reflect.DeepEqual(tc.output, out) { t.Fatalf("expected '%v' but got '%v'", tc.input, out) } }) } } msgp-1.2.5/msgp/write_test.go000066400000000000000000000205501472554463600161750ustar00rootroot00000000000000package msgp import ( "bytes" "math" "math/rand" "testing" "time" ) var ( tint8 int8 = 126 // cannot be most fix* types tint16 int16 = 150 // cannot be int8 tint32 int32 = math.MaxInt16 + 100 // cannot be int16 tint64 int64 = math.MaxInt32 + 100 // cannot be int32 tuint16 uint32 = 300 // cannot be uint8 tuint32 uint32 = math.MaxUint16 + 100 // cannot be uint16 tuint64 uint64 = math.MaxUint32 + 100 // cannot be uint32 ) func RandBytes(sz int) []byte { out := make([]byte, sz) rng := rand.New(rand.NewSource(time.Now().UnixNano())) for i := range out { out[i] = byte(rng.Uint32()) } return out } func TestWriteMapHeader(t *testing.T) { tests := []struct { Sz uint32 Outbytes []byte }{ {0, []byte{mfixmap}}, {1, []byte{mfixmap | byte(1)}}, {100, []byte{mmap16, byte(uint16(100) >> 8), byte(uint16(100))}}, { tuint32, []byte{ mmap32, byte(tuint32 >> 24), byte(tuint32 >> 16), byte(tuint32 >> 8), byte(tuint32), }, }, } var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, test := range tests { buf.Reset() err = wr.WriteMapHeader(test.Sz) if err != nil { t.Error(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if !bytes.Equal(buf.Bytes(), test.Outbytes) { t.Errorf("Expected bytes %x; got %x", test.Outbytes, buf.Bytes()) } } } func BenchmarkWriteMapHeader(b *testing.B) { wr := NewWriter(Nowhere) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { wr.WriteMapHeader(0) wr.WriteMapHeader(8) wr.WriteMapHeader(tuint16) wr.WriteMapHeader(tuint32) } } func TestWriteArrayHeader(t *testing.T) { tests := []struct { Sz uint32 Outbytes []byte }{ {0, []byte{mfixarray}}, {1, []byte{mfixarray | byte(1)}}, {tuint16, []byte{marray16, byte(tuint16 >> 8), byte(tuint16)}}, {tuint32, []byte{marray32, byte(tuint32 >> 24), byte(tuint32 >> 16), byte(tuint32 >> 8), byte(tuint32)}}, } var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, test := range tests { buf.Reset() err = wr.WriteArrayHeader(test.Sz) if err != nil { t.Error(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if !bytes.Equal(buf.Bytes(), test.Outbytes) { t.Errorf("Expected bytes %x; got %x", test.Outbytes, buf.Bytes()) } } } func TestReadWriteStringHeader(t *testing.T) { sizes := []uint32{0, 5, 8, 19, 150, tuint16, tuint32} var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, sz := range sizes { buf.Reset() err = wr.WriteStringHeader(sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } var nsz uint32 nsz, err = NewReader(&buf).ReadStringHeader() if err != nil { t.Fatal(err) } if nsz != sz { t.Errorf("put in size %d but got out size %d", sz, nsz) } } } func TestReadWriteBytesHeader(t *testing.T) { sizes := []uint32{0, 5, 8, 19, 150, tuint16, tuint32} var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, sz := range sizes { buf.Reset() err = wr.WriteBytesHeader(sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } var nsz uint32 nsz, err = NewReader(&buf).ReadBytesHeader() if err != nil { t.Fatal(err) } if nsz != sz { t.Errorf("put in size %d but got out size %d", sz, nsz) } } } func BenchmarkWriteArrayHeader(b *testing.B) { wr := NewWriter(Nowhere) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { wr.WriteArrayHeader(0) wr.WriteArrayHeader(16) wr.WriteArrayHeader(tuint16) wr.WriteArrayHeader(tuint32) } } func TestWriteNil(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) err := wr.WriteNil() if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mnil { t.Errorf("Expected %x; wrote %x", mnil, bts[0]) } } func TestWriteFloat64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() flt := (rand.Float64() - 0.5) * math.MaxFloat64 err := wr.WriteFloat64(flt) if err != nil { t.Errorf("Error with %f: %s", flt, err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mfloat64 { t.Errorf("Leading byte was %x and not %x", bts[0], mfloat64) } } } func TestReadWriterDuration(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() dur := time.Duration(rand.Int63()) err := wr.WriteDuration(dur) if err != nil { t.Errorf("Error with %v: %s", dur, err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mint64 { t.Errorf("Leading byte was %x and not %x", bts[0], mint64) } wr := NewReader(&buf) d, err := wr.ReadDuration() if err != nil { t.Errorf("Error reading duration: %v", err) } if d != dur { t.Errorf("Got duration %v, want %v", d, dur) } } } func BenchmarkWriteFloat64(b *testing.B) { f := rand.Float64() wr := NewWriter(Nowhere) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteFloat64(f) } } func TestWriteFloat32(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() flt := (rand.Float32() - 0.5) * math.MaxFloat32 err := wr.WriteFloat32(flt) if err != nil { t.Errorf("Error with %f: %s", flt, err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mfloat32 { t.Errorf("Leading byte was %x and not %x", bts[0], mfloat64) } } } func BenchmarkWriteFloat32(b *testing.B) { f := rand.Float32() wr := NewWriter(Nowhere) b.SetBytes(5) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteFloat32(f) } } func TestWriteInt64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() num := (rand.Int63n(math.MaxInt64)) - (math.MaxInt64 / 2) err := wr.WriteInt64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() > 9 { t.Errorf("buffer length should be <= 9; it's %d", buf.Len()) } } } func BenchmarkWriteInt64(b *testing.B) { wr := NewWriter(Nowhere) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteInt64(int64(tint64)) } } func TestWriteUint64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() num := uint64(rand.Int63n(math.MaxInt64)) err := wr.WriteUint64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() > 9 { t.Errorf("buffer length should be <= 9; it's %d", buf.Len()) } } } func BenchmarkWriteUint64(b *testing.B) { wr := NewWriter(Nowhere) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteUint64(uint64(tuint64)) } } func TestWriteBytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) sizes := []int{0, 1, 225, int(tuint32)} for _, size := range sizes { buf.Reset() bts := RandBytes(size) err := wr.WriteBytes(bts) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() < len(bts) { t.Errorf("somehow, %d bytes were encoded in %d bytes", len(bts), buf.Len()) } } } func benchwrBytes(size uint32, b *testing.B) { bts := RandBytes(int(size)) wr := NewWriter(Nowhere) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteBytes(bts) } } func BenchmarkWrite16Bytes(b *testing.B) { benchwrBytes(16, b) } func BenchmarkWrite256Bytes(b *testing.B) { benchwrBytes(256, b) } func BenchmarkWrite2048Bytes(b *testing.B) { benchwrBytes(2048, b) } func TestWriteTime(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) tm := time.Now() err := wr.WriteTime(tm) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() != 15 { t.Errorf("expected time.Time to be %d bytes; got %d", 15, buf.Len()) } newt, err := NewReader(&buf).ReadTime() if err != nil { t.Fatal(err) } if !newt.Equal(tm) { t.Errorf("in/out not equal; %s in and %s out", tm, newt) } } func BenchmarkWriteTime(b *testing.B) { t := time.Now() wr := NewWriter(Nowhere) b.SetBytes(15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteTime(t) } } msgp-1.2.5/parse/000077500000000000000000000000001472554463600136175ustar00rootroot00000000000000msgp-1.2.5/parse/directives.go000066400000000000000000000111331472554463600163060ustar00rootroot00000000000000package parse import ( "fmt" "go/ast" "go/parser" "strings" "github.com/tinylib/msgp/gen" ) const linePrefix = "//msgp:" // func(args, fileset) type directive func([]string, *FileSet) error // func(passName, args, printer) type passDirective func(gen.Method, []string, *gen.Printer) error // map of all recognized directives // // to add a directive, define a func([]string, *FileSet) error // and then add it to this list. var directives = map[string]directive{ "shim": applyShim, "replace": replace, "ignore": ignore, "tuple": astuple, "compactfloats": compactfloats, "clearomitted": clearomitted, "newtime": newtime, } // map of all recognized directives which will be applied // before process() is called // // to add an early directive, define a func([]string, *FileSet) error // and then add it to this list. var earlyDirectives = map[string]directive{ "tag": tag, "pointer": pointer, } var passDirectives = map[string]passDirective{ "ignore": passignore, } func passignore(m gen.Method, text []string, p *gen.Printer) error { pushstate(m.String()) for _, a := range text { p.ApplyDirective(m, gen.IgnoreTypename(a)) infof("ignoring %s\n", a) } popstate() return nil } // find all comment lines that begin with //msgp: func yieldComments(c []*ast.CommentGroup) []string { var out []string for _, cg := range c { for _, line := range cg.List { if strings.HasPrefix(line.Text, linePrefix) { out = append(out, strings.TrimPrefix(line.Text, linePrefix)) } } } return out } //msgp:shim {Type} as:{NewType} using:{toFunc/fromFunc} mode:{Mode} func applyShim(text []string, f *FileSet) error { if len(text) < 4 || len(text) > 5 { return fmt.Errorf("shim directive should have 3 or 4 arguments; found %d", len(text)-1) } name := text[1] be := gen.Ident(strings.TrimPrefix(strings.TrimSpace(text[2]), "as:")) // parse as::{base} if name[0] == '*' { name = name[1:] be.Needsref(true) } be.Alias(name) usestr := strings.TrimPrefix(strings.TrimSpace(text[3]), "using:") // parse using::{method/method} methods := strings.Split(usestr, "/") if len(methods) != 2 { return fmt.Errorf("expected 2 using::{} methods; found %d (%q)", len(methods), text[3]) } be.ShimToBase = methods[0] be.ShimFromBase = methods[1] if len(text) == 5 { modestr := strings.TrimPrefix(strings.TrimSpace(text[4]), "mode:") // parse mode::{mode} switch modestr { case "cast": be.ShimMode = gen.Cast case "convert": be.ShimMode = gen.Convert default: return fmt.Errorf("invalid shim mode; found %s, expected 'cast' or 'convert", modestr) } } infof("%s -> %s\n", name, be.Value.String()) f.findShim(name, be, true) return nil } //msgp:replace {Type} with:{NewType} func replace(text []string, f *FileSet) error { if len(text) != 3 { return fmt.Errorf("replace directive should have only 2 arguments; found %d", len(text)-1) } name := text[1] replacement := strings.TrimPrefix(strings.TrimSpace(text[2]), "with:") expr, err := parser.ParseExpr(replacement) if err != nil { return err } e := f.parseExpr(expr) e.AlwaysPtr(&f.pointerRcv) if be, ok := e.(*gen.BaseElem); ok { be.Convert = true be.Alias(name) if be.Value == gen.IDENT { be.ShimToBase = "(*" + replacement + ")" be.Needsref(true) } } infof("%s -> %s\n", name, replacement) f.findShim(name, e, false) return nil } //msgp:ignore {TypeA} {TypeB}... func ignore(text []string, f *FileSet) error { if len(text) < 2 { return nil } for _, item := range text[1:] { name := strings.TrimSpace(item) if _, ok := f.Identities[name]; ok { delete(f.Identities, name) infof("ignoring %s\n", name) } } return nil } //msgp:tuple {TypeA} {TypeB}... func astuple(text []string, f *FileSet) error { if len(text) < 2 { return nil } for _, item := range text[1:] { name := strings.TrimSpace(item) if el, ok := f.Identities[name]; ok { if st, ok := el.(*gen.Struct); ok { st.AsTuple = true infof(name) } else { warnf("%s: only structs can be tuples\n", name) } } } return nil } //msgp:tag {tagname} func tag(text []string, f *FileSet) error { if len(text) != 2 { return nil } f.tagName = strings.TrimSpace(text[1]) return nil } //msgp:pointer func pointer(text []string, f *FileSet) error { f.pointerRcv = true return nil } //msgp:compactfloats func compactfloats(text []string, f *FileSet) error { f.CompactFloats = true return nil } //msgp:clearomitted func clearomitted(text []string, f *FileSet) error { f.ClearOmitted = true return nil } //msgp:newtime func newtime(text []string, f *FileSet) error { f.NewTime = true return nil } msgp-1.2.5/parse/getast.go000066400000000000000000000335621472554463600154460ustar00rootroot00000000000000package parse import ( "fmt" "go/ast" "go/parser" "go/token" "os" "reflect" "sort" "strings" "github.com/tinylib/msgp/gen" ) // A FileSet is the in-memory representation of a // parsed file. type FileSet struct { Package string // package name Specs map[string]ast.Expr // type specs in file Identities map[string]gen.Elem // processed from specs Directives []string // raw preprocessor directives Imports []*ast.ImportSpec // imports CompactFloats bool // Use smaller floats when feasible ClearOmitted bool // Set omitted fields to zero value NewTime bool // Set to use -1 extension for time.Time tagName string // tag to read field names from pointerRcv bool // generate with pointer receivers. } // File parses a file at the relative path // provided and produces a new *FileSet. // If you pass in a path to a directory, the entire // directory will be parsed. // If unexport is false, only exported identifiers are included in the FileSet. // If the resulting FileSet would be empty, an error is returned. func File(name string, unexported bool) (*FileSet, error) { pushstate(name) defer popstate() fs := &FileSet{ Specs: make(map[string]ast.Expr), Identities: make(map[string]gen.Elem), } fset := token.NewFileSet() finfo, err := os.Stat(name) if err != nil { return nil, err } if finfo.IsDir() { pkgs, err := parser.ParseDir(fset, name, nil, parser.ParseComments) if err != nil { return nil, err } if len(pkgs) != 1 { return nil, fmt.Errorf("multiple packages in directory: %s", name) } var one *ast.Package for _, nm := range pkgs { one = nm break } fs.Package = one.Name for _, fl := range one.Files { pushstate(fl.Name.Name) fs.Directives = append(fs.Directives, yieldComments(fl.Comments)...) if !unexported { ast.FileExports(fl) } fs.getTypeSpecs(fl) popstate() } } else { f, err := parser.ParseFile(fset, name, nil, parser.ParseComments) if err != nil { return nil, err } fs.Package = f.Name.Name fs.Directives = yieldComments(f.Comments) if !unexported { ast.FileExports(f) } fs.getTypeSpecs(f) } if len(fs.Specs) == 0 { return nil, fmt.Errorf("no definitions in %s", name) } fs.applyEarlyDirectives() fs.process() fs.applyDirectives() fs.propInline() return fs, nil } // applyDirectives applies all of the directives that // are known to the parser. additional method-specific // directives remain in f.Directives func (f *FileSet) applyDirectives() { newdirs := make([]string, 0, len(f.Directives)) for _, d := range f.Directives { chunks := strings.Split(d, " ") if len(chunks) > 0 { if fn, ok := directives[chunks[0]]; ok { pushstate(chunks[0]) err := fn(chunks, f) if err != nil { warnf("directive error: %s", err) } popstate() } else { newdirs = append(newdirs, d) } } } f.Directives = newdirs } // applyEarlyDirectives applies all early directives needed before process() is called. // additional directives remain in f.Directives for future processing func (f *FileSet) applyEarlyDirectives() { newdirs := make([]string, 0, len(f.Directives)) for _, d := range f.Directives { parts := strings.Split(d, " ") if len(parts) == 0 { continue } if fn, ok := earlyDirectives[parts[0]]; ok { pushstate(parts[0]) err := fn(parts, f) if err != nil { warnf("early directive error: %s", err) } popstate() } else { newdirs = append(newdirs, d) } } f.Directives = newdirs } // A linkset is a graph of unresolved // identities. // // Since gen.Ident can only represent // one level of type indirection (e.g. Foo -> uint8), // type declarations like `type Foo Bar` // aren't resolve-able until we've processed // everything else. // // The goal of this dependency resolution // is to distill the type declaration // into just one level of indirection. // In other words, if we have: // // type A uint64 // type B A // type C B // type D C // // ... then we want to end up // figuring out that D is just a uint64. type linkset map[string]*gen.BaseElem func (f *FileSet) resolve(ls linkset) { progress := true for progress && len(ls) > 0 { progress = false for name, elem := range ls { real, ok := f.Identities[elem.TypeName()] if ok { // copy the old type descriptor, // alias it to the new value, // and insert it into the resolved // identities list progress = true nt := real.Copy() nt.Alias(name) f.Identities[name] = nt delete(ls, name) } } } // what's left can't be resolved for name, elem := range ls { warnf("couldn't resolve type %s (%s)\n", name, elem.TypeName()) } } // process takes the contents of f.Specs and // uses them to populate f.Identities func (f *FileSet) process() { deferred := make(linkset) parse: for name, def := range f.Specs { pushstate(name) el := f.parseExpr(def) if el == nil { warnf("failed to parse") popstate() continue parse } el.AlwaysPtr(&f.pointerRcv) // push unresolved identities into // the graph of links and resolve after // we've handled every possible named type. if be, ok := el.(*gen.BaseElem); ok && be.Value == gen.IDENT { deferred[name] = be popstate() continue parse } el.Alias(name) f.Identities[name] = el popstate() } if len(deferred) > 0 { f.resolve(deferred) } } func strToMethod(s string) gen.Method { switch s { case "encode": return gen.Encode case "decode": return gen.Decode case "test": return gen.Test case "size": return gen.Size case "marshal": return gen.Marshal case "unmarshal": return gen.Unmarshal default: return 0 } } func (f *FileSet) applyDirs(p *gen.Printer) { // apply directives of the form // // //msgp:encode ignore {{TypeName}} // loop: for _, d := range f.Directives { chunks := strings.Split(d, " ") if len(chunks) > 1 { for i := range chunks { chunks[i] = strings.TrimSpace(chunks[i]) } m := strToMethod(chunks[0]) if m == 0 { warnf("unknown pass name: %q\n", chunks[0]) continue loop } if fn, ok := passDirectives[chunks[1]]; ok { pushstate(chunks[1]) err := fn(m, chunks[2:], p) if err != nil { warnf("error applying directive: %s\n", err) } popstate() } else { warnf("unrecognized directive %q\n", chunks[1]) } } else { warnf("empty directive: %q\n", d) } } p.CompactFloats = f.CompactFloats p.ClearOmitted = f.ClearOmitted p.NewTime = f.NewTime } func (f *FileSet) PrintTo(p *gen.Printer) error { f.applyDirs(p) names := make([]string, 0, len(f.Identities)) for name := range f.Identities { names = append(names, name) } sort.Strings(names) for _, name := range names { el := f.Identities[name] el.SetVarname("z") pushstate(el.TypeName()) err := p.Print(el) popstate() if err != nil { return err } } return nil } // getTypeSpecs extracts all of the *ast.TypeSpecs in the file // into fs.Identities, but does not set the actual element func (fs *FileSet) getTypeSpecs(f *ast.File) { // collect all imports... fs.Imports = append(fs.Imports, f.Imports...) // check all declarations... for i := range f.Decls { // for GenDecls... if g, ok := f.Decls[i].(*ast.GenDecl); ok { // and check the specs... for _, s := range g.Specs { // for ast.TypeSpecs.... if ts, ok := s.(*ast.TypeSpec); ok { switch ts.Type.(type) { // this is the list of parse-able // type specs case *ast.StructType, *ast.ArrayType, *ast.StarExpr, *ast.MapType, *ast.Ident: fs.Specs[ts.Name.Name] = ts.Type } } } } } } func fieldName(f *ast.Field) string { switch len(f.Names) { case 0: return stringify(f.Type) case 1: return f.Names[0].Name default: return f.Names[0].Name + " (and others)" } } func (fs *FileSet) parseFieldList(fl *ast.FieldList) []gen.StructField { if fl == nil || fl.NumFields() == 0 { return nil } out := make([]gen.StructField, 0, fl.NumFields()) for _, field := range fl.List { pushstate(fieldName(field)) fds := fs.getField(field) if len(fds) > 0 { out = append(out, fds...) } else { warnf("ignored") } popstate() } return out } // translate *ast.Field into []gen.StructField func (fs *FileSet) getField(f *ast.Field) []gen.StructField { sf := make([]gen.StructField, 1) var extension, flatten bool // parse tag; otherwise field name is field tag if f.Tag != nil { var body string if fs.tagName != "" { body = reflect.StructTag(strings.Trim(f.Tag.Value, "`")).Get(fs.tagName) } if body == "" { body = reflect.StructTag(strings.Trim(f.Tag.Value, "`")).Get("msg") } if body == "" { body = reflect.StructTag(strings.Trim(f.Tag.Value, "`")).Get("msgpack") } tags := strings.Split(body, ",") if len(tags) >= 2 { switch tags[1] { case "extension": extension = true case "flatten": flatten = true } } // ignore "-" fields if tags[0] == "-" { return nil } sf[0].FieldTag = tags[0] sf[0].FieldTagParts = tags sf[0].RawTag = f.Tag.Value } ex := fs.parseExpr(f.Type) if ex == nil { return nil } // parse field name switch len(f.Names) { case 0: if flatten { return fs.getFieldsFromEmbeddedStruct(f.Type) } else { sf[0].FieldName = embedded(f.Type) } case 1: sf[0].FieldName = f.Names[0].Name default: // this is for a multiple in-line declaration, // e.g. type A struct { One, Two int } sf = sf[0:0] for _, nm := range f.Names { sf = append(sf, gen.StructField{ FieldTag: nm.Name, FieldName: nm.Name, FieldElem: ex.Copy(), }) } return sf } sf[0].FieldElem = ex if sf[0].FieldTag == "" { sf[0].FieldTag = sf[0].FieldName if len(sf[0].FieldTagParts) <= 1 { sf[0].FieldTagParts = []string{sf[0].FieldTag} } else { sf[0].FieldTagParts = append([]string{sf[0].FieldName}, sf[0].FieldTagParts[1:]...) } } // validate extension if extension { switch ex := ex.(type) { case *gen.Ptr: if b, ok := ex.Value.(*gen.BaseElem); ok { b.Value = gen.Ext } else { warnf("couldn't cast to extension.") return nil } case *gen.BaseElem: ex.Value = gen.Ext default: warnf("couldn't cast to extension.") return nil } } return sf } func (fs *FileSet) getFieldsFromEmbeddedStruct(f ast.Expr) []gen.StructField { switch f := f.(type) { case *ast.Ident: s := fs.Specs[f.Name] switch s := s.(type) { case *ast.StructType: return fs.parseFieldList(s.Fields) default: return nil } default: // other possibilities are disallowed return nil } } // extract embedded field name // // so, for a struct like // // type A struct { // io.Writer // } // // we want "Writer" func embedded(f ast.Expr) string { switch f := f.(type) { case *ast.Ident: return f.Name case *ast.StarExpr: return embedded(f.X) case *ast.SelectorExpr: return f.Sel.Name default: // other possibilities are disallowed return "" } } // stringify a field type name func stringify(e ast.Expr) string { switch e := e.(type) { case *ast.Ident: return e.Name case *ast.StarExpr: return "*" + stringify(e.X) case *ast.SelectorExpr: return stringify(e.X) + "." + e.Sel.Name case *ast.ArrayType: if e.Len == nil { return "[]" + stringify(e.Elt) } return fmt.Sprintf("[%s]%s", stringify(e.Len), stringify(e.Elt)) case *ast.InterfaceType: if e.Methods == nil || e.Methods.NumFields() == 0 { return "interface{}" } } return "" } // recursively translate ast.Expr to gen.Elem; nil means type not supported // expected input types: // - *ast.MapType (map[T]J) // - *ast.Ident (name) // - *ast.ArrayType ([(sz)]T) // - *ast.StarExpr (*T) // - *ast.StructType (struct {}) // - *ast.SelectorExpr (a.B) // - *ast.InterfaceType (interface {}) func (fs *FileSet) parseExpr(e ast.Expr) gen.Elem { switch e := e.(type) { case *ast.MapType: if k, ok := e.Key.(*ast.Ident); ok && k.Name == "string" { if in := fs.parseExpr(e.Value); in != nil { return &gen.Map{Value: in} } } return nil case *ast.Ident: b := gen.Ident(e.Name) // work to resolve this expression // can be done later, once we've resolved // everything else. if b.Value == gen.IDENT { if _, ok := fs.Specs[e.Name]; !ok { warnf("non-local identifier: %s\n", e.Name) } } return b case *ast.ArrayType: // special case for []byte if e.Len == nil { if i, ok := e.Elt.(*ast.Ident); ok && i.Name == "byte" { return &gen.BaseElem{Value: gen.Bytes} } } // return early if we don't know // what the slice element type is els := fs.parseExpr(e.Elt) if els == nil { return nil } // array and not a slice if e.Len != nil { switch s := e.Len.(type) { case *ast.BasicLit: return &gen.Array{ Size: s.Value, Els: els, } case *ast.Ident: return &gen.Array{ Size: s.String(), Els: els, } case *ast.SelectorExpr: return &gen.Array{ Size: stringify(s), Els: els, } default: return nil } } return &gen.Slice{Els: els} case *ast.StarExpr: if v := fs.parseExpr(e.X); v != nil { return &gen.Ptr{Value: v} } return nil case *ast.StructType: return &gen.Struct{Fields: fs.parseFieldList(e.Fields)} case *ast.SelectorExpr: return gen.Ident(stringify(e)) case *ast.InterfaceType: // support `interface{}` if len(e.Methods.List) == 0 { return &gen.BaseElem{Value: gen.Intf} } return nil default: // other types not supported return nil } } var Logf func(s string, v ...interface{}) func infof(s string, v ...interface{}) { if Logf != nil { pushstate(s) Logf("info: "+strings.Join(logctx, ": "), v...) popstate() } } func warnf(s string, v ...interface{}) { if Logf != nil { pushstate(s) Logf("warn: "+strings.Join(logctx, ": "), v...) popstate() } } var logctx []string // push logging state func pushstate(s string) { logctx = append(logctx, s) } // pop logging state func popstate() { logctx = logctx[:len(logctx)-1] } msgp-1.2.5/parse/inline.go000066400000000000000000000076721472554463600154400ustar00rootroot00000000000000package parse import ( "sort" "github.com/tinylib/msgp/gen" ) // This file defines when and how we // propagate type information from // one type declaration to another. // After the processing pass, every // non-primitive type is marshalled/unmarshalled/etc. // through a function call. Here, we propagate // the type information into the caller's type // tree *if* the child type is simple enough. // // For example, types like // // type A [4]int // // will get pushed into parent methods, // whereas types like // // type B [3]map[string]struct{A, B [4]string} // // will not. // this is an approximate measure // of the number of children in a node const maxComplex = 5 // begin recursive search for identities with the // given name and replace them with e func (f *FileSet) findShim(id string, e gen.Elem, addID bool) { for name, el := range f.Identities { pushstate(name) switch el := el.(type) { case *gen.Struct: for i := range el.Fields { f.nextShim(&el.Fields[i].FieldElem, id, e) } case *gen.Array: f.nextShim(&el.Els, id, e) case *gen.Slice: f.nextShim(&el.Els, id, e) case *gen.Map: f.nextShim(&el.Value, id, e) case *gen.Ptr: f.nextShim(&el.Value, id, e) } popstate() } if addID { f.Identities[id] = e } } func (f *FileSet) nextShim(ref *gen.Elem, id string, e gen.Elem) { if (*ref).TypeName() == id { vn := (*ref).Varname() *ref = e.Copy() (*ref).SetVarname(vn) } else { switch el := (*ref).(type) { case *gen.Struct: for i := range el.Fields { f.nextShim(&el.Fields[i].FieldElem, id, e) } case *gen.Array: f.nextShim(&el.Els, id, e) case *gen.Slice: f.nextShim(&el.Els, id, e) case *gen.Map: f.nextShim(&el.Value, id, e) case *gen.Ptr: f.nextShim(&el.Value, id, e) } } } // propInline identifies and inlines candidates func (f *FileSet) propInline() { type gelem struct { name string el gen.Elem } all := make([]gelem, 0, len(f.Identities)) for name, el := range f.Identities { all = append(all, gelem{name: name, el: el}) } // make sure we process inlining determinstically: // start with the least-complex elems; // use identifier names as a tie-breaker sort.Slice(all, func(i, j int) bool { ig, jg := &all[i], &all[j] ic, jc := ig.el.Complexity(), jg.el.Complexity() return ic < jc || (ic == jc && ig.name < jg.name) }) for i := range all { name := all[i].name pushstate(name) switch el := all[i].el.(type) { case *gen.Struct: for i := range el.Fields { f.nextInline(&el.Fields[i].FieldElem, name) } case *gen.Array: f.nextInline(&el.Els, name) case *gen.Slice: f.nextInline(&el.Els, name) case *gen.Map: f.nextInline(&el.Value, name) case *gen.Ptr: f.nextInline(&el.Value, name) } popstate() } } const fatalloop = `detected infinite recursion in inlining loop! Please file a bug at github.com/tinylib/msgp/issues! Thanks! ` func (f *FileSet) nextInline(ref *gen.Elem, root string) { switch el := (*ref).(type) { case *gen.BaseElem: // ensure that we're not inlining // a type into itself typ := el.TypeName() if el.Value == gen.IDENT && typ != root { if node, ok := f.Identities[typ]; ok && node.Complexity() < maxComplex { infof("inlining %s\n", typ) // This should never happen; it will cause // infinite recursion. if node == *ref { panic(fatalloop) } *ref = node.Copy() f.nextInline(ref, node.TypeName()) } else if !ok && !el.Resolved() { // this is the point at which we're sure that // we've got a type that isn't a primitive, // a library builtin, or a processed type warnf("unresolved identifier: %s\n", typ) } } case *gen.Struct: for i := range el.Fields { f.nextInline(&el.Fields[i].FieldElem, root) } case *gen.Array: f.nextInline(&el.Els, root) case *gen.Slice: f.nextInline(&el.Els, root) case *gen.Map: f.nextInline(&el.Value, root) case *gen.Ptr: f.nextInline(&el.Value, root) default: panic("bad elem type") } } msgp-1.2.5/printer/000077500000000000000000000000001472554463600141705ustar00rootroot00000000000000msgp-1.2.5/printer/print.go000066400000000000000000000065451472554463600156650ustar00rootroot00000000000000package printer import ( "bytes" "fmt" "io" "os" "strings" "github.com/tinylib/msgp/gen" "github.com/tinylib/msgp/parse" "golang.org/x/tools/imports" ) var Logf func(s string, v ...interface{}) // PrintFile prints the methods for the provided list // of elements to the given file name and canonical // package path. func PrintFile(file string, f *parse.FileSet, mode gen.Method) error { out, tests, err := generate(f, mode) if err != nil { return err } // we'll run goimports on the main file // in another goroutine, and run it here // for the test file. empirically, this // takes about the same amount of time as // doing them in serial when GOMAXPROCS=1, // and faster otherwise. res := goformat(file, out.Bytes()) if tests != nil { testfile := strings.TrimSuffix(file, ".go") + "_test.go" err = format(testfile, tests.Bytes()) if err != nil { return err } if Logf != nil { Logf("Wrote and formatted \"%s\"\n", testfile) } } err = <-res if err != nil { os.WriteFile(file+".broken", out.Bytes(), os.ModePerm) if Logf != nil { Logf("Error: %s. Wrote broken output to %s\n", err, file+".broken") } return err } return nil } func format(file string, data []byte) error { out, err := imports.Process(file, data, nil) if err != nil { return err } return os.WriteFile(file, out, 0o600) } func goformat(file string, data []byte) <-chan error { out := make(chan error, 1) go func(file string, data []byte, end chan error) { end <- format(file, data) if Logf != nil { Logf("Wrote and formatted \"%s\"\n", file) } }(file, data, out) return out } func dedupImports(imp []string) []string { m := make(map[string]struct{}) for i := range imp { m[imp[i]] = struct{}{} } r := []string{} for k := range m { r = append(r, k) } return r } func generate(f *parse.FileSet, mode gen.Method) (*bytes.Buffer, *bytes.Buffer, error) { outbuf := bytes.NewBuffer(make([]byte, 0, 4096)) writePkgHeader(outbuf, f.Package) myImports := []string{"github.com/tinylib/msgp/msgp"} for _, imp := range f.Imports { if imp.Name != nil { // have an alias, include it. myImports = append(myImports, imp.Name.Name+` `+imp.Path.Value) } else { myImports = append(myImports, imp.Path.Value) } } dedup := dedupImports(myImports) writeImportHeader(outbuf, dedup...) var testbuf *bytes.Buffer var testwr io.Writer if mode&gen.Test == gen.Test { testbuf = bytes.NewBuffer(make([]byte, 0, 4096)) writePkgHeader(testbuf, f.Package) if mode&(gen.Encode|gen.Decode) != 0 { writeImportHeader(testbuf, "bytes", "github.com/tinylib/msgp/msgp", "testing") } else { writeImportHeader(testbuf, "github.com/tinylib/msgp/msgp", "testing") } testwr = testbuf } return outbuf, testbuf, f.PrintTo(gen.NewPrinter(mode, outbuf, testwr)) } func writePkgHeader(b *bytes.Buffer, name string) { b.WriteString("package ") b.WriteString(name) b.WriteByte('\n') // write generated code marker // https://github.com/tinylib/msgp/issues/229 // https://golang.org/s/generatedcode b.WriteString("// Code generated by github.com/tinylib/msgp DO NOT EDIT.\n\n") } func writeImportHeader(b *bytes.Buffer, imports ...string) { b.WriteString("import (\n") for _, im := range imports { if im[len(im)-1] == '"' { // support aliased imports fmt.Fprintf(b, "\t%s\n", im) } else { fmt.Fprintf(b, "\t%q\n", im) } } b.WriteString(")\n\n") } msgp-1.2.5/tinygotest/000077500000000000000000000000001472554463600147165ustar00rootroot00000000000000msgp-1.2.5/tinygotest/.gitignore000066400000000000000000000001071472554463600167040ustar00rootroot00000000000000*_gen.go *_gen_test.go *.out *.bin *.hex *.wasm *.wasi nativebin tmp* msgp-1.2.5/tinygotest/testdata/000077500000000000000000000000001472554463600165275ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/empty/000077500000000000000000000000001472554463600176655ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/empty/main.go000066400000000000000000000001651472554463600211420ustar00rootroot00000000000000package main // baseline file just so we can compare the size func main() { println("successfully did nothing") } msgp-1.2.5/tinygotest/testdata/roundtrip/000077500000000000000000000000001472554463600205555ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/roundtrip/main.go000066400000000000000000000073611472554463600220370ustar00rootroot00000000000000package main import ( "bytes" "github.com/tinylib/msgp/msgp" ) //go:generate msgp type SomeStruct struct { A string `msg:"a"` } type EmbeddedStruct struct { A string `msg:"a"` } // Example provides a decent variety of types and features and // lets us test for basic functionality in TinyGo. type Example struct { Interface interface{} `msg:"interface"` Any any `msg:"any"` Int64 int64 `msg:"int64"` Uint64 uint64 `msg:"uint64"` Int32 int32 `msg:"int32"` Uint32 uint32 `msg:"uint32"` Int16 int32 `msg:"int16"` Uint16 uint32 `msg:"uint16"` Int8 int32 `msg:"int8"` Byte byte `msg:"byte"` Float64 float64 `msg:"float64"` Float32 float32 `msg:"float32"` String string `msg:"string"` ByteSlice []byte `msg:"byte_slice"` StringSlice []string `msg:"string_slice"` IntArray [2]int `msg:"int_array"` SomeStruct SomeStruct `msg:"some_struct"` EmbeddedStruct Omitted string `msg:"-"` OmitEmptyString string `msg:"omit_empty_string,omitempty"` } // Setup populuates the struct with test data func (e *Example) Setup() { e.Interface = 10 e.Any = "any" e.Int64 = 10 e.Uint64 = 11 e.Int32 = 12 e.Uint32 = 13 e.Int16 = 14 e.Uint16 = 15 e.Int8 = 16 e.Byte = 17 e.Float64 = 18.1 e.Float32 = 19.2 e.String = "astr" e.ByteSlice = []byte("bstr") e.StringSlice = []string{"a", "b"} e.IntArray = [...]int{20, 21} e.SomeStruct = SomeStruct{A: "x"} e.EmbeddedStruct = EmbeddedStruct{A: "y"} e.Omitted = "nope" e.OmitEmptyString = "here" } func (e *Example) Eq(e2 *Example) bool { if int64(e.Interface.(int)) != e2.Interface.(int64) { return false } if e.Any.(string) != e2.Any.(string) { return false } if e.Int64 != e2.Int64 { return false } if e.Uint64 != e2.Uint64 { return false } if e.Int32 != e2.Int32 { return false } if e.Uint32 != e2.Uint32 { return false } if e.Int16 != e2.Int16 { return false } if e.Uint16 != e2.Uint16 { return false } if e.Int8 != e2.Int8 { return false } if e.Byte != e2.Byte { return false } if e.Float64 != e2.Float64 { return false } if e.Float32 != e2.Float32 { return false } if e.String != e2.String { return false } if bytes.Compare(e.ByteSlice, e2.ByteSlice) != 0 { return false } if len(e.StringSlice) != len(e2.StringSlice) { return false } for i := 0; i < len(e.StringSlice); i++ { if e.StringSlice[i] != e2.StringSlice[i] { return false } } if len(e.IntArray) != len(e2.IntArray) { return false } for i := 0; i < len(e.IntArray); i++ { if e.IntArray[i] != e2.IntArray[i] { return false } } if e.SomeStruct.A != e2.SomeStruct.A { return false } if e.EmbeddedStruct.A != e2.EmbeddedStruct.A { return false } if e.Omitted != e2.Omitted { return false } if e.OmitEmptyString != e2.OmitEmptyString { return false } return true } var buf [256]byte func main() { var e Example e.Setup() b, err := e.MarshalMsg(buf[:0]) if err != nil { panic(err) } b1 := b var e2 Example _, err = e2.UnmarshalMsg(b) if err != nil { panic(err) } println("marshal/unmarshal done: ", &e2) e.Omitted = "" if !e.Eq(&e2) { panic("comparison after marshal/unmarhsal failed") } var wbuf bytes.Buffer mw := msgp.NewWriterSize(&wbuf, 64) e.Omitted = "other" err = e.EncodeMsg(mw) if err != nil { panic(err) } mw.Flush() e2 = Example{} mr := msgp.NewReaderSize(bytes.NewReader(wbuf.Bytes()), 64) err = e2.DecodeMsg(mr) if err != nil { panic(err) } println("writer/reader done: ", &e2) e.Omitted = "" if !e.Eq(&e2) { panic("comparison after writer/reader failed") } if bytes.Compare(wbuf.Bytes(), b1) != 0 { panic("writer and marshal produced different results") } } msgp-1.2.5/tinygotest/testdata/simple_bytes_append/000077500000000000000000000000001472554463600225555ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/simple_bytes_append/main.go000066400000000000000000000006601472554463600240320ustar00rootroot00000000000000package main import "github.com/tinylib/msgp/msgp" type Example struct { I int S string } var buf [64]byte func main() { e := Example{ I: 1, S: "2", } b := buf[:0] b = msgp.AppendMapHeader(b, 2) b = msgp.AppendString(b, "i") b = msgp.AppendInt(b, e.I) b = msgp.AppendString(b, "s") b = msgp.AppendString(b, e.S) println("done, len(b):", len(b)) for i := 0; i < len(b); i++ { print(b[i], " ") } println() } msgp-1.2.5/tinygotest/testdata/simple_bytes_read/000077500000000000000000000000001472554463600222215ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/simple_bytes_read/main.go000066400000000000000000000014741472554463600235020ustar00rootroot00000000000000package main import "github.com/tinylib/msgp/msgp" type Example struct { I int S string } func main() { b := []byte{130, 161, 105, 1, 161, 115, 161, 50} e := Example{} sz, b, err := msgp.ReadMapHeaderBytes(b) if err != nil { panic(err) } if sz != 2 { panic("bad sz") } for i := 0; i < int(sz); i++ { var sb []byte var i32 int32 var err error sb, b, err = msgp.ReadStringZC(b) switch string(sb) { case "i": i32, b, err = msgp.ReadInt32Bytes(b) if err != nil { panic(err) } e.I = int(i32) case "s": sb, b, err = msgp.ReadStringZC(b) if err != nil { panic(err) } e.S = string(sb) default: panic("unexpected field:" + string(sb)) } } if len(b) != 0 { panic("unexpected extra") } if e.I != 1 || e.S != "2" { panic("not equal") } println("done") } msgp-1.2.5/tinygotest/testdata/simple_marshal/000077500000000000000000000000001472554463600215275ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/simple_marshal/main.go000066400000000000000000000004701472554463600230030ustar00rootroot00000000000000package main //go:generate msgp type Example struct { I int `msg:"i"` S string `msg:"s"` } func main() { e := Example{ I: 1, S: "2", } b, err := e.MarshalMsg(nil) if err != nil { panic(err) } println("done, len(b):", len(b)) for i := 0; i < len(b); i++ { print(b[i], " ") } println() } msgp-1.2.5/tinygotest/testdata/simple_roundtrip/000077500000000000000000000000001472554463600221265ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/simple_roundtrip/main.go000066400000000000000000000011361472554463600234020ustar00rootroot00000000000000package main //go:generate msgp type Example struct { I int `msg:"i"` S string `msg:"s"` // NOTE: floats omitted intentionally, many MCUs don't have // native support for it so the binary size bloats just due to // the software float implementation } func main() { e := Example{ I: 1, S: "2", } b, err := e.MarshalMsg(nil) if err != nil { panic(err) } var e2 Example extra, err := e2.UnmarshalMsg(b) if err != nil { panic(err) } if len(extra) != 0 { panic("unexpected extra") } if e.I != e2.I || e.S != e2.S { panic("not equal") } println("done, len(b):", len(b)) } msgp-1.2.5/tinygotest/testdata/simple_unmarshal/000077500000000000000000000000001472554463600220725ustar00rootroot00000000000000msgp-1.2.5/tinygotest/testdata/simple_unmarshal/main.go000066400000000000000000000006111472554463600233430ustar00rootroot00000000000000package main //go:generate msgp type Example struct { I int `msg:"i"` S string `msg:"s"` } func main() { b := []byte{130, 161, 105, 1, 161, 115, 161, 50} var e2 Example extra, err := e2.UnmarshalMsg(b) if err != nil { panic(err) } if len(extra) != 0 { panic("unexpected extra") } if e2.I != 1 || e2.S != "2" { panic("not equal") } println("done, len(b):", len(b)) } msgp-1.2.5/tinygotest/tinygo_test.go000066400000000000000000000155251472554463600176250ustar00rootroot00000000000000//go:build amd64 || darwin package tinygotest // NOTE: this is intended to be run with `go test` not `tinygo test`, // as it then in turn performs various `tinygo build` commands and // verifies the output. import ( "bytes" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" ) // empty gives us a baseline to compare sizes (and to prove that tinygo itself isn't broken) func TestEmptyRun(t *testing.T) { // build and run native executable tinygoBuild(t, "empty") run(t, "empty", "nativebin") } func TestEmptyBuild(t *testing.T) { // build for various environments tinygoBuild(t, "empty", buildOnlyTargets...) reportSizes(t, "empty") } // roundtrip provides reasonable coverage for various cases, but is // fairly bloated in terms of size compared to simpler examples func TestRoundtripRun(t *testing.T) { // build and run native executable goGenerate(t, "roundtrip") tinygoBuild(t, "roundtrip") run(t, "roundtrip", "nativebin") } func TestRoundtripBuild(t *testing.T) { // build for various environments goGenerate(t, "roundtrip") tinygoBuild(t, "roundtrip", buildOnlyTargets...) reportSizes(t, "roundtrip") } // simple_roundtrip is a minimal marshal+unmarshal example and should show // the baseline size when only using minimal functionality and marshal+unmarshal func TestSimpleRoundtripRun(t *testing.T) { // build and run native executable goGenerate(t, "simple_roundtrip") tinygoBuild(t, "simple_roundtrip") run(t, "simple_roundtrip", "nativebin") } func TestSimpleRoundtripBuild(t *testing.T) { // build for various environments goGenerate(t, "simple_roundtrip") tinygoBuild(t, "simple_roundtrip", buildOnlyTargets...) sizes := reportSizes(t, "simple_roundtrip") // this was ~13k at the time this test was written, // if it bloats up to over 20k, we assume something is wrong // (e.g. "fmt" or other large package being using) sz := sizes["arduino-nano33.bin"] if sz > 20000 { t.Errorf("arduino-nano33.bin is larger than expected: %d", sz) } } // simple_marshal is just the MarshalMsg part func TestSimpleMarshalRun(t *testing.T) { // build and run native executable goGenerate(t, "simple_marshal") tinygoBuild(t, "simple_marshal") run(t, "simple_marshal", "nativebin") } func TestSimpleMarshalBuild(t *testing.T) { // build for various environments goGenerate(t, "simple_marshal") tinygoBuild(t, "simple_marshal", buildOnlyTargets...) reportSizes(t, "simple_marshal") } // simple_unmarshal is just the UnmarshalMsg part func TestSimpleUnmarshalRun(t *testing.T) { // build and run native executable goGenerate(t, "simple_unmarshal") tinygoBuild(t, "simple_unmarshal") run(t, "simple_unmarshal", "nativebin") } func TestSimpleUnmarshalBuild(t *testing.T) { // build for various environments goGenerate(t, "simple_unmarshal") tinygoBuild(t, "simple_unmarshal", buildOnlyTargets...) reportSizes(t, "simple_unmarshal") } // simple_bytes_append is AppendX() methods without code generation func TestSimpleBytesAppendRun(t *testing.T) { // build and run native executable goGenerate(t, "simple_bytes_append") tinygoBuild(t, "simple_bytes_append") run(t, "simple_bytes_append", "nativebin") } func TestSimpleBytesAppendBuild(t *testing.T) { // build for various environments goGenerate(t, "simple_bytes_append") tinygoBuild(t, "simple_bytes_append", buildOnlyTargets...) reportSizes(t, "simple_bytes_append") } // simple_bytes_append is ReadX() methods without code generation func TestSimpleBytesReadRun(t *testing.T) { // build and run native executable goGenerate(t, "simple_bytes_read") tinygoBuild(t, "simple_bytes_read") run(t, "simple_bytes_read", "nativebin") } func TestSimpleBytesReadBuild(t *testing.T) { // build for various environments goGenerate(t, "simple_bytes_read") tinygoBuild(t, "simple_bytes_read", buildOnlyTargets...) reportSizes(t, "simple_bytes_read") } // do builds for these tinygo boards/environments to make sure // it at least compiles var buildOnlyTargets = []string{ "arduino-nano33", // ARM Cortex-M0, SAMD21 "feather-m4", // ARM Cortex-M4, SAMD51 "wasm", // WebAssembly // "arduino-nano", // AVR - roundtrip build currently fails with: could not store type code number inside interface type code // "esp32-coreboard-v2", // ESP - xtensa seems to require additional setup, currently errors with: No available targets are compatible with triple "xtensa" } func run(t *testing.T, dir, exe string, args ...string) { t.Helper() if runtime.GOOS == "windows" { exe += ".exe" } cmd := exec.Command("./" + exe) wd, err := os.Getwd() if err != nil { t.Fatal(err) } cmd.Dir = filepath.Join(wd, "testdata", dir) cmd.Args = args b, err := cmd.CombinedOutput() if len(bytes.TrimSpace(b)) > 0 { t.Logf("%s: %s %v; output: %s", dir, exe, args, b) } if err != nil { t.Fatal(err) } } func goGenerate(t *testing.T, dir string) { t.Helper() t.Logf("%s: go generate", dir) cmd := exec.Command("go", "generate") wd, err := os.Getwd() if err != nil { t.Fatal(err) } cmd.Dir = filepath.Join(wd, "testdata", dir) b, err := cmd.CombinedOutput() if len(bytes.TrimSpace(b)) > 0 { t.Logf("%s: go generate output: %s", dir, b) } if err != nil { t.Fatal(err) } } func tinygoBuild(t *testing.T, dir string, targets ...string) { t.Helper() wd, err := os.Getwd() if err != nil { t.Fatal(err) } dirabs := filepath.Join(wd, "testdata", dir) // no targets specified implies just build the native executable if len(targets) == 0 { targets = []string{""} } for _, tgt := range targets { ext := ".bin" if tgt == "wasm" { ext = ".wasm" } dst := tgt + ext if tgt == "" { dst = "nativebin" if runtime.GOOS == "windows" { dst += ".exe" } } var args []string if tgt == "" { // empty target means the native platform args = []string{"build", "-o=" + dst, "."} } else { args = []string{"build", "-target=" + tgt, "-o=" + dst, "."} } t.Logf("%s: tinygo %v", dir, args) cmd := exec.Command("tinygo", args...) cmd.Dir = dirabs b, err := cmd.CombinedOutput() if len(bytes.TrimSpace(b)) > 0 { t.Logf("%s: tinygo build %v; output: %s", dir, args, b) } if err != nil { // See https://github.com/tinygo-org/tinygo/issues/3977 if strings.Contains(string(b), "could not find wasm-opt") { t.Skipf("skipping wasm test because wasm-opt is not installed") } t.Fatal(err) } } } var spacePad = strings.Repeat(" ", 64) func reportSizes(t *testing.T, dir string) (ret map[string]int64) { ret = make(map[string]int64) var fnl []string for _, gl := range []string{"*.bin", "*.wasm"} { fnl2, err := filepath.Glob(filepath.Join("testdata", dir, gl)) if err != nil { t.Fatal(err) } fnl = append(fnl, fnl2...) } for _, fn := range fnl { st, err := os.Stat(fn) if err != nil { t.Fatal(err) } _, jfn := filepath.Split(fn) t.Logf("size report - %s %6d bytes", (jfn + spacePad)[:24], st.Size()) ret[jfn] = st.Size() } return ret }