pax_global_header00006660000000000000000000000064126125333750014521gustar00rootroot0000000000000052 comment=0cea1fa86e8403be1284013014f87ab942056de8 msgp-1.0-beta/000077500000000000000000000000001261253337500132405ustar00rootroot00000000000000msgp-1.0-beta/.gitignore000066400000000000000000000001321261253337500152240ustar00rootroot00000000000000_generated/generated.go _generated/generated_test.go msgp/defgen_test.go msgp/cover.out *~msgp-1.0-beta/.travis.yml000066400000000000000000000001571261253337500153540ustar00rootroot00000000000000language: go go: - 1.4 - 1.5 - tip env: - GIMME_ARCH=amd64 - GIMME_ARCH=386 script: "make travis" msgp-1.0-beta/LICENSE000066400000000000000000000021711261253337500142460ustar00rootroot00000000000000Copyright (c) 2014 Philip Hofer Portions Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.msgp-1.0-beta/Makefile000066400000000000000000000017341261253337500147050ustar00rootroot00000000000000 # NOTE: This Makefile is only necessary if you # plan on developing the msgp tool and library. # Installation can still be performed with a # normal `go install`. # generated integration test files GGEN = ./_generated/generated.go ./_generated/generated_test.go # generated unit test files MGEN = ./msgp/defgen_test.go SHELL := /bin/bash BIN = $(GOBIN)/msgp .PHONY: clean wipe install get-deps bench all $(BIN): */*.go @go install ./... install: $(BIN) $(GGEN): ./_generated/def.go go generate ./_generated $(MGEN): ./msgp/defs_test.go go generate ./msgp test: all go test -v ./msgp go test -v ./_generated bench: all go test -bench . ./msgp go test -bench . ./_generated clean: $(RM) $(GGEN) $(MGEN) wipe: clean $(RM) $(BIN) get-deps: go get -d -t ./... all: install $(GGEN) $(MGEN) # travis CI enters here travis: go get -d -t ./... go build -o "$${GOPATH%%:*}/bin/msgp" . go generate ./msgp go generate ./_generated go test ./msgp go test ./_generated msgp-1.0-beta/README.md000066400000000000000000000125211261253337500145200ustar00rootroot00000000000000MessagePack Code Generator [![Build Status](https://travis-ci.org/tinylib/msgp.svg?branch=master)](https://travis-ci.org/tinylib/msgp) ======= [![forthebadge](http://forthebadge.com/badges/uses-badges.svg)](http://forthebadge.com) [![forthebadge](http://forthebadge.com/badges/ages-12.svg)](http://forthebadge.com) This is a code generation tool and serialization library for [MessagePack](http://msgpack.org). It is targeted at the `go generate` [tool](http://tip.golang.org/cmd/go/#hdr-Generate_Go_files_by_processing_source). You can read more about MessagePack [in the wiki](http://github.com/tinylib/msgp/wiki), or at [msgpack.org](http://msgpack.org). ### Why? - Use Go as your schema language - Speeeeeed (400MB/s on modern hardware) - [JSON interop](http://godoc.org/github.com/tinylib/msgp/msgp#CopyToJSON) - [User-defined extensions](http://github.com/tinylib/msgp/wiki/Using-Extensions) - Type safety - Encoding flexibility ### Quickstart Note: you need at least go 1.3 to compile this package, and at least go 1.4 to use `go generate`. In a source file, include the following directive: ```go //go:generate msgp ``` The `msgp` command will generate serialization methods for all exported type declarations in the file. You can [read more about the code generation options here](http://github.com/tinylib/msgp/wiki/Using-the-Code-Generator). ### Use Field names can be set in much the same way as the `encoding/json` package. For example: ```go type Person struct { Name string `msg:"name"` Address string `msg:"address"` Age int `msg:"age"` Hidden string `msg:"-"` // this field is ignored unexported bool // this field is also ignored } ``` By default, the code generator will satisfy `msgp.Sizer`, `msgp.Encodable`, `msgp.Decodable`, `msgp.Marshaler`, and `msgp.Unmarshaler`. Carefully-designed applications can use these methods to do marshalling/unmarshalling with zero allocations. While `msgp.Marshaler` and `msgp.Unmarshaler` are quite similar to the standard library's `json.Marshaler` and `json.Unmarshaler`, `msgp.Encodable` and `msgp.Decodable` are useful for stream serialization. (`*msgp.Writer` and `*msgp.Reader` are essentially protocol-aware versions of `*bufio.Writer` and `*bufio.Reader`, respectively.) ### Features - Extremely fast generated code - Test and benchmark generation - JSON interoperability (see `msgp.CopyToJSON() and msgp.UnmarshalAsJSON()`) - Support for complex type declarations - Native support for Go's `time.Time`, `complex64`, and `complex128` types - Generation of both `[]byte`-oriented and `io.Reader/io.Writer`-oriented methods - Support for arbitrary type system extensions - [Preprocessor directives](http://github.com/tinylib/msgp/wiki/Preprocessor-Directives) Consider the following: ```go const Eight = 8 type MyInt int type Data []byte type Struct struct { Which map[string]*MyInt `msg:"which"` Other Data `msg:"other"` Nums [Eight]float64 `msg:"nums"` } ``` As long as the declarations of `MyInt` and `Data` are in the same file as `Struct`, the parser will determine that the type information for `MyInt` and `Data` can be passed into the definition of `Struct` before its methods are generated. #### Extensions MessagePack supports defining your own types through "extensions," which are just a tuple of the data "type" (`int8`) and the raw binary. You [can see a worked example in the wiki.](http://github.com/tinylib/msgp/wiki/Using-Extensions) ### Status Alpha. I _will_ break stuff. There is an open milestone for Beta stability (targeted for January.) Only the `/msgp` sub-directory will have a stability guarantee. You can read more about how `msgp` maps MessagePack types onto Go types [in the wiki](http://github.com/tinylib/msgp/wiki). Here some of the known limitations/restrictions: - Identifiers from outside the processed source file are assumed (optimistically) to satisfy the generator's interfaces. If this isn't the case, your code will fail to compile. - Like most serializers, `chan` and `func` fields are ignored, as well as non-exported fields. - Encoding of `interface{}` is limited to built-ins or types that have explicit encoding methods. - _Maps must have `string` keys._ This is intentional (as it preserves JSON interop.) Although non-string map keys are not forbidden by the MessagePack standard, many serializers impose this restriction. (It also means *any* well-formed `struct` can be de-serialized into a `map[string]interface{}`.) The only exception to this rule is that the deserializers will allow you to read map keys encoded as `bin` types, due to the fact that some legacy encodings permitted this. (However, those values will still be cast to Go `string`s, and they will be converted to `str` types when re-encoded. It is the responsibility of the user to ensure that map keys are UTF-8 safe in this case.) The same rules hold true for JSON translation. If the output compiles, then there's a pretty good chance things are fine. (Plus, we generate tests for you.) *Please, please, please* file an issue if you think the generator is writing broken code. ### Performance If you like benchmarks, see [here.](https://github.com/alecthomas/go_serialization_benchmarks) As one might expect, the generated methods that deal with `[]byte` are faster, but the `io.Reader/Writer` methods are generally more memory-efficient for large (> 2KB) objects. msgp-1.0-beta/_generated/000077500000000000000000000000001261253337500153355ustar00rootroot00000000000000msgp-1.0-beta/_generated/def.go000066400000000000000000000071271261253337500164310ustar00rootroot00000000000000package _generated import ( "github.com/tinylib/msgp/msgp" "os" "time" ) //go:generate msgp -o generated.go // All of the struct // definitions in this // file are fed to the code // generator when `make test` is // called, followed by an // invocation of `go test -v` in this // directory. A simple way of testing // a struct definition is // by adding it to this file. type Block [32]byte // tests edge-cases with // compiling size compilation. type X struct { Values [32]byte // should compile to 32*msgp.ByteSize; encoded as Bin More Block // should be identical to the above Others [][32]int32 // should compile to len(x.Others)*32*msgp.Int32Size Matrix [][]int32 // should not optimize ManyFixed []Fixed } // test fixed-size struct // size compilation type Fixed struct { A float64 B bool } type TestType struct { F *float64 `msg:"float"` Els map[string]string `msg:"elements"` Obj struct { // test anonymous struct ValueA string `msg:"value_a"` ValueB []byte `msg:"value_b"` } `msg:"object"` Child *TestType `msg:"child"` Time time.Time `msg:"time"` Any interface{} `msg:"any"` Appended msgp.Raw `msg:"appended"` Num msgp.Number `msg:"num"` } //msgp:tuple TestBench type TestBench struct { Name string BirthDay time.Time Phone string Siblings int Spouse bool Money float64 } //msgp:tuple TestFast type TestFast struct { Lat, Long, Alt float64 // test inline decl Data []byte } // Test nested aliases type FastAlias TestFast type AliasContainer struct { Fast FastAlias } // Test dependency resolution type IntA int type IntB IntA type IntC IntB type TestHidden struct { A string B []float64 Bad func(string) bool // This results in a warning: field "Bad" unsupported } type Embedded struct { *Embedded // test embedded field Children []Embedded PtrChildren []*Embedded Other string } const eight = 8 type Things struct { Cmplx complex64 `msg:"complex"` // test slices Vals []int32 `msg:"values"` Arr [msgp.ExtensionPrefixSize]float64 `msg:"arr"` // test const array and *ast.SelectorExpr as array size Arr2 [4]float64 `msg:"arr2"` // test basic lit array Ext *msgp.RawExtension `msg:"ext,extension"` // test extension Oext msgp.RawExtension `msg:"oext,extension"` // test extension reference } type MyEnum byte const ( A MyEnum = iota B C D invalid ) // test shim directive (below) //msgp:shim MyEnum as:string using:(MyEnum).String/myenumStr //msgp:shim *os.File as:string using:filetostr/filefromstr func filetostr(f *os.File) string { return f.Name() } func filefromstr(s string) *os.File { f, _ := os.Open(s) return f } func (m MyEnum) String() string { switch m { case A: return "A" case B: return "B" case C: return "C" case D: return "D" default: return "" } } func myenumStr(s string) MyEnum { switch s { case "A": return A case "B": return B case "C": return C case "D": return D default: return invalid } } // test pass-specific directive //msgp:decode ignore Insane type Insane [3]map[string]struct{ A, B CustomInt } type Custom struct { Bts CustomBytes `msg:"bts"` Mp map[string]*Embedded `msg:"mp"` Enums []MyEnum `msg:"enums"` // test explicit enum shim Some FileHandle `msg:file_handle` } type Files []*os.File type FileHandle struct { Relevent Files `msg:"files"` Name string `msg:"name"` } type CustomInt int type CustomBytes []byte msgp-1.0-beta/_generated/gen_test.go000066400000000000000000000046771261253337500175120ustar00rootroot00000000000000package _generated import ( "bytes" "github.com/tinylib/msgp/msgp" "reflect" "testing" "time" ) // benchmark encoding a small, "fast" type. // the point here is to see how much garbage // is generated intrinsically by the encoding/ // decoding process as opposed to the nature // of the struct. func BenchmarkFastEncode(b *testing.B) { v := &TestFast{ Lat: 40.12398, Long: -41.9082, Alt: 201.08290, Data: []byte("whaaaaargharbl"), } var buf bytes.Buffer msgp.Encode(&buf, v) en := msgp.NewWriter(msgp.Nowhere) b.SetBytes(int64(buf.Len())) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { v.EncodeMsg(en) } en.Flush() } // benchmark decoding a small, "fast" type. // the point here is to see how much garbage // is generated intrinsically by the encoding/ // decoding process as opposed to the nature // of the struct. func BenchmarkFastDecode(b *testing.B) { v := &TestFast{ Lat: 40.12398, Long: -41.9082, Alt: 201.08290, Data: []byte("whaaaaargharbl"), } var buf bytes.Buffer msgp.Encode(&buf, v) dc := msgp.NewReader(msgp.NewEndlessReader(buf.Bytes(), b)) b.SetBytes(int64(buf.Len())) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { v.DecodeMsg(dc) } } // This covers the following cases: // - Recursive types // - Non-builtin identifiers (and recursive types) // - time.Time // - map[string]string // - anonymous structs // func Test1EncodeDecode(t *testing.T) { f := 32.00 tt := &TestType{ F: &f, Els: map[string]string{ "thing_one": "one", "thing_two": "two", }, Obj: struct { ValueA string `msg:"value_a"` ValueB []byte `msg:"value_b"` }{ ValueA: "here's the first inner value", ValueB: []byte("here's the second inner value"), }, Child: nil, Time: time.Now(), Appended: msgp.Raw([]byte{0xc0}), // 'nil' } var buf bytes.Buffer err := msgp.Encode(&buf, tt) if err != nil { t.Fatal(err) } tnew := new(TestType) err = msgp.Decode(&buf, tnew) if err != nil { t.Error(err) } if !reflect.DeepEqual(tt, tnew) { t.Logf("in: %v", tt) t.Logf("out: %v", tnew) t.Fatal("objects not equal") } tanother := new(TestType) buf.Reset() msgp.Encode(&buf, tt) var left []byte left, err = tanother.UnmarshalMsg(buf.Bytes()) if err != nil { t.Error(err) } if len(left) > 0 { t.Errorf("%d bytes left", len(left)) } if !reflect.DeepEqual(tt, tanother) { t.Logf("in: %v", tt) t.Logf("out: %v", tanother) t.Fatal("objects not equal") } } msgp-1.0-beta/gen/000077500000000000000000000000001261253337500140115ustar00rootroot00000000000000msgp-1.0-beta/gen/decode.go000066400000000000000000000105601261253337500155650ustar00rootroot00000000000000package gen import ( "io" "strconv" ) const ( structArraySizeVar = "ssz" structMapSizeVar = "isz" mapSizeVar = "msz" sliceSizeVar = "xsz" arraySizeVar = "asz" ) func decode(w io.Writer) *decodeGen { return &decodeGen{ p: printer{w: w}, hasfield: false, } } type decodeGen struct { passes p printer hasfield bool } func (d *decodeGen) Method() Method { return Decode } func (d *decodeGen) needsField() { if d.hasfield { return } d.p.print("\nvar field []byte; _ = field") d.hasfield = true } func (d *decodeGen) Execute(p Elem) error { p = d.applyall(p) if p == nil { return nil } d.hasfield = false if !d.p.ok() { return d.p.err } if !IsPrintable(p) { return nil } d.p.comment("DecodeMsg implements msgp.Decodable") d.p.printf("\nfunc (%s %s) DecodeMsg(dc *msgp.Reader) (err error) {", p.Varname(), methodReceiver(p)) next(d, p) d.p.nakedReturn() unsetReceiver(p) return d.p.err } func (d *decodeGen) gStruct(s *Struct) { if !d.p.ok() { return } if s.AsTuple { d.structAsTuple(s) } else { d.structAsMap(s) } return } func (d *decodeGen) assignAndCheck(name string, typ string) { if !d.p.ok() { return } d.p.printf("\n%s, err = dc.Read%s()", name, typ) d.p.print(errcheck) } func (d *decodeGen) structAsTuple(s *Struct) { nfields := len(s.Fields) d.p.print("\n{") d.p.declare(structArraySizeVar, u32) d.assignAndCheck(structArraySizeVar, arrayHeader) d.p.arrayCheck(strconv.Itoa(nfields), structArraySizeVar) d.p.print("\n}") for i := range s.Fields { if !d.p.ok() { return } next(d, s.Fields[i].FieldElem) } } func (d *decodeGen) structAsMap(s *Struct) { d.needsField() d.p.declare(structMapSizeVar, u32) d.assignAndCheck(structMapSizeVar, mapHeader) d.p.print("\nfor isz > 0 {\nisz--") d.assignAndCheck("field", mapKey) d.p.print("\nswitch msgp.UnsafeString(field) {") for i := range s.Fields { d.p.printf("\ncase \"%s\":", s.Fields[i].FieldTag) next(d, s.Fields[i].FieldElem) if !d.p.ok() { return } } d.p.print("\ndefault:\nerr = dc.Skip()") d.p.print(errcheck) d.p.closeblock() // close switch d.p.closeblock() // close for loop } func (d *decodeGen) gBase(b *BaseElem) { if !d.p.ok() { return } // open block for 'tmp' if b.Convert { d.p.printf("\n{ var tmp %s", b.BaseType()) } vname := b.Varname() // e.g. "z.FieldOne" bname := b.BaseName() // e.g. "Float64" // handle special cases // for object type. switch b.Value { case Bytes: if b.Convert { d.p.printf("\ntmp, err = dc.ReadBytes([]byte(%s))", vname) } else { d.p.printf("\n%s, err = dc.ReadBytes(%s)", vname, vname) } case IDENT: d.p.printf("\nerr = %s.DecodeMsg(dc)", vname) case Ext: d.p.printf("\nerr = dc.ReadExtension(%s)", vname) default: if b.Convert { d.p.printf("\ntmp, err = dc.Read%s()", bname) } else { d.p.printf("\n%s, err = dc.Read%s()", vname, bname) } } // close block for 'tmp' if b.Convert { d.p.printf("\n%s = %s(tmp)\n}", vname, b.FromBase()) } d.p.print(errcheck) } func (d *decodeGen) gMap(m *Map) { if !d.p.ok() { return } // resize or allocate map d.p.declare(mapSizeVar, u32) d.assignAndCheck(mapSizeVar, mapHeader) d.p.resizeMap(mapSizeVar, m) // for element in map, read string/value // pair and assign d.p.print("\nfor msz > 0 {\nmsz--") d.p.declare(m.Keyidx, "string") d.p.declare(m.Validx, m.Value.TypeName()) d.assignAndCheck(m.Keyidx, stringTyp) next(d, m.Value) d.p.mapAssign(m) d.p.closeblock() } func (d *decodeGen) gSlice(s *Slice) { if !d.p.ok() { return } d.p.declare(sliceSizeVar, u32) d.assignAndCheck(sliceSizeVar, arrayHeader) d.p.resizeSlice(sliceSizeVar, s) d.p.rangeBlock(s.Index, s.Varname(), d, s.Els) } func (d *decodeGen) gArray(a *Array) { if !d.p.ok() { return } // special case if we have [const]byte if be, ok := a.Els.(*BaseElem); ok && (be.Value == Byte || be.Value == Uint8) { d.p.printf("\nerr = dc.ReadExactBytes(%s[:])", a.Varname()) d.p.print(errcheck) return } d.p.declare(arraySizeVar, u32) d.assignAndCheck(arraySizeVar, arrayHeader) d.p.arrayCheck(a.Size, arraySizeVar) d.p.rangeBlock(a.Index, a.Varname(), d, a.Els) } func (d *decodeGen) gPtr(p *Ptr) { if !d.p.ok() { return } d.p.print("\nif dc.IsNil() {") d.p.print("\nerr = dc.ReadNil()") d.p.print(errcheck) d.p.printf("\n%s = nil\n} else {", p.Varname()) d.p.initPtr(p) next(d, p.Value) d.p.closeblock() } msgp-1.0-beta/gen/elem.go000066400000000000000000000264721261253337500152750ustar00rootroot00000000000000package gen import ( "fmt" "math/rand" "strings" ) const ( idxChars = "abcdefghijlkmnopqrstuvwxyz" idxLen = 3 ) // generate a random index variable name func randIdx() string { bts := make([]byte, idxLen) for i := range bts { bts[i] = idxChars[rand.Intn(len(idxChars))] } return string(bts) } // This code defines the type declaration tree. // // Consider the following: // // type Marshaler struct { // Thing1 *float64 `msg:"thing1"` // Body []byte `msg:"body"` // } // // A parser using this generator as a backend // should parse the above into: // // var val Elem = &Ptr{ // name: "z", // Value: &Struct{ // Name: "Marshaler", // Fields: []StructField{ // { // FieldTag: "thing1", // FieldElem: &Ptr{ // name: "z.Thing1", // Value: &BaseElem{ // name: "*z.Thing1", // Value: Float64, // Convert: false, // }, // }, // }, // { // FieldTag: "body", // FieldElem: &BaseElem{ // name: "z.Body", // Value: Bytes, // Convert: false, // }, // }, // }, // }, // } // Base is one of the // base types type Primitive uint8 // this is effectively the // list of currently available // ReadXxxx / WriteXxxx methods. const ( Invalid Primitive = iota Bytes String Float32 Float64 Complex64 Complex128 Uint Uint8 Uint16 Uint32 Uint64 Byte Int Int8 Int16 Int32 Int64 Bool Intf // interface{} Time // time.Time Ext // extension IDENT // IDENT means an unrecognized identifier ) // all of the recognized identities // that map to primitive types var primitives = map[string]Primitive{ "[]byte": Bytes, "string": String, "float32": Float32, "float64": Float64, "complex64": Complex64, "complex128": Complex128, "uint": Uint, "uint8": Uint8, "uint16": Uint16, "uint32": Uint32, "uint64": Uint64, "byte": Byte, "int": Int, "int8": Int8, "int16": Int16, "int32": Int32, "int64": Int64, "bool": Bool, "interface{}": Intf, "time.Time": Time, "msgp.Extension": Ext, } // types built into the library // that satisfy all of the // interfaces. var builtins = map[string]struct{}{ "msgp.Raw": struct{}{}, "msgp.Number": struct{}{}, } // common data/methods for every Elem type common struct{ vname, alias string } func (c *common) SetVarname(s string) { c.vname = s } func (c *common) Varname() string { return c.vname } func (c *common) Alias(typ string) { c.alias = typ } func (c *common) hidden() {} func IsPrintable(e Elem) bool { if be, ok := e.(*BaseElem); ok && !be.Printable() { return false } return true } // Elem is a go type capable of being // serialized into MessagePack. It is // implemented by *Ptr, *Struct, *Array, // *Slice, *Map, and *BaseElem. type Elem interface { // SetVarname sets this nodes // variable name and recursively // sets the names of all its children. // In general, this should only be // called on the parent of the tree. SetVarname(s string) // Varname returns the variable // name of the element. Varname() string // TypeName is the canonical // go type name of the node // e.g. "string", "int", "map[string]float64" // OR the alias name, if it has been set. TypeName() string // Alias sets a type (alias) name Alias(typ string) // Copy should perform a deep copy of the object Copy() Elem // Complexity returns a measure of the // complexity of element (greater than // or equal to 1.) Complexity() int hidden() } // Ident returns the *BaseElem that corresponds // to the provided identity. func Ident(id string) *BaseElem { p, ok := primitives[id] if ok { return &BaseElem{Value: p} } be := &BaseElem{Value: IDENT} be.Alias(id) return be } type Array struct { common Index string // index variable name Size string // array size Els Elem // child } func (a *Array) SetVarname(s string) { a.common.SetVarname(s) ridx: a.Index = randIdx() // try to avoid using the same // index as a parent slice if strings.Contains(a.Varname(), a.Index) { goto ridx } a.Els.SetVarname(fmt.Sprintf("%s[%s]", a.Varname(), a.Index)) } func (a *Array) TypeName() string { if a.common.alias != "" { return a.common.alias } a.common.Alias(fmt.Sprintf("[%s]%s", a.Size, a.Els.TypeName())) return a.common.alias } func (a *Array) Copy() Elem { b := *a b.Els = a.Els.Copy() return &b } func (a *Array) Complexity() int { return 1 + a.Els.Complexity() } // Map is a map[string]Elem type Map struct { common Keyidx string // key variable name Validx string // value variable name Value Elem // value element } func (m *Map) SetVarname(s string) { m.common.SetVarname(s) ridx: m.Keyidx = randIdx() m.Validx = randIdx() // just in case if m.Keyidx == m.Validx { goto ridx } m.Value.SetVarname(m.Validx) } func (m *Map) TypeName() string { if m.common.alias != "" { return m.common.alias } m.common.Alias("map[string]" + m.Value.TypeName()) return m.common.alias } func (m *Map) Copy() Elem { g := *m g.Value = m.Value.Copy() return &g } func (m *Map) Complexity() int { return 2 + m.Value.Complexity() } type Slice struct { common Index string Els Elem // The type of each element } func (s *Slice) SetVarname(a string) { s.common.SetVarname(a) s.Index = randIdx() s.Els.SetVarname(fmt.Sprintf("%s[%s]", s.Varname(), s.Index)) } func (s *Slice) TypeName() string { if s.common.alias != "" { return s.common.alias } s.common.Alias("[]" + s.Els.TypeName()) return s.common.alias } func (s *Slice) Copy() Elem { z := *s z.Els = s.Els.Copy() return &z } func (s *Slice) Complexity() int { return 1 + s.Els.Complexity() } type Ptr struct { common Value Elem } func (s *Ptr) SetVarname(a string) { s.common.SetVarname(a) // struct fields are dereferenced // automatically... switch x := s.Value.(type) { case *Struct: // struct fields are automatically dereferenced x.SetVarname(a) return case *BaseElem: // identities have pointer receivers if x.Value == IDENT { x.SetVarname(a) } else { x.SetVarname("*" + a) } return default: s.Value.SetVarname("*" + a) return } } func (s *Ptr) TypeName() string { if s.common.alias != "" { return s.common.alias } s.common.Alias("*" + s.Value.TypeName()) return s.common.alias } func (s *Ptr) Copy() Elem { v := *s v.Value = s.Value.Copy() return &v } func (s *Ptr) Complexity() int { return 1 + s.Value.Complexity() } func (s *Ptr) Needsinit() bool { if be, ok := s.Value.(*BaseElem); ok && be.needsref { return false } return true } type Struct struct { common Fields []StructField // field list AsTuple bool // write as an array instead of a map } func (s *Struct) TypeName() string { if s.common.alias != "" { return s.common.alias } str := "struct{\n" for i := range s.Fields { str += s.Fields[i].FieldName + " " + s.Fields[i].FieldElem.TypeName() + ";\n" } str += "}" s.common.Alias(str) return s.common.alias } func (s *Struct) SetVarname(a string) { s.common.SetVarname(a) writeStructFields(s.Fields, a) } func (s *Struct) Copy() Elem { g := *s g.Fields = make([]StructField, len(s.Fields)) copy(g.Fields, s.Fields) for i := range s.Fields { g.Fields[i].FieldElem = s.Fields[i].FieldElem.Copy() } return &g } func (s *Struct) Complexity() int { c := 1 for i := range s.Fields { c += s.Fields[i].FieldElem.Complexity() } return c } type StructField struct { FieldTag string // the string inside the `msg:""` tag FieldName string // the name of the struct field FieldElem Elem // the field type } // BaseElem is an element that // can be represented by a primitive // MessagePack type. type BaseElem struct { common ShimToBase string // shim to base type, or empty ShimFromBase string // shim from base type, or empty Value Primitive // Type of element Convert bool // should we do an explicit conversion? mustinline bool // must inline; not printable needsref bool // needs reference for shim } func (s *BaseElem) Printable() bool { return !s.mustinline } func (s *BaseElem) Alias(typ string) { s.common.Alias(typ) if s.Value != IDENT { s.Convert = true } if strings.Contains(typ, ".") { s.mustinline = true } } func (s *BaseElem) SetVarname(a string) { // extensions whose parents // are not pointers need to // be explicitly referenced if s.Value == Ext || s.needsref { if strings.HasPrefix(a, "*") { s.common.SetVarname(a[1:]) return } s.common.SetVarname("&" + a) return } s.common.SetVarname(a) } // TypeName returns the syntactically correct Go // type name for the base element. func (s *BaseElem) TypeName() string { if s.common.alias != "" { return s.common.alias } s.common.Alias(s.BaseType()) return s.common.alias } // ToBase, used if Convert==true, is used as tmp = {{ToBase}}({{Varname}}) func (s *BaseElem) ToBase() string { if s.ShimToBase != "" { return s.ShimToBase } return s.BaseType() } // FromBase, used if Convert==true, is used as {{Varname}} = {{FromBase}}(tmp) func (s *BaseElem) FromBase() string { if s.ShimFromBase != "" { return s.ShimFromBase } return s.TypeName() } // BaseName returns the string form of the // base type (e.g. Float64, Ident, etc) func (s *BaseElem) BaseName() string { // time is a special case; // we strip the package prefix if s.Value == Time { return "Time" } return s.Value.String() } func (s *BaseElem) BaseType() string { switch s.Value { case IDENT: return s.TypeName() // exceptions to the naming/capitalization // rule: case Intf: return "interface{}" case Bytes: return "[]byte" case Time: return "time.Time" case Ext: return "msgp.Extension" // everything else is base.String() with // the first letter as lowercase default: return strings.ToLower(s.BaseName()) } } func (s *BaseElem) Needsref(b bool) { s.needsref = b } func (s *BaseElem) Copy() Elem { g := *s return &g } func (s *BaseElem) Complexity() int { if s.Convert && !s.mustinline { return 2 } // we need to return 1 if !printable(), // in order to make sure that stuff gets // inlined appropriately return 1 } // Resolved returns whether or not // the type of the element is // a primitive or a builtin provided // by the package. func (s *BaseElem) Resolved() bool { if s.Value == IDENT { _, ok := builtins[s.TypeName()] return ok } return true } func (k Primitive) String() string { switch k { case String: return "String" case Bytes: return "Bytes" case Float32: return "Float32" case Float64: return "Float64" case Complex64: return "Complex64" case Complex128: return "Complex128" case Uint: return "Uint" case Uint8: return "Uint8" case Uint16: return "Uint16" case Uint32: return "Uint32" case Uint64: return "Uint64" case Byte: return "Byte" case Int: return "Int" case Int8: return "Int8" case Int16: return "Int16" case Int32: return "Int32" case Int64: return "Int64" case Bool: return "Bool" case Intf: return "Intf" case Time: return "time.Time" case Ext: return "Extension" case IDENT: return "Ident" default: return "INVALID" } } // writeStructFields is a trampoline for writeBase for // all of the fields in a struct func writeStructFields(s []StructField, name string) { for i := range s { s[i].FieldElem.SetVarname(fmt.Sprintf("%s.%s", name, s[i].FieldName)) } } msgp-1.0-beta/gen/encode.go000066400000000000000000000070101261253337500155730ustar00rootroot00000000000000package gen import ( "fmt" "github.com/tinylib/msgp/msgp" "io" ) func encode(w io.Writer) *encodeGen { return &encodeGen{ p: printer{w: w}, } } type encodeGen struct { passes p printer fuse []byte } func (e *encodeGen) Method() Method { return Encode } func (e *encodeGen) Apply(dirs []string) error { return nil } func (e *encodeGen) writeAndCheck(typ string, argfmt string, arg interface{}) { e.p.printf("\nerr = en.Write%s(%s)", typ, fmt.Sprintf(argfmt, arg)) e.p.print(errcheck) } func (e *encodeGen) fuseHook() { if len(e.fuse) > 0 { e.appendraw(e.fuse) e.fuse = e.fuse[:0] } } func (e *encodeGen) Fuse(b []byte) { if len(e.fuse) > 0 { e.fuse = append(e.fuse, b...) } else { e.fuse = b } } func (e *encodeGen) Execute(p Elem) error { if !e.p.ok() { return e.p.err } p = e.applyall(p) if p == nil { return nil } if !IsPrintable(p) { return nil } e.p.comment("EncodeMsg implements msgp.Encodable") e.p.printf("\nfunc (%s %s) EncodeMsg(en *msgp.Writer) (err error) {", p.Varname(), imutMethodReceiver(p)) next(e, p) e.p.nakedReturn() return e.p.err } func (e *encodeGen) gStruct(s *Struct) { if !e.p.ok() { return } if s.AsTuple { e.tuple(s) } else { e.structmap(s) } return } func (e *encodeGen) tuple(s *Struct) { nfields := len(s.Fields) data := msgp.AppendArrayHeader(nil, uint32(nfields)) e.p.printf("\n// array header, size %d", nfields) e.Fuse(data) for i := range s.Fields { if !e.p.ok() { return } next(e, s.Fields[i].FieldElem) } } func (e *encodeGen) appendraw(bts []byte) { e.p.print("\nerr = en.Append(") for i, b := range bts { if i != 0 { e.p.print(", ") } e.p.printf("0x%x", b) } e.p.print(")\nif err != nil { return err }") } func (e *encodeGen) structmap(s *Struct) { nfields := len(s.Fields) data := msgp.AppendMapHeader(nil, uint32(nfields)) e.p.printf("\n// map header, size %d", nfields) e.Fuse(data) for i := range s.Fields { if !e.p.ok() { return } data = msgp.AppendString(nil, s.Fields[i].FieldTag) e.p.printf("\n// write %q", s.Fields[i].FieldTag) e.Fuse(data) next(e, s.Fields[i].FieldElem) } } func (e *encodeGen) gMap(m *Map) { if !e.p.ok() { return } e.fuseHook() vname := m.Varname() e.writeAndCheck(mapHeader, lenAsUint32, vname) e.p.printf("\nfor %s, %s := range %s {", m.Keyidx, m.Validx, vname) e.writeAndCheck(stringTyp, literalFmt, m.Keyidx) next(e, m.Value) e.p.closeblock() } func (e *encodeGen) gPtr(s *Ptr) { if !e.p.ok() { return } e.fuseHook() e.p.printf("\nif %s == nil { err = en.WriteNil(); if err != nil { return; } } else {", s.Varname()) next(e, s.Value) e.p.closeblock() } func (e *encodeGen) gSlice(s *Slice) { if !e.p.ok() { return } e.fuseHook() e.writeAndCheck(arrayHeader, lenAsUint32, s.Varname()) e.p.rangeBlock(s.Index, s.Varname(), e, s.Els) } func (e *encodeGen) gArray(a *Array) { if !e.p.ok() { return } e.fuseHook() // shortcut for [const]byte if be, ok := a.Els.(*BaseElem); ok && (be.Value == Byte || be.Value == Uint8) { e.p.printf("\nerr = en.WriteBytes(%s[:])", a.Varname()) e.p.print(errcheck) return } e.writeAndCheck(arrayHeader, literalFmt, a.Size) e.p.rangeBlock(a.Index, a.Varname(), e, a.Els) } func (e *encodeGen) gBase(b *BaseElem) { if !e.p.ok() { return } e.fuseHook() vname := b.Varname() if b.Convert { vname = tobaseConvert(b) } if b.Value == IDENT { // unknown identity e.p.printf("\nerr = %s.EncodeMsg(en)", vname) e.p.print(errcheck) } else { // typical case e.writeAndCheck(b.BaseName(), literalFmt, vname) } } msgp-1.0-beta/gen/marshal.go000066400000000000000000000072551261253337500160000ustar00rootroot00000000000000package gen import ( "fmt" "github.com/tinylib/msgp/msgp" "io" ) func marshal(w io.Writer) *marshalGen { return &marshalGen{ p: printer{w: w}, } } type marshalGen struct { passes p printer fuse []byte } func (m *marshalGen) Method() Method { return Marshal } func (m *marshalGen) Apply(dirs []string) error { return nil } func (m *marshalGen) Execute(p Elem) error { if !m.p.ok() { return m.p.err } p = m.applyall(p) if p == nil { return nil } if !IsPrintable(p) { return nil } m.p.comment("MarshalMsg implements msgp.Marshaler") // save the vname before // calling methodReceiver so // that z.Msgsize() is printed correctly c := p.Varname() m.p.printf("\nfunc (%s %s) MarshalMsg(b []byte) (o []byte, err error) {", p.Varname(), imutMethodReceiver(p)) m.p.printf("\no = msgp.Require(b, %s.Msgsize())", c) next(m, p) m.p.nakedReturn() return m.p.err } func (m *marshalGen) rawAppend(typ string, argfmt string, arg interface{}) { m.p.printf("\no = msgp.Append%s(o, %s)", typ, fmt.Sprintf(argfmt, arg)) } func (m *marshalGen) fuseHook() { if len(m.fuse) > 0 { m.rawbytes(m.fuse) m.fuse = m.fuse[:0] } } func (m *marshalGen) Fuse(b []byte) { if len(m.fuse) == 0 { m.fuse = b } else { m.fuse = append(m.fuse, b...) } } func (m *marshalGen) gStruct(s *Struct) { if !m.p.ok() { return } if s.AsTuple { m.tuple(s) } else { m.mapstruct(s) } return } func (m *marshalGen) tuple(s *Struct) { data := make([]byte, 0, 5) data = msgp.AppendArrayHeader(data, uint32(len(s.Fields))) m.p.printf("\n// array header, size %d", len(s.Fields)) m.Fuse(data) for i := range s.Fields { if !m.p.ok() { return } next(m, s.Fields[i].FieldElem) } } func (m *marshalGen) mapstruct(s *Struct) { data := make([]byte, 0, 64) data = msgp.AppendMapHeader(data, uint32(len(s.Fields))) m.p.printf("\n// map header, size %d", len(s.Fields)) m.Fuse(data) for i := range s.Fields { if !m.p.ok() { return } data = msgp.AppendString(nil, s.Fields[i].FieldTag) m.p.printf("\n// string %q", s.Fields[i].FieldTag) m.Fuse(data) next(m, s.Fields[i].FieldElem) } } // append raw data func (m *marshalGen) rawbytes(bts []byte) { m.p.print("\no = append(o, ") for _, b := range bts { m.p.printf("0x%x,", b) } m.p.print(")") } func (m *marshalGen) gMap(s *Map) { if !m.p.ok() { return } m.fuseHook() vname := s.Varname() m.rawAppend(mapHeader, lenAsUint32, vname) m.p.printf("\nfor %s, %s := range %s {", s.Keyidx, s.Validx, vname) m.rawAppend(stringTyp, literalFmt, s.Keyidx) next(m, s.Value) m.p.closeblock() } func (m *marshalGen) gSlice(s *Slice) { if !m.p.ok() { return } m.fuseHook() vname := s.Varname() m.rawAppend(arrayHeader, lenAsUint32, vname) m.p.rangeBlock(s.Index, vname, m, s.Els) } func (m *marshalGen) gArray(a *Array) { if !m.p.ok() { return } m.fuseHook() if be, ok := a.Els.(*BaseElem); ok && be.Value == Byte { m.rawAppend("Bytes", "%s[:]", a.Varname()) return } m.rawAppend(arrayHeader, literalFmt, a.Size) m.p.rangeBlock(a.Index, a.Varname(), m, a.Els) } func (m *marshalGen) gPtr(p *Ptr) { if !m.p.ok() { return } m.fuseHook() m.p.printf("\nif %s == nil {\no = msgp.AppendNil(o)\n} else {", p.Varname()) next(m, p.Value) m.p.closeblock() } func (m *marshalGen) gBase(b *BaseElem) { if !m.p.ok() { return } m.fuseHook() vname := b.Varname() if b.Convert { vname = tobaseConvert(b) } var echeck bool switch b.Value { case IDENT: echeck = true m.p.printf("\no, err = %s.MarshalMsg(o)", vname) case Intf, Ext: echeck = true m.p.printf("\no, err = msgp.Append%s(o, %s)", b.BaseName(), vname) default: m.rawAppend(b.BaseName(), literalFmt, vname) } if echeck { m.p.print(errcheck) } } msgp-1.0-beta/gen/size.go000066400000000000000000000123711261253337500153160ustar00rootroot00000000000000package gen import ( "fmt" "github.com/tinylib/msgp/msgp" "io" "strconv" ) type sizeState uint8 const ( // need to write "s = ..." assign sizeState = iota // need to write "s += ..." add // can just append "+ ..." expr ) func sizes(w io.Writer) *sizeGen { return &sizeGen{ p: printer{w: w}, state: assign, } } type sizeGen struct { passes p printer state sizeState } func (s *sizeGen) Method() Method { return Size } func (s *sizeGen) Apply(dirs []string) error { return nil } func builtinSize(typ string) string { return "msgp." + typ + "Size" } // this lets us chain together addition // operations where possible func (s *sizeGen) addConstant(sz string) { if !s.p.ok() { return } switch s.state { case assign: s.p.print("\ns = " + sz) s.state = expr return case add: s.p.print("\ns += " + sz) s.state = expr return case expr: s.p.print(" + " + sz) return } panic("unknown size state") } func (s *sizeGen) Execute(p Elem) error { if !s.p.ok() { return s.p.err } p = s.applyall(p) if p == nil { return nil } if !IsPrintable(p) { return nil } s.p.printf("\nfunc (%s %s) Msgsize() (s int) {", p.Varname(), imutMethodReceiver(p)) s.state = assign next(s, p) s.p.nakedReturn() return s.p.err } func (s *sizeGen) gStruct(st *Struct) { if !s.p.ok() { return } nfields := uint32(len(st.Fields)) if st.AsTuple { data := msgp.AppendArrayHeader(nil, nfields) s.addConstant(strconv.Itoa(len(data))) for i := range st.Fields { if !s.p.ok() { return } next(s, st.Fields[i].FieldElem) } } else { data := msgp.AppendMapHeader(nil, nfields) s.addConstant(strconv.Itoa(len(data))) for i := range st.Fields { data = data[:0] data = msgp.AppendString(data, st.Fields[i].FieldTag) s.addConstant(strconv.Itoa(len(data))) next(s, st.Fields[i].FieldElem) } } } func (s *sizeGen) gPtr(p *Ptr) { s.state = add // inner must use add s.p.printf("\nif %s == nil {\ns += msgp.NilSize\n} else {", p.Varname()) next(s, p.Value) s.state = add // closing block; reset to add s.p.closeblock() } func (s *sizeGen) gSlice(sl *Slice) { if !s.p.ok() { return } s.addConstant(builtinSize(arrayHeader)) // if the slice's element is a fixed size // (e.g. float64, [32]int, etc.), then // print the length times the element size directly if str, ok := fixedsizeExpr(sl.Els); ok { s.addConstant(fmt.Sprintf("(%s * (%s))", lenExpr(sl), str)) return } // add inside the range block, and immediately after s.state = add s.p.rangeBlock(sl.Index, sl.Varname(), s, sl.Els) s.state = add } func (s *sizeGen) gArray(a *Array) { if !s.p.ok() { return } s.addConstant(builtinSize(arrayHeader)) // if the array's children are a fixed // size, we can compile an expression // that always represents the array's wire size if str, ok := fixedsizeExpr(a); ok { s.addConstant(str) return } s.state = add s.p.rangeBlock(a.Index, a.Varname(), s, a.Els) s.state = add } func (s *sizeGen) gMap(m *Map) { s.addConstant(builtinSize(mapHeader)) vn := m.Varname() s.p.printf("\nif %s != nil {", vn) s.p.printf("\nfor %s, %s := range %s {", m.Keyidx, m.Validx, vn) s.p.printf("\n_ = %s", m.Validx) // we may not use the value s.p.printf("\ns += msgp.StringPrefixSize + len(%s)", m.Keyidx) s.state = expr next(s, m.Value) s.p.closeblock() s.p.closeblock() s.state = add } func (s *sizeGen) gBase(b *BaseElem) { if !s.p.ok() { return } s.addConstant(basesizeExpr(b)) } // returns "len(slice)" func lenExpr(sl *Slice) string { return "len(" + sl.Varname() + ")" } // is a given primitive always the same (max) // size on the wire? func fixedSize(p Primitive) bool { switch p { case Intf, Ext, IDENT, Bytes, String: return false default: return true } } // strip reference from string func stripRef(s string) string { if s[0] == '&' { return s[1:] } return s } // return a fixed-size expression, if possible. // only possible for *BaseElem and *Array. // returns (expr, ok) func fixedsizeExpr(e Elem) (string, bool) { switch e := e.(type) { case *Array: if str, ok := fixedsizeExpr(e.Els); ok { return fmt.Sprintf("(%s * (%s))", e.Size, str), true } case *BaseElem: if fixedSize(e.Value) { return builtinSize(e.BaseName()), true } case *Struct: var str string for _, f := range e.Fields { if fs, ok := fixedsizeExpr(f.FieldElem); ok { if str == "" { str = fs } else { str += "+" + fs } } else { return "", false } } var hdrlen int mhdr := msgp.AppendMapHeader(nil, uint32(len(e.Fields))) hdrlen += len(mhdr) var strbody []byte for _, f := range e.Fields { strbody = msgp.AppendString(strbody[:0], f.FieldTag) hdrlen += len(strbody) } return fmt.Sprintf("%d + %s", hdrlen, str), true } return "", false } // print size expression of a variable name func basesizeExpr(b *BaseElem) string { vname := b.Varname() if b.Convert { vname = tobaseConvert(b) } switch b.Value { case Ext: return "msgp.ExtensionPrefixSize + " + stripRef(vname) + ".Len()" case Intf: return "msgp.GuessSize(" + vname + ")" case IDENT: return vname + ".Msgsize()" case Bytes: return "msgp.BytesPrefixSize + len(" + vname + ")" case String: return "msgp.StringPrefixSize + len(" + vname + ")" default: return builtinSize(b.BaseName()) } } msgp-1.0-beta/gen/spec.go000066400000000000000000000174711261253337500153040ustar00rootroot00000000000000package gen import ( "fmt" "io" ) const ( errcheck = "\nif err != nil { return }" lenAsUint32 = "uint32(len(%s))" literalFmt = "%s" intFmt = "%d" quotedFmt = `"%s"` mapHeader = "MapHeader" arrayHeader = "ArrayHeader" mapKey = "MapKeyPtr" stringTyp = "String" u32 = "uint32" ) // Method is a bitfield representing something that the // generator knows how to print. type Method uint8 // are the bits in 'f' set in 'm'? func (m Method) isset(f Method) bool { return (m&f == f) } // String implements fmt.Stringer func (m Method) String() string { switch m { case 0, invalidmeth: return "" case Decode: return "decode" case Encode: return "encode" case Marshal: return "marshal" case Unmarshal: return "unmarshal" case Size: return "size" case Test: return "test" default: // return e.g. "decode+encode+test" modes := [...]Method{Decode, Encode, Marshal, Unmarshal, Size, Test} any := false nm := "" for _, mm := range modes { if m.isset(mm) { if any { nm += "+" + mm.String() } else { nm += mm.String() any = true } } } return nm } } func strtoMeth(s string) Method { switch s { case "encode": return Encode case "decode": return Decode case "marshal": return Marshal case "unmarshal": return Unmarshal case "size": return Size case "test": return Test default: return 0 } } const ( Decode Method = 1 << iota // msgp.Decodable Encode // msgp.Encodable Marshal // msgp.Marshaler Unmarshal // msgp.Unmarshaler Size // msgp.Sizer Test // generate tests invalidmeth // this isn't a method encodetest = Encode | Decode | Test // tests for Encodable and Decodable marshaltest = Marshal | Unmarshal | Test // tests for Marshaler and Unmarshaler ) type Printer struct { gens []generator } func NewPrinter(m Method, out io.Writer, tests io.Writer) *Printer { if m.isset(Test) && tests == nil { panic("cannot print tests with 'nil' tests argument!") } gens := make([]generator, 0, 7) if m.isset(Decode) { gens = append(gens, decode(out)) } if m.isset(Encode) { gens = append(gens, encode(out)) } if m.isset(Marshal) { gens = append(gens, marshal(out)) } if m.isset(Unmarshal) { gens = append(gens, unmarshal(out)) } if m.isset(Size) { gens = append(gens, sizes(out)) } if m.isset(marshaltest) { gens = append(gens, mtest(tests)) } if m.isset(encodetest) { gens = append(gens, etest(tests)) } if len(gens) == 0 { panic("NewPrinter called with invalid method flags") } return &Printer{gens: gens} } // TransformPass is a pass that transforms individual // elements. (Note that if the returned is different from // the argument, it should not point to the same objects.) type TransformPass func(Elem) Elem // IgnoreTypename is a pass that just ignores // types of a given name. func IgnoreTypename(name string) TransformPass { return func(e Elem) Elem { if e.TypeName() == name { return nil } return e } } // ApplyDirective applies a directive to a named pass // and all of its dependents. func (p *Printer) ApplyDirective(pass Method, t TransformPass) { for _, g := range p.gens { if g.Method().isset(pass) { g.Add(t) } } } // Print prints an Elem. func (p *Printer) Print(e Elem) error { for _, g := range p.gens { err := g.Execute(e) if err != nil { return err } } return nil } // generator is the interface through // which code is generated. type generator interface { Method() Method Add(p TransformPass) Execute(Elem) error // execute writes the method for the provided object. } type passes []TransformPass func (p *passes) Add(t TransformPass) { *p = append(*p, t) } func (p *passes) applyall(e Elem) Elem { for _, t := range *p { e = t(e) if e == nil { return nil } } return e } type traversal interface { gMap(*Map) gSlice(*Slice) gArray(*Array) gPtr(*Ptr) gBase(*BaseElem) gStruct(*Struct) } // type-switch dispatch to the correct // method given the type of 'e' func next(t traversal, e Elem) { switch e := e.(type) { case *Map: t.gMap(e) case *Struct: t.gStruct(e) case *Slice: t.gSlice(e) case *Array: t.gArray(e) case *Ptr: t.gPtr(e) case *BaseElem: t.gBase(e) default: panic("bad element type") } } // possibly-immutable method receiver func imutMethodReceiver(p Elem) string { switch e := p.(type) { case *Struct: // TODO(HACK): actually do real math here. if len(e.Fields) <= 3 { for i := range e.Fields { if be, ok := e.Fields[i].FieldElem.(*BaseElem); !ok || (be.Value == IDENT || be.Value == Bytes) { goto nope } } return p.TypeName() } nope: return "*" + p.TypeName() // gets dereferenced automatically case *Array: return "*" + p.TypeName() // everything else can be // by-value. default: return p.TypeName() } } // if necessary, wraps a type // so that its method receiver // is of the write type. func methodReceiver(p Elem) string { switch p.(type) { // structs and arrays are // dereferenced automatically, // so no need to alter varname case *Struct, *Array: return "*" + p.TypeName() // set variable name to // *varname default: p.SetVarname("(*" + p.Varname() + ")") return "*" + p.TypeName() } } func unsetReceiver(p Elem) { switch p.(type) { case *Struct, *Array: default: p.SetVarname("z") } } // shared utility for generators type printer struct { w io.Writer err error } // writes "var {{name}} {{typ}};" func (p *printer) declare(name string, typ string) { p.printf("\nvar %s %s", name, typ) } // does: // // if m != nil && size > 0 { // m = make(type, size) // } else if len(m) > 0 { // for key, _ := range m { delete(m, key) } // } // func (p *printer) resizeMap(size string, m *Map) { vn := m.Varname() if !p.ok() { return } p.printf("\nif %s == nil && %s > 0 {", vn, size) p.printf("\n%s = make(%s, %s)", vn, m.TypeName(), size) p.printf("\n} else if len(%s) > 0 {", vn) p.clearMap(vn) p.closeblock() } // assign key to value based on varnames func (p *printer) mapAssign(m *Map) { if !p.ok() { return } p.printf("\n%s[%s] = %s", m.Varname(), m.Keyidx, m.Validx) } // clear map keys func (p *printer) clearMap(name string) { p.printf("\nfor key, _ := range %[1]s { delete(%[1]s, key) }", name) } func (p *printer) resizeSlice(size string, s *Slice) { p.printf("\nif cap(%[1]s) >= int(%[2]s) { %[1]s = %[1]s[:%[2]s] } else { %[1]s = make(%[3]s, %[2]s) }", s.Varname(), size, s.TypeName()) } func (p *printer) arrayCheck(want string, got string) { p.printf("\nif %[1]s != %[2]s { err = msgp.ArrayError{Wanted: %[2]s, Got: %[1]s}; return }", got, want) } func (p *printer) closeblock() { p.print("\n}") } // does: // // for idx := range iter { // {{generate inner}} // } // func (p *printer) rangeBlock(idx string, iter string, t traversal, inner Elem) { p.printf("\n for %s := range %s {", idx, iter) next(t, inner) p.closeblock() } func (p *printer) nakedReturn() { if p.ok() { p.print("\nreturn\n}\n") } } func (p *printer) comment(s string) { p.print("\n// " + s) } func (p *printer) printf(format string, args ...interface{}) { if p.err == nil { _, p.err = fmt.Fprintf(p.w, format, args...) } } func (p *printer) print(format string) { if p.err == nil { _, p.err = io.WriteString(p.w, format) } } func (p *printer) initPtr(pt *Ptr) { if pt.Needsinit() { vname := pt.Varname() p.printf("\nif %s == nil { %s = new(%s); }", vname, vname, pt.Value.TypeName()) } } func (p *printer) ok() bool { return p.err == nil } func tobaseConvert(b *BaseElem) string { return b.ToBase() + "(" + b.Varname() + ")" } msgp-1.0-beta/gen/testgen.go000066400000000000000000000065351261253337500160220ustar00rootroot00000000000000package gen import ( "io" "text/template" ) var ( marshalTestTempl = template.New("MarshalTest") encodeTestTempl = template.New("EncodeTest") ) // TODO(philhofer): // for simplicity's sake, right now // we can only generate tests for types // that can be initialized with the // "Type{}" syntax. // we should support all the types. func mtest(w io.Writer) *mtestGen { return &mtestGen{w: w} } type mtestGen struct { passes w io.Writer } func (m *mtestGen) Execute(p Elem) error { p = m.applyall(p) if p != nil && IsPrintable(p) { switch p.(type) { case *Struct, *Array, *Slice, *Map: return marshalTestTempl.Execute(m.w, p) } } return nil } func (m *mtestGen) Method() Method { return marshaltest } type etestGen struct { passes w io.Writer } func etest(w io.Writer) *etestGen { return &etestGen{w: w} } func (e *etestGen) Execute(p Elem) error { p = e.applyall(p) if p != nil && IsPrintable(p) { switch p.(type) { case *Struct, *Array, *Slice, *Map: return encodeTestTempl.Execute(e.w, p) } } return nil } func (e *etestGen) Method() Method { return encodetest } func init() { template.Must(marshalTestTempl.Parse(`func TestMarshalUnmarshal{{.TypeName}}(t *testing.T) { v := {{.TypeName}}{} bts, err := v.MarshalMsg(nil) if err != nil { t.Fatal(err) } left, err := v.UnmarshalMsg(bts) if err != nil { t.Fatal(err) } if len(left) > 0 { t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) } left, err = msgp.Skip(bts) if err != nil { t.Fatal(err) } if len(left) > 0 { t.Errorf("%d bytes left over after Skip(): %q", len(left), left) } } func BenchmarkMarshalMsg{{.TypeName}}(b *testing.B) { v := {{.TypeName}}{} b.ReportAllocs() b.ResetTimer() for i:=0; i m { t.Logf("WARNING: Msgsize() for %v is inaccurate", v) } vn := {{.TypeName}}{} err := msgp.Decode(&buf, &vn) if err != nil { t.Error(err) } buf.Reset() msgp.Encode(&buf, &v) err = msgp.NewReader(&buf).Skip() if err != nil { t.Error(err) } } func BenchmarkEncode{{.TypeName}}(b *testing.B) { v := {{.TypeName}}{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) en := msgp.NewWriter(msgp.Nowhere) b.ReportAllocs() b.ResetTimer() for i:=0; i 0 {") u.p.print("\nisz--; field, bts, err = msgp.ReadMapKeyZC(bts)") u.p.print(errcheck) u.p.print("\nswitch msgp.UnsafeString(field) {") for i := range s.Fields { if !u.p.ok() { return } u.p.printf("\ncase \"%s\":", s.Fields[i].FieldTag) next(u, s.Fields[i].FieldElem) } u.p.print("\ndefault:\nbts, err = msgp.Skip(bts)") u.p.print(errcheck) u.p.print("\n}\n}") // close switch and for loop } func (u *unmarshalGen) gBase(b *BaseElem) { if !u.p.ok() { return } refname := b.Varname() // assigned to lowered := b.Varname() // passed as argument if b.Convert { // begin 'tmp' block refname = "tmp" lowered = b.ToBase() + "(" + lowered + ")" u.p.printf("\n{\nvar tmp %s", b.BaseType()) } switch b.Value { case Bytes: u.p.printf("\n%s, bts, err = msgp.ReadBytesBytes(bts, %s)", refname, lowered) case Ext: u.p.printf("\nbts, err = msgp.ReadExtensionBytes(bts, %s)", lowered) case IDENT: u.p.printf("\nbts, err = %s.UnmarshalMsg(bts)", lowered) default: u.p.printf("\n%s, bts, err = msgp.Read%sBytes(bts)", refname, b.BaseName()) } if b.Convert { // close 'tmp' block u.p.printf("\n%s = %s(tmp)\n}", b.Varname(), b.FromBase()) } u.p.print(errcheck) } func (u *unmarshalGen) gArray(a *Array) { if !u.p.ok() { return } // special case for [const]byte objects // see decode.go for symmetry if be, ok := a.Els.(*BaseElem); ok && be.Value == Byte { u.p.printf("\nbts, err = msgp.ReadExactBytes(bts, %s[:])", a.Varname()) u.p.print(errcheck) return } u.p.declare(arraySizeVar, u32) u.assignAndCheck(arraySizeVar, arrayHeader) u.p.arrayCheck(a.Size, arraySizeVar) u.p.rangeBlock(a.Index, a.Varname(), u, a.Els) } func (u *unmarshalGen) gSlice(s *Slice) { if !u.p.ok() { return } u.p.declare(sliceSizeVar, u32) u.assignAndCheck(sliceSizeVar, arrayHeader) u.p.resizeSlice(sliceSizeVar, s) u.p.rangeBlock(s.Index, s.Varname(), u, s.Els) } func (u *unmarshalGen) gMap(m *Map) { if !u.p.ok() { return } u.p.declare(mapSizeVar, u32) u.assignAndCheck(mapSizeVar, mapHeader) // allocate or clear map u.p.resizeMap(mapSizeVar, m) // loop and get key,value u.p.print("\nfor msz > 0 {") u.p.printf("\nvar %s string; var %s %s; msz--", m.Keyidx, m.Validx, m.Value.TypeName()) u.assignAndCheck(m.Keyidx, stringTyp) next(u, m.Value) u.p.mapAssign(m) u.p.closeblock() } func (u *unmarshalGen) gPtr(p *Ptr) { u.p.printf("\nif msgp.IsNil(bts) { bts, err = msgp.ReadNilBytes(bts); if err != nil { return }; %s = nil; } else { ", p.Varname()) u.p.initPtr(p) next(u, p.Value) u.p.closeblock() } msgp-1.0-beta/main.go000066400000000000000000000066131261253337500145210ustar00rootroot00000000000000// msgp is a code generation tool for // creating methods to serialize and de-serialize // Go data structures to and from MessagePack. // // This package is targeted at the `go generate` tool. // To use it, include the following directive in a // go source file with types requiring source generation: // // //go:generate msgp // // The go generate tool should set the proper environment variables for // the generator to execute without any command-line flags. However, the // following options are supported, if you need them: // // -o = output file name (default is {input}_gen.go) // -file = input file name (or directory; default is $GOFILE, which is set by the `go generate` command) // -io = satisfy the `msgp.Decodable` and `msgp.Encodable` interfaces (default is true) // -marshal = satisfy the `msgp.Marshaler` and `msgp.Unmarshaler` interfaces (default is true) // -tests = generate tests and benchmarks (default is true) // // For more information, please read README.md, and the wiki at github.com/tinylib/msgp // package main import ( "flag" "fmt" "os" "path/filepath" "strings" "github.com/tinylib/msgp/gen" "github.com/tinylib/msgp/parse" "github.com/tinylib/msgp/printer" "github.com/ttacon/chalk" ) var ( out = flag.String("o", "", "output file") file = flag.String("file", "", "input file") encode = flag.Bool("io", true, "create Encode and Decode methods") marshal = flag.Bool("marshal", true, "create Marshal and Unmarshal methods") tests = flag.Bool("tests", true, "create tests and benchmarks") unexported = flag.Bool("unexported", false, "also process unexported types") ) func main() { flag.Parse() // GOFILE is set by go generate if *file == "" { *file = os.Getenv("GOFILE") if *file == "" { fmt.Println(chalk.Red.Color("No file to parse.")) os.Exit(1) } } var mode gen.Method if *encode { mode |= (gen.Encode | gen.Decode | gen.Size) } if *marshal { mode |= (gen.Marshal | gen.Unmarshal | gen.Size) } if *tests { mode |= gen.Test } if mode&^gen.Test == 0 { fmt.Println(chalk.Red.Color("No methods to generate; -io=false && -marshal=false")) os.Exit(1) } if err := Run(*file, mode, *unexported); err != nil { fmt.Println(chalk.Red.Color(err.Error())) os.Exit(1) } } // Run writes all methods using the associated file or path, e.g. // // err := msgp.Run("path/to/myfile.go", gen.Size|gen.Marshal|gen.Unmarshal|gen.Test, false) // func Run(gofile string, mode gen.Method, unexported bool) error { if mode&^gen.Test == 0 { return nil } fmt.Println(chalk.Magenta.Color("======== MessagePack Code Generator =======")) fmt.Printf(chalk.Magenta.Color(">>> Input: \"%s\"\n"), gofile) fs, err := parse.File(gofile, unexported) if err != nil { return err } if len(fs.Identities) == 0 { fmt.Println(chalk.Magenta.Color("No types requiring code generation were found!")) return nil } return printer.PrintFile(newFilename(gofile, fs.Package), fs, mode) } // picks a new file name based on input flags and input filename(s). func newFilename(old string, pkg string) string { if *out != "" { if pre := strings.TrimPrefix(*out, old); len(pre) > 0 && !strings.HasSuffix(*out, ".go") { return filepath.Join(old, *out) } return *out } if fi, err := os.Stat(old); err == nil && fi.IsDir() { old = filepath.Join(old, pkg) } // new file name is old file name + _gen.go return strings.TrimSuffix(old, ".go") + "_gen.go" } msgp-1.0-beta/msgp/000077500000000000000000000000001261253337500142065ustar00rootroot00000000000000msgp-1.0-beta/msgp/advise_linux.go000066400000000000000000000006421261253337500172310ustar00rootroot00000000000000// +build linux,!appengine package msgp import ( "os" "syscall" ) func adviseRead(mem []byte) { syscall.Madvise(mem, syscall.MADV_SEQUENTIAL|syscall.MADV_WILLNEED) } func adviseWrite(mem []byte) { syscall.Madvise(mem, syscall.MADV_SEQUENTIAL) } func fallocate(f *os.File, sz int64) error { err := syscall.Fallocate(int(f.Fd()), 0, 0, sz) if err == syscall.ENOTSUP { return f.Truncate(sz) } return err } msgp-1.0-beta/msgp/advise_other.go000066400000000000000000000003411261253337500172070ustar00rootroot00000000000000// +build !linux appengine package msgp import ( "os" ) // TODO: darwin, BSD support func adviseRead(mem []byte) {} func adviseWrite(mem []byte) {} func fallocate(f *os.File, sz int64) error { return f.Truncate(sz) } msgp-1.0-beta/msgp/appengine.go000066400000000000000000000003471261253337500165070ustar00rootroot00000000000000// +build appengine package msgp // let's just assume appengine // uses 64-bit hardware... const smallint = false func UnsafeString(b []byte) string { return string(b) } func UnsafeBytes(s string) []byte { return []byte(s) } msgp-1.0-beta/msgp/circular.go000066400000000000000000000014621261253337500163440ustar00rootroot00000000000000package msgp type timer interface { StartTimer() StopTimer() } // EndlessReader is an io.Reader // that loops over the same data // endlessly. It is used for benchmarking. type EndlessReader struct { tb timer data []byte offset int } // NewEndlessReader returns a new endless reader func NewEndlessReader(b []byte, tb timer) *EndlessReader { return &EndlessReader{tb: tb, data: b, offset: 0} } // Read implements io.Reader. In practice, it // always returns (len(p), nil), although it // fills the supplied slice while the benchmark // timer is stopped. func (c *EndlessReader) Read(p []byte) (int, error) { c.tb.StopTimer() var n int l := len(p) m := len(c.data) for n < l { nn := copy(p[n:], c.data[c.offset:]) n += nn c.offset += nn c.offset %= m } c.tb.StartTimer() return n, nil } msgp-1.0-beta/msgp/defs.go000066400000000000000000000061701261253337500154620ustar00rootroot00000000000000// This package is the support library for the msgp code generator (http://github.com/tinylib/msgp). // // This package defines the utilites used by the msgp code generator for encoding and decoding MessagePack // from []byte and io.Reader/io.Writer types. Much of this package is devoted to helping the msgp code // generator implement the Marshaler/Unmarshaler and Encodable/Decodable interfaces. // // This package defines four "families" of functions: // - AppendXxxx() appends an object to a []byte in MessagePack encoding. // - ReadXxxxBytes() reads an object from a []byte and returns the remaining bytes. // - (*Writer).WriteXxxx() writes an object to the buffered *Writer type. // - (*Reader).ReadXxxx() reads an object from a buffered *Reader type. // // Once a type has satisfied the `Encodable` and `Decodable` interfaces, // it can be written and read from arbitrary `io.Writer`s and `io.Reader`s using // msgp.Encode(io.Writer, msgp.Encodable) // and // msgp.Decode(io.Reader, msgp.Decodable) // // There are also methods for converting MessagePack to JSON without // an explicit de-serialization step. // // For additional tips, tricks, and gotchas, please visit // the wiki at http://github.com/tinylib/msgp package msgp const last4 = 0x0f const first4 = 0xf0 const last5 = 0x1f const first3 = 0xe0 const last7 = 0x7f func isfixint(b byte) bool { return b>>7 == 0 } func isnfixint(b byte) bool { return b&first3 == mnfixint } func isfixmap(b byte) bool { return b&first4 == mfixmap } func isfixarray(b byte) bool { return b&first4 == mfixarray } func isfixstr(b byte) bool { return b&first3 == mfixstr } func wfixint(u uint8) byte { return u & last7 } func rfixint(b byte) uint8 { return b } func wnfixint(i int8) byte { return byte(i) | mnfixint } func rnfixint(b byte) int8 { return int8(b) } func rfixmap(b byte) uint8 { return b & last4 } func wfixmap(u uint8) byte { return mfixmap | (u & last4) } func rfixstr(b byte) uint8 { return b & last5 } func wfixstr(u uint8) byte { return (u & last5) | mfixstr } func rfixarray(b byte) uint8 { return (b & last4) } func wfixarray(u uint8) byte { return (u & last4) | mfixarray } // These are all the byte // prefixes defined by the // msgpack standard const ( // 0XXXXXXX mfixint uint8 = 0x00 // 111XXXXX mnfixint uint8 = 0xe0 // 1000XXXX mfixmap uint8 = 0x80 // 1001XXXX mfixarray uint8 = 0x90 // 101XXXXX mfixstr uint8 = 0xa0 mnil uint8 = 0xc0 mfalse uint8 = 0xc2 mtrue uint8 = 0xc3 mbin8 uint8 = 0xc4 mbin16 uint8 = 0xc5 mbin32 uint8 = 0xc6 mext8 uint8 = 0xc7 mext16 uint8 = 0xc8 mext32 uint8 = 0xc9 mfloat32 uint8 = 0xca mfloat64 uint8 = 0xcb muint8 uint8 = 0xcc muint16 uint8 = 0xcd muint32 uint8 = 0xce muint64 uint8 = 0xcf mint8 uint8 = 0xd0 mint16 uint8 = 0xd1 mint32 uint8 = 0xd2 mint64 uint8 = 0xd3 mfixext1 uint8 = 0xd4 mfixext2 uint8 = 0xd5 mfixext4 uint8 = 0xd6 mfixext8 uint8 = 0xd7 mfixext16 uint8 = 0xd8 mstr8 uint8 = 0xd9 mstr16 uint8 = 0xda mstr32 uint8 = 0xdb marray16 uint8 = 0xdc marray32 uint8 = 0xdd mmap16 uint8 = 0xde mmap32 uint8 = 0xdf ) msgp-1.0-beta/msgp/defs_test.go000066400000000000000000000003461261253337500165200ustar00rootroot00000000000000package msgp_test //go:generate msgp -o=defgen_test.go -tests=false type Blobs []Blob type Blob struct { Name string `msg:"name"` Float float64 `msg:"float"` Bytes []byte `msg:"bytes"` Amount int64 `msg:"amount"` } msgp-1.0-beta/msgp/edit.go000066400000000000000000000127661261253337500154760ustar00rootroot00000000000000package msgp import ( "math" ) // Locate returns a []byte pointing to the field // in a messagepack map with the provided key. (The returned []byte // points to a sub-slice of 'raw'; Locate does no allocations.) If the // key doesn't exist in the map, a zero-length []byte will be returned. func Locate(key string, raw []byte) []byte { s, n := locate(raw, key) return raw[s:n] } // Replace takes a key ("key") in a messagepack map ("raw") // and replaces its value with the one provided and returns // the new []byte. The returned []byte may point to the same // memory as "raw". Replace makes no effort to evaluate the validity // of the contents of 'val'. It may use up to the full capacity of 'raw.' // Replace returns 'nil' if the field doesn't exist or if the object in 'raw' // is not a map. func Replace(key string, raw []byte, val []byte) []byte { start, end := locate(raw, key) if start == end { return nil } return replace(raw, start, end, val, true) } // CopyReplace works similarly to Replace except that the returned // byte slice does not point to the same memory as 'raw'. CopyReplace // returns 'nil' if the field doesn't exist or 'raw' isn't a map. func CopyReplace(key string, raw []byte, val []byte) []byte { start, end := locate(raw, key) if start == end { return nil } return replace(raw, start, end, val, false) } // Remove removes a key-value pair from 'raw'. It returns // 'raw' unchanged if the key didn't exist. func Remove(key string, raw []byte) []byte { start, end := locateKV(raw, key) if start == end { return raw } raw = raw[:start+copy(raw[start:], raw[end:])] return resizeMap(raw, -1) } // HasKey returns whether the map in 'raw' has // a field with key 'key' func HasKey(key string, raw []byte) bool { sz, bts, err := ReadMapHeaderBytes(raw) if err != nil { return false } var field []byte for i := uint32(0); i < sz; i++ { field, bts, err = ReadStringZC(bts) if err != nil { return false } if UnsafeString(field) == key { return true } } return false } func replace(raw []byte, start int, end int, val []byte, inplace bool) []byte { ll := end - start // length of segment to replace lv := len(val) if inplace { extra := lv - ll // fastest case: we're doing // a 1:1 replacement if extra == 0 { copy(raw[start:], val) return raw } else if extra < 0 { // 'val' smaller than replaced value // copy in place and shift back x := copy(raw[start:], val) y := copy(raw[start+x:], raw[end:]) return raw[:start+x+y] } else if extra < cap(raw)-len(raw) { // 'val' less than (cap-len) extra bytes // copy in place and shift forward raw = raw[0 : len(raw)+extra] // shift end forward copy(raw[end+extra:], raw[end:]) copy(raw[start:], val) return raw } } // we have to allocate new space out := make([]byte, len(raw)+len(val)-ll) x := copy(out, raw[:start]) y := copy(out[x:], val) copy(out[x+y:], raw[end:]) return out } // locate does a naive O(n) search for the map key; returns start, end // (returns 0,0 on error) func locate(raw []byte, key string) (start int, end int) { var ( sz uint32 bts []byte field []byte err error ) sz, bts, err = ReadMapHeaderBytes(raw) if err != nil { return } // loop and locate field for i := uint32(0); i < sz; i++ { field, bts, err = ReadStringZC(bts) if err != nil { return 0, 0 } if UnsafeString(field) == key { // start location l := len(raw) start = l - len(bts) bts, err = Skip(bts) if err != nil { return 0, 0 } end = l - len(bts) return } bts, err = Skip(bts) if err != nil { return 0, 0 } } return 0, 0 } // locate key AND value func locateKV(raw []byte, key string) (start int, end int) { var ( sz uint32 bts []byte field []byte err error ) sz, bts, err = ReadMapHeaderBytes(raw) if err != nil { return 0, 0 } for i := uint32(0); i < sz; i++ { tmp := len(bts) field, bts, err = ReadStringZC(bts) if err != nil { return 0, 0 } if UnsafeString(field) == key { start = len(raw) - tmp bts, err = Skip(bts) if err != nil { return 0, 0 } end = len(raw) - len(bts) return } bts, err = Skip(bts) if err != nil { return 0, 0 } } return 0, 0 } // delta is delta on map size func resizeMap(raw []byte, delta int64) []byte { var sz int64 switch raw[0] { case mmap16: sz = int64(big.Uint16(raw[1:])) if sz+delta <= math.MaxUint16 { big.PutUint16(raw[1:], uint16(sz+delta)) return raw } if cap(raw)-len(raw) >= 2 { raw = raw[0 : len(raw)+2] copy(raw[5:], raw[3:]) big.PutUint32(raw[1:], uint32(sz+delta)) return raw } n := make([]byte, 0, len(raw)+5) n = AppendMapHeader(n, uint32(sz+delta)) return append(n, raw[3:]...) case mmap32: sz = int64(big.Uint32(raw[1:])) big.PutUint32(raw[1:], uint32(sz+delta)) return raw default: sz = int64(rfixmap(raw[0])) if sz+delta < 16 { raw[0] = wfixmap(uint8(sz + delta)) return raw } else if sz+delta <= math.MaxUint16 { if cap(raw)-len(raw) >= 2 { raw = raw[0 : len(raw)+2] copy(raw[3:], raw[1:]) raw[0] = mmap16 big.PutUint16(raw[1:], uint16(sz+delta)) return raw } n := make([]byte, 0, len(raw)+5) n = AppendMapHeader(n, uint32(sz+delta)) return append(n, raw[1:]...) } if cap(raw)-len(raw) >= 4 { raw = raw[0 : len(raw)+4] copy(raw[5:], raw[1:]) raw[0] = mmap32 big.PutUint32(raw[1:], uint32(sz+delta)) return raw } n := make([]byte, 0, len(raw)+5) n = AppendMapHeader(n, uint32(sz+delta)) return append(n, raw[1:]...) } } msgp-1.0-beta/msgp/edit_test.go000066400000000000000000000106121261253337500165210ustar00rootroot00000000000000package msgp import ( "bytes" "reflect" "testing" ) func TestRemove(t *testing.T) { var buf bytes.Buffer w := NewWriter(&buf) w.WriteMapHeader(3) w.WriteString("first") w.WriteFloat64(-3.1) w.WriteString("second") w.WriteString("DELETE ME!!!") w.WriteString("third") w.WriteBytes([]byte("blah")) w.Flush() raw := Remove("second", buf.Bytes()) m, _, err := ReadMapStrIntfBytes(raw, nil) if err != nil { t.Fatal(err) } if len(m) != 2 { t.Errorf("expected %d fields; found %d", 2, len(m)) } if _, ok := m["first"]; !ok { t.Errorf("field %q not found", "first") } if _, ok := m["third"]; !ok { t.Errorf("field %q not found", "third") } if _, ok := m["second"]; ok { t.Errorf("field %q (deleted field) still present", "second") } } func TestLocate(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(2) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(2.0) en.Flush() field := Locate("thing_one", buf.Bytes()) if len(field) == 0 { t.Fatal("field not found") } if !HasKey("thing_one", buf.Bytes()) { t.Fatal("field not found") } var zbuf bytes.Buffer w := NewWriter(&zbuf) w.WriteString("value_one") w.Flush() if !bytes.Equal(zbuf.Bytes(), field) { t.Errorf("got %q; wanted %q", field, zbuf.Bytes()) } zbuf.Reset() w.WriteFloat64(2.0) w.Flush() field = Locate("thing_two", buf.Bytes()) if len(field) == 0 { t.Fatal("field not found") } if !bytes.Equal(zbuf.Bytes(), field) { t.Errorf("got %q; wanted %q", field, zbuf.Bytes()) } field = Locate("nope", buf.Bytes()) if len(field) != 0 { t.Fatalf("wanted a zero-length returned slice") } } func TestReplace(t *testing.T) { // there are 4 cases that need coverage: // - new value is smaller than old value // - new value is the same size as the old value // - new value is larger than old, but fits within cap(b) // - new value is larger than old, and doesn't fit within cap(b) var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(3) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(2.0) en.WriteString("some_bytes") en.WriteBytes([]byte("here are some bytes")) en.Flush() // same-size replacement var fbuf bytes.Buffer w := NewWriter(&fbuf) w.WriteFloat64(4.0) w.Flush() // replace 2.0 with 4.0 in field two raw := Replace("thing_two", buf.Bytes(), fbuf.Bytes()) if len(raw) == 0 { t.Fatal("field not found") } var err error m := make(map[string]interface{}) m, _, err = ReadMapStrIntfBytes(raw, m) if err != nil { t.Logf("%q", raw) t.Fatal(err) } if !reflect.DeepEqual(m["thing_two"], 4.0) { t.Errorf("wanted %v; got %v", 4.0, m["thing_two"]) } // smaller-size replacement // replace 2.0 with []byte("hi!") fbuf.Reset() w.WriteBytes([]byte("hi!")) w.Flush() raw = Replace("thing_two", raw, fbuf.Bytes()) if len(raw) == 0 { t.Fatal("field not found") } m, _, err = ReadMapStrIntfBytes(raw, m) if err != nil { t.Logf("%q", raw) t.Fatal(err) } if !reflect.DeepEqual(m["thing_two"], []byte("hi!")) { t.Errorf("wanted %v; got %v", []byte("hi!"), m["thing_two"]) } // larger-size replacement fbuf.Reset() w.WriteBytes([]byte("some even larger bytes than before")) w.Flush() raw = Replace("some_bytes", raw, fbuf.Bytes()) if len(raw) == 0 { t.Logf("%q", raw) t.Fatal(err) } m, _, err = ReadMapStrIntfBytes(raw, m) if err != nil { t.Logf("%q", raw) t.Fatal(err) } if !reflect.DeepEqual(m["some_bytes"], []byte("some even larger bytes than before")) { t.Errorf("wanted %v; got %v", []byte("hello there!"), m["some_bytes"]) } // identical in-place replacement field := Locate("some_bytes", raw) newraw := CopyReplace("some_bytes", raw, field) if !bytes.Equal(newraw, raw) { t.Logf("in: %q", raw) t.Logf("out: %q", newraw) t.Error("bytes not equal after copyreplace") } } func BenchmarkLocate(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(3) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(2.0) en.WriteString("thing_three") en.WriteBytes([]byte("hello!")) en.Flush() raw := buf.Bytes() // bytes/s will be the number of bytes traversed per unit of time field := Locate("thing_three", raw) b.SetBytes(int64(len(raw) - len(field))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { Locate("thing_three", raw) } } msgp-1.0-beta/msgp/elsize.go000066400000000000000000000065671261253337500160460ustar00rootroot00000000000000package msgp // size of every object on the wire, // plus type information. gives us // constant-time type information // for traversing composite objects. // var sizes = [256]bytespec{ mnil: {size: 1, extra: constsize, typ: NilType}, mfalse: {size: 1, extra: constsize, typ: BoolType}, mtrue: {size: 1, extra: constsize, typ: BoolType}, mbin8: {size: 2, extra: extra8, typ: BinType}, mbin16: {size: 3, extra: extra16, typ: BinType}, mbin32: {size: 5, extra: extra32, typ: BinType}, mext8: {size: 3, extra: extra8, typ: ExtensionType}, mext16: {size: 4, extra: extra16, typ: ExtensionType}, mext32: {size: 6, extra: extra32, typ: ExtensionType}, mfloat32: {size: 5, extra: constsize, typ: Float32Type}, mfloat64: {size: 9, extra: constsize, typ: Float64Type}, muint8: {size: 2, extra: constsize, typ: UintType}, muint16: {size: 3, extra: constsize, typ: UintType}, muint32: {size: 5, extra: constsize, typ: UintType}, muint64: {size: 9, extra: constsize, typ: UintType}, mint8: {size: 2, extra: constsize, typ: IntType}, mint16: {size: 3, extra: constsize, typ: IntType}, mint32: {size: 5, extra: constsize, typ: IntType}, mint64: {size: 9, extra: constsize, typ: IntType}, mfixext1: {size: 3, extra: constsize, typ: ExtensionType}, mfixext2: {size: 4, extra: constsize, typ: ExtensionType}, mfixext4: {size: 6, extra: constsize, typ: ExtensionType}, mfixext8: {size: 10, extra: constsize, typ: ExtensionType}, mfixext16: {size: 18, extra: constsize, typ: ExtensionType}, mstr8: {size: 2, extra: extra8, typ: StrType}, mstr16: {size: 3, extra: extra16, typ: StrType}, mstr32: {size: 5, extra: extra32, typ: StrType}, marray16: {size: 3, extra: array16v, typ: ArrayType}, marray32: {size: 5, extra: array32v, typ: ArrayType}, mmap16: {size: 3, extra: map16v, typ: MapType}, mmap32: {size: 5, extra: map32v, typ: MapType}, } func init() { // set up fixed fields // fixint for i := mfixint; i < 0x80; i++ { sizes[i] = bytespec{size: 1, extra: constsize, typ: IntType} } // nfixint for i := uint16(mnfixint); i < 0x100; i++ { sizes[uint8(i)] = bytespec{size: 1, extra: constsize, typ: IntType} } // fixstr gets constsize, // since the prefix yields the size for i := mfixstr; i < 0xc0; i++ { sizes[i] = bytespec{size: 1 + rfixstr(i), extra: constsize, typ: StrType} } // fixmap for i := mfixmap; i < 0x90; i++ { sizes[i] = bytespec{size: 1, extra: varmode(2 * rfixmap(i)), typ: MapType} } // fixarray for i := mfixarray; i < 0xa0; i++ { sizes[i] = bytespec{size: 1, extra: varmode(rfixarray(i)), typ: ArrayType} } } // a valid bytespsec has // non-zero 'size' and // non-zero 'typ' type bytespec struct { size uint8 // prefix size information extra varmode // extra size information typ Type // type _ byte // makes bytespec 4 bytes (yes, this matters) } // size mode // if positive, # elements for composites type varmode int8 const ( constsize varmode = 0 // constant size (size bytes + uint8(varmode) objects) extra8 = -1 // has uint8(p[1]) extra bytes extra16 = -2 // has be16(p[1:]) extra bytes extra32 = -3 // has be32(p[1:]) extra bytes map16v = -4 // use map16 map32v = -5 // use map32 array16v = -6 // use array16 array32v = -7 // use array32 ) func getType(v byte) Type { return sizes[v].typ } msgp-1.0-beta/msgp/errors.go000066400000000000000000000077261261253337500160650ustar00rootroot00000000000000package msgp import ( "fmt" "reflect" ) var ( // ErrShortBytes is returned when the // slice being decoded is too short to // contain the contents of the message ErrShortBytes error = errShort{} // this error is only returned // if we reach code that should // be unreachable fatal error = errFatal{} ) // Error is the interface satisfied // by all of the errors that originate // from this package. type Error interface { error // Resumable returns whether // or not the error means that // the stream of data is malformed // and the information is unrecoverable. Resumable() bool } type errShort struct{} func (e errShort) Error() string { return "msgp: too few bytes left to read object" } func (e errShort) Resumable() bool { return false } type errFatal struct{} func (f errFatal) Error() string { return "msgp: fatal decoding error (unreachable code)" } func (f errFatal) Resumable() bool { return false } // ArrayError is an error returned // when decoding a fix-sized array // of the wrong size type ArrayError struct { Wanted uint32 Got uint32 } // Error implements the error interface func (a ArrayError) Error() string { return fmt.Sprintf("msgp: wanted array of size %d; got %d", a.Wanted, a.Got) } // Resumable is always 'true' for ArrayErrors func (a ArrayError) Resumable() bool { return true } // IntOverflow is returned when a call // would downcast an integer to a type // with too few bits to hold its value. type IntOverflow struct { Value int64 // the value of the integer FailedBitsize int // the bit size that the int64 could not fit into } // Error implements the error interface func (i IntOverflow) Error() string { return fmt.Sprintf("msgp: %d overflows int%d", i.Value, i.FailedBitsize) } // Resumable is always 'true' for overflows func (i IntOverflow) Resumable() bool { return true } // UintOverflow is returned when a call // would downcast an unsigned integer to a type // with too few bits to hold its value type UintOverflow struct { Value uint64 // value of the uint FailedBitsize int // the bit size that couldn't fit the value } // Error implements the error interface func (u UintOverflow) Error() string { return fmt.Sprintf("msgp: %d overflows uint%d", u.Value, u.FailedBitsize) } // Resumable is always 'true' for overflows func (u UintOverflow) Resumable() bool { return true } // A TypeError is returned when a particular // decoding method is unsuitable for decoding // a particular MessagePack value. type TypeError struct { Method Type // Type expected by method Encoded Type // Type actually encoded } // Error implements the error interface func (t TypeError) Error() string { return fmt.Sprintf("msgp: attempted to decode type %q with method for %q", t.Encoded, t.Method) } // Resumable returns 'true' for TypeErrors func (t TypeError) Resumable() bool { return true } // returns either InvalidPrefixError or // TypeError depending on whether or not // the prefix is recognized func badPrefix(want Type, lead byte) error { t := sizes[lead].typ if t == InvalidType { return InvalidPrefixError(lead) } return TypeError{Method: want, Encoded: t} } // InvalidPrefixError is returned when a bad encoding // uses a prefix that is not recognized in the MessagePack standard. // This kind of error is unrecoverable. type InvalidPrefixError byte // Error implements the error interface func (i InvalidPrefixError) Error() string { return fmt.Sprintf("msgp: unrecognized type prefix 0x%x", byte(i)) } // Resumable returns 'false' for InvalidPrefixErrors func (i InvalidPrefixError) Resumable() bool { return false } // ErrUnsupportedType is returned // when a bad argument is supplied // to a function that takes `interface{}`. type ErrUnsupportedType struct { T reflect.Type } // Error implements error func (e *ErrUnsupportedType) Error() string { return fmt.Sprintf("msgp: type %q not supported", e.T) } // Resumable returns 'true' for ErrUnsupportedType func (e *ErrUnsupportedType) Resumable() bool { return true } msgp-1.0-beta/msgp/extension.go000066400000000000000000000271421261253337500165570ustar00rootroot00000000000000package msgp import ( "fmt" "math" ) const ( // Complex64Extension is the extension number used for complex64 Complex64Extension = 3 // Complex128Extension is the extension number used for complex128 Complex128Extension = 4 // TimeExtension is the extension number used for time.Time TimeExtension = 5 ) // our extensions live here var extensionReg = make(map[int8]func() Extension) // RegisterExtension registers extensions so that they // can be initialized and returned by methods that // decode `interface{}` values. This should only // be called during initialization. f() should return // a newly-initialized zero value of the extension. Keep in // mind that extensions 3, 4, and 5 are reserved for // complex64, complex128, and time.Time, respectively, // and that MessagePack reserves extension types from -127 to -1. // // For example, if you wanted to register a user-defined struct: // // msgp.RegisterExtension(10, func() msgp.Extension { &MyExtension{} }) // // RegisterExtension will panic if you call it multiple times // with the same 'typ' argument, or if you use a reserved // type (3, 4, or 5). func RegisterExtension(typ int8, f func() Extension) { switch typ { case Complex64Extension, Complex128Extension, TimeExtension: panic(fmt.Sprint("msgp: forbidden extension type:", typ)) } if _, ok := extensionReg[typ]; ok { panic(fmt.Sprint("msgp: RegisterExtension() called with typ", typ, "more than once")) } extensionReg[typ] = f } // ExtensionTypeError is an error type returned // when there is a mis-match between an extension type // and the type encoded on the wire type ExtensionTypeError struct { Got int8 Want int8 } // Error implements the error interface func (e ExtensionTypeError) Error() string { return fmt.Sprintf("msgp: error decoding extension: wanted type %d; got type %d", e.Want, e.Got) } // Resumable returns 'true' for ExtensionTypeErrors func (e ExtensionTypeError) Resumable() bool { return true } func errExt(got int8, wanted int8) error { return ExtensionTypeError{Got: got, Want: wanted} } // Extension is the interface fulfilled // by types that want to define their // own binary encoding. type Extension interface { // ExtensionType should return // a int8 that identifies the concrete // type of the extension. (Types <0 are // officially reserved by the MessagePack // specifications.) ExtensionType() int8 // Len should return the length // of the data to be encoded Len() int // MarshalBinaryTo should copy // the data into the supplied slice, // assuming that the slice has length Len() MarshalBinaryTo([]byte) error UnmarshalBinary([]byte) error } // RawExtension implements the Extension interface type RawExtension struct { Data []byte Type int8 } // ExtensionType implements Extension.ExtensionType, and returns r.Type func (r *RawExtension) ExtensionType() int8 { return r.Type } // Len implements Extension.Len, and returns len(r.Data) func (r *RawExtension) Len() int { return len(r.Data) } // MarshalBinaryTo implements Extension.MarshalBinaryTo, // and returns a copy of r.Data func (r *RawExtension) MarshalBinaryTo(d []byte) error { copy(d, r.Data) return nil } // UnmarshalBinary implements Extension.UnmarshalBinary, // and sets r.Data to the contents of the provided slice func (r *RawExtension) UnmarshalBinary(b []byte) error { if cap(r.Data) >= len(b) { r.Data = r.Data[0:len(b)] } else { r.Data = make([]byte, len(b)) } copy(r.Data, b) return nil } // WriteExtension writes an extension type to the writer func (mw *Writer) WriteExtension(e Extension) error { l := e.Len() var err error switch l { case 0: o, err := mw.require(3) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = 0 mw.buf[o+2] = byte(e.ExtensionType()) case 1: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext1 mw.buf[o+1] = byte(e.ExtensionType()) case 2: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext2 mw.buf[o+1] = byte(e.ExtensionType()) case 4: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext4 mw.buf[o+1] = byte(e.ExtensionType()) case 8: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext8 mw.buf[o+1] = byte(e.ExtensionType()) case 16: o, err := mw.require(2) if err != nil { return err } mw.buf[o] = mfixext16 mw.buf[o+1] = byte(e.ExtensionType()) default: switch { case l < math.MaxUint8: o, err := mw.require(3) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = byte(uint8(l)) mw.buf[o+2] = byte(e.ExtensionType()) case l < math.MaxUint16: o, err := mw.require(4) if err != nil { return err } mw.buf[o] = mext16 big.PutUint16(mw.buf[o+1:], uint16(l)) mw.buf[o+3] = byte(e.ExtensionType()) default: o, err := mw.require(6) if err != nil { return err } mw.buf[o] = mext32 big.PutUint32(mw.buf[o+1:], uint32(l)) mw.buf[o+5] = byte(e.ExtensionType()) } } // we can only write directly to the // buffer if we're sure that it // fits the object if l <= mw.bufsize() { o, err := mw.require(l) if err != nil { return err } return e.MarshalBinaryTo(mw.buf[o:]) } // here we create a new buffer // just large enough for the body // and save it as the write buffer err = mw.flush() if err != nil { return err } buf := make([]byte, l) err = e.MarshalBinaryTo(buf) if err != nil { return err } mw.buf = buf mw.wloc = l return nil } // peek at the extension type, assuming the next // kind to be read is Extension func (m *Reader) peekExtensionType() (int8, error) { p, err := m.R.Peek(2) if err != nil { return 0, err } spec := sizes[p[0]] if spec.typ != ExtensionType { return 0, badPrefix(ExtensionType, p[0]) } if spec.extra == constsize { return int8(p[1]), nil } size := spec.size p, err = m.R.Peek(int(size)) if err != nil { return 0, err } return int8(p[size-1]), nil } // peekExtension peeks at the extension encoding type // (must guarantee at least 1 byte in 'b') func peekExtension(b []byte) (int8, error) { spec := sizes[b[0]] size := spec.size if spec.typ != ExtensionType { return 0, badPrefix(ExtensionType, b[0]) } if len(b) < int(size) { return 0, ErrShortBytes } // for fixed extensions, // the type information is in // the second byte if spec.extra == constsize { return int8(b[1]), nil } // otherwise, it's in the last // part of the prefix return int8(b[size-1]), nil } // ReadExtension reads the next object from the reader // as an extension. ReadExtension will fail if the next // object in the stream is not an extension, or if // e.Type() is not the same as the wire type. func (m *Reader) ReadExtension(e Extension) (err error) { var p []byte p, err = m.R.Peek(2) if err != nil { return } lead := p[0] var read int var off int switch lead { case mfixext1: if int8(p[1]) != e.ExtensionType() { err = errExt(int8(p[1]), e.ExtensionType()) return } p, err = m.R.Peek(3) if err != nil { return } err = e.UnmarshalBinary(p[2:]) if err == nil { _, err = m.R.Skip(3) } return case mfixext2: if int8(p[1]) != e.ExtensionType() { err = errExt(int8(p[1]), e.ExtensionType()) return } p, err = m.R.Peek(4) if err != nil { return } err = e.UnmarshalBinary(p[2:]) if err == nil { _, err = m.R.Skip(4) } return case mfixext4: if int8(p[1]) != e.ExtensionType() { err = errExt(int8(p[1]), e.ExtensionType()) return } p, err = m.R.Peek(6) if err != nil { return } err = e.UnmarshalBinary(p[2:]) if err == nil { _, err = m.R.Skip(6) } return case mfixext8: if int8(p[1]) != e.ExtensionType() { err = errExt(int8(p[1]), e.ExtensionType()) return } p, err = m.R.Peek(10) if err != nil { return } err = e.UnmarshalBinary(p[2:]) if err == nil { _, err = m.R.Skip(10) } return case mfixext16: if int8(p[1]) != e.ExtensionType() { err = errExt(int8(p[1]), e.ExtensionType()) return } p, err = m.R.Peek(18) if err != nil { return } err = e.UnmarshalBinary(p[2:]) if err == nil { _, err = m.R.Skip(18) } return case mext8: p, err = m.R.Peek(3) if err != nil { return } if int8(p[2]) != e.ExtensionType() { err = errExt(int8(p[2]), e.ExtensionType()) return } read = int(uint8(p[1])) off = 3 case mext16: p, err = m.R.Peek(4) if err != nil { return } if int8(p[3]) != e.ExtensionType() { err = errExt(int8(p[3]), e.ExtensionType()) return } read = int(big.Uint16(p[1:])) off = 4 case mext32: p, err = m.R.Peek(6) if err != nil { return } if int8(p[5]) != e.ExtensionType() { err = errExt(int8(p[5]), e.ExtensionType()) return } read = int(big.Uint32(p[1:])) off = 6 default: err = badPrefix(ExtensionType, lead) return } p, err = m.R.Peek(read + off) if err != nil { return } err = e.UnmarshalBinary(p[off:]) if err == nil { _, err = m.R.Skip(read + off) } return } // AppendExtension appends a MessagePack extension to the provided slice func AppendExtension(b []byte, e Extension) ([]byte, error) { l := e.Len() var o []byte var n int switch l { case 0: o, n = ensure(b, 3) o[n] = mext8 o[n+1] = 0 o[n+2] = byte(e.ExtensionType()) return o[:n+3], nil case 1: o, n = ensure(b, 3) o[n] = mfixext1 o[n+1] = byte(e.ExtensionType()) n += 2 case 2: o, n = ensure(b, 4) o[n] = mfixext2 o[n+1] = byte(e.ExtensionType()) n += 2 case 4: o, n = ensure(b, 6) o[n] = mfixext4 o[n+1] = byte(e.ExtensionType()) n += 2 case 8: o, n = ensure(b, 10) o[n] = mfixext8 o[n+1] = byte(e.ExtensionType()) n += 2 case 16: o, n = ensure(b, 18) o[n] = mfixext16 o[n+1] = byte(e.ExtensionType()) n += 2 } switch { case l < math.MaxUint8: o, n = ensure(b, l+3) o[n] = mext8 o[n+1] = byte(uint8(l)) o[n+2] = byte(e.ExtensionType()) n += 3 case l < math.MaxUint16: o, n = ensure(b, l+4) o[n] = mext16 big.PutUint16(o[n+1:], uint16(l)) o[n+3] = byte(e.ExtensionType()) n += 4 default: o, n = ensure(b, l+6) o[n] = mext32 big.PutUint32(o[n+1:], uint32(l)) o[n+5] = byte(e.ExtensionType()) n += 6 } return o, e.MarshalBinaryTo(o[n:]) } // ReadExtensionBytes reads an extension from 'b' into 'e' // and returns any remaining bytes. // Possible errors: // - ErrShortBytes ('b' not long enough) // - ExtensionTypeErorr{} (wire type not the same as e.Type()) // - TypeErorr{} (next object not an extension) // - InvalidPrefixError // - An umarshal error returned from e.UnmarshalBinary func ReadExtensionBytes(b []byte, e Extension) ([]byte, error) { l := len(b) if l < 3 { return b, ErrShortBytes } lead := b[0] var ( sz int // size of 'data' off int // offset of 'data' typ int8 ) switch lead { case mfixext1: typ = int8(b[1]) sz = 1 off = 2 case mfixext2: typ = int8(b[1]) sz = 2 off = 2 case mfixext4: typ = int8(b[1]) sz = 4 off = 2 case mfixext8: typ = int8(b[1]) sz = 8 off = 2 case mfixext16: typ = int8(b[1]) sz = 16 off = 2 case mext8: sz = int(uint8(b[1])) typ = int8(b[2]) off = 3 if sz == 0 { return b[3:], e.UnmarshalBinary(b[3:3]) } case mext16: if l < 4 { return b, ErrShortBytes } sz = int(big.Uint16(b[1:])) typ = int8(b[3]) off = 4 case mext32: if l < 6 { return b, ErrShortBytes } sz = int(big.Uint32(b[1:])) typ = int8(b[5]) off = 6 default: return b, badPrefix(ExtensionType, lead) } if typ != e.ExtensionType() { return b, errExt(typ, e.ExtensionType()) } // the data of the extension starts // at 'off' and is 'sz' bytes long if len(b[off:]) < sz { return b, ErrShortBytes } tot := off + sz return b[tot:], e.UnmarshalBinary(b[off:tot]) } msgp-1.0-beta/msgp/extension_test.go000066400000000000000000000017361261253337500176170ustar00rootroot00000000000000package msgp import ( "bytes" "math/rand" "testing" "time" ) var extSizes = [...]int{0, 1, 2, 4, 8, 16, int(tint8), int(tuint16), int(tuint32)} func randomExt() RawExtension { e := RawExtension{} e.Type = int8(rand.Int()) e.Data = RandBytes(extSizes[rand.Intn(len(extSizes))]) return e } func TestReadWriteExtension(t *testing.T) { rand.Seed(time.Now().Unix()) var buf bytes.Buffer en := NewWriter(&buf) dc := NewReader(&buf) for i := 0; i < 25; i++ { buf.Reset() e := randomExt() en.WriteExtension(&e) en.Flush() err := dc.ReadExtension(&e) if err != nil { t.Errorf("error with extension (length %d): %s", len(buf.Bytes()), err) } } } func TestReadWriteExtensionBytes(t *testing.T) { var bts []byte rand.Seed(time.Now().Unix()) for i := 0; i < 24; i++ { e := randomExt() bts, _ = AppendExtension(bts[0:0], &e) _, err := ReadExtensionBytes(bts, &e) if err != nil { t.Errorf("error with extension (length %d): %s", len(bts), err) } } } msgp-1.0-beta/msgp/file.go000066400000000000000000000045201261253337500154550ustar00rootroot00000000000000// +build linux,!appengine darwin dragonfly freebsd netbsd openbsd package msgp import ( "os" "syscall" ) // ReadFile reads a file into 'dst' using // a read-only memory mapping. Consequently, // the file must be mmap-able, and the // Unmarshaler should never write to // the source memory. (Methods generated // by the msgp tool obey that constraint, but // user-defined implementations may not.) // // Reading and writing through file mappings // is only efficient for large files; small // files are best read and written using // the ordinary streaming interfaces. // func ReadFile(dst Unmarshaler, file *os.File) error { stat, err := file.Stat() if err != nil { return err } data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { return err } adviseRead(data) _, err = dst.UnmarshalMsg(data) uerr := syscall.Munmap(data) if err == nil { err = uerr } return err } // MarshalSizer is the combination // of the Marshaler and Sizer // interfaces. type MarshalSizer interface { Marshaler Sizer } // WriteFile writes a file from 'src' using // memory mapping. It overwrites the entire // contents of the previous file. // The mapping size is calculated // using the `Msgsize()` method // of 'src', so it must produce a result // equal to or greater than the actual encoded // size of the object. Otherwise, // a fault (SIGBUS) will occur. // // Reading and writing through file mappings // is only efficient for large files; small // files are best read and written using // the ordinary streaming interfaces. // // NOTE: The performance of this call // is highly OS- and filesystem-dependent. // Users should take care to test that this // performs as expected in a production environment. // (Linux users should run a kernel and filesystem // that support fallocate(2) for the best results.) func WriteFile(src MarshalSizer, file *os.File) error { sz := src.Msgsize() err := fallocate(file, int64(sz)) if err != nil { return err } data, err := syscall.Mmap(int(file.Fd()), 0, sz, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) if err != nil { return err } adviseWrite(data) chunk := data[:0] chunk, err = src.MarshalMsg(chunk) if err != nil { return err } uerr := syscall.Munmap(data) if uerr != nil { return uerr } return file.Truncate(int64(len(chunk))) } msgp-1.0-beta/msgp/file_port.go000066400000000000000000000013621261253337500165220ustar00rootroot00000000000000// +build windows appengine package msgp import ( "io/ioutil" "os" ) // MarshalSizer is the combination // of the Marshaler and Sizer // interfaces. type MarshalSizer interface { Marshaler Sizer } func ReadFile(dst Unmarshaler, file *os.File) error { if u, ok := dst.(Decodable); ok { return u.DecodeMsg(NewReader(file)) } data, err := ioutil.ReadAll(file) if err != nil { return err } _, err = dst.UnmarshalMsg(data) return err } func WriteFile(src MarshalSizer, file *os.File) error { if e, ok := src.(Encodable); ok { w := NewWriter(file) err := e.EncodeMsg(w) if err == nil { err = w.Flush() } return err } raw, err := src.MarshalMsg(nil) if err != nil { return err } _, err = file.Write(raw) return err } msgp-1.0-beta/msgp/file_test.go000066400000000000000000000041051261253337500165130ustar00rootroot00000000000000// +build linux darwin dragonfly freebsd netbsd openbsd package msgp_test import ( "bytes" "crypto/rand" "github.com/tinylib/msgp/msgp" prand "math/rand" "os" "testing" ) type rawBytes []byte func (r rawBytes) MarshalMsg(b []byte) ([]byte, error) { return msgp.AppendBytes(b, []byte(r)), nil } func (r rawBytes) Msgsize() int { return msgp.BytesPrefixSize + len(r) } func (r *rawBytes) UnmarshalMsg(b []byte) ([]byte, error) { tmp, out, err := msgp.ReadBytesBytes(b, (*(*[]byte)(r))[:0]) *r = rawBytes(tmp) return out, err } func TestReadWriteFile(t *testing.T) { t.Parallel() f, err := os.Create("tmpfile") if err != nil { t.Fatal(err) } defer func() { f.Close() os.Remove("tmpfile") }() data := make([]byte, 1024*1024) rand.Read(data) err = msgp.WriteFile(rawBytes(data), f) if err != nil { t.Fatal(err) } var out rawBytes f.Seek(0, os.SEEK_SET) err = msgp.ReadFile(&out, f) if err != nil { t.Fatal(err) } if !bytes.Equal([]byte(out), []byte(data)) { t.Fatal("Input and output not equal.") } } var blobstrings = []string{"", "a string", "a longer string here!"} var blobfloats = []float64{0.0, -1.0, 1.0, 3.1415926535} var blobints = []int64{0, 1, -1, 80000, 1 << 30} var blobbytes = [][]byte{[]byte{}, []byte("hello"), []byte("{\"is_json\":true,\"is_compact\":\"unable to determine\"}")} func BenchmarkWriteReadFile(b *testing.B) { // let's not run out of disk space... if b.N > 10000000 { b.N = 10000000 } fname := "bench-tmpfile" f, err := os.Create(fname) if err != nil { b.Fatal(err) } defer func(f *os.File, name string) { f.Close() os.Remove(name) }(f, fname) data := make(Blobs, b.N) for i := range data { data[i].Name = blobstrings[prand.Intn(len(blobstrings))] data[i].Float = blobfloats[prand.Intn(len(blobfloats))] data[i].Amount = blobints[prand.Intn(len(blobints))] data[i].Bytes = blobbytes[prand.Intn(len(blobbytes))] } b.SetBytes(int64(data.Msgsize() / b.N)) b.ResetTimer() err = msgp.WriteFile(data, f) if err != nil { b.Fatal(err) } err = msgp.ReadFile(&data, f) if err != nil { b.Fatal(err) } } msgp-1.0-beta/msgp/floatbench_test.go000066400000000000000000000007521261253337500177050ustar00rootroot00000000000000package msgp import ( "testing" ) func BenchmarkReadWriteFloat32(b *testing.B) { var f float32 = 3.9081 bts := AppendFloat32([]byte{}, f) b.ResetTimer() for i := 0; i < b.N; i++ { bts = AppendFloat32(bts[0:0], f) f, bts, _ = ReadFloat32Bytes(bts) } } func BenchmarkReadWriteFloat64(b *testing.B) { var f float64 = 3.9081 bts := AppendFloat64([]byte{}, f) b.ResetTimer() for i := 0; i < b.N; i++ { bts = AppendFloat64(bts[0:0], f) f, bts, _ = ReadFloat64Bytes(bts) } } msgp-1.0-beta/msgp/integers.go000066400000000000000000000071361261253337500163640ustar00rootroot00000000000000package msgp /* ---------------------------------- integer encoding utilities (inline-able) TODO(tinylib): there are faster, albeit non-portable solutions to the code below. implement byteswap? ---------------------------------- */ func putMint64(b []byte, i int64) { b[0] = mint64 b[1] = byte(i >> 56) b[2] = byte(i >> 48) b[3] = byte(i >> 40) b[4] = byte(i >> 32) b[5] = byte(i >> 24) b[6] = byte(i >> 16) b[7] = byte(i >> 8) b[8] = byte(i) } func getMint64(b []byte) int64 { return (int64(b[1]) << 56) | (int64(b[2]) << 48) | (int64(b[3]) << 40) | (int64(b[4]) << 32) | (int64(b[5]) << 24) | (int64(b[6]) << 16) | (int64(b[7]) << 8) | (int64(b[8])) } func putMint32(b []byte, i int32) { b[0] = mint32 b[1] = byte(i >> 24) b[2] = byte(i >> 16) b[3] = byte(i >> 8) b[4] = byte(i) } func getMint32(b []byte) int32 { return (int32(b[1]) << 24) | (int32(b[2]) << 16) | (int32(b[3]) << 8) | (int32(b[4])) } func putMint16(b []byte, i int16) { b[0] = mint16 b[1] = byte(i >> 8) b[2] = byte(i) } func getMint16(b []byte) (i int16) { return (int16(b[1]) << 8) | int16(b[2]) } func putMint8(b []byte, i int8) { b[0] = mint8 b[1] = byte(i) } func getMint8(b []byte) (i int8) { return int8(b[1]) } func putMuint64(b []byte, u uint64) { b[0] = muint64 b[1] = byte(u >> 56) b[2] = byte(u >> 48) b[3] = byte(u >> 40) b[4] = byte(u >> 32) b[5] = byte(u >> 24) b[6] = byte(u >> 16) b[7] = byte(u >> 8) b[8] = byte(u) } func getMuint64(b []byte) uint64 { return (uint64(b[1]) << 56) | (uint64(b[2]) << 48) | (uint64(b[3]) << 40) | (uint64(b[4]) << 32) | (uint64(b[5]) << 24) | (uint64(b[6]) << 16) | (uint64(b[7]) << 8) | (uint64(b[8])) } func putMuint32(b []byte, u uint32) { b[0] = muint32 b[1] = byte(u >> 24) b[2] = byte(u >> 16) b[3] = byte(u >> 8) b[4] = byte(u) } func getMuint32(b []byte) uint32 { return (uint32(b[1]) << 24) | (uint32(b[2]) << 16) | (uint32(b[3]) << 8) | (uint32(b[4])) } func putMuint16(b []byte, u uint16) { b[0] = muint16 b[1] = byte(u >> 8) b[2] = byte(u) } func getMuint16(b []byte) uint16 { return (uint16(b[1]) << 8) | uint16(b[2]) } func putMuint8(b []byte, u uint8) { b[0] = muint8 b[1] = byte(u) } func getMuint8(b []byte) uint8 { return uint8(b[1]) } func getUnix(b []byte) (sec int64, nsec int32) { sec = (int64(b[0]) << 56) | (int64(b[1]) << 48) | (int64(b[2]) << 40) | (int64(b[3]) << 32) | (int64(b[4]) << 24) | (int64(b[5]) << 16) | (int64(b[6]) << 8) | (int64(b[7])) nsec = (int32(b[8]) << 24) | (int32(b[9]) << 16) | (int32(b[10]) << 8) | (int32(b[11])) return } func putUnix(b []byte, sec int64, nsec int32) { b[0] = byte(sec >> 56) b[1] = byte(sec >> 48) b[2] = byte(sec >> 40) b[3] = byte(sec >> 32) b[4] = byte(sec >> 24) b[5] = byte(sec >> 16) b[6] = byte(sec >> 8) b[7] = byte(sec) b[8] = byte(nsec >> 24) b[9] = byte(nsec >> 16) b[10] = byte(nsec >> 8) b[11] = byte(nsec) } /* ----------------------------- prefix utilities ----------------------------- */ // write prefix and uint8 func prefixu8(b []byte, pre byte, sz uint8) { b[0] = pre b[1] = byte(sz) } // write prefix and big-endian uint16 func prefixu16(b []byte, pre byte, sz uint16) { b[0] = pre b[1] = byte(sz >> 8) b[2] = byte(sz) } // write prefix and big-endian uint32 func prefixu32(b []byte, pre byte, sz uint32) { b[0] = pre b[1] = byte(sz >> 24) b[2] = byte(sz >> 16) b[3] = byte(sz >> 8) b[4] = byte(sz) } func prefixu64(b []byte, pre byte, sz uint64) { b[0] = pre b[1] = byte(sz >> 56) b[2] = byte(sz >> 48) b[3] = byte(sz >> 40) b[4] = byte(sz >> 32) b[5] = byte(sz >> 24) b[6] = byte(sz >> 16) b[7] = byte(sz >> 8) b[8] = byte(sz) } msgp-1.0-beta/msgp/json.go000066400000000000000000000220541261253337500155110ustar00rootroot00000000000000package msgp import ( "bufio" "encoding/base64" "encoding/json" "io" "strconv" "unicode/utf8" ) var ( null = []byte("null") hex = []byte("0123456789abcdef") ) var defuns [_maxtype]func(jsWriter, *Reader) (int, error) // note: there is an initialization loop if // this isn't set up during init() func init() { // since none of these functions are inline-able, // there is not much of a penalty to the indirect // call. however, this is best expressed as a jump-table... defuns = [_maxtype]func(jsWriter, *Reader) (int, error){ StrType: rwString, BinType: rwBytes, MapType: rwMap, ArrayType: rwArray, Float64Type: rwFloat64, Float32Type: rwFloat32, BoolType: rwBool, IntType: rwInt, UintType: rwUint, NilType: rwNil, ExtensionType: rwExtension, Complex64Type: rwExtension, Complex128Type: rwExtension, TimeType: rwTime, } } // this is the interface // used to write json type jsWriter interface { io.Writer io.ByteWriter WriteString(string) (int, error) } // CopyToJSON reads MessagePack from 'src' and copies it // as JSON to 'dst' until EOF. func CopyToJSON(dst io.Writer, src io.Reader) (n int64, err error) { r := NewReader(src) n, err = r.WriteToJSON(dst) freeR(r) return } // WriteToJSON translates MessagePack from 'r' and writes it as // JSON to 'w' until the underlying reader returns io.EOF. It returns // the number of bytes written, and an error if it stopped before EOF. func (r *Reader) WriteToJSON(w io.Writer) (n int64, err error) { var j jsWriter var bf *bufio.Writer if jsw, ok := w.(jsWriter); ok { j = jsw } else { bf = bufio.NewWriter(w) j = bf } var nn int for err == nil { nn, err = rwNext(j, r) n += int64(nn) } if err != io.EOF { if bf != nil { bf.Flush() } return } err = nil if bf != nil { err = bf.Flush() } return } func rwNext(w jsWriter, src *Reader) (int, error) { t, err := src.NextType() if err != nil { return 0, err } return defuns[t](w, src) } func rwMap(dst jsWriter, src *Reader) (n int, err error) { var comma bool var sz uint32 var field []byte sz, err = src.ReadMapHeader() if err != nil { return } if sz == 0 { return dst.WriteString("{}") } err = dst.WriteByte('{') if err != nil { return } n++ var nn int for i := uint32(0); i < sz; i++ { if comma { err = dst.WriteByte(',') if err != nil { return } n++ } field, err = src.ReadMapKeyPtr() if err != nil { return } nn, err = rwquoted(dst, field) n += nn if err != nil { return } err = dst.WriteByte(':') if err != nil { return } n++ nn, err = rwNext(dst, src) n += nn if err != nil { return } if !comma { comma = true } } err = dst.WriteByte('}') if err != nil { return } n++ return } func rwArray(dst jsWriter, src *Reader) (n int, err error) { err = dst.WriteByte('[') if err != nil { return } var sz uint32 var nn int sz, err = src.ReadArrayHeader() if err != nil { return } comma := false for i := uint32(0); i < sz; i++ { if comma { err = dst.WriteByte(',') if err != nil { return } n++ } nn, err = rwNext(dst, src) n += nn if err != nil { return } comma = true } err = dst.WriteByte(']') if err != nil { return } n++ return } func rwNil(dst jsWriter, src *Reader) (int, error) { err := src.ReadNil() if err != nil { return 0, err } return dst.Write(null) } func rwFloat32(dst jsWriter, src *Reader) (int, error) { f, err := src.ReadFloat32() if err != nil { return 0, err } src.scratch = strconv.AppendFloat(src.scratch[:0], float64(f), 'f', -1, 64) return dst.Write(src.scratch) } func rwFloat64(dst jsWriter, src *Reader) (int, error) { f, err := src.ReadFloat64() if err != nil { return 0, err } src.scratch = strconv.AppendFloat(src.scratch[:0], f, 'f', -1, 32) return dst.Write(src.scratch) } func rwInt(dst jsWriter, src *Reader) (int, error) { i, err := src.ReadInt64() if err != nil { return 0, err } src.scratch = strconv.AppendInt(src.scratch[:0], i, 10) return dst.Write(src.scratch) } func rwUint(dst jsWriter, src *Reader) (int, error) { u, err := src.ReadUint64() if err != nil { return 0, err } src.scratch = strconv.AppendUint(src.scratch[:0], u, 10) return dst.Write(src.scratch) } func rwBool(dst jsWriter, src *Reader) (int, error) { b, err := src.ReadBool() if err != nil { return 0, err } if b { return dst.WriteString("true") } return dst.WriteString("false") } func rwTime(dst jsWriter, src *Reader) (int, error) { t, err := src.ReadTime() if err != nil { return 0, err } bts, err := t.MarshalJSON() if err != nil { return 0, err } return dst.Write(bts) } func rwExtension(dst jsWriter, src *Reader) (n int, err error) { et, err := src.peekExtensionType() if err != nil { return 0, err } // registered extensions can override // the JSON encoding if j, ok := extensionReg[et]; ok { var bts []byte e := j() err = src.ReadExtension(e) if err != nil { return } bts, err = json.Marshal(e) if err != nil { return } return dst.Write(bts) } e := RawExtension{} e.Type = et err = src.ReadExtension(&e) if err != nil { return } var nn int err = dst.WriteByte('{') if err != nil { return } n++ nn, err = dst.WriteString(`"type:"`) n += nn if err != nil { return } src.scratch = strconv.AppendInt(src.scratch[0:0], int64(e.Type), 10) nn, err = dst.Write(src.scratch) n += nn if err != nil { return } nn, err = dst.WriteString(`,"data":"`) n += nn if err != nil { return } enc := base64.NewEncoder(base64.StdEncoding, dst) nn, err = enc.Write(e.Data) n += nn if err != nil { return } err = enc.Close() if err != nil { return } nn, err = dst.WriteString(`"}`) n += nn return } func rwString(dst jsWriter, src *Reader) (n int, err error) { var p []byte p, err = src.R.Peek(1) if err != nil { return } lead := p[0] var read int if isfixstr(lead) { read = int(rfixstr(lead)) src.R.Skip(1) goto write } switch lead { case mstr8: p, err = src.R.Next(2) if err != nil { return } read = int(uint8(p[1])) case mstr16: p, err = src.R.Next(3) if err != nil { return } read = int(big.Uint16(p[1:])) case mstr32: p, err = src.R.Next(5) if err != nil { return } read = int(big.Uint32(p[1:])) default: err = badPrefix(StrType, lead) return } write: p, err = src.R.Next(read) if err != nil { return } n, err = rwquoted(dst, p) return } func rwBytes(dst jsWriter, src *Reader) (n int, err error) { var nn int err = dst.WriteByte('"') if err != nil { return } n++ src.scratch, err = src.ReadBytes(src.scratch[:0]) if err != nil { return } enc := base64.NewEncoder(base64.StdEncoding, dst) nn, err = enc.Write(src.scratch) n += nn if err != nil { return } err = enc.Close() if err != nil { return } err = dst.WriteByte('"') if err != nil { return } n++ return } // Below (c) The Go Authors, 2009-2014 // Subject to the BSD-style license found at http://golang.org // // see: encoding/json/encode.go:(*encodeState).stringbytes() func rwquoted(dst jsWriter, s []byte) (n int, err error) { var nn int err = dst.WriteByte('"') if err != nil { return } n++ start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { i++ continue } if start < i { nn, err = dst.Write(s[start:i]) n += nn if err != nil { return } } switch b { case '\\', '"': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte(b) if err != nil { return } n++ case '\n': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte('n') if err != nil { return } n++ case '\r': err = dst.WriteByte('\\') if err != nil { return } n++ err = dst.WriteByte('r') if err != nil { return } n++ default: nn, err = dst.WriteString(`\u00`) n += nn if err != nil { return } err = dst.WriteByte(hex[b>>4]) if err != nil { return } n++ err = dst.WriteByte(hex[b&0xF]) if err != nil { return } n++ } i++ start = i continue } c, size := utf8.DecodeRune(s[i:]) if c == utf8.RuneError && size == 1 { if start < i { nn, err = dst.Write(s[start:i]) n += nn if err != nil { return } nn, err = dst.WriteString(`\ufffd`) n += nn if err != nil { return } i += size start = i continue } } if c == '\u2028' || c == '\u2029' { if start < i { nn, err = dst.Write(s[start:i]) n += nn if err != nil { return } nn, err = dst.WriteString(`\u202`) n += nn if err != nil { return } err = dst.WriteByte(hex[c&0xF]) if err != nil { return } n++ } } i += size } if start < len(s) { nn, err = dst.Write(s[start:]) n += nn if err != nil { return } } err = dst.WriteByte('"') if err != nil { return } n++ return } msgp-1.0-beta/msgp/json_bytes.go000066400000000000000000000203701261253337500167160ustar00rootroot00000000000000package msgp import ( "bufio" "encoding/base64" "encoding/json" "io" "strconv" "time" ) var unfuns [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error) func init() { // NOTE(pmh): this is best expressed as a jump table, // but gc doesn't do that yet. revisit post-go1.5. unfuns = [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error){ StrType: rwStringBytes, BinType: rwBytesBytes, MapType: rwMapBytes, ArrayType: rwArrayBytes, Float64Type: rwFloat64Bytes, Float32Type: rwFloat32Bytes, BoolType: rwBoolBytes, IntType: rwIntBytes, UintType: rwUintBytes, NilType: rwNullBytes, ExtensionType: rwExtensionBytes, Complex64Type: rwExtensionBytes, Complex128Type: rwExtensionBytes, TimeType: rwTimeBytes, } } // UnmarshalAsJSON takes raw messagepack and writes // it as JSON to 'w'. If an error is returned, the // bytes not translated will also be returned. If // no errors are encountered, the length of the returned // slice will be zero. func UnmarshalAsJSON(w io.Writer, msg []byte) ([]byte, error) { var ( scratch []byte cast bool dst jsWriter err error ) if jsw, ok := w.(jsWriter); ok { dst = jsw cast = true } else { dst = bufio.NewWriterSize(w, 512) } for len(msg) > 0 && err == nil { msg, scratch, err = writeNext(dst, msg, scratch) } if !cast && err == nil { err = dst.(*bufio.Writer).Flush() } return msg, err } func writeNext(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { if len(msg) < 1 { return msg, scratch, ErrShortBytes } t := getType(msg[0]) if t == InvalidType { return msg, scratch, InvalidPrefixError(msg[0]) } if t == ExtensionType { et, err := peekExtension(msg) if err != nil { return nil, scratch, err } if et == TimeExtension { t = TimeType } } return unfuns[t](w, msg, scratch) } func rwArrayBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { sz, msg, err := ReadArrayHeaderBytes(msg) if err != nil { return msg, scratch, err } err = w.WriteByte('[') if err != nil { return msg, scratch, err } for i := uint32(0); i < sz; i++ { if i != 0 { err = w.WriteByte(',') if err != nil { return msg, scratch, err } } msg, scratch, err = writeNext(w, msg, scratch) if err != nil { return msg, scratch, err } } err = w.WriteByte(']') return msg, scratch, err } func rwMapBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { sz, msg, err := ReadMapHeaderBytes(msg) if err != nil { return msg, scratch, err } err = w.WriteByte('{') if err != nil { return msg, scratch, err } for i := uint32(0); i < sz; i++ { if i != 0 { err = w.WriteByte(',') if err != nil { return msg, scratch, err } } msg, scratch, err = rwMapKeyBytes(w, msg, scratch) if err != nil { return msg, scratch, err } err = w.WriteByte(':') if err != nil { return msg, scratch, err } msg, scratch, err = writeNext(w, msg, scratch) if err != nil { return msg, scratch, err } } err = w.WriteByte('}') return msg, scratch, err } func rwMapKeyBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { msg, scratch, err := rwStringBytes(w, msg, scratch) if err != nil { if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { return rwBytesBytes(w, msg, scratch) } } return msg, scratch, err } func rwStringBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { str, msg, err := ReadStringZC(msg) if err != nil { return msg, scratch, err } _, err = rwquoted(w, str) return msg, scratch, err } func rwBytesBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { bts, msg, err := ReadBytesZC(msg) if err != nil { return msg, scratch, err } l := base64.StdEncoding.EncodedLen(len(bts)) if cap(scratch) >= l { scratch = scratch[0:l] } else { scratch = make([]byte, l) } base64.StdEncoding.Encode(scratch, bts) err = w.WriteByte('"') if err != nil { return msg, scratch, err } _, err = w.Write(scratch) if err != nil { return msg, scratch, err } err = w.WriteByte('"') return msg, scratch, err } func rwNullBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { msg, err := ReadNilBytes(msg) if err != nil { return msg, scratch, err } _, err = w.Write(null) return msg, scratch, err } func rwBoolBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { b, msg, err := ReadBoolBytes(msg) if err != nil { return msg, scratch, err } if b { _, err = w.WriteString("true") return msg, scratch, err } _, err = w.WriteString("false") return msg, scratch, err } func rwIntBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { i, msg, err := ReadInt64Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendInt(scratch[0:0], i, 10) _, err = w.Write(scratch) return msg, scratch, err } func rwUintBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { u, msg, err := ReadUint64Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendUint(scratch[0:0], u, 10) _, err = w.Write(scratch) return msg, scratch, err } func rwFloatBytes(w jsWriter, msg []byte, f64 bool, scratch []byte) ([]byte, []byte, error) { var f float64 var err error var sz int if f64 { sz = 64 f, msg, err = ReadFloat64Bytes(msg) } else { sz = 32 var v float32 v, msg, err = ReadFloat32Bytes(msg) f = float64(v) } if err != nil { return msg, scratch, err } scratch = strconv.AppendFloat(scratch, f, 'f', -1, sz) _, err = w.Write(scratch) return msg, scratch, err } func rwFloat32Bytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { var f float32 var err error f, msg, err = ReadFloat32Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendFloat(scratch[:0], float64(f), 'f', -1, 32) _, err = w.Write(scratch) return msg, scratch, err } func rwFloat64Bytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { var f float64 var err error f, msg, err = ReadFloat64Bytes(msg) if err != nil { return msg, scratch, err } scratch = strconv.AppendFloat(scratch[:0], f, 'f', -1, 64) _, err = w.Write(scratch) return msg, scratch, err } func rwTimeBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { var t time.Time var err error t, msg, err = ReadTimeBytes(msg) if err != nil { return msg, scratch, err } bts, err := t.MarshalJSON() if err != nil { return msg, scratch, err } _, err = w.Write(bts) return msg, scratch, err } func rwExtensionBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { var err error var et int8 et, err = peekExtension(msg) if err != nil { return msg, scratch, err } // if it's time.Time if et == TimeExtension { var tm time.Time tm, msg, err = ReadTimeBytes(msg) if err != nil { return msg, scratch, err } bts, err := tm.MarshalJSON() if err != nil { return msg, scratch, err } _, err = w.Write(bts) return msg, scratch, err } // if the extension is registered, // use its canonical JSON form if f, ok := extensionReg[et]; ok { e := f() msg, err = ReadExtensionBytes(msg, e) if err != nil { return msg, scratch, err } bts, err := json.Marshal(e) if err != nil { return msg, scratch, err } _, err = w.Write(bts) return msg, scratch, err } // otherwise, write `{"type": , "data": ""}` r := RawExtension{} r.Type = et msg, err = ReadExtensionBytes(msg, &r) if err != nil { return msg, scratch, err } scratch, err = writeExt(w, r, scratch) return msg, scratch, err } func writeExt(w jsWriter, r RawExtension, scratch []byte) ([]byte, error) { _, err := w.WriteString(`{"type":`) if err != nil { return scratch, err } scratch = strconv.AppendInt(scratch[0:0], int64(r.Type), 10) _, err = w.Write(scratch) if err != nil { return scratch, err } _, err = w.WriteString(`,"data":"`) if err != nil { return scratch, err } l := base64.StdEncoding.EncodedLen(len(r.Data)) if cap(scratch) >= l { scratch = scratch[0:l] } else { scratch = make([]byte, l) } base64.StdEncoding.Encode(scratch, r.Data) _, err = w.Write(scratch) if err != nil { return scratch, err } _, err = w.WriteString(`"}`) return scratch, err } msgp-1.0-beta/msgp/json_bytes_test.go000066400000000000000000000045351261253337500177620ustar00rootroot00000000000000package msgp import ( "bytes" "encoding/json" "testing" "time" ) func TestUnmarshalJSON(t *testing.T) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(5) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_map") enc.WriteMapHeader(2) // INNER enc.WriteString("cmplx") enc.WriteComplex64(complex(1.0, 1.0)) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("an extension") enc.WriteExtension(&RawExtension{Type: 1, Data: []byte("blaaahhh")}) enc.WriteString("some bytes") enc.WriteBytes([]byte("here are some bytes")) enc.WriteString("now") enc.WriteTime(time.Now()) enc.Flush() var js bytes.Buffer _, err := UnmarshalAsJSON(&js, buf.Bytes()) if err != nil { t.Logf("%s", js.Bytes()) t.Fatal(err) } mp := make(map[string]interface{}) err = json.Unmarshal(js.Bytes(), &mp) if err != nil { t.Log(js.String()) t.Fatalf("Error unmarshaling: %s", err) } if len(mp) != 5 { t.Errorf("map length should be %d, not %d", 5, len(mp)) } so, ok := mp["thing_1"] if !ok || so != "a string object" { t.Errorf("expected %q; got %q", "a string object", so) } if _, ok := mp["now"]; !ok { t.Error(`"now" field doesn't exist`) } c, ok := mp["a_map"] if !ok { t.Error(`"a_map" field doesn't exist`) } else { if m, ok := c.(map[string]interface{}); ok { if _, ok := m["cmplx"]; !ok { t.Error(`"a_map.cmplx" doesn't exist`) } } else { t.Error(`can't type-assert "c" to map[string]interface{}`) } } t.Logf("JSON: %s", js.Bytes()) } func BenchmarkUnmarshalAsJSON(b *testing.B) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(4) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_first_map") enc.WriteMapHeader(2) enc.WriteString("float_a") enc.WriteFloat32(1.0) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("an array") enc.WriteArrayHeader(2) enc.WriteBool(true) enc.WriteUint(2089) enc.WriteString("a_second_map") enc.WriteMapStrStr(map[string]string{ "internal_one": "blah", "internal_two": "blahhh...", }) enc.Flush() var js bytes.Buffer bts := buf.Bytes() _, err := UnmarshalAsJSON(&js, bts) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(js.Bytes()))) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { js.Reset() UnmarshalAsJSON(&js, bts) } } msgp-1.0-beta/msgp/json_test.go000066400000000000000000000055731261253337500165570ustar00rootroot00000000000000package msgp import ( "bytes" "encoding/json" "reflect" "testing" ) func TestCopyJSON(t *testing.T) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(5) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_map") enc.WriteMapHeader(2) enc.WriteString("float_a") enc.WriteFloat32(1.0) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("some bytes") enc.WriteBytes([]byte("here are some bytes")) enc.WriteString("a bool") enc.WriteBool(true) enc.WriteString("a map") enc.WriteMapStrStr(map[string]string{ "internal_one": "blah", "internal_two": "blahhh...", }) enc.Flush() var js bytes.Buffer _, err := CopyToJSON(&js, &buf) if err != nil { t.Fatal(err) } mp := make(map[string]interface{}) err = json.Unmarshal(js.Bytes(), &mp) if err != nil { t.Log(js.String()) t.Fatalf("Error unmarshaling: %s", err) } if len(mp) != 5 { t.Errorf("map length should be %d, not %d", 4, len(mp)) } so, ok := mp["thing_1"] if !ok || so != "a string object" { t.Errorf("expected %q; got %q", "a string object", so) } in, ok := mp["a map"] if !ok { t.Error("no key 'a map'") } if inm, ok := in.(map[string]interface{}); !ok { t.Error("inner map not type-assertable to map[string]interface{}") } else { inm1, ok := inm["internal_one"] if !ok || !reflect.DeepEqual(inm1, "blah") { t.Errorf("inner map field %q should be %q, not %q", "internal_one", "blah", inm1) } } } func BenchmarkCopyToJSON(b *testing.B) { var buf bytes.Buffer enc := NewWriter(&buf) enc.WriteMapHeader(4) enc.WriteString("thing_1") enc.WriteString("a string object") enc.WriteString("a_first_map") enc.WriteMapHeader(2) enc.WriteString("float_a") enc.WriteFloat32(1.0) enc.WriteString("int_b") enc.WriteInt64(-100) enc.WriteString("an array") enc.WriteArrayHeader(2) enc.WriteBool(true) enc.WriteUint(2089) enc.WriteString("a_second_map") enc.WriteMapStrStr(map[string]string{ "internal_one": "blah", "internal_two": "blahhh...", }) enc.Flush() var js bytes.Buffer bts := buf.Bytes() _, err := CopyToJSON(&js, &buf) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(js.Bytes()))) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { js.Reset() CopyToJSON(&js, bytes.NewReader(bts)) } } func BenchmarkStdlibJSON(b *testing.B) { obj := map[string]interface{}{ "thing_1": "a string object", "a_first_map": map[string]interface{}{ "float_a": float32(1.0), "float_b": -100, }, "an array": []interface{}{ "part_A", "part_B", }, "a_second_map": map[string]interface{}{ "internal_one": "blah", "internal_two": "blahhh...", }, } var js bytes.Buffer err := json.NewEncoder(&js).Encode(&obj) if err != nil { b.Fatal(err) } b.SetBytes(int64(len(js.Bytes()))) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { js.Reset() json.NewEncoder(&js).Encode(&obj) } } msgp-1.0-beta/msgp/number.go000066400000000000000000000127131261253337500160310ustar00rootroot00000000000000package msgp import ( "math" "strconv" ) // The portable parts of the Number implementation // Number can be // an int64, uint64, float32, // or float64 internally. // It can decode itself // from any of the native // messagepack number types. // The zero-value of Number // is Int(0). Using the equality // operator with Number compares // both the type and the value // of the number. type Number struct { // internally, this // is just a tagged union. // the raw bits of the number // are stored the same way regardless. bits uint64 typ Type } // AsInt sets the number to an int64. func (n *Number) AsInt(i int64) { // we always store int(0) // as {0, InvalidType} in // order to preserve // the behavior of the == operator if i == 0 { n.typ = InvalidType n.bits = 0 return } n.typ = IntType n.bits = uint64(i) } // AsUint sets the number to a uint64. func (n *Number) AsUint(u uint64) { n.typ = UintType n.bits = u } // AsFloat32 sets the value of the number // to a float32. func (n *Number) AsFloat32(f float32) { n.typ = Float32Type n.bits = uint64(math.Float32bits(f)) } // AsFloat64 sets the value of the // number to a float64. func (n *Number) AsFloat64(f float64) { n.typ = Float64Type n.bits = math.Float64bits(f) } // Int casts the number as an int64, and // returns whether or not that was the // underlying type. func (n *Number) Int() (int64, bool) { return int64(n.bits), n.typ == IntType || n.typ == InvalidType } // Uint casts the number as a uint64, and returns // whether or not that was the underlying type. func (n *Number) Uint() (uint64, bool) { return n.bits, n.typ == UintType } // Float casts the number to a float64, and // returns whether or not that was the underlying // type (either a float64 or a float32). func (n *Number) Float() (float64, bool) { switch n.typ { case Float32Type: return float64(math.Float32frombits(uint32(n.bits))), true case Float64Type: return math.Float64frombits(n.bits), true default: return 0.0, false } } // Type will return one of: // Float64Type, Float32Type, UintType, or IntType. func (n *Number) Type() Type { if n.typ == InvalidType { return IntType } return n.typ } // DecodeMsg implements msgp.Decodable func (n *Number) DecodeMsg(r *Reader) error { typ, err := r.NextType() if err != nil { return err } switch typ { case Float32Type: f, err := r.ReadFloat32() if err != nil { return err } n.AsFloat32(f) return nil case Float64Type: f, err := r.ReadFloat64() if err != nil { return err } n.AsFloat64(f) return nil case IntType: i, err := r.ReadInt64() if err != nil { return err } n.AsInt(i) return nil case UintType: u, err := r.ReadUint64() if err != nil { return err } n.AsUint(u) return nil default: return TypeError{Encoded: typ, Method: IntType} } } // UnmarshalMsg implements msgp.Unmarshaler func (n *Number) UnmarshalMsg(b []byte) ([]byte, error) { typ := NextType(b) switch typ { case IntType: i, o, err := ReadInt64Bytes(b) if err != nil { return b, err } n.AsInt(i) return o, nil case UintType: u, o, err := ReadUint64Bytes(b) if err != nil { return b, err } n.AsUint(u) return o, nil case Float64Type: f, o, err := ReadFloat64Bytes(b) if err != nil { return b, err } n.AsFloat64(f) return o, nil case Float32Type: f, o, err := ReadFloat32Bytes(b) if err != nil { return b, err } n.AsFloat32(f) return o, nil default: return b, TypeError{Method: IntType, Encoded: typ} } } // MarshalMsg implements msgp.Marshaler func (n *Number) MarshalMsg(b []byte) ([]byte, error) { switch n.typ { case IntType: return AppendInt64(b, int64(n.bits)), nil case UintType: return AppendUint64(b, uint64(n.bits)), nil case Float64Type: return AppendFloat64(b, math.Float64frombits(n.bits)), nil case Float32Type: return AppendFloat32(b, math.Float32frombits(uint32(n.bits))), nil default: return AppendInt64(b, 0), nil } } // EncodeMsg implements msgp.Encodable func (n *Number) EncodeMsg(w *Writer) error { switch n.typ { case IntType: return w.WriteInt64(int64(n.bits)) case UintType: return w.WriteUint64(n.bits) case Float64Type: return w.WriteFloat64(math.Float64frombits(n.bits)) case Float32Type: return w.WriteFloat32(math.Float32frombits(uint32(n.bits))) default: return w.WriteInt64(0) } } // Msgsize implements msgp.Sizer func (n *Number) Msgsize() int { switch n.typ { case Float32Type: return Float32Size case Float64Type: return Float64Size case IntType: return Int64Size case UintType: return Uint64Size default: return 1 // fixint(0) } } // MarshalJSON implements json.Marshaler func (n *Number) MarshalJSON() ([]byte, error) { t := n.Type() if t == InvalidType { return []byte{'0'}, nil } out := make([]byte, 0, 32) switch t { case Float32Type, Float64Type: f, _ := n.Float() return strconv.AppendFloat(out, f, 'f', -1, 64), nil case IntType: i, _ := n.Int() return strconv.AppendInt(out, i, 10), nil case UintType: u, _ := n.Uint() return strconv.AppendUint(out, u, 10), nil default: panic("(*Number).typ is invalid") } } // String implements fmt.Stringer func (n *Number) String() string { switch n.typ { case InvalidType: return "0" case Float32Type, Float64Type: f, _ := n.Float() return strconv.FormatFloat(f, 'f', -1, 64) case IntType: i, _ := n.Int() return strconv.FormatInt(i, 10) case UintType: u, _ := n.Uint() return strconv.FormatUint(u, 10) default: panic("(*Number).typ is invalid") } } msgp-1.0-beta/msgp/number_test.go000066400000000000000000000035341261253337500170710ustar00rootroot00000000000000package msgp import ( "bytes" "testing" ) func TestNumber(t *testing.T) { n := Number{} if n.Type() != IntType { t.Errorf("expected zero-value type to be %s; got %s", IntType, n.Type()) } if n.String() != "0" { t.Errorf("expected Number{}.String() to be \"0\" but got %q", n.String()) } n.AsInt(248) i, ok := n.Int() if !ok || i != 248 || n.Type() != IntType || n.String() != "248" { t.Errorf("%d in; %d out!", 248, i) } n.AsFloat64(3.141) f, ok := n.Float() if !ok || f != 3.141 || n.Type() != Float64Type || n.String() != "3.141" { t.Errorf("%f in; %f out!", 3.141, f) } n.AsUint(40000) u, ok := n.Uint() if !ok || u != 40000 || n.Type() != UintType || n.String() != "40000" { t.Errorf("%d in; %d out!", 40000, u) } nums := []interface{}{ float64(3.14159), int64(-29081), uint64(90821983), float32(3.141), } var dat []byte var buf bytes.Buffer wr := NewWriter(&buf) for _, n := range nums { dat, _ = AppendIntf(dat, n) wr.WriteIntf(n) } wr.Flush() mout := make([]Number, len(nums)) dout := make([]Number, len(nums)) rd := NewReader(&buf) unm := dat for i := range nums { var err error unm, err = mout[i].UnmarshalMsg(unm) if err != nil { t.Fatal("unmarshal error:", err) } err = dout[i].DecodeMsg(rd) if err != nil { t.Fatal("decode error:", err) } if mout[i] != dout[i] { t.Errorf("for %#v, got %#v from unmarshal and %#v from decode", nums[i], mout[i], dout[i]) } } buf.Reset() var odat []byte for i := range nums { var err error odat, err = mout[i].MarshalMsg(odat) if err != nil { t.Fatal("marshal error:", err) } err = dout[i].EncodeMsg(wr) } wr.Flush() if !bytes.Equal(dat, odat) { t.Errorf("marshal: expected output %#v; got %#v", dat, odat) } if !bytes.Equal(dat, buf.Bytes()) { t.Errorf("encode: expected output %#v; got %#v", dat, buf.Bytes()) } } msgp-1.0-beta/msgp/raw_test.go000066400000000000000000000031431261253337500163660ustar00rootroot00000000000000package msgp import ( "bytes" "testing" "time" ) // all standard interfaces type allifaces interface { Encodable Decodable Marshaler Unmarshaler Sizer } func TestRaw(t *testing.T) { bts := make([]byte, 0, 512) bts = AppendMapHeader(bts, 3) bts = AppendString(bts, "key_one") bts = AppendFloat64(bts, -1.0) bts = AppendString(bts, "key_two") bts = AppendString(bts, "value_two") bts = AppendString(bts, "key_three") bts = AppendTime(bts, time.Now()) var r Raw // verify that Raw satisfies // the interfaces we want it to var _ allifaces = &r // READ TESTS extra, err := r.UnmarshalMsg(bts) if err != nil { t.Fatal("error from UnmarshalMsg:", err) } if len(extra) != 0 { t.Errorf("expected 0 bytes left; found %d", len(extra)) } if !bytes.Equal([]byte(r), bts) { t.Fatal("value of raw and input slice are not equal after UnmarshalMsg") } r = r[:0] var buf bytes.Buffer buf.Write(bts) rd := NewReader(&buf) err = r.DecodeMsg(rd) if err != nil { t.Fatal("error from DecodeMsg:", err) } if !bytes.Equal([]byte(r), bts) { t.Fatal("value of raw and input slice are not equal after DecodeMsg") } // WRITE TESTS buf.Reset() wr := NewWriter(&buf) err = r.EncodeMsg(wr) if err != nil { t.Fatal("error from EncodeMsg:", err) } wr.Flush() if !bytes.Equal(buf.Bytes(), bts) { t.Fatal("value of buf.Bytes() and input slice are not equal after EncodeMsg") } var outsl []byte outsl, err = r.MarshalMsg(outsl) if err != nil { t.Fatal("error from MarshalMsg:", err) } if !bytes.Equal(outsl, bts) { t.Fatal("value of output and input of MarshalMsg are not equal.") } } msgp-1.0-beta/msgp/read.go000066400000000000000000000556441261253337500154660ustar00rootroot00000000000000package msgp import ( "io" "math" "sync" "time" "github.com/philhofer/fwd" ) // where we keep old *Readers var readerPool = sync.Pool{New: func() interface{} { return &Reader{} }} // Type is a MessagePack wire type, // including this package's built-in // extension types. type Type byte // MessagePack Types // // The zero value of Type // is InvalidType. const ( InvalidType Type = iota // MessagePack built-in types StrType BinType MapType ArrayType Float64Type Float32Type BoolType IntType UintType NilType ExtensionType // pseudo-types provided // by extensions Complex64Type Complex128Type TimeType _maxtype ) // String implements fmt.Stringer func (t Type) String() string { switch t { case StrType: return "str" case BinType: return "bin" case MapType: return "map" case ArrayType: return "array" case Float64Type: return "float64" case Float32Type: return "float32" case BoolType: return "bool" case UintType: return "uint" case IntType: return "int" case ExtensionType: return "ext" case NilType: return "nil" default: return "" } } func freeR(m *Reader) { readerPool.Put(m) } // Unmarshaler is the interface fulfilled // by objects that know how to unmarshal // themselves from MessagePack. // UnmarshalMsg unmarshals the object // from binary, returing any leftover // bytes and any errors encountered. type Unmarshaler interface { UnmarshalMsg([]byte) ([]byte, error) } // Decodable is the interface fulfilled // by objects that know how to read // themselves from a *Reader. type Decodable interface { DecodeMsg(*Reader) error } // Decode decodes 'd' from 'r'. func Decode(r io.Reader, d Decodable) error { rd := NewReader(r) err := d.DecodeMsg(rd) freeR(rd) return err } // NewReader returns a *Reader that // reads from the provided reader. The // reader will be buffered. func NewReader(r io.Reader) *Reader { p := readerPool.Get().(*Reader) if p.R == nil { p.R = fwd.NewReader(r) } else { p.R.Reset(r) } return p } // NewReaderSize returns a *Reader with a buffer of the given size. // (This is vastly preferable to passing the decoder a reader that is already buffered.) func NewReaderSize(r io.Reader, sz int) *Reader { return &Reader{R: fwd.NewReaderSize(r, sz)} } // Reader wraps an io.Reader and provides // methods to read MessagePack-encoded values // from it. Readers are buffered. type Reader struct { // R is the buffered reader // that the Reader uses // to decode MessagePack. // The Reader itself // is stateless; all the // buffering is done // within R. R *fwd.Reader scratch []byte } // Read implements `io.Reader` func (m *Reader) Read(p []byte) (int, error) { return m.R.Read(p) } // ReadFull implements `io.ReadFull` func (m *Reader) ReadFull(p []byte) (int, error) { return m.R.ReadFull(p) } // Reset resets the underlying reader. func (m *Reader) Reset(r io.Reader) { m.R.Reset(r) } // Buffered returns the number of bytes currently in the read buffer. func (m *Reader) Buffered() int { return m.R.Buffered() } // BufferSize returns the capacity of the read buffer. func (m *Reader) BufferSize() int { return m.R.BufferSize() } // NextType returns the next object type to be decoded. func (m *Reader) NextType() (Type, error) { p, err := m.R.Peek(1) if err != nil { return InvalidType, err } t := getType(p[0]) if t == InvalidType { return t, InvalidPrefixError(p[0]) } if t == ExtensionType { v, err := m.peekExtensionType() if err != nil { return InvalidType, err } switch v { case Complex64Extension: return Complex64Type, nil case Complex128Extension: return Complex128Type, nil case TimeExtension: return TimeType, nil } } return t, nil } // IsNil returns whether or not // the next byte is a null messagepack byte func (m *Reader) IsNil() bool { p, err := m.R.Peek(1) return err == nil && p[0] == mnil } // returns (obj size, obj elements, error) // only maps and arrays have non-zero obj elements // // use uintptr b/c it's guaranteed to be large enough // to hold whatever we can fit in memory. func getNextSize(r *fwd.Reader) (uintptr, uintptr, error) { b, err := r.Peek(1) if err != nil { return 0, 0, err } lead := b[0] spec := &sizes[lead] size, mode := spec.size, spec.extra if size == 0 { return 0, 0, InvalidPrefixError(lead) } if mode >= 0 { return uintptr(size), uintptr(mode), nil } b, err = r.Peek(int(size)) if err != nil { return 0, 0, err } switch mode { case extra8: return uintptr(size) + uintptr(b[1]), 0, nil case extra16: return uintptr(size) + uintptr(big.Uint16(b[1:])), 0, nil case extra32: return uintptr(size) + uintptr(big.Uint32(b[1:])), 0, nil case map16v: return uintptr(size), 2 * uintptr(big.Uint16(b[1:])), nil case map32v: return uintptr(size), 2 * uintptr(big.Uint32(b[1:])), nil case array16v: return uintptr(size), uintptr(big.Uint16(b[1:])), nil case array32v: return uintptr(size), uintptr(big.Uint32(b[1:])), nil default: return 0, 0, fatal } } // Skip skips over the next object, regardless of // its type. If it is an array or map, the whole array // or map will be skipped. func (m *Reader) Skip() error { var ( v uintptr // bytes o uintptr // objects err error p []byte ) // we can use the faster // method if we have enough // buffered data if m.R.Buffered() >= 5 { p, err = m.R.Peek(5) if err != nil { return err } v, o, err = getSize(p) if err != nil { return err } } else { v, o, err = getNextSize(m.R) if err != nil { return err } } // 'v' is always non-zero // if err == nil _, err = m.R.Skip(int(v)) if err != nil { return err } // for maps and slices, skip elements for x := uintptr(0); x < o; x++ { err = m.Skip() if err != nil { return err } } return nil } // ReadMapHeader reads the next object // as a map header and returns the size // of the map and the number of bytes written. // It will return a TypeError{} if the next // object is not a map. func (m *Reader) ReadMapHeader() (sz uint32, err error) { var p []byte var lead byte p, err = m.R.Peek(1) if err != nil { return } lead = p[0] if isfixmap(lead) { sz = uint32(rfixmap(lead)) _, err = m.R.Skip(1) return } switch lead { case mmap16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case mmap32: p, err = m.R.Next(5) if err != nil { return } sz = big.Uint32(p[1:]) return default: err = badPrefix(MapType, lead) return } } // ReadMapKey reads either a 'str' or 'bin' field from // the reader and returns the value as a []byte. It uses // scratch for storage if it is large enough. func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) { out, err := m.ReadStringAsBytes(scratch) if err != nil { if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { return m.ReadBytes(scratch) } return nil, err } return out, nil } // MapKeyPtr returns a []byte pointing to the contents // of a valid map key. The key cannot be empty, and it // must be shorter than the total buffer size of the // *Reader. Additionally, the returned slice is only // valid until the next *Reader method call. Users // should exercise extreme care when using this // method; writing into the returned slice may // corrupt future reads. func (m *Reader) ReadMapKeyPtr() ([]byte, error) { p, err := m.R.Peek(1) if err != nil { return nil, err } lead := p[0] var read int if isfixstr(lead) { read = int(rfixstr(lead)) m.R.Skip(1) goto fill } switch lead { case mstr8, mbin8: p, err = m.R.Next(2) if err != nil { return nil, err } read = int(p[1]) case mstr16, mbin16: p, err = m.R.Next(3) if err != nil { return nil, err } read = int(big.Uint16(p[1:])) case mstr32, mbin32: p, err = m.R.Next(5) if err != nil { return nil, err } read = int(big.Uint32(p[1:])) default: return nil, badPrefix(StrType, lead) } fill: if read == 0 { return nil, ErrShortBytes } return m.R.Next(read) } // ReadArrayHeader reads the next object as an // array header and returns the size of the array // and the number of bytes read. func (m *Reader) ReadArrayHeader() (sz uint32, err error) { var lead byte var p []byte p, err = m.R.Peek(1) if err != nil { return } lead = p[0] if isfixarray(lead) { sz = uint32(rfixarray(lead)) _, err = m.R.Skip(1) return } switch lead { case marray16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case marray32: p, err = m.R.Next(5) if err != nil { return } sz = big.Uint32(p[1:]) return default: err = badPrefix(ArrayType, lead) return } } // ReadNil reads a 'nil' MessagePack byte from the reader func (m *Reader) ReadNil() error { p, err := m.R.Peek(1) if err != nil { return err } if p[0] != mnil { return badPrefix(NilType, p[0]) } _, err = m.R.Skip(1) return err } // ReadFloat64 reads a float64 from the reader. // (If the value on the wire is encoded as a float32, // it will be up-cast to a float64.) func (m *Reader) ReadFloat64() (f float64, err error) { var p []byte p, err = m.R.Peek(9) if err != nil { // we'll allow a coversion from float32 to float64, // since we don't lose any precision if err == io.EOF && len(p) > 0 && p[0] == mfloat32 { ef, err := m.ReadFloat32() return float64(ef), err } return } if p[0] != mfloat64 { // see above if p[0] == mfloat32 { ef, err := m.ReadFloat32() return float64(ef), err } err = badPrefix(Float64Type, p[0]) return } f = math.Float64frombits(getMuint64(p)) _, err = m.R.Skip(9) return } // ReadFloat32 reads a float32 from the reader func (m *Reader) ReadFloat32() (f float32, err error) { var p []byte p, err = m.R.Peek(5) if err != nil { return } if p[0] != mfloat32 { err = badPrefix(Float32Type, p[0]) return } f = math.Float32frombits(getMuint32(p)) _, err = m.R.Skip(5) return } // ReadBool reads a bool from the reader func (m *Reader) ReadBool() (b bool, err error) { var p []byte p, err = m.R.Peek(1) if err != nil { return } switch p[0] { case mtrue: b = true case mfalse: default: err = badPrefix(BoolType, p[0]) return } _, err = m.R.Skip(1) return } // ReadInt64 reads an int64 from the reader func (m *Reader) ReadInt64() (i int64, err error) { var p []byte var lead byte p, err = m.R.Peek(1) if err != nil { return } lead = p[0] if isfixint(lead) { i = int64(rfixint(lead)) _, err = m.R.Skip(1) return } else if isnfixint(lead) { i = int64(rnfixint(lead)) _, err = m.R.Skip(1) return } switch lead { case mint8: p, err = m.R.Next(2) if err != nil { return } i = int64(getMint8(p)) return case mint16: p, err = m.R.Next(3) if err != nil { return } i = int64(getMint16(p)) return case mint32: p, err = m.R.Next(5) if err != nil { return } i = int64(getMint32(p)) return case mint64: p, err = m.R.Next(9) if err != nil { return } i = getMint64(p) return default: err = badPrefix(IntType, lead) return } } // ReadInt32 reads an int32 from the reader func (m *Reader) ReadInt32() (i int32, err error) { var in int64 in, err = m.ReadInt64() if in > math.MaxInt32 || in < math.MinInt32 { err = IntOverflow{Value: in, FailedBitsize: 32} return } i = int32(in) return } // ReadInt16 reads an int16 from the reader func (m *Reader) ReadInt16() (i int16, err error) { var in int64 in, err = m.ReadInt64() if in > math.MaxInt16 || in < math.MinInt16 { err = IntOverflow{Value: in, FailedBitsize: 16} return } i = int16(in) return } // ReadInt8 reads an int8 from the reader func (m *Reader) ReadInt8() (i int8, err error) { var in int64 in, err = m.ReadInt64() if in > math.MaxInt8 || in < math.MinInt8 { err = IntOverflow{Value: in, FailedBitsize: 8} return } i = int8(in) return } // ReadInt reads an int from the reader func (m *Reader) ReadInt() (i int, err error) { if smallint { var in int32 in, err = m.ReadInt32() i = int(in) return } var in int64 in, err = m.ReadInt64() i = int(in) return } // ReadUint64 reads a uint64 from the reader func (m *Reader) ReadUint64() (u uint64, err error) { var p []byte var lead byte p, err = m.R.Peek(1) if err != nil { return } lead = p[0] if isfixint(lead) { u = uint64(rfixint(lead)) _, err = m.R.Skip(1) return } switch lead { case muint8: p, err = m.R.Next(2) if err != nil { return } u = uint64(getMuint8(p)) return case muint16: p, err = m.R.Next(3) if err != nil { return } u = uint64(getMuint16(p)) return case muint32: p, err = m.R.Next(5) if err != nil { return } u = uint64(getMuint32(p)) return case muint64: p, err = m.R.Next(9) if err != nil { return } u = getMuint64(p) return default: err = badPrefix(UintType, lead) return } } // ReadUint32 reads a uint32 from the reader func (m *Reader) ReadUint32() (u uint32, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint32 { err = UintOverflow{Value: in, FailedBitsize: 32} return } u = uint32(in) return } // ReadUint16 reads a uint16 from the reader func (m *Reader) ReadUint16() (u uint16, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint16 { err = UintOverflow{Value: in, FailedBitsize: 16} return } u = uint16(in) return } // ReadUint8 reads a uint8 from the reader func (m *Reader) ReadUint8() (u uint8, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint8 { err = UintOverflow{Value: in, FailedBitsize: 8} return } u = uint8(in) return } // ReadUint reads a uint from the reader func (m *Reader) ReadUint() (u uint, err error) { if smallint { var un uint32 un, err = m.ReadUint32() u = uint(un) return } var un uint64 un, err = m.ReadUint64() u = uint(un) return } // ReadByte is analagous to ReadUint8. // // NOTE: this is *not* an implementation // of io.ByteReader. func (m *Reader) ReadByte() (b byte, err error) { var in uint64 in, err = m.ReadUint64() if in > math.MaxUint8 { err = UintOverflow{Value: in, FailedBitsize: 8} return } b = byte(in) return } // ReadBytes reads a MessagePack 'bin' object // from the reader and returns its value. It may // use 'scratch' for storage if it is non-nil. func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) { var p []byte var lead byte p, err = m.R.Peek(2) if err != nil { return } lead = p[0] var read int64 switch lead { case mbin8: read = int64(p[1]) m.R.Skip(2) case mbin16: p, err = m.R.Next(3) if err != nil { return } read = int64(big.Uint16(p[1:])) case mbin32: p, err = m.R.Next(5) if err != nil { return } read = int64(big.Uint32(p[1:])) default: err = badPrefix(BinType, lead) return } if int64(cap(scratch)) < read { b = make([]byte, read) } else { b = scratch[0:read] } _, err = m.R.ReadFull(b) return } // ReadBytesHeader reads the size header // of a MessagePack 'bin' object. The user // is responsible for dealing with the next // 'sz' bytes from the reader in an application-specific // way. func (m *Reader) ReadBytesHeader() (sz uint32, err error) { var p []byte p, err = m.R.Peek(1) if err != nil { return } switch p[0] { case mbin8: p, err = m.R.Next(2) if err != nil { return } sz = uint32(p[1]) return case mbin16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case mbin32: p, err = m.R.Next(5) if err != nil { return } sz = uint32(big.Uint32(p[1:])) return default: err = badPrefix(BinType, p[0]) return } } // ReadExactBytes reads a MessagePack 'bin'-encoded // object off of the wire into the provided slice. An // ArrayError will be returned if the object is not // exactly the length of the input slice. func (m *Reader) ReadExactBytes(into []byte) error { p, err := m.R.Peek(2) if err != nil { return err } lead := p[0] var read int64 // bytes to read var skip int // prefix size to skip switch lead { case mbin8: read = int64(p[1]) skip = 2 case mbin16: p, err = m.R.Peek(3) if err != nil { return err } read = int64(big.Uint16(p[1:])) skip = 3 case mbin32: p, err = m.R.Peek(5) if err != nil { return err } read = int64(big.Uint32(p[1:])) skip = 5 default: return badPrefix(BinType, lead) } if read != int64(len(into)) { return ArrayError{Wanted: uint32(len(into)), Got: uint32(read)} } m.R.Skip(skip) _, err = m.R.ReadFull(into) return err } // ReadStringAsBytes reads a MessagePack 'str' (utf-8) string // and returns its value as bytes. It may use 'scratch' for storage // if it is non-nil. func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) { var p []byte var lead byte p, err = m.R.Peek(1) if err != nil { return } lead = p[0] var read int64 if isfixstr(lead) { read = int64(rfixstr(lead)) m.R.Skip(1) goto fill } switch lead { case mstr8: p, err = m.R.Next(2) if err != nil { return } read = int64(uint8(p[1])) case mstr16: p, err = m.R.Next(3) if err != nil { return } read = int64(big.Uint16(p[1:])) case mstr32: p, err = m.R.Next(5) if err != nil { return } read = int64(big.Uint32(p[1:])) default: err = badPrefix(StrType, lead) return } fill: if int64(cap(scratch)) < read { b = make([]byte, read) } else { b = scratch[0:read] } _, err = m.R.ReadFull(b) return } // ReadStringHeader reads a string header // off of the wire. The user is then responsible // for dealing with the next 'sz' bytes from // the reader in an application-specific manner. func (m *Reader) ReadStringHeader() (sz uint32, err error) { var p []byte p, err = m.R.Peek(1) if err != nil { return } lead := p[0] if isfixstr(lead) { sz = uint32(rfixstr(lead)) m.R.Skip(1) return } switch lead { case mstr8: p, err = m.R.Next(2) if err != nil { return } sz = uint32(p[1]) return case mstr16: p, err = m.R.Next(3) if err != nil { return } sz = uint32(big.Uint16(p[1:])) return case mstr32: p, err = m.R.Next(5) if err != nil { return } sz = big.Uint32(p[1:]) return default: err = badPrefix(StrType, lead) return } } // ReadString reads a utf-8 string from the reader func (m *Reader) ReadString() (s string, err error) { var p []byte var lead byte var read int64 p, err = m.R.Peek(1) if err != nil { return } lead = p[0] if isfixstr(lead) { read = int64(rfixstr(lead)) m.R.Skip(1) goto fill } switch lead { case mstr8: p, err = m.R.Next(2) if err != nil { return } read = int64(uint8(p[1])) case mstr16: p, err = m.R.Next(3) if err != nil { return } read = int64(big.Uint16(p[1:])) case mstr32: p, err = m.R.Next(5) if err != nil { return } read = int64(big.Uint32(p[1:])) default: err = badPrefix(StrType, lead) return } fill: if read == 0 { s, err = "", nil return } // reading into the memory // that will become the string // itself has vastly superior // worst-case performance, because // the reader buffer doesn't have // to be large enough to hold the string. // the idea here is to make it more // difficult for someone malicious // to cause the system to run out of // memory by sending very large strings. // // NOTE: this works because the argument // passed to (*fwd.Reader).ReadFull escapes // to the heap; its argument may, in turn, // be passed to the underlying reader, and // thus escape analysis *must* conclude that // 'out' escapes. out := make([]byte, read) _, err = m.R.ReadFull(out) if err != nil { return } s = UnsafeString(out) return } // ReadComplex64 reads a complex64 from the reader func (m *Reader) ReadComplex64() (f complex64, err error) { var p []byte p, err = m.R.Peek(10) if err != nil { return } if p[0] != mfixext8 { err = badPrefix(Complex64Type, p[0]) return } if int8(p[1]) != Complex64Extension { err = errExt(int8(p[1]), Complex64Extension) return } f = complex(math.Float32frombits(big.Uint32(p[2:])), math.Float32frombits(big.Uint32(p[6:]))) _, err = m.R.Skip(10) return } // ReadComplex128 reads a complex128 from the reader func (m *Reader) ReadComplex128() (f complex128, err error) { var p []byte p, err = m.R.Peek(18) if err != nil { return } if p[0] != mfixext16 { err = badPrefix(Complex128Type, p[0]) return } if int8(p[1]) != Complex128Extension { err = errExt(int8(p[1]), Complex128Extension) return } f = complex(math.Float64frombits(big.Uint64(p[2:])), math.Float64frombits(big.Uint64(p[10:]))) _, err = m.R.Skip(18) return } // ReadMapStrIntf reads a MessagePack map into a map[string]interface{}. // (You must pass a non-nil map into the function.) func (m *Reader) ReadMapStrIntf(mp map[string]interface{}) (err error) { var sz uint32 sz, err = m.ReadMapHeader() if err != nil { return } for key := range mp { delete(mp, key) } for i := uint32(0); i < sz; i++ { var key string var val interface{} key, err = m.ReadString() if err != nil { return } val, err = m.ReadIntf() if err != nil { return } mp[key] = val } return } // ReadTime reads a time.Time object from the reader. // The returned time's location will be set to time.Local. func (m *Reader) ReadTime() (t time.Time, err error) { var p []byte p, err = m.R.Peek(15) if err != nil { return } if p[0] != mext8 || p[1] != 12 { err = badPrefix(TimeType, p[0]) return } if int8(p[2]) != TimeExtension { err = errExt(int8(p[2]), TimeExtension) return } sec, nsec := getUnix(p[3:]) t = time.Unix(sec, int64(nsec)).Local() _, err = m.R.Skip(15) return } // ReadIntf reads out the next object as a raw interface{}. // Arrays are decoded as []interface{}, and maps are decoded // as map[string]interface{}. Integers are decoded as int64 // and unsigned integers are decoded as uint64. func (m *Reader) ReadIntf() (i interface{}, err error) { var t Type t, err = m.NextType() if err != nil { return } switch t { case BoolType: i, err = m.ReadBool() return case IntType: i, err = m.ReadInt64() return case UintType: i, err = m.ReadUint64() return case BinType: i, err = m.ReadBytes(nil) return case StrType: i, err = m.ReadString() return case Complex64Type: i, err = m.ReadComplex64() return case Complex128Type: i, err = m.ReadComplex128() return case TimeType: i, err = m.ReadTime() return case ExtensionType: var t int8 t, err = m.peekExtensionType() if err != nil { return } f, ok := extensionReg[t] if ok { e := f() err = m.ReadExtension(e) i = e return } var e RawExtension e.Type = t err = m.ReadExtension(&e) i = &e return case MapType: mp := make(map[string]interface{}) err = m.ReadMapStrIntf(mp) i = mp return case NilType: err = m.ReadNil() i = nil return case Float32Type: i, err = m.ReadFloat32() return case Float64Type: i, err = m.ReadFloat64() return case ArrayType: var sz uint32 sz, err = m.ReadArrayHeader() if err != nil { return } out := make([]interface{}, int(sz)) for j := range out { out[j], err = m.ReadIntf() if err != nil { return } } i = out return default: return nil, fatal // unreachable } } msgp-1.0-beta/msgp/read_bytes.go000066400000000000000000000537141261253337500166700ustar00rootroot00000000000000package msgp import ( "bytes" "encoding/binary" "math" "time" ) var big = binary.BigEndian // NextType returns the type of the next // object in the slice. If the length // of the input is zero, it returns // InvalidType. func NextType(b []byte) Type { if len(b) == 0 { return InvalidType } spec := sizes[b[0]] t := spec.typ if t == ExtensionType && len(b) > int(spec.size) { var tp int8 if spec.extra == constsize { tp = int8(b[1]) } else { tp = int8(b[spec.size-1]) } switch tp { case TimeExtension: return TimeType case Complex128Extension: return Complex128Type case Complex64Extension: return Complex64Type default: return ExtensionType } } return t } // IsNil returns true if len(b)>0 and // the leading byte is a 'nil' MessagePack // byte; false otherwise func IsNil(b []byte) bool { if len(b) != 0 && b[0] == mnil { return true } return false } // Raw is raw MessagePack. // Raw allows you to read and write // data without interpreting its contents. type Raw []byte // MarshalMsg implements msgp.Marshaler. // It appends the raw contents of 'raw' // to the provided byte slice. If 'raw' // is 0 bytes, 'nil' will be appended instead. func (r Raw) MarshalMsg(b []byte) ([]byte, error) { i := len(r) if i == 0 { return AppendNil(b), nil } o, l := ensure(b, i) copy(o[l:], []byte(r)) return o, nil } // UnmarshalMsg implements msgp.Unmarshaler. // It sets the contents of *Raw to be the next // object in the provided byte slice. func (r *Raw) UnmarshalMsg(b []byte) ([]byte, error) { l := len(b) out, err := Skip(b) if err != nil { return b, err } rlen := l - len(out) if cap(*r) < rlen { *r = make(Raw, rlen) } else { *r = (*r)[0:rlen] } copy(*r, b[:rlen]) return out, nil } // EncodeMsg implements msgp.Encodable. // It writes the raw bytes to the writer. // If r is empty, it writes 'nil' instead. func (r Raw) EncodeMsg(w *Writer) error { if len(r) == 0 { return w.WriteNil() } _, err := w.Write([]byte(r)) return err } // DecodeMsg implements msgp.Decodable. // It sets the value of *Raw to be the // next object on the wire. func (r *Raw) DecodeMsg(f *Reader) error { *r = (*r)[:0] return appendNext(f, (*[]byte)(r)) } // Msgsize implements msgp.Sizer func (r Raw) Msgsize() int { l := len(r) if l == 0 { return 1 // for 'nil' } return l } func appendNext(f *Reader, d *[]byte) error { amt, o, err := getNextSize(f.R) if err != nil { return err } var i int *d, i = ensure(*d, int(amt)) _, err = f.R.ReadFull((*d)[i:]) if err != nil { return err } for o > 0 { err = appendNext(f, d) if err != nil { return err } o-- } return nil } // MarshalJSON implements json.Marshaler func (r *Raw) MarshalJSON() ([]byte, error) { var buf bytes.Buffer _, err := UnmarshalAsJSON(&buf, []byte(*r)) return buf.Bytes(), err } // ReadMapHeaderBytes reads a map header size // from 'b' and returns the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a map) func ReadMapHeaderBytes(b []byte) (sz uint32, o []byte, err error) { l := len(b) if l < 1 { err = ErrShortBytes return } lead := b[0] if isfixmap(lead) { sz = uint32(rfixmap(lead)) o = b[1:] return } switch lead { case mmap16: if l < 3 { err = ErrShortBytes return } sz = uint32(big.Uint16(b[1:])) o = b[3:] return case mmap32: if l < 5 { err = ErrShortBytes return } sz = big.Uint32(b[1:]) o = b[5:] return default: err = badPrefix(MapType, lead) return } } // ReadMapKeyZC attempts to read a map key // from 'b' and returns the key bytes and the remaining bytes // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a str or bin) func ReadMapKeyZC(b []byte) ([]byte, []byte, error) { o, b, err := ReadStringZC(b) if err != nil { if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { return ReadBytesZC(b) } return nil, b, err } return o, b, nil } // ReadArrayHeaderBytes attempts to read // the array header size off of 'b' and return // the size and remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not an array) func ReadArrayHeaderBytes(b []byte) (sz uint32, o []byte, err error) { if len(b) < 1 { return 0, nil, ErrShortBytes } lead := b[0] if isfixarray(lead) { sz = uint32(rfixarray(lead)) o = b[1:] return } switch lead { case marray16: if len(b) < 3 { err = ErrShortBytes return } sz = uint32(big.Uint16(b[1:])) o = b[3:] return case marray32: if len(b) < 5 { err = ErrShortBytes return } sz = big.Uint32(b[1:]) o = b[5:] return default: err = badPrefix(ArrayType, lead) return } } // ReadNilBytes tries to read a "nil" byte // off of 'b' and return the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a 'nil') // - InvalidPrefixError func ReadNilBytes(b []byte) ([]byte, error) { if len(b) < 1 { return nil, ErrShortBytes } if b[0] != mnil { return b, badPrefix(NilType, b[0]) } return b[1:], nil } // ReadFloat64Bytes tries to read a float64 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a float64) func ReadFloat64Bytes(b []byte) (f float64, o []byte, err error) { if len(b) < 9 { if len(b) >= 5 && b[0] == mfloat32 { var tf float32 tf, o, err = ReadFloat32Bytes(b) f = float64(tf) return } err = ErrShortBytes return } if b[0] != mfloat64 { if b[0] == mfloat32 { var tf float32 tf, o, err = ReadFloat32Bytes(b) f = float64(tf) return } err = badPrefix(Float64Type, b[0]) return } f = math.Float64frombits(getMuint64(b)) o = b[9:] return } // ReadFloat32Bytes tries to read a float64 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a float32) func ReadFloat32Bytes(b []byte) (f float32, o []byte, err error) { if len(b) < 5 { err = ErrShortBytes return } if b[0] != mfloat32 { err = TypeError{Method: Float32Type, Encoded: getType(b[0])} return } f = math.Float32frombits(getMuint32(b)) o = b[5:] return } // ReadBoolBytes tries to read a float64 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a bool) func ReadBoolBytes(b []byte) (bool, []byte, error) { if len(b) < 1 { return false, b, ErrShortBytes } switch b[0] { case mtrue: return true, b[1:], nil case mfalse: return false, b[1:], nil default: return false, b, badPrefix(BoolType, b[0]) } } // ReadInt64Bytes tries to read an int64 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError (not a int) func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { l := len(b) if l < 1 { return 0, nil, ErrShortBytes } lead := b[0] if isfixint(lead) { i = int64(rfixint(lead)) o = b[1:] return } if isnfixint(lead) { i = int64(rnfixint(lead)) o = b[1:] return } switch lead { case mint8: if l < 2 { err = ErrShortBytes return } i = int64(getMint8(b)) o = b[2:] return case mint16: if l < 3 { err = ErrShortBytes return } i = int64(getMint16(b)) o = b[3:] return case mint32: if l < 5 { err = ErrShortBytes return } i = int64(getMint32(b)) o = b[5:] return case mint64: if l < 9 { err = ErrShortBytes return } i = getMint64(b) o = b[9:] return default: err = badPrefix(IntType, lead) return } } // ReadInt32Bytes tries to read an int32 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a int) // - IntOverflow{} (value doesn't fit in int32) func ReadInt32Bytes(b []byte) (int32, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt32 || i < math.MinInt32 { return 0, o, IntOverflow{Value: i, FailedBitsize: 32} } return int32(i), o, err } // ReadInt16Bytes tries to read an int16 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a int) // - IntOverflow{} (value doesn't fit in int16) func ReadInt16Bytes(b []byte) (int16, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt16 || i < math.MinInt16 { return 0, o, IntOverflow{Value: i, FailedBitsize: 16} } return int16(i), o, err } // ReadInt8Bytes tries to read an int16 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a int) // - IntOverflow{} (value doesn't fit in int8) func ReadInt8Bytes(b []byte) (int8, []byte, error) { i, o, err := ReadInt64Bytes(b) if i > math.MaxInt8 || i < math.MinInt8 { return 0, o, IntOverflow{Value: i, FailedBitsize: 8} } return int8(i), o, err } // ReadIntBytes tries to read an int // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a int) // - IntOverflow{} (value doesn't fit in int; 32-bit platforms only) func ReadIntBytes(b []byte) (int, []byte, error) { if smallint { i, b, err := ReadInt32Bytes(b) return int(i), b, err } i, b, err := ReadInt64Bytes(b) return int(i), b, err } // ReadUint64Bytes tries to read a uint64 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a uint) func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { l := len(b) if l < 1 { return 0, nil, ErrShortBytes } lead := b[0] if isfixint(lead) { u = uint64(rfixint(lead)) o = b[1:] return } switch lead { case muint8: if l < 2 { err = ErrShortBytes return } u = uint64(getMuint8(b)) o = b[2:] return case muint16: if l < 3 { err = ErrShortBytes return } u = uint64(getMuint16(b)) o = b[3:] return case muint32: if l < 5 { err = ErrShortBytes return } u = uint64(getMuint32(b)) o = b[5:] return case muint64: if l < 9 { err = ErrShortBytes return } u = getMuint64(b) o = b[9:] return default: err = badPrefix(UintType, lead) return } } // ReadUint32Bytes tries to read a uint32 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a uint) // - UintOverflow{} (value too large for uint32) func ReadUint32Bytes(b []byte) (uint32, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint32 { return 0, nil, UintOverflow{Value: v, FailedBitsize: 32} } return uint32(v), o, err } // ReadUint16Bytes tries to read a uint16 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a uint) // - UintOverflow{} (value too large for uint16) func ReadUint16Bytes(b []byte) (uint16, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint16 { return 0, nil, UintOverflow{Value: v, FailedBitsize: 16} } return uint16(v), o, err } // ReadUint8Bytes tries to read a uint8 // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a uint) // - UintOverflow{} (value too large for uint8) func ReadUint8Bytes(b []byte) (uint8, []byte, error) { v, o, err := ReadUint64Bytes(b) if v > math.MaxUint8 { return 0, nil, UintOverflow{Value: v, FailedBitsize: 8} } return uint8(v), o, err } // ReadUintBytes tries to read a uint // from 'b' and return the value and the remaining bytes. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a uint) // - UintOverflow{} (value too large for uint; 32-bit platforms only) func ReadUintBytes(b []byte) (uint, []byte, error) { if smallint { u, b, err := ReadUint32Bytes(b) return uint(u), b, err } u, b, err := ReadUint64Bytes(b) return uint(u), b, err } // ReadByteBytes is analagous to ReadUint8Bytes func ReadByteBytes(b []byte) (byte, []byte, error) { return ReadUint8Bytes(b) } // ReadBytesBytes reads a 'bin' object // from 'b' and returns its vaue and // the remaining bytes in 'b'. // Possible errors: // - ErrShortBytes (too few bytes) // - TypeError{} (not a 'bin' object) func ReadBytesBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { return readBytesBytes(b, scratch, false) } func readBytesBytes(b []byte, scratch []byte, zc bool) (v []byte, o []byte, err error) { l := len(b) if l < 1 { return nil, nil, ErrShortBytes } lead := b[0] var read int switch lead { case mbin8: if l < 2 { err = ErrShortBytes return } read = int(b[1]) b = b[2:] case mbin16: if l < 3 { err = ErrShortBytes return } read = int(big.Uint16(b[1:])) b = b[3:] case mbin32: if l < 5 { err = ErrShortBytes return } read = int(big.Uint32(b[1:])) b = b[5:] default: err = badPrefix(BinType, lead) return } if len(b) < read { err = ErrShortBytes return } // zero-copy if zc { v = b[0:read] o = b[read:] return } if cap(scratch) >= read { v = scratch[0:read] } else { v = make([]byte, read) } o = b[copy(v, b):] return } // ReadBytesZC extracts the messagepack-encoded // binary field without copying. The returned []byte // points to the same memory as the input slice. // Possible errors: // - ErrShortBytes (b not long enough) // - TypeError{} (object not 'bin') func ReadBytesZC(b []byte) (v []byte, o []byte, err error) { return readBytesBytes(b, nil, true) } func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { l := len(b) if l < 1 { err = ErrShortBytes return } lead := b[0] var read uint32 var skip int switch lead { case mbin8: if l < 2 { err = ErrShortBytes return } read = uint32(b[1]) skip = 2 case mbin16: if l < 3 { err = ErrShortBytes return } read = uint32(big.Uint16(b[1:])) skip = 3 case mbin32: if l < 5 { err = ErrShortBytes return } read = uint32(big.Uint32(b[1:])) skip = 5 default: err = badPrefix(BinType, lead) return } if read != uint32(len(into)) { err = ArrayError{Wanted: uint32(len(into)), Got: read} return } o = b[skip+copy(into, b[skip:]):] return } // ReadStringZC reads a messagepack string field // without copying. The returned []byte points // to the same memory as the input slice. // Possible errors: // - ErrShortBytes (b not long enough) // - TypeError{} (object not 'str') func ReadStringZC(b []byte) (v []byte, o []byte, err error) { l := len(b) if l < 1 { return nil, nil, ErrShortBytes } lead := b[0] var read int if isfixstr(lead) { read = int(rfixstr(lead)) b = b[1:] } else { switch lead { case mstr8: if l < 2 { err = ErrShortBytes return } read = int(b[1]) b = b[2:] case mstr16: if l < 3 { err = ErrShortBytes return } read = int(big.Uint16(b[1:])) b = b[3:] case mstr32: if l < 5 { err = ErrShortBytes return } read = int(big.Uint32(b[1:])) b = b[5:] default: err = TypeError{Method: StrType, Encoded: getType(lead)} return } } if len(b) < read { err = ErrShortBytes return } v = b[0:read] o = b[read:] return } // ReadStringBytes reads a 'str' object // from 'b' and returns its value and the // remaining bytes in 'b'. // Possible errors: // - ErrShortBytes (b not long enough) // - TypeError{} (not 'str' type) // - InvalidPrefixError func ReadStringBytes(b []byte) (string, []byte, error) { v, o, err := ReadStringZC(b) return string(v), o, err } // ReadStringAsBytes reads a 'str' object // into a slice of bytes. 'v' is the value of // the 'str' object, which may reside in memory // pointed to by 'scratch.' 'o' is the remaining bytes // in 'b.'' // Possible errors: // - ErrShortBytes (b not long enough) // - TypeError{} (not 'str' type) // - InvalidPrefixError (unknown type marker) func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { var tmp []byte tmp, o, err = ReadStringZC(b) v = append(scratch[:0], tmp...) return } // ReadComplex128Bytes reads a complex128 // extension object from 'b' and returns the // remaining bytes. // Possible errors: // - ErrShortBytes (not enough bytes in 'b') // - TypeError{} (object not a complex128) // - InvalidPrefixError // - ExtensionTypeError{} (object an extension of the correct size, but not a complex128) func ReadComplex128Bytes(b []byte) (c complex128, o []byte, err error) { if len(b) < 18 { err = ErrShortBytes return } if b[0] != mfixext16 { err = badPrefix(Complex128Type, b[0]) return } if int8(b[1]) != Complex128Extension { err = errExt(int8(b[1]), Complex128Extension) return } c = complex(math.Float64frombits(big.Uint64(b[2:])), math.Float64frombits(big.Uint64(b[10:]))) o = b[18:] return } // ReadComplex64Bytes reads a complex64 // extension object from 'b' and returns the // remaining bytes. // Possible errors: // - ErrShortBytes (not enough bytes in 'b') // - TypeError{} (object not a complex64) // - ExtensionTypeError{} (object an extension of the correct size, but not a complex64) func ReadComplex64Bytes(b []byte) (c complex64, o []byte, err error) { if len(b) < 10 { err = ErrShortBytes return } if b[0] != mfixext8 { err = badPrefix(Complex64Type, b[0]) return } if b[1] != Complex64Extension { err = errExt(int8(b[1]), Complex64Extension) return } c = complex(math.Float32frombits(big.Uint32(b[2:])), math.Float32frombits(big.Uint32(b[6:]))) o = b[10:] return } // ReadTimeBytes reads a time.Time // extension object from 'b' and returns the // remaining bytes. // Possible errors: // - ErrShortBytes (not enough bytes in 'b') // - TypeError{} (object not a complex64) // - ExtensionTypeError{} (object an extension of the correct size, but not a time.Time) func ReadTimeBytes(b []byte) (t time.Time, o []byte, err error) { if len(b) < 15 { err = ErrShortBytes return } if b[0] != mext8 || b[1] != 12 { err = badPrefix(TimeType, b[0]) return } if int8(b[2]) != TimeExtension { err = errExt(int8(b[2]), TimeExtension) return } sec, nsec := getUnix(b[3:]) t = time.Unix(sec, int64(nsec)).Local() o = b[15:] return } // ReadMapStrIntfBytes reads a map[string]interface{} // out of 'b' and returns the map and remaining bytes. // If 'old' is non-nil, the values will be read into that map. func ReadMapStrIntfBytes(b []byte, old map[string]interface{}) (v map[string]interface{}, o []byte, err error) { var sz uint32 o = b sz, o, err = ReadMapHeaderBytes(o) if err != nil { return } if old != nil { for key := range old { delete(old, key) } v = old } else { v = make(map[string]interface{}, int(sz)) } for z := uint32(0); z < sz; z++ { if len(o) < 1 { err = ErrShortBytes return } var key []byte key, o, err = ReadMapKeyZC(o) if err != nil { return } var val interface{} val, o, err = ReadIntfBytes(o) if err != nil { return } v[string(key)] = val } return } // ReadIntfBytes attempts to read // the next object out of 'b' as a raw interface{} and // return the remaining bytes. func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) { if len(b) < 1 { err = ErrShortBytes return } k := NextType(b) switch k { case MapType: i, o, err = ReadMapStrIntfBytes(b, nil) return case ArrayType: var sz uint32 sz, o, err = ReadArrayHeaderBytes(b) if err != nil { return } j := make([]interface{}, int(sz)) i = j for d := range j { j[d], o, err = ReadIntfBytes(o) if err != nil { return } } return case Float32Type: i, o, err = ReadFloat32Bytes(b) return case Float64Type: i, o, err = ReadFloat64Bytes(b) return case IntType: i, o, err = ReadInt64Bytes(b) return case UintType: i, o, err = ReadUint64Bytes(b) return case BoolType: i, o, err = ReadBoolBytes(b) return case TimeType: i, o, err = ReadTimeBytes(b) return case Complex64Type: i, o, err = ReadComplex64Bytes(b) return case Complex128Type: i, o, err = ReadComplex128Bytes(b) return case ExtensionType: var t int8 t, err = peekExtension(b) if err != nil { return } // use a user-defined extension, // if it's been registered f, ok := extensionReg[t] if ok { e := f() o, err = ReadExtensionBytes(b, e) i = e return } // last resort is a raw extension e := RawExtension{} e.Type = int8(t) o, err = ReadExtensionBytes(b, &e) i = &e return case NilType: o, err = ReadNilBytes(b) return case BinType: i, o, err = ReadBytesBytes(b, nil) return case StrType: i, o, err = ReadStringBytes(b) return default: err = InvalidPrefixError(b[0]) return } } // Skip skips the next object in 'b' and // returns the remaining bytes. If the object // is a map or array, all of its elements // will be skipped. // Possible Errors: // - ErrShortBytes (not enough bytes in b) // - InvalidPrefixError (bad encoding) func Skip(b []byte) ([]byte, error) { sz, asz, err := getSize(b) if err != nil { return b, err } if uintptr(len(b)) < sz { return b, ErrShortBytes } b = b[sz:] for asz > 0 { b, err = Skip(b) if err != nil { return b, err } asz-- } return b, nil } // returns (skip N bytes, skip M objects, error) func getSize(b []byte) (uintptr, uintptr, error) { l := len(b) if l == 0 { return 0, 0, ErrShortBytes } lead := b[0] spec := &sizes[lead] // get type information size, mode := spec.size, spec.extra if size == 0 { return 0, 0, InvalidPrefixError(lead) } if mode >= 0 { // fixed composites return uintptr(size), uintptr(mode), nil } if l < int(size) { return 0, 0, ErrShortBytes } switch mode { case extra8: return uintptr(size) + uintptr(b[1]), 0, nil case extra16: return uintptr(size) + uintptr(big.Uint16(b[1:])), 0, nil case extra32: return uintptr(size) + uintptr(big.Uint32(b[1:])), 0, nil case map16v: return uintptr(size), 2 * uintptr(big.Uint16(b[1:])), nil case map32v: return uintptr(size), 2 * uintptr(big.Uint32(b[1:])), nil case array16v: return uintptr(size), uintptr(big.Uint16(b[1:])), nil case array32v: return uintptr(size), uintptr(big.Uint32(b[1:])), nil default: return 0, 0, fatal } } msgp-1.0-beta/msgp/read_bytes_test.go000066400000000000000000000236401261253337500177220ustar00rootroot00000000000000package msgp import ( "bytes" "reflect" "testing" "time" ) func TestReadMapHeaderBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []uint32{0, 1, 5, 49082} for i, v := range tests { buf.Reset() en.WriteMapHeader(v) en.Flush() out, left, err := ReadMapHeaderBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func BenchmarkReadMapHeaderBytes(b *testing.B) { sizes := []uint32{1, 100, tuint16, tuint32} buf := make([]byte, 0, 5*len(sizes)) for _, sz := range sizes { buf = AppendMapHeader(buf, sz) } b.SetBytes(int64(len(buf) / len(sizes))) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadMapHeaderBytes(buf) if len(buf) == 0 { buf = o } } } func TestReadArrayHeaderBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []uint32{0, 1, 5, 49082} for i, v := range tests { buf.Reset() en.WriteArrayHeader(v) en.Flush() out, left, err := ReadArrayHeaderBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func BenchmarkReadArrayHeaderBytes(b *testing.B) { sizes := []uint32{1, 100, tuint16, tuint32} buf := make([]byte, 0, 5*len(sizes)) for _, sz := range sizes { buf = AppendArrayHeader(buf, sz) } b.SetBytes(int64(len(buf) / len(sizes))) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadArrayHeaderBytes(buf) if len(buf) == 0 { buf = o } } } func TestReadNilBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteNil() en.Flush() left, err := ReadNilBytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } } func BenchmarkReadNilByte(b *testing.B) { buf := []byte{mnil} b.SetBytes(1) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadNilBytes(buf) } } func TestReadFloat64Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteFloat64(3.14159) en.Flush() out, left, err := ReadFloat64Bytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != 3.14159 { t.Errorf("%f in; %f out", 3.14159, out) } } func BenchmarkReadFloat64Bytes(b *testing.B) { f := float64(3.14159) buf := make([]byte, 0, 9) buf = AppendFloat64(buf, f) b.SetBytes(int64(len(buf))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadFloat64Bytes(buf) } } func TestReadFloat32Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteFloat32(3.1) en.Flush() out, left, err := ReadFloat32Bytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != 3.1 { t.Errorf("%f in; %f out", 3.1, out) } } func BenchmarkReadFloat32Bytes(b *testing.B) { f := float32(3.14159) buf := make([]byte, 0, 5) buf = AppendFloat32(buf, f) b.SetBytes(int64(len(buf))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadFloat32Bytes(buf) } } func TestReadBoolBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []bool{true, false} for i, v := range tests { buf.Reset() en.WriteBool(v) en.Flush() out, left, err := ReadBoolBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%t in; %t out", v, out) } } } func BenchmarkReadBoolBytes(b *testing.B) { buf := []byte{mtrue, mfalse, mtrue, mfalse} b.SetBytes(1) b.ReportAllocs() b.ResetTimer() o := buf for i := 0; i < b.N; i++ { _, buf, _ = ReadBoolBytes(buf) if len(buf) == 0 { buf = o } } } func TestReadInt64Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []int64{-5, -30, 0, 1, 127, 300, 40921, 34908219} for i, v := range tests { buf.Reset() en.WriteInt64(v) en.Flush() out, left, err := ReadInt64Bytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func TestReadUint64Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []uint64{0, 1, 127, 300, 40921, 34908219} for i, v := range tests { buf.Reset() en.WriteUint64(v) en.Flush() out, left, err := ReadUint64Bytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%d in; %d out", v, out) } } } func TestReadBytesBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := [][]byte{[]byte{}, []byte("some bytes"), []byte("some more bytes")} var scratch []byte for i, v := range tests { buf.Reset() en.WriteBytes(v) en.Flush() out, left, err := ReadBytesBytes(buf.Bytes(), scratch) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !bytes.Equal(out, v) { t.Errorf("%q in; %q out", v, out) } } } func TestReadZCBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := [][]byte{[]byte{}, []byte("some bytes"), []byte("some more bytes")} for i, v := range tests { buf.Reset() en.WriteBytes(v) en.Flush() out, left, err := ReadBytesZC(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !bytes.Equal(out, v) { t.Errorf("%q in; %q out", v, out) } } } func TestReadZCString(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []string{"", "hello", "here's another string......"} for i, v := range tests { buf.Reset() en.WriteString(v) en.Flush() out, left, err := ReadStringZC(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if string(out) != v { t.Errorf("%q in; %q out", v, out) } } } func TestReadStringBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []string{"", "hello", "here's another string......"} for i, v := range tests { buf.Reset() en.WriteString(v) en.Flush() out, left, err := ReadStringBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%q in; %q out", v, out) } } } func TestReadComplex128Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []complex128{complex(0, 0), complex(12.8, 32.0)} for i, v := range tests { buf.Reset() en.WriteComplex128(v) en.Flush() out, left, err := ReadComplex128Bytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%f in; %f out", v, out) } } } func TestReadComplex64Bytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := []complex64{complex(0, 0), complex(12.8, 32.0)} for i, v := range tests { buf.Reset() en.WriteComplex64(v) en.Flush() out, left, err := ReadComplex64Bytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if out != v { t.Errorf("%f in; %f out", v, out) } } } func TestReadTimeBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) now := time.Now() en.WriteTime(now) en.Flush() out, left, err := ReadTimeBytes(buf.Bytes()) if err != nil { t.Fatal(err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !now.Equal(out) { t.Errorf("%s in; %s out", now, out) } } func BenchmarkReadTimeBytes(b *testing.B) { data := AppendTime(nil, time.Now()) b.SetBytes(15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ReadTimeBytes(data) } } func TestReadIntfBytes(t *testing.T) { var buf bytes.Buffer en := NewWriter(&buf) tests := make([]interface{}, 0, 10) tests = append(tests, float64(3.5)) tests = append(tests, int64(-49082)) tests = append(tests, uint64(34908)) tests = append(tests, string("hello!")) tests = append(tests, []byte("blah.")) tests = append(tests, map[string]interface{}{ "key_one": 3.5, "key_two": "hi.", }) for i, v := range tests { buf.Reset() if err := en.WriteIntf(v); err != nil { t.Fatal(err) } en.Flush() out, left, err := ReadIntfBytes(buf.Bytes()) if err != nil { t.Errorf("test case %d: %s", i, err) } if len(left) != 0 { t.Errorf("expected 0 bytes left; found %d", len(left)) } if !reflect.DeepEqual(v, out) { t.Errorf("ReadIntf(): %v in; %v out", v, out) } } } func BenchmarkSkipBytes(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(6) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(3.14159) en.WriteString("some_bytes") en.WriteBytes([]byte("nkl4321rqw908vxzpojnlk2314rqew098-s09123rdscasd")) en.WriteString("the_time") en.WriteTime(time.Now()) en.WriteString("what?") en.WriteBool(true) en.WriteString("ext") en.WriteExtension(&RawExtension{Type: 55, Data: []byte("raw data!!!")}) en.Flush() bts := buf.Bytes() b.SetBytes(int64(len(bts))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := Skip(bts) if err != nil { b.Fatal(err) } } } msgp-1.0-beta/msgp/read_test.go000066400000000000000000000330241261253337500165110ustar00rootroot00000000000000package msgp import ( "bytes" "io" "math" "math/rand" "reflect" "testing" "time" ) func TestSanity(t *testing.T) { if !isfixint(0) { t.Fatal("WUT.") } } func TestReadIntf(t *testing.T) { // NOTE: if you include cases // with, say, int32s, the test // will fail, b/c integers are // always read out as int64, and // unsigned integers as uint64 var testCases = []interface{}{ float64(128.032), float32(9082.092), int64(-40), uint64(9082981), time.Now(), "hello!", []byte("hello!"), map[string]interface{}{ "thing-1": "thing-1-value", "thing-2": int64(800), "thing-3": []byte("some inner bytes..."), "thing-4": false, }, } var buf bytes.Buffer var v interface{} dec := NewReader(&buf) enc := NewWriter(&buf) for i, ts := range testCases { buf.Reset() err := enc.WriteIntf(ts) if err != nil { t.Errorf("Test case %d: %s", i, err) continue } err = enc.Flush() if err != nil { t.Fatal(err) } v, err = dec.ReadIntf() if err != nil { t.Errorf("Test case: %d: %s", i, err) } if !reflect.DeepEqual(v, ts) { t.Errorf("%v in; %v out", ts, v) } } } func TestReadMapHeader(t *testing.T) { tests := []struct { Sz uint32 }{ {0}, {1}, {tuint16}, {tuint32}, } var buf bytes.Buffer var sz uint32 var err error wr := NewWriter(&buf) rd := NewReader(&buf) for i, test := range tests { buf.Reset() err = wr.WriteMapHeader(test.Sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } sz, err = rd.ReadMapHeader() if err != nil { t.Errorf("Test case %d: got error %s", i, err) } if sz != test.Sz { t.Errorf("Test case %d: wrote size %d; got size %d", i, test.Sz, sz) } } } func BenchmarkReadMapHeader(b *testing.B) { sizes := []uint32{0, 1, tuint16, tuint32} data := make([]byte, 0, len(sizes)*5) for _, d := range sizes { data = AppendMapHeader(data, d) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(sizes))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { rd.ReadMapHeader() } } func TestReadArrayHeader(t *testing.T) { tests := []struct { Sz uint32 }{ {0}, {1}, {tuint16}, {tuint32}, } var buf bytes.Buffer var sz uint32 var err error wr := NewWriter(&buf) rd := NewReader(&buf) for i, test := range tests { buf.Reset() err = wr.WriteArrayHeader(test.Sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } sz, err = rd.ReadArrayHeader() if err != nil { t.Errorf("Test case %d: got error %s", i, err) } if sz != test.Sz { t.Errorf("Test case %d: wrote size %d; got size %d", i, test.Sz, sz) } } } func BenchmarkReadArrayHeader(b *testing.B) { sizes := []uint32{0, 1, tuint16, tuint32} data := make([]byte, 0, len(sizes)*5) for _, d := range sizes { data = AppendArrayHeader(data, d) } rd := NewReader(NewEndlessReader(data, b)) b.ReportAllocs() b.SetBytes(int64(len(data) / len(sizes))) b.ResetTimer() for i := 0; i < b.N; i++ { rd.ReadArrayHeader() } } func TestReadNil(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) wr.WriteNil() wr.Flush() err := rd.ReadNil() if err != nil { t.Fatal(err) } } func BenchmarkReadNil(b *testing.B) { data := AppendNil(nil) rd := NewReader(NewEndlessReader(data, b)) b.ReportAllocs() b.SetBytes(1) b.ResetTimer() for i := 0; i < b.N; i++ { err := rd.ReadNil() if err != nil { b.Fatal(err) } } } func TestReadFloat64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 100; i++ { buf.Reset() flt := (rand.Float64() - 0.5) * math.MaxFloat64 err := wr.WriteFloat64(flt) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadFloat64() if err != nil { t.Errorf("Error reading %f: %s", flt, err) continue } if out != flt { t.Errorf("Put in %f but got out %f", flt, out) } } } func BenchmarkReadFloat64(b *testing.B) { fs := []float64{rand.Float64(), rand.Float64(), rand.Float64(), rand.Float64()} data := make([]byte, 0, 9*len(fs)) for _, f := range fs { data = AppendFloat64(data, f) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadFloat64() if err != nil { b.Fatal(err) } } } func TestReadFloat32(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 10000; i++ { buf.Reset() flt := (rand.Float32() - 0.5) * math.MaxFloat32 err := wr.WriteFloat32(flt) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadFloat32() if err != nil { t.Errorf("Error reading %f: %s", flt, err) continue } if out != flt { t.Errorf("Put in %f but got out %f", flt, out) } } } func BenchmarkReadFloat32(b *testing.B) { fs := []float32{rand.Float32(), rand.Float32(), rand.Float32(), rand.Float32()} data := make([]byte, 0, 5*len(fs)) for _, f := range fs { data = AppendFloat32(data, f) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(5) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadFloat32() if err != nil { b.Fatal(err) } } } func TestReadInt64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) ints := []int64{-100000, -5000, -5, 0, 8, 240, int64(tuint16), int64(tuint32), int64(tuint64)} for i, num := range ints { buf.Reset() err := wr.WriteInt64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadInt64() if err != nil { t.Fatal(err) } if out != num { t.Errorf("Test case %d: put %d in and got %d out", i, num, out) } } } func BenchmarkReadInt64(b *testing.B) { is := []int64{0, 1, 65000, rand.Int63()} data := make([]byte, 0, 9*len(is)) for _, n := range is { data = AppendInt64(data, n) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(is))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadInt64() if err != nil { b.Fatal(err) } } } func TestReadUint64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) ints := []uint64{0, 8, 240, uint64(tuint16), uint64(tuint32), uint64(tuint64)} for i, num := range ints { buf.Reset() err := wr.WriteUint64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadUint64() if out != num { t.Errorf("Test case %d: put %d in and got %d out", i, num, out) } } } func BenchmarkReadUint64(b *testing.B) { us := []uint64{0, 1, 10000, uint64(rand.Uint32() * 4)} data := make([]byte, 0, 9*len(us)) for _, n := range us { data = AppendUint64(data, n) } rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data) / len(us))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadUint64() if err != nil { b.Fatal(err) } } } func TestReadBytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) sizes := []int{0, 1, 225, int(tuint32)} var scratch []byte for i, size := range sizes { buf.Reset() bts := RandBytes(size) err := wr.WriteBytes(bts) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadBytes(scratch) if err != nil { t.Errorf("test case %d: %s", i, err) continue } if !bytes.Equal(bts, out) { t.Errorf("test case %d: Bytes not equal.", i) } } } func benchBytes(size uint32, b *testing.B) { data := make([]byte, 0, size+5) data = AppendBytes(data, RandBytes(int(size))) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() var scratch []byte var err error for i := 0; i < b.N; i++ { scratch, err = rd.ReadBytes(scratch) if err != nil { b.Fatal(err) } } } func BenchmarkRead16Bytes(b *testing.B) { benchBytes(16, b) } func BenchmarkRead256Bytes(b *testing.B) { benchBytes(256, b) } // This particular case creates // an object larger than the default // read buffer size, so it's a decent // indicator of worst-case performance. func BenchmarkRead2048Bytes(b *testing.B) { benchBytes(2048, b) } func TestReadString(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) sizes := []int{0, 1, 225, int(math.MaxUint16 + 5)} for i, size := range sizes { buf.Reset() in := string(RandBytes(size)) err := wr.WriteString(in) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadString() if err != nil { t.Errorf("test case %d: %s", i, err) } if out != in { t.Errorf("test case %d: strings not equal.", i) t.Errorf("string (len = %d) in; string (len = %d) out", size, len(out)) } } } func benchString(size uint32, b *testing.B) { str := string(RandBytes(int(size))) data := make([]byte, 0, len(str)+5) data = AppendString(data, str) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadString() if err != nil { b.Fatal(err) } } } func benchStringAsBytes(size uint32, b *testing.B) { str := string(RandBytes(int(size))) data := make([]byte, 0, len(str)+5) data = AppendString(data, str) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() var scratch []byte var err error for i := 0; i < b.N; i++ { scratch, err = rd.ReadStringAsBytes(scratch) if err != nil { b.Fatal(err) } } } func BenchmarkRead16StringAsBytes(b *testing.B) { benchStringAsBytes(16, b) } func BenchmarkRead256StringAsBytes(b *testing.B) { benchStringAsBytes(256, b) } func BenchmarkRead16String(b *testing.B) { benchString(16, b) } func BenchmarkRead256String(b *testing.B) { benchString(256, b) } func TestReadComplex64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 100; i++ { buf.Reset() f := complex(rand.Float32()*math.MaxFloat32, rand.Float32()*math.MaxFloat32) wr.WriteComplex64(f) err := wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadComplex64() if err != nil { t.Error(err) continue } if out != f { t.Errorf("Wrote %f; read %f", f, out) } } } func BenchmarkReadComplex64(b *testing.B) { f := complex(rand.Float32(), rand.Float32()) data := AppendComplex64(nil, f) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadComplex64() if err != nil { b.Fatal(err) } } } func TestReadComplex128(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) for i := 0; i < 10; i++ { buf.Reset() f := complex(rand.Float64()*math.MaxFloat64, rand.Float64()*math.MaxFloat64) wr.WriteComplex128(f) err := wr.Flush() if err != nil { t.Fatal(err) } out, err := rd.ReadComplex128() if err != nil { t.Error(err) continue } if out != f { t.Errorf("Wrote %f; read %f", f, out) } } } func BenchmarkReadComplex128(b *testing.B) { f := complex(rand.Float64(), rand.Float64()) data := AppendComplex128(nil, f) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadComplex128() if err != nil { b.Fatal(err) } } } func TestTime(t *testing.T) { var buf bytes.Buffer now := time.Now() en := NewWriter(&buf) dc := NewReader(&buf) err := en.WriteTime(now) if err != nil { t.Fatal(err) } err = en.Flush() if err != nil { t.Fatal(err) } out, err := dc.ReadTime() if err != nil { t.Fatal(err) } // check for equivalence if !now.Equal(out) { t.Fatalf("%s in; %s out", now, out) } // check for time.Local zone if now != out { t.Error("returned time.Time not set to time.Local") } } func BenchmarkReadTime(b *testing.B) { t := time.Now() data := AppendTime(nil, t) rd := NewReader(NewEndlessReader(data, b)) b.SetBytes(int64(len(data))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := rd.ReadTime() if err != nil { b.Fatal(err) } } } func TestSkip(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) rd := NewReader(&buf) wr.WriteMapHeader(4) wr.WriteString("key_1") wr.WriteBytes([]byte("value_1")) wr.WriteString("key_2") wr.WriteFloat64(2.0) wr.WriteString("key_3") wr.WriteComplex128(3.0i) wr.WriteString("key_4") wr.WriteInt64(49080432189) wr.Flush() // this should skip the whole map err := rd.Skip() if err != nil { t.Fatal(err) } tp, err := rd.NextType() if err != io.EOF { t.Errorf("expected %q; got %q", io.EOF, err) t.Errorf("returned type %q", tp) } } func BenchmarkSkip(b *testing.B) { var buf bytes.Buffer en := NewWriter(&buf) en.WriteMapHeader(6) en.WriteString("thing_one") en.WriteString("value_one") en.WriteString("thing_two") en.WriteFloat64(3.14159) en.WriteString("some_bytes") en.WriteBytes([]byte("nkl4321rqw908vxzpojnlk2314rqew098-s09123rdscasd")) en.WriteString("the_time") en.WriteTime(time.Now()) en.WriteString("what?") en.WriteBool(true) en.WriteString("ext") en.WriteExtension(&RawExtension{Type: 55, Data: []byte("raw data!!!")}) en.Flush() bts := buf.Bytes() b.SetBytes(int64(len(bts))) b.ReportAllocs() b.ResetTimer() rd := NewReader(NewEndlessReader(bts, b)) for i := 0; i < b.N; i++ { err := rd.Skip() if err != nil { b.Fatal(err) } } } msgp-1.0-beta/msgp/size.go000066400000000000000000000013321261253337500155060ustar00rootroot00000000000000package msgp // The sizes provided // are the worst-case // encoded sizes for // each type. For variable- // length types ([]byte, string), // the total encoded size is // the prefix size plus the // length of the object. const ( Int64Size = 9 IntSize = Int64Size UintSize = Int64Size Int8Size = 2 Int16Size = 3 Int32Size = 5 Uint8Size = 2 ByteSize = Uint8Size Uint16Size = 3 Uint32Size = 5 Uint64Size = Int64Size Float64Size = 9 Float32Size = 5 Complex64Size = 10 Complex128Size = 18 TimeSize = 15 BoolSize = 1 NilSize = 1 MapHeaderSize = 5 ArrayHeaderSize = 5 BytesPrefixSize = 5 StringPrefixSize = 5 ExtensionPrefixSize = 6 ) msgp-1.0-beta/msgp/unsafe.go000066400000000000000000000017421261253337500160220ustar00rootroot00000000000000// +build !appengine package msgp import ( "reflect" "unsafe" ) // NOTE: // all of the definition in this file // should be repeated in appengine.go, // but without using unsafe const ( // spec says int and uint are always // the same size, but that int/uint // size may not be machine word size smallint = unsafe.Sizeof(int(0)) == 4 ) // UnsafeString returns the byte slice as a volatile string // THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. // THIS IS EVIL CODE. // YOU HAVE BEEN WARNED. func UnsafeString(b []byte) string { return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: uintptr(unsafe.Pointer(&b[0])), Len: len(b)})) } // UnsafeBytes returns the string as a byte slice // THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. // THIS IS EVIL CODE. // YOU HAVE BEEN WARNED. func UnsafeBytes(s string) []byte { return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ Len: len(s), Cap: len(s), Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data, })) } msgp-1.0-beta/msgp/write.go000066400000000000000000000456511261253337500157020ustar00rootroot00000000000000package msgp import ( "errors" "fmt" "io" "math" "reflect" "sync" "time" ) // Sizer is an interface implemented // by types that can estimate their // size when MessagePack encoded. // This interface is optional, but // encoding/marshaling implementations // may use this as a way to pre-allocate // memory for serialization. type Sizer interface { Msgsize() int } var ( // Nowhere is an io.Writer to nowhere Nowhere io.Writer = nwhere{} btsType = reflect.TypeOf(([]byte)(nil)) writerPool = sync.Pool{ New: func() interface{} { return &Writer{buf: make([]byte, 2048)} }, } ) func popWriter(w io.Writer) *Writer { wr := writerPool.Get().(*Writer) wr.Reset(w) return wr } func pushWriter(wr *Writer) { wr.w = nil wr.wloc = 0 writerPool.Put(wr) } // freeW frees a writer for use // by other processes. It is not necessary // to call freeW on a writer. However, maintaining // a reference to a *Writer after calling freeW on // it will cause undefined behavior. func freeW(w *Writer) { pushWriter(w) } // Require ensures that cap(old)-len(old) >= extra. func Require(old []byte, extra int) []byte { l := len(old) c := cap(old) r := l + extra if c >= r { return old } else if l == 0 { return make([]byte, 0, extra) } // the new size is the greater // of double the old capacity // and the sum of the old length // and the number of new bytes // necessary. c <<= 1 if c < r { c = r } n := make([]byte, l, c) copy(n, old) return n } // nowhere writer type nwhere struct{} func (n nwhere) Write(p []byte) (int, error) { return len(p), nil } // Marshaler is the interface implemented // by types that know how to marshal themselves // as MessagePack. MarshalMsg appends the marshalled // form of the object to the provided // byte slice, returning the extended // slice and any errors encountered. type Marshaler interface { MarshalMsg([]byte) ([]byte, error) } // Encodable is the interface implemented // by types that know how to write themselves // as MessagePack using a *msgp.Writer. type Encodable interface { EncodeMsg(*Writer) error } // Writer is a buffered writer // that can be used to write // MessagePack objects to an io.Writer. // You must call *Writer.Flush() in order // to flush all of the buffered data // to the underlying writer. type Writer struct { w io.Writer buf []byte wloc int } // NewWriter returns a new *Writer. func NewWriter(w io.Writer) *Writer { if wr, ok := w.(*Writer); ok { return wr } return popWriter(w) } // NewWriterSize returns a writer with a custom buffer size. func NewWriterSize(w io.Writer, sz int) *Writer { // we must be able to require() 18 // contiguous bytes, so that is the // practical minimum buffer size if sz < 18 { sz = 18 } return &Writer{ w: w, buf: make([]byte, sz), } } // Encode encodes an Encodable to an io.Writer. func Encode(w io.Writer, e Encodable) error { wr := NewWriter(w) err := e.EncodeMsg(wr) if err == nil { err = wr.Flush() } freeW(wr) return err } func (mw *Writer) flush() error { if mw.wloc == 0 { return nil } n, err := mw.w.Write(mw.buf[:mw.wloc]) if err != nil { if n > 0 { mw.wloc = copy(mw.buf, mw.buf[n:mw.wloc]) } return err } mw.wloc = 0 return nil } // Flush flushes all of the buffered // data to the underlying writer. func (mw *Writer) Flush() error { return mw.flush() } // Buffered returns the number bytes in the write buffer func (mw *Writer) Buffered() int { return len(mw.buf) - mw.wloc } func (mw *Writer) avail() int { return len(mw.buf) - mw.wloc } func (mw *Writer) bufsize() int { return len(mw.buf) } // NOTE: this should only be called with // a number that is guaranteed to be less than // len(mw.buf). typically, it is called with a constant. // // NOTE: this is a hot code path func (mw *Writer) require(n int) (int, error) { c := len(mw.buf) wl := mw.wloc if c-wl < n { if err := mw.flush(); err != nil { return 0, err } wl = mw.wloc } mw.wloc += n return wl, nil } func (mw *Writer) Append(b ...byte) error { if mw.avail() < len(b) { err := mw.flush() if err != nil { return err } } mw.wloc += copy(mw.buf[mw.wloc:], b) return nil } // push one byte onto the buffer // // NOTE: this is a hot code path func (mw *Writer) push(b byte) error { if mw.wloc == len(mw.buf) { if err := mw.flush(); err != nil { return err } } mw.buf[mw.wloc] = b mw.wloc++ return nil } func (mw *Writer) prefix8(b byte, u uint8) error { const need = 2 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu8(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } func (mw *Writer) prefix16(b byte, u uint16) error { const need = 3 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu16(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } func (mw *Writer) prefix32(b byte, u uint32) error { const need = 5 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu32(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } func (mw *Writer) prefix64(b byte, u uint64) error { const need = 9 if len(mw.buf)-mw.wloc < need { if err := mw.flush(); err != nil { return err } } prefixu64(mw.buf[mw.wloc:], b, u) mw.wloc += need return nil } // Write implements io.Writer, and writes // data directly to the buffer. func (mw *Writer) Write(p []byte) (int, error) { l := len(p) if mw.avail() < l { if err := mw.flush(); err != nil { return 0, err } if l > len(mw.buf) { return mw.w.Write(p) } } mw.wloc += copy(mw.buf[mw.wloc:], p) return l, nil } // implements io.WriteString func (mw *Writer) writeString(s string) error { l := len(s) if mw.avail() < l { if err := mw.flush(); err != nil { return err } if l > len(mw.buf) { _, err := io.WriteString(mw.w, s) return err } } mw.wloc += copy(mw.buf[mw.wloc:], s) return nil } // Reset changes the underlying writer used by the Writer func (mw *Writer) Reset(w io.Writer) { mw.buf = mw.buf[:cap(mw.buf)] mw.w = w mw.wloc = 0 } // WriteMapHeader writes a map header of the given // size to the writer func (mw *Writer) WriteMapHeader(sz uint32) error { switch { case sz <= 15: return mw.push(wfixmap(uint8(sz))) case sz <= math.MaxUint16: return mw.prefix16(mmap16, uint16(sz)) default: return mw.prefix32(mmap32, sz) } } // WriteArrayHeader writes an array header of the // given size to the writer func (mw *Writer) WriteArrayHeader(sz uint32) error { switch { case sz <= 15: return mw.push(wfixarray(uint8(sz))) case sz <= math.MaxUint16: return mw.prefix16(marray16, uint16(sz)) default: return mw.prefix32(marray32, sz) } } // WriteNil writes a nil byte to the buffer func (mw *Writer) WriteNil() error { return mw.push(mnil) } // WriteFloat64 writes a float64 to the writer func (mw *Writer) WriteFloat64(f float64) error { return mw.prefix64(mfloat64, math.Float64bits(f)) } // WriteFloat32 writes a float32 to the writer func (mw *Writer) WriteFloat32(f float32) error { return mw.prefix32(mfloat32, math.Float32bits(f)) } // WriteInt64 writes an int64 to the writer func (mw *Writer) WriteInt64(i int64) error { if i >= 0 { switch { case i <= math.MaxInt8: return mw.push(wfixint(uint8(i))) case i <= math.MaxInt16: return mw.prefix16(mint16, uint16(i)) case i <= math.MaxInt32: return mw.prefix32(mint32, uint32(i)) default: return mw.prefix64(mint64, uint64(i)) } } switch { case i >= -32: return mw.push(wnfixint(int8(i))) case i >= math.MinInt8: return mw.prefix8(mint8, uint8(i)) case i >= math.MinInt16: return mw.prefix16(mint16, uint16(i)) case i >= math.MinInt32: return mw.prefix32(mint32, uint32(i)) default: return mw.prefix64(mint64, uint64(i)) } } // WriteInt8 writes an int8 to the writer func (mw *Writer) WriteInt8(i int8) error { return mw.WriteInt64(int64(i)) } // WriteInt16 writes an int16 to the writer func (mw *Writer) WriteInt16(i int16) error { return mw.WriteInt64(int64(i)) } // WriteInt32 writes an int32 to the writer func (mw *Writer) WriteInt32(i int32) error { return mw.WriteInt64(int64(i)) } // WriteInt writes an int to the writer func (mw *Writer) WriteInt(i int) error { return mw.WriteInt64(int64(i)) } // WriteUint64 writes a uint64 to the writer func (mw *Writer) WriteUint64(u uint64) error { switch { case u <= (1<<7)-1: return mw.push(wfixint(uint8(u))) case u <= math.MaxUint8: return mw.prefix8(muint8, uint8(u)) case u <= math.MaxUint16: return mw.prefix16(muint16, uint16(u)) case u <= math.MaxUint32: return mw.prefix32(muint32, uint32(u)) default: return mw.prefix64(muint64, u) } } // WriteByte is analagous to WriteUint8 func (mw *Writer) WriteByte(u byte) error { return mw.WriteUint8(uint8(u)) } // WriteUint8 writes a uint8 to the writer func (mw *Writer) WriteUint8(u uint8) error { return mw.WriteUint64(uint64(u)) } // WriteUint16 writes a uint16 to the writer func (mw *Writer) WriteUint16(u uint16) error { return mw.WriteUint64(uint64(u)) } // WriteUint32 writes a uint32 to the writer func (mw *Writer) WriteUint32(u uint32) error { return mw.WriteUint64(uint64(u)) } // WriteUint writes a uint to the writer func (mw *Writer) WriteUint(u uint) error { return mw.WriteUint64(uint64(u)) } // WriteBytes writes binary as 'bin' to the writer func (mw *Writer) WriteBytes(b []byte) error { sz := uint32(len(b)) var err error switch { case sz <= math.MaxUint8: err = mw.prefix8(mbin8, uint8(sz)) case sz <= math.MaxUint16: err = mw.prefix16(mbin16, uint16(sz)) default: err = mw.prefix32(mbin32, sz) } if err != nil { return err } _, err = mw.Write(b) return err } // WriteBytesHeader writes just the size header // of a MessagePack 'bin' object. The user is responsible // for then writing 'sz' more bytes into the stream. func (mw *Writer) WriteBytesHeader(sz uint32) error { switch { case sz <= math.MaxUint8: return mw.prefix8(mbin8, uint8(sz)) case sz <= math.MaxUint16: return mw.prefix16(mbin16, uint16(sz)) default: return mw.prefix32(mbin32, sz) } } // WriteBool writes a bool to the writer func (mw *Writer) WriteBool(b bool) error { if b { return mw.push(mtrue) } return mw.push(mfalse) } // WriteString writes a messagepack string to the writer. // (This is NOT an implementation of io.StringWriter) func (mw *Writer) WriteString(s string) error { sz := uint32(len(s)) var err error switch { case sz <= 31: err = mw.push(wfixstr(uint8(sz))) case sz <= math.MaxUint8: err = mw.prefix8(mstr8, uint8(sz)) case sz <= math.MaxUint16: err = mw.prefix16(mstr16, uint16(sz)) default: err = mw.prefix32(mstr32, sz) } if err != nil { return err } return mw.writeString(s) } // WriteStringHeader writes just the string size // header of a MessagePack 'str' object. The user // is responsible for writing 'sz' more valid UTF-8 // bytes to the stream. func (mw *Writer) WriteStringHeader(sz uint32) error { switch { case sz <= 31: return mw.push(wfixstr(uint8(sz))) case sz <= math.MaxUint8: return mw.prefix8(mstr8, uint8(sz)) case sz <= math.MaxUint16: return mw.prefix16(mstr16, uint16(sz)) default: return mw.prefix32(mstr32, sz) } } // WriteStringFromBytes writes a 'str' object // from a []byte. func (mw *Writer) WriteStringFromBytes(str []byte) error { sz := uint32(len(str)) var err error switch { case sz <= 31: err = mw.push(wfixstr(uint8(sz))) case sz <= math.MaxUint8: err = mw.prefix8(mstr8, uint8(sz)) case sz <= math.MaxUint16: err = mw.prefix16(mstr16, uint16(sz)) default: err = mw.prefix32(mstr32, sz) } if err != nil { return err } _, err = mw.Write(str) return err } // WriteComplex64 writes a complex64 to the writer func (mw *Writer) WriteComplex64(f complex64) error { o, err := mw.require(10) if err != nil { return err } mw.buf[o] = mfixext8 mw.buf[o+1] = Complex64Extension big.PutUint32(mw.buf[o+2:], math.Float32bits(real(f))) big.PutUint32(mw.buf[o+6:], math.Float32bits(imag(f))) return nil } // WriteComplex128 writes a complex128 to the writer func (mw *Writer) WriteComplex128(f complex128) error { o, err := mw.require(18) if err != nil { return err } mw.buf[o] = mfixext16 mw.buf[o+1] = Complex128Extension big.PutUint64(mw.buf[o+2:], math.Float64bits(real(f))) big.PutUint64(mw.buf[o+10:], math.Float64bits(imag(f))) return nil } // WriteMapStrStr writes a map[string]string to the writer func (mw *Writer) WriteMapStrStr(mp map[string]string) (err error) { err = mw.WriteMapHeader(uint32(len(mp))) if err != nil { return } for key, val := range mp { err = mw.WriteString(key) if err != nil { return } err = mw.WriteString(val) if err != nil { return } } return nil } // WriteMapStrIntf writes a map[string]interface to the writer func (mw *Writer) WriteMapStrIntf(mp map[string]interface{}) (err error) { err = mw.WriteMapHeader(uint32(len(mp))) if err != nil { return } for key, val := range mp { err = mw.WriteString(key) if err != nil { return } err = mw.WriteIntf(val) if err != nil { return } } return } // WriteTime writes a time.Time object to the wire. // // Time is encoded as Unix time, which means that // location (time zone) data is removed from the object. // The encoded object itself is 12 bytes: 8 bytes for // a big-endian 64-bit integer denoting seconds // elapsed since "zero" Unix time, followed by 4 bytes // for a big-endian 32-bit signed integer denoting // the nanosecond offset of the time. This encoding // is intended to ease portability accross languages. // (Note that this is *not* the standard time.Time // binary encoding, because its implementation relies // heavily on the internal representation used by the // time package.) func (mw *Writer) WriteTime(t time.Time) error { t = t.UTC() o, err := mw.require(15) if err != nil { return err } mw.buf[o] = mext8 mw.buf[o+1] = 12 mw.buf[o+2] = TimeExtension putUnix(mw.buf[o+3:], t.Unix(), int32(t.Nanosecond())) return nil } // WriteIntf writes the concrete type of 'v'. // WriteIntf will error if 'v' is not one of the following: // - A bool, float, string, []byte, int, uint, or complex // - A map of supported types (with string keys) // - An array or slice of supported types // - A pointer to a supported type // - A type that satisfies the msgp.Encodable interface // - A type that satisfies the msgp.Extension interface func (mw *Writer) WriteIntf(v interface{}) error { if v == nil { return mw.WriteNil() } switch v := v.(type) { // preferred interfaces case Encodable: return v.EncodeMsg(mw) case Extension: return mw.WriteExtension(v) // concrete types case bool: return mw.WriteBool(v) case float32: return mw.WriteFloat32(v) case float64: return mw.WriteFloat64(v) case complex64: return mw.WriteComplex64(v) case complex128: return mw.WriteComplex128(v) case uint8: return mw.WriteUint8(v) case uint16: return mw.WriteUint16(v) case uint32: return mw.WriteUint32(v) case uint64: return mw.WriteUint64(v) case uint: return mw.WriteUint(v) case int8: return mw.WriteInt8(v) case int16: return mw.WriteInt16(v) case int32: return mw.WriteInt32(v) case int64: return mw.WriteInt64(v) case int: return mw.WriteInt(v) case string: return mw.WriteString(v) case []byte: return mw.WriteBytes(v) case map[string]string: return mw.WriteMapStrStr(v) case map[string]interface{}: return mw.WriteMapStrIntf(v) case time.Time: return mw.WriteTime(v) } val := reflect.ValueOf(v) if !isSupported(val.Kind()) || !val.IsValid() { return fmt.Errorf("msgp: type %s not supported", val) } switch val.Kind() { case reflect.Ptr: if val.IsNil() { return mw.WriteNil() } return mw.WriteIntf(val.Elem().Interface()) case reflect.Slice: return mw.writeSlice(val) case reflect.Map: return mw.writeMap(val) } return &ErrUnsupportedType{val.Type()} } func (mw *Writer) writeMap(v reflect.Value) (err error) { if v.Elem().Kind() != reflect.String { return errors.New("msgp: map keys must be strings") } ks := v.MapKeys() err = mw.WriteMapHeader(uint32(len(ks))) if err != nil { return } for _, key := range ks { val := v.MapIndex(key) err = mw.WriteString(key.String()) if err != nil { return } err = mw.WriteIntf(val.Interface()) if err != nil { return } } return } func (mw *Writer) writeSlice(v reflect.Value) (err error) { // is []byte if v.Type().ConvertibleTo(btsType) { return mw.WriteBytes(v.Bytes()) } sz := uint32(v.Len()) err = mw.WriteArrayHeader(sz) if err != nil { return } for i := uint32(0); i < sz; i++ { err = mw.WriteIntf(v.Index(int(i)).Interface()) if err != nil { return } } return } func (mw *Writer) writeStruct(v reflect.Value) error { if enc, ok := v.Interface().(Encodable); ok { return enc.EncodeMsg(mw) } return fmt.Errorf("msgp: unsupported type: %s", v.Type()) } func (mw *Writer) writeVal(v reflect.Value) error { if !isSupported(v.Kind()) { return fmt.Errorf("msgp: msgp/enc: type %q not supported", v.Type()) } // shortcut for nil values if v.IsNil() { return mw.WriteNil() } switch v.Kind() { case reflect.Bool: return mw.WriteBool(v.Bool()) case reflect.Float32, reflect.Float64: return mw.WriteFloat64(v.Float()) case reflect.Complex64, reflect.Complex128: return mw.WriteComplex128(v.Complex()) case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: return mw.WriteInt64(v.Int()) case reflect.Interface, reflect.Ptr: if v.IsNil() { mw.WriteNil() } return mw.writeVal(v.Elem()) case reflect.Map: return mw.writeMap(v) case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint8: return mw.WriteUint64(v.Uint()) case reflect.String: return mw.WriteString(v.String()) case reflect.Slice, reflect.Array: return mw.writeSlice(v) case reflect.Struct: return mw.writeStruct(v) } return fmt.Errorf("msgp: msgp/enc: type %q not supported", v.Type()) } // is the reflect.Kind encodable? func isSupported(k reflect.Kind) bool { switch k { case reflect.Func, reflect.Chan, reflect.Invalid, reflect.UnsafePointer: return false default: return true } } // GuessSize guesses the size of the underlying // value of 'i'. If the underlying value is not // a simple builtin (or []byte), GuessSize defaults // to 512. func GuessSize(i interface{}) int { if i == nil { return NilSize } switch i := i.(type) { case Sizer: return i.Msgsize() case Extension: return ExtensionPrefixSize + i.Len() case float64: return Float64Size case float32: return Float32Size case uint8, uint16, uint32, uint64, uint: return UintSize case int8, int16, int32, int64, int: return IntSize case []byte: return BytesPrefixSize + len(i) case string: return StringPrefixSize + len(i) case complex64: return Complex64Size case complex128: return Complex128Size case bool: return BoolSize case map[string]interface{}: s := MapHeaderSize for key, val := range i { s += StringPrefixSize + len(key) + GuessSize(val) } return s case map[string]string: s := MapHeaderSize for key, val := range i { s += 2*StringPrefixSize + len(key) + len(val) } return s default: return 512 } } msgp-1.0-beta/msgp/write_bytes.go000066400000000000000000000231741261253337500171040ustar00rootroot00000000000000package msgp import ( "math" "reflect" "time" ) // ensure 'sz' extra bytes in 'b' btw len(b) and cap(b) func ensure(b []byte, sz int) ([]byte, int) { l := len(b) c := cap(b) if c-l < sz { o := make([]byte, (2*c)+sz) // exponential growth n := copy(o, b) return o[:n+sz], n } return b[:l+sz], l } // AppendMapHeader appends a map header with the // given size to the slice func AppendMapHeader(b []byte, sz uint32) []byte { switch { case sz <= 15: return append(b, wfixmap(uint8(sz))) case sz <= math.MaxUint16: o, n := ensure(b, 3) prefixu16(o[n:], mmap16, uint16(sz)) return o default: o, n := ensure(b, 5) prefixu32(o[n:], mmap32, sz) return o } } // AppendArrayHeader appends an array header with // the given size to the slice func AppendArrayHeader(b []byte, sz uint32) []byte { switch { case sz <= 15: return append(b, wfixarray(uint8(sz))) case sz <= math.MaxUint16: o, n := ensure(b, 3) prefixu16(o[n:], marray16, uint16(sz)) return o default: o, n := ensure(b, 5) prefixu32(o[n:], marray32, sz) return o } } // AppendNil appends a 'nil' byte to the slice func AppendNil(b []byte) []byte { return append(b, mnil) } // AppendFloat64 appends a float64 to the slice func AppendFloat64(b []byte, f float64) []byte { o, n := ensure(b, Float64Size) prefixu64(o[n:], mfloat64, math.Float64bits(f)) return o } // AppendFloat32 appends a float32 to the slice func AppendFloat32(b []byte, f float32) []byte { o, n := ensure(b, Float32Size) prefixu32(o[n:], mfloat32, math.Float32bits(f)) return o } // AppendInt64 appends an int64 to the slice func AppendInt64(b []byte, i int64) []byte { if i >= 0 { switch { case i <= math.MaxInt8: return append(b, wfixint(uint8(i))) case i <= math.MaxInt16: o, n := ensure(b, 3) putMint16(o[n:], int16(i)) return o case i <= math.MaxInt32: o, n := ensure(b, 5) putMint32(o[n:], int32(i)) return o default: o, n := ensure(b, 9) putMint64(o[n:], i) return o } } switch { case i >= -32: return append(b, wnfixint(int8(i))) case i >= math.MinInt8: o, n := ensure(b, 2) putMint8(o[n:], int8(i)) return o case i >= math.MinInt16: o, n := ensure(b, 3) putMint16(o[n:], int16(i)) return o case i >= math.MinInt32: o, n := ensure(b, 5) putMint32(o[n:], int32(i)) return o default: o, n := ensure(b, 9) putMint64(o[n:], i) return o } } // AppendInt appends an int to the slice func AppendInt(b []byte, i int) []byte { return AppendInt64(b, int64(i)) } // AppendInt8 appends an int8 to the slice func AppendInt8(b []byte, i int8) []byte { return AppendInt64(b, int64(i)) } // AppendInt16 appends an int16 to the slice func AppendInt16(b []byte, i int16) []byte { return AppendInt64(b, int64(i)) } // AppendInt32 appends an int32 to the slice func AppendInt32(b []byte, i int32) []byte { return AppendInt64(b, int64(i)) } // AppendUint64 appends a uint64 to the slice func AppendUint64(b []byte, u uint64) []byte { switch { case u <= (1<<7)-1: return append(b, wfixint(uint8(u))) case u <= math.MaxUint8: o, n := ensure(b, 2) putMuint8(o[n:], uint8(u)) return o case u <= math.MaxUint16: o, n := ensure(b, 3) putMuint16(o[n:], uint16(u)) return o case u <= math.MaxUint32: o, n := ensure(b, 5) putMuint32(o[n:], uint32(u)) return o default: o, n := ensure(b, 9) putMuint64(o[n:], u) return o } } // AppendUint appends a uint to the slice func AppendUint(b []byte, u uint) []byte { return AppendUint64(b, uint64(u)) } // AppendUint8 appends a uint8 to the slice func AppendUint8(b []byte, u uint8) []byte { return AppendUint64(b, uint64(u)) } // AppendByte is analagous to AppendUint8 func AppendByte(b []byte, u byte) []byte { return AppendUint8(b, uint8(u)) } // AppendUint16 appends a uint16 to the slice func AppendUint16(b []byte, u uint16) []byte { return AppendUint64(b, uint64(u)) } // AppendUint32 appends a uint32 to the slice func AppendUint32(b []byte, u uint32) []byte { return AppendUint64(b, uint64(u)) } // AppendBytes appends bytes to the slice as MessagePack 'bin' data func AppendBytes(b []byte, bts []byte) []byte { sz := len(bts) var o []byte var n int switch { case sz <= math.MaxUint8: o, n = ensure(b, 2+sz) prefixu8(o[n:], mbin8, uint8(sz)) n += 2 case sz <= math.MaxUint16: o, n = ensure(b, 3+sz) prefixu16(o[n:], mbin16, uint16(sz)) n += 3 default: o, n = ensure(b, 5+sz) prefixu32(o[n:], mbin32, uint32(sz)) n += 5 } return o[:n+copy(o[n:], bts)] } // AppendBool appends a bool to the slice func AppendBool(b []byte, t bool) []byte { if t { return append(b, mtrue) } return append(b, mfalse) } // AppendString appends a string as a MessagePack 'str' to the slice func AppendString(b []byte, s string) []byte { sz := len(s) var n int var o []byte switch { case sz <= 31: o, n = ensure(b, 1+sz) o[n] = wfixstr(uint8(sz)) n++ case sz <= math.MaxUint8: o, n = ensure(b, 2+sz) prefixu8(o[n:], mstr8, uint8(sz)) n += 2 case sz <= math.MaxUint16: o, n = ensure(b, 3+sz) prefixu16(o[n:], mstr16, uint16(sz)) n += 3 default: o, n = ensure(b, 5+sz) prefixu32(o[n:], mstr32, uint32(sz)) n += 5 } return o[:n+copy(o[n:], s)] } // AppendStringFromBytes appends a []byte // as a MessagePack 'str' to the slice 'b.' func AppendStringFromBytes(b []byte, str []byte) []byte { sz := len(str) var n int var o []byte switch { case sz <= 31: o, n = ensure(b, 1+sz) o[n] = wfixstr(uint8(sz)) n++ case sz <= math.MaxUint8: o, n = ensure(b, 2+sz) prefixu8(o[n:], mstr8, uint8(sz)) n += 2 case sz <= math.MaxUint16: o, n = ensure(b, 3+sz) prefixu16(o[n:], mstr16, uint16(sz)) n += 3 default: o, n = ensure(b, 5+sz) prefixu32(o[n:], mstr32, uint32(sz)) n += 5 } return o[:n+copy(o[n:], str)] } // AppendComplex64 appends a complex64 to the slice as a MessagePack extension func AppendComplex64(b []byte, c complex64) []byte { o, n := ensure(b, Complex64Size) o[n] = mfixext8 o[n+1] = Complex64Extension big.PutUint32(o[n+2:], math.Float32bits(real(c))) big.PutUint32(o[n+6:], math.Float32bits(imag(c))) return o } // AppendComplex128 appends a complex128 to the slice as a MessagePack extension func AppendComplex128(b []byte, c complex128) []byte { o, n := ensure(b, Complex128Size) o[n] = mfixext16 o[n+1] = Complex128Extension big.PutUint64(o[n+2:], math.Float64bits(real(c))) big.PutUint64(o[n+10:], math.Float64bits(imag(c))) return o } // AppendTime appends a time.Time to the slice as a MessagePack extension func AppendTime(b []byte, t time.Time) []byte { o, n := ensure(b, TimeSize) t = t.UTC() o[n] = mext8 o[n+1] = 12 o[n+2] = TimeExtension putUnix(o[n+3:], t.Unix(), int32(t.Nanosecond())) return o } // AppendMapStrStr appends a map[string]string to the slice // as a MessagePack map with 'str'-type keys and values func AppendMapStrStr(b []byte, m map[string]string) []byte { sz := uint32(len(m)) b = AppendMapHeader(b, sz) for key, val := range m { b = AppendString(b, key) b = AppendString(b, val) } return b } // AppendMapStrIntf appends a map[string]interface{} to the slice // as a MessagePack map with 'str'-type keys. func AppendMapStrIntf(b []byte, m map[string]interface{}) ([]byte, error) { sz := uint32(len(m)) b = AppendMapHeader(b, sz) var err error for key, val := range m { b = AppendString(b, key) b, err = AppendIntf(b, val) if err != nil { return b, err } } return b, nil } // AppendIntf appends the concrete type of 'i' to the // provided []byte. 'i' must be one of the following: // - 'nil' // - A bool, float, string, []byte, int, uint, or complex // - A map[string]interface{} or map[string]string // - A []T, where T is another supported type // - A *T, where T is another supported type // - A type that satisfieds the msgp.Marshaler interface // - A type that satisfies the msgp.Extension interface func AppendIntf(b []byte, i interface{}) ([]byte, error) { if i == nil { return AppendNil(b), nil } // all the concrete types // for which we have methods switch i := i.(type) { case Marshaler: return i.MarshalMsg(b) case Extension: return AppendExtension(b, i) case bool: return AppendBool(b, i), nil case float32: return AppendFloat32(b, i), nil case float64: return AppendFloat64(b, i), nil case complex64: return AppendComplex64(b, i), nil case complex128: return AppendComplex128(b, i), nil case string: return AppendString(b, i), nil case []byte: return AppendBytes(b, i), nil case int8: return AppendInt8(b, i), nil case int16: return AppendInt16(b, i), nil case int32: return AppendInt32(b, i), nil case int64: return AppendInt64(b, i), nil case int: return AppendInt64(b, int64(i)), nil case uint: return AppendUint64(b, uint64(i)), nil case uint8: return AppendUint8(b, i), nil case uint16: return AppendUint16(b, i), nil case uint32: return AppendUint32(b, i), nil case uint64: return AppendUint64(b, i), nil case time.Time: return AppendTime(b, i), nil case map[string]interface{}: return AppendMapStrIntf(b, i) case map[string]string: return AppendMapStrStr(b, i), nil case []interface{}: b = AppendArrayHeader(b, uint32(len(i))) var err error for _, k := range i { b, err = AppendIntf(b, k) if err != nil { return b, err } } return b, nil } var err error v := reflect.ValueOf(i) switch v.Kind() { case reflect.Array, reflect.Slice: l := v.Len() b = AppendArrayHeader(b, uint32(l)) for i := 0; i < l; i++ { b, err = AppendIntf(b, v.Index(i).Interface()) if err != nil { return b, err } } return b, nil case reflect.Ptr: if v.IsNil() { return AppendNil(b), err } b, err = AppendIntf(b, v.Elem().Interface()) return b, err default: return b, &ErrUnsupportedType{T: v.Type()} } } msgp-1.0-beta/msgp/write_bytes_test.go000066400000000000000000000160201261253337500201330ustar00rootroot00000000000000package msgp import ( "bytes" "math" "testing" "time" ) func TestIssue116(t *testing.T) { data := AppendInt64(nil, math.MinInt64) i, _, err := ReadInt64Bytes(data) if err != nil { t.Fatal(err) } if i != math.MinInt64 { t.Errorf("put %d in and got %d out", int64(math.MinInt64), i) } var buf bytes.Buffer w := NewWriter(&buf) w.WriteInt64(math.MinInt64) w.Flush() i, err = NewReader(&buf).ReadInt64() if err != nil { t.Fatal(err) } if i != math.MinInt64 { t.Errorf("put %d in and got %d out", int64(math.MinInt64), i) } } func TestAppendMapHeader(t *testing.T) { szs := []uint32{0, 1, uint32(tint8), uint32(tint16), tuint32} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range szs { buf.Reset() en.WriteMapHeader(sz) en.Flush() bts = AppendMapHeader(bts[0:0], sz) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for size %d, encoder wrote %q and append wrote %q", sz, buf.Bytes(), bts) } } } func BenchmarkAppendMapHeader(b *testing.B) { buf := make([]byte, 0, 9) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { AppendMapHeader(buf[:0], 0) AppendMapHeader(buf[:0], uint32(tint8)) AppendMapHeader(buf[:0], tuint16) AppendMapHeader(buf[:0], tuint32) } } func TestAppendArrayHeader(t *testing.T) { szs := []uint32{0, 1, uint32(tint8), uint32(tint16), tuint32} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range szs { buf.Reset() en.WriteArrayHeader(sz) en.Flush() bts = AppendArrayHeader(bts[0:0], sz) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for size %d, encoder wrote %q and append wrote %q", sz, buf.Bytes(), bts) } } } func BenchmarkAppendArrayHeader(b *testing.B) { buf := make([]byte, 0, 9) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { AppendArrayHeader(buf[:0], 0) AppendArrayHeader(buf[:0], uint32(tint8)) AppendArrayHeader(buf[:0], tuint16) AppendArrayHeader(buf[:0], tuint32) } } func TestAppendNil(t *testing.T) { var bts []byte bts = AppendNil(bts[0:0]) if bts[0] != mnil { t.Fatal("bts[0] is not 'nil'") } } func TestAppendFloat64(t *testing.T) { f := float64(3.14159) var buf bytes.Buffer en := NewWriter(&buf) var bts []byte en.WriteFloat64(f) en.Flush() bts = AppendFloat64(bts[0:0], f) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for float %f, encoder wrote %q; append wrote %q", f, buf.Bytes(), bts) } } func BenchmarkAppendFloat64(b *testing.B) { f := float64(3.14159) buf := make([]byte, 0, 9) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendFloat64(buf[0:0], f) } } func TestAppendFloat32(t *testing.T) { f := float32(3.14159) var buf bytes.Buffer en := NewWriter(&buf) var bts []byte en.WriteFloat32(f) en.Flush() bts = AppendFloat32(bts[0:0], f) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for float %f, encoder wrote %q; append wrote %q", f, buf.Bytes(), bts) } } func BenchmarkAppendFloat32(b *testing.B) { f := float32(3.14159) buf := make([]byte, 0, 5) b.SetBytes(5) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendFloat32(buf[0:0], f) } } func TestAppendInt64(t *testing.T) { is := []int64{0, 1, -5, -50, int64(tint16), int64(tint32), int64(tint64)} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, i := range is { buf.Reset() en.WriteInt64(i) en.Flush() bts = AppendInt64(bts[0:0], i) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for int64 %d, encoder wrote %q; append wrote %q", i, buf.Bytes(), bts) } } } func BenchmarkAppendInt64(b *testing.B) { is := []int64{0, 1, -5, -50, int64(tint16), int64(tint32), int64(tint64)} l := len(is) buf := make([]byte, 0, 9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendInt64(buf[0:0], is[i%l]) } } func TestAppendUint64(t *testing.T) { us := []uint64{0, 1, uint64(tuint16), uint64(tuint32), tuint64} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, u := range us { buf.Reset() en.WriteUint64(u) en.Flush() bts = AppendUint64(bts[0:0], u) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for uint64 %d, encoder wrote %q; append wrote %q", u, buf.Bytes(), bts) } } } func BenchmarkAppendUint64(b *testing.B) { us := []uint64{0, 1, 15, uint64(tuint16), uint64(tuint32), tuint64} buf := make([]byte, 0, 9) b.ReportAllocs() b.ResetTimer() l := len(us) for i := 0; i < b.N; i++ { AppendUint64(buf[0:0], us[i%l]) } } func TestAppendBytes(t *testing.T) { sizes := []int{0, 1, 225, int(tuint32)} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range sizes { buf.Reset() b := RandBytes(sz) en.WriteBytes(b) en.Flush() bts = AppendBytes(b[0:0], b) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for bytes of length %d, encoder wrote %d bytes and append wrote %d bytes", sz, buf.Len(), len(bts)) } } } func benchappendBytes(size uint32, b *testing.B) { bts := RandBytes(int(size)) buf := make([]byte, 0, len(bts)+5) b.SetBytes(int64(len(bts) + 5)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendBytes(buf[0:0], bts) } } func BenchmarkAppend16Bytes(b *testing.B) { benchappendBytes(16, b) } func BenchmarkAppend256Bytes(b *testing.B) { benchappendBytes(256, b) } func BenchmarkAppend2048Bytes(b *testing.B) { benchappendBytes(2048, b) } func TestAppendString(t *testing.T) { sizes := []int{0, 1, 225, int(tuint32)} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, sz := range sizes { buf.Reset() s := string(RandBytes(sz)) en.WriteString(s) en.Flush() bts = AppendString(bts[0:0], s) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for string of length %d, encoder wrote %d bytes and append wrote %d bytes", sz, buf.Len(), len(bts)) t.Errorf("WriteString prefix: %x", buf.Bytes()[0:5]) t.Errorf("Appendstring prefix: %x", bts[0:5]) } } } func benchappendString(size uint32, b *testing.B) { str := string(RandBytes(int(size))) buf := make([]byte, 0, len(str)+5) b.SetBytes(int64(len(str) + 5)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendString(buf[0:0], str) } } func BenchmarkAppend16String(b *testing.B) { benchappendString(16, b) } func BenchmarkAppend256String(b *testing.B) { benchappendString(256, b) } func BenchmarkAppend2048String(b *testing.B) { benchappendString(2048, b) } func TestAppendBool(t *testing.T) { vs := []bool{true, false} var buf bytes.Buffer en := NewWriter(&buf) var bts []byte for _, v := range vs { buf.Reset() en.WriteBool(v) en.Flush() bts = AppendBool(bts[0:0], v) if !bytes.Equal(buf.Bytes(), bts) { t.Errorf("for %t, encoder wrote %q and append wrote %q", v, buf.Bytes(), bts) } } } func BenchmarkAppendBool(b *testing.B) { vs := []bool{true, false} buf := make([]byte, 0, 1) b.SetBytes(1) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendBool(buf[0:0], vs[i%2]) } } func BenchmarkAppendTime(b *testing.B) { t := time.Now() b.SetBytes(15) buf := make([]byte, 0, 15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { AppendTime(buf[0:0], t) } } msgp-1.0-beta/msgp/write_test.go000066400000000000000000000173001261253337500167270ustar00rootroot00000000000000package msgp import ( "bytes" "math" "math/rand" "testing" "time" ) var ( tint8 int8 = 126 // cannot be most fix* types tint16 int16 = 150 // cannot be int8 tint32 int32 = math.MaxInt16 + 100 // cannot be int16 tint64 int64 = math.MaxInt32 + 100 // cannot be int32 tuint16 uint32 = 300 // cannot be uint8 tuint32 uint32 = math.MaxUint16 + 100 // cannot be uint16 tuint64 uint64 = math.MaxUint32 + 100 // cannot be uint32 ) func RandBytes(sz int) []byte { out := make([]byte, sz) for i := range out { out[i] = byte(rand.Int63n(math.MaxInt64) % 256) } return out } func TestWriteMapHeader(t *testing.T) { tests := []struct { Sz uint32 Outbytes []byte }{ {0, []byte{mfixmap}}, {1, []byte{mfixmap | byte(1)}}, {100, []byte{mmap16, byte(uint16(100) >> 8), byte(uint16(100))}}, {tuint32, []byte{mmap32, byte(tuint32 >> 24), byte(tuint32 >> 16), byte(tuint32 >> 8), byte(tuint32), }, }, } var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, test := range tests { buf.Reset() err = wr.WriteMapHeader(test.Sz) if err != nil { t.Error(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if !bytes.Equal(buf.Bytes(), test.Outbytes) { t.Errorf("Expected bytes %x; got %x", test.Outbytes, buf.Bytes()) } } } func BenchmarkWriteMapHeader(b *testing.B) { wr := NewWriter(Nowhere) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { wr.WriteMapHeader(0) wr.WriteMapHeader(8) wr.WriteMapHeader(tuint16) wr.WriteMapHeader(tuint32) } } func TestWriteArrayHeader(t *testing.T) { tests := []struct { Sz uint32 Outbytes []byte }{ {0, []byte{mfixarray}}, {1, []byte{mfixarray | byte(1)}}, {tuint16, []byte{marray16, byte(tuint16 >> 8), byte(tuint16)}}, {tuint32, []byte{marray32, byte(tuint32 >> 24), byte(tuint32 >> 16), byte(tuint32 >> 8), byte(tuint32)}}, } var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, test := range tests { buf.Reset() err = wr.WriteArrayHeader(test.Sz) if err != nil { t.Error(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if !bytes.Equal(buf.Bytes(), test.Outbytes) { t.Errorf("Expected bytes %x; got %x", test.Outbytes, buf.Bytes()) } } } func TestReadWriteStringHeader(t *testing.T) { sizes := []uint32{0, 5, 8, 19, 150, tuint16, tuint32} var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, sz := range sizes { buf.Reset() err = wr.WriteStringHeader(sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } var nsz uint32 nsz, err = NewReader(&buf).ReadStringHeader() if err != nil { t.Fatal(err) } if nsz != sz { t.Errorf("put in size %d but got out size %d", sz, nsz) } } } func TestReadWriteBytesHeader(t *testing.T) { sizes := []uint32{0, 5, 8, 19, 150, tuint16, tuint32} var buf bytes.Buffer var err error wr := NewWriter(&buf) for _, sz := range sizes { buf.Reset() err = wr.WriteBytesHeader(sz) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } var nsz uint32 nsz, err = NewReader(&buf).ReadBytesHeader() if err != nil { t.Fatal(err) } if nsz != sz { t.Errorf("put in size %d but got out size %d", sz, nsz) } } } func BenchmarkWriteArrayHeader(b *testing.B) { wr := NewWriter(Nowhere) N := b.N / 4 b.ReportAllocs() b.ResetTimer() for i := 0; i < N; i++ { wr.WriteArrayHeader(0) wr.WriteArrayHeader(16) wr.WriteArrayHeader(tuint16) wr.WriteArrayHeader(tuint32) } } func TestWriteNil(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) err := wr.WriteNil() if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mnil { t.Errorf("Expected %x; wrote %x", mnil, bts[0]) } } func TestWriteFloat64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() flt := (rand.Float64() - 0.5) * math.MaxFloat64 err := wr.WriteFloat64(flt) if err != nil { t.Errorf("Error with %f: %s", flt, err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mfloat64 { t.Errorf("Leading byte was %x and not %x", bts[0], mfloat64) } } } func BenchmarkWriteFloat64(b *testing.B) { f := rand.Float64() wr := NewWriter(Nowhere) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteFloat64(f) } } func TestWriteFloat32(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() flt := (rand.Float32() - 0.5) * math.MaxFloat32 err := wr.WriteFloat32(flt) if err != nil { t.Errorf("Error with %f: %s", flt, err) } err = wr.Flush() if err != nil { t.Fatal(err) } bts := buf.Bytes() if bts[0] != mfloat32 { t.Errorf("Leading byte was %x and not %x", bts[0], mfloat64) } } } func BenchmarkWriteFloat32(b *testing.B) { f := rand.Float32() wr := NewWriter(Nowhere) b.SetBytes(5) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteFloat32(f) } } func TestWriteInt64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() num := (rand.Int63n(math.MaxInt64)) - (math.MaxInt64 / 2) err := wr.WriteInt64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() > 9 { t.Errorf("buffer length should be <= 9; it's %d", buf.Len()) } } } func BenchmarkWriteInt64(b *testing.B) { wr := NewWriter(Nowhere) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteInt64(int64(tint64)) } } func TestWriteUint64(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) for i := 0; i < 10000; i++ { buf.Reset() num := uint64(rand.Int63n(math.MaxInt64)) err := wr.WriteUint64(num) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() > 9 { t.Errorf("buffer length should be <= 9; it's %d", buf.Len()) } } } func BenchmarkWriteUint64(b *testing.B) { wr := NewWriter(Nowhere) b.SetBytes(9) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteUint64(uint64(tuint64)) } } func TestWriteBytes(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) sizes := []int{0, 1, 225, int(tuint32)} for _, size := range sizes { buf.Reset() bts := RandBytes(size) err := wr.WriteBytes(bts) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() < len(bts) { t.Errorf("somehow, %d bytes were encoded in %d bytes", len(bts), buf.Len()) } } } func benchwrBytes(size uint32, b *testing.B) { bts := RandBytes(int(size)) wr := NewWriter(Nowhere) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteBytes(bts) } } func BenchmarkWrite16Bytes(b *testing.B) { benchwrBytes(16, b) } func BenchmarkWrite256Bytes(b *testing.B) { benchwrBytes(256, b) } func BenchmarkWrite2048Bytes(b *testing.B) { benchwrBytes(2048, b) } func TestWriteTime(t *testing.T) { var buf bytes.Buffer wr := NewWriter(&buf) tm := time.Now() err := wr.WriteTime(tm) if err != nil { t.Fatal(err) } err = wr.Flush() if err != nil { t.Fatal(err) } if buf.Len() != 15 { t.Errorf("expected time.Time to be %d bytes; got %d", 15, buf.Len()) } newt, err := NewReader(&buf).ReadTime() if err != nil { t.Fatal(err) } if !newt.Equal(tm) { t.Errorf("in/out not equal; %s in and %s out", tm, newt) } } func BenchmarkWriteTime(b *testing.B) { t := time.Now() wr := NewWriter(Nowhere) b.SetBytes(15) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { wr.WriteTime(t) } } msgp-1.0-beta/parse/000077500000000000000000000000001261253337500143525ustar00rootroot00000000000000msgp-1.0-beta/parse/directives.go000066400000000000000000000051221261253337500170420ustar00rootroot00000000000000package parse import ( "fmt" "github.com/tinylib/msgp/gen" "go/ast" "strings" ) const linePrefix = "//msgp:" // func(args, fileset) type directive func([]string, *FileSet) error // func(passName, args, printer) type passDirective func(gen.Method, []string, *gen.Printer) error // map of all recognized directives // // to add a directive, define a func([]string, *FileSet) error // and then add it to this list. var directives = map[string]directive{ "shim": applyShim, "ignore": ignore, "tuple": astuple, } var passDirectives = map[string]passDirective{ "ignore": passignore, } func passignore(m gen.Method, text []string, p *gen.Printer) error { pushstate(m.String()) for _, a := range text { p.ApplyDirective(m, gen.IgnoreTypename(a)) infof("ignoring %s\n", a) } popstate() return nil } // find all comment lines that begin with //msgp: func yieldComments(c []*ast.CommentGroup) []string { var out []string for _, cg := range c { for _, line := range cg.List { if strings.HasPrefix(line.Text, linePrefix) { out = append(out, strings.TrimPrefix(line.Text, linePrefix)) } } } return out } //msgp:shim {Type} as:{Newtype} using:{toFunc/fromFunc} func applyShim(text []string, f *FileSet) error { if len(text) != 4 { return fmt.Errorf("shim directive should have 3 arguments; found %d", len(text)-1) } name := text[1] be := gen.Ident(strings.TrimPrefix(strings.TrimSpace(text[2]), "as:")) // parse as::{base} if name[0] == '*' { name = name[1:] be.Needsref(true) } be.Alias(name) usestr := strings.TrimPrefix(strings.TrimSpace(text[3]), "using:") // parse using::{method/method} methods := strings.Split(usestr, "/") if len(methods) != 2 { return fmt.Errorf("expected 2 using::{} methods; found %d (%q)", len(methods), text[3]) } be.ShimToBase = methods[0] be.ShimFromBase = methods[1] infof("%s -> %s\n", name, be.Value.String()) f.Identities[name] = be return nil } //msgp:ignore {TypeA} {TypeB}... func ignore(text []string, f *FileSet) error { if len(text) < 2 { return nil } for _, item := range text[1:] { name := strings.TrimSpace(item) if _, ok := f.Identities[name]; ok { delete(f.Identities, name) infof("ignoring %s\n", name) } } return nil } //msgp:tuple {TypeA} {TypeB}... func astuple(text []string, f *FileSet) error { if len(text) < 2 { return nil } for _, item := range text[1:] { name := strings.TrimSpace(item) if el, ok := f.Identities[name]; ok { if st, ok := el.(*gen.Struct); ok { st.AsTuple = true infoln(name) } else { warnf("%s: only structs can be tuples\n", name) } } } return nil } msgp-1.0-beta/parse/getast.go000066400000000000000000000301231261253337500161670ustar00rootroot00000000000000package parse import ( "fmt" "go/ast" "go/parser" "go/token" "os" "reflect" "sort" "strings" "github.com/tinylib/msgp/gen" "github.com/ttacon/chalk" ) // A FileSet is the in-memory representation of a // parsed file. type FileSet struct { Package string // package name Specs map[string]ast.Expr // type specs in file Identities map[string]gen.Elem // processed from specs Directives []string // raw preprocessor directives } // File parses a file at the relative path // provided and produces a new *FileSet. // If you pass in a path to a directory, the entire // directory will be parsed. // If unexport is false, only exported identifiers are included in the FileSet. // If the resulting FileSet would be empty, an error is returned. func File(name string, unexported bool) (*FileSet, error) { pushstate(name) defer popstate() fs := &FileSet{ Specs: make(map[string]ast.Expr), Identities: make(map[string]gen.Elem), } fset := token.NewFileSet() finfo, err := os.Stat(name) if err != nil { return nil, err } if finfo.IsDir() { pkgs, err := parser.ParseDir(fset, name, nil, parser.ParseComments) if err != nil { return nil, err } if len(pkgs) != 1 { return nil, fmt.Errorf("multiple packages in directory: %s", name) } var one *ast.Package for _, nm := range pkgs { one = nm break } fs.Package = one.Name for _, fl := range one.Files { pushstate(fl.Name.Name) fs.Directives = append(fs.Directives, yieldComments(fl.Comments)...) if !unexported { ast.FileExports(fl) } fs.getTypeSpecs(fl) popstate() } } else { f, err := parser.ParseFile(fset, name, nil, parser.ParseComments) if err != nil { return nil, err } fs.Package = f.Name.Name fs.Directives = yieldComments(f.Comments) if !unexported { ast.FileExports(f) } fs.getTypeSpecs(f) } if len(fs.Specs) == 0 { return nil, fmt.Errorf("no definitions in %s", name) } fs.process() fs.applyDirectives() fs.propInline() return fs, nil } // applyDirectives applies all of the directives that // are known to the parser. additional method-specific // directives remain in f.Directives func (f *FileSet) applyDirectives() { newdirs := make([]string, 0, len(f.Directives)) for _, d := range f.Directives { chunks := strings.Split(d, " ") if len(chunks) > 0 { if fn, ok := directives[chunks[0]]; ok { pushstate(chunks[0]) err := fn(chunks, f) if err != nil { warnln(err.Error()) } popstate() } else { newdirs = append(newdirs, d) } } } f.Directives = newdirs } // A linkset is a graph of unresolved // identities. // // Since gen.Ident can only represent // one level of type indirection (e.g. Foo -> uint8), // type declarations like `type Foo Bar` // aren't resolve-able until we've processed // everything else. // // The goal of this dependency resolution // is to distill the type declaration // into just one level of indirection. // In other words, if we have: // // type A uint64 // type B A // type C B // type D C // // ... then we want to end up // figuring out that D is just a uint64. type linkset map[string]*gen.BaseElem func (f *FileSet) resolve(ls linkset) { progress := true for progress && len(ls) > 0 { progress = false for name, elem := range ls { real, ok := f.Identities[elem.TypeName()] if ok { // copy the old type descriptor, // alias it to the new value, // and insert it into the resolved // identities list progress = true nt := real.Copy() nt.Alias(name) f.Identities[name] = nt delete(ls, name) } } } // what's left can't be resolved for name, elem := range ls { warnf("couldn't resolve type %s (%s)\n", name, elem.TypeName()) } } // process takes the contents of f.Specs and // uses them to populate f.Identities func (f *FileSet) process() { deferred := make(linkset) parse: for name, def := range f.Specs { pushstate(name) el := f.parseExpr(def) if el == nil { warnln("failed to parse") popstate() continue parse } // push unresolved identities into // the graph of links and resolve after // we've handled every possible named type. if be, ok := el.(*gen.BaseElem); ok && be.Value == gen.IDENT { deferred[name] = be popstate() continue parse } el.Alias(name) f.Identities[name] = el popstate() } if len(deferred) > 0 { f.resolve(deferred) } } func strToMethod(s string) gen.Method { switch s { case "encode": return gen.Encode case "decode": return gen.Decode case "test": return gen.Test case "size": return gen.Size case "marshal": return gen.Marshal case "unmarshal": return gen.Unmarshal default: return 0 } } func (f *FileSet) applyDirs(p *gen.Printer) { // apply directives of the form // // //msgp:encode ignore {{TypeName}} // loop: for _, d := range f.Directives { chunks := strings.Split(d, " ") if len(chunks) > 1 { for i := range chunks { chunks[i] = strings.TrimSpace(chunks[i]) } m := strToMethod(chunks[0]) if m == 0 { warnf("unknown pass name: %q\n", chunks[0]) continue loop } if fn, ok := passDirectives[chunks[1]]; ok { pushstate(chunks[1]) err := fn(m, chunks[2:], p) if err != nil { warnf("error applying directive: %s\n", err) } popstate() } else { warnf("unrecognized directive %q\n", chunks[1]) } } else { warnf("empty directive: %q\n", d) } } } func (f *FileSet) PrintTo(p *gen.Printer) error { f.applyDirs(p) names := make([]string, 0, len(f.Identities)) for name := range f.Identities { names = append(names, name) } sort.Strings(names) for _, name := range names { el := f.Identities[name] el.SetVarname("z") pushstate(el.TypeName()) err := p.Print(el) popstate() if err != nil { return err } } return nil } // getTypeSpecs extracts all of the *ast.TypeSpecs in the file // into fs.Identities, but does not set the actual element func (fs *FileSet) getTypeSpecs(f *ast.File) { // check all declarations... for i := range f.Decls { // for GenDecls... if g, ok := f.Decls[i].(*ast.GenDecl); ok { // and check the specs... for _, s := range g.Specs { // for ast.TypeSpecs.... if ts, ok := s.(*ast.TypeSpec); ok { switch ts.Type.(type) { // this is the list of parse-able // type specs case *ast.StructType, *ast.ArrayType, *ast.StarExpr, *ast.MapType, *ast.Ident: fs.Specs[ts.Name.Name] = ts.Type } } } } } } func fieldName(f *ast.Field) string { switch len(f.Names) { case 0: return stringify(f.Type) case 1: return f.Names[0].Name default: return f.Names[0].Name + " (and others)" } } func (fs *FileSet) parseFieldList(fl *ast.FieldList) []gen.StructField { if fl == nil || fl.NumFields() == 0 { return nil } out := make([]gen.StructField, 0, fl.NumFields()) for _, field := range fl.List { pushstate(fieldName(field)) fds := fs.getField(field) if len(fds) > 0 { out = append(out, fds...) } else { warnln("ignored.") } popstate() } return out } // translate *ast.Field into []gen.StructField func (fs *FileSet) getField(f *ast.Field) []gen.StructField { sf := make([]gen.StructField, 1) var extension bool // parse tag; otherwise field name is field tag if f.Tag != nil { body := reflect.StructTag(strings.Trim(f.Tag.Value, "`")).Get("msg") tags := strings.Split(body, ",") if len(tags) == 2 && tags[1] == "extension" { extension = true } // ignore "-" fields if tags[0] == "-" { return nil } sf[0].FieldTag = tags[0] } ex := fs.parseExpr(f.Type) if ex == nil { return nil } // parse field name switch len(f.Names) { case 0: sf[0].FieldName = embedded(f.Type) case 1: sf[0].FieldName = f.Names[0].Name default: // this is for a multiple in-line declaration, // e.g. type A struct { One, Two int } sf = sf[0:0] for _, nm := range f.Names { sf = append(sf, gen.StructField{ FieldTag: nm.Name, FieldName: nm.Name, FieldElem: ex.Copy(), }) } return sf } sf[0].FieldElem = ex if sf[0].FieldTag == "" { sf[0].FieldTag = sf[0].FieldName } // validate extension if extension { switch ex := ex.(type) { case *gen.Ptr: if b, ok := ex.Value.(*gen.BaseElem); ok { b.Value = gen.Ext } else { warnln("couldn't cast to extension.") return nil } case *gen.BaseElem: ex.Value = gen.Ext default: warnln("couldn't cast to extension.") return nil } } return sf } // extract embedded field name // // so, for a struct like // // type A struct { // io.Writer // } // // we want "Writer" func embedded(f ast.Expr) string { switch f := f.(type) { case *ast.Ident: return f.Name case *ast.StarExpr: return embedded(f.X) case *ast.SelectorExpr: return f.Sel.Name default: // other possibilities are disallowed return "" } } // stringify a field type name func stringify(e ast.Expr) string { switch e := e.(type) { case *ast.Ident: return e.Name case *ast.StarExpr: return "*" + stringify(e.X) case *ast.SelectorExpr: return stringify(e.X) + "." + e.Sel.Name case *ast.ArrayType: if e.Len == nil { return "[]" + stringify(e.Elt) } return fmt.Sprintf("[%s]%s", stringify(e.Len), stringify(e.Elt)) case *ast.InterfaceType: if e.Methods == nil || e.Methods.NumFields() == 0 { return "interface{}" } } return "" } // recursively translate ast.Expr to gen.Elem; nil means type not supported // expected input types: // - *ast.MapType (map[T]J) // - *ast.Ident (name) // - *ast.ArrayType ([(sz)]T) // - *ast.StarExpr (*T) // - *ast.StructType (struct {}) // - *ast.SelectorExpr (a.B) // - *ast.InterfaceType (interface {}) func (fs *FileSet) parseExpr(e ast.Expr) gen.Elem { switch e := e.(type) { case *ast.MapType: if k, ok := e.Key.(*ast.Ident); ok && k.Name == "string" { if in := fs.parseExpr(e.Value); in != nil { return &gen.Map{Value: in} } } return nil case *ast.Ident: b := gen.Ident(e.Name) // work to resove this expression // can be done later, once we've resolved // everything else. if b.Value == gen.IDENT { if _, ok := fs.Specs[e.Name]; !ok { warnf("non-local identifier: %s\n", e.Name) } } return b case *ast.ArrayType: // special case for []byte if e.Len == nil { if i, ok := e.Elt.(*ast.Ident); ok && i.Name == "byte" { return &gen.BaseElem{Value: gen.Bytes} } } // return early if we don't know // what the slice element type is els := fs.parseExpr(e.Elt) if els == nil { return nil } // array and not a slice if e.Len != nil { switch s := e.Len.(type) { case *ast.BasicLit: return &gen.Array{ Size: s.Value, Els: els, } case *ast.Ident: return &gen.Array{ Size: s.String(), Els: els, } case *ast.SelectorExpr: return &gen.Array{ Size: stringify(s), Els: els, } default: return nil } } return &gen.Slice{Els: els} case *ast.StarExpr: if v := fs.parseExpr(e.X); v != nil { return &gen.Ptr{Value: v} } return nil case *ast.StructType: if fields := fs.parseFieldList(e.Fields); len(fields) > 0 { return &gen.Struct{Fields: fields} } return nil case *ast.SelectorExpr: return gen.Ident(stringify(e)) case *ast.InterfaceType: // support `interface{}` if len(e.Methods.List) == 0 { return &gen.BaseElem{Value: gen.Intf} } return nil default: // other types not supported return nil } } func infof(s string, v ...interface{}) { pushstate(s) fmt.Printf(chalk.Green.Color(strings.Join(logctx, ": ")), v...) popstate() } func infoln(s string) { pushstate(s) fmt.Println(chalk.Green.Color(strings.Join(logctx, ": "))) popstate() } func warnf(s string, v ...interface{}) { pushstate(s) fmt.Printf(chalk.Yellow.Color(strings.Join(logctx, ": ")), v...) popstate() } func warnln(s string) { pushstate(s) fmt.Println(chalk.Yellow.Color(strings.Join(logctx, ": "))) popstate() } func fatalf(s string, v ...interface{}) { pushstate(s) fmt.Printf(chalk.Red.Color(strings.Join(logctx, ": ")), v...) popstate() } var logctx []string // push logging state func pushstate(s string) { logctx = append(logctx, s) } // pop logging state func popstate() { logctx = logctx[:len(logctx)-1] } msgp-1.0-beta/parse/inline.go000066400000000000000000000046441261253337500161670ustar00rootroot00000000000000package parse import ( "github.com/tinylib/msgp/gen" ) // This file defines when and how we // propagate type information from // one type declaration to another. // After the processing pass, every // non-primitive type is marshalled/unmarshalled/etc. // through a function call. Here, we propagate // the type information into the caller's type // tree *if* the child type is simple enough. // // For example, types like // // type A [4]int // // will get pushed into parent methods, // whereas types like // // type B [3]map[string]struct{A, B [4]string} // // will not. // this is an approximate measure // of the number of children in a node const maxComplex = 5 // propInline identifies and inlines candidates func (f *FileSet) propInline() { for name, el := range f.Identities { pushstate(name) switch el := el.(type) { case *gen.Struct: for i := range el.Fields { f.nextInline(&el.Fields[i].FieldElem, name) } case *gen.Array: f.nextInline(&el.Els, name) case *gen.Slice: f.nextInline(&el.Els, name) case *gen.Map: f.nextInline(&el.Value, name) case *gen.Ptr: f.nextInline(&el.Value, name) } popstate() } } const fatalloop = `detected infinite recursion in inlining loop! Please file a bug at github.com/tinylib/msgp/issues! Thanks! ` func (f *FileSet) nextInline(ref *gen.Elem, root string) { switch el := (*ref).(type) { case *gen.BaseElem: // ensure that we're not inlining // a type into itself typ := el.TypeName() if el.Value == gen.IDENT && typ != root { if node, ok := f.Identities[typ]; ok && node.Complexity() < maxComplex { infof("inlining %s\n", typ) // This should never happen; it will cause // infinite recursion. if node == *ref { panic(fatalloop) } // inline bottom-up so as not to miss // other inlining opportunities. f.nextInline(&node, root) *ref = node.Copy() } else if !ok && !el.Resolved() { // this is the point at which we're sure that // we've got a type that isn't a primitive, // a library builtin, or a processed type warnf("unresolved identifier: %s\n", typ) } } case *gen.Struct: for i := range el.Fields { f.nextInline(&el.Fields[i].FieldElem, root) } case *gen.Array: f.nextInline(&el.Els, root) case *gen.Slice: f.nextInline(&el.Els, root) case *gen.Map: f.nextInline(&el.Value, root) case *gen.Ptr: f.nextInline(&el.Value, root) default: panic("bad elem type") } } msgp-1.0-beta/printer/000077500000000000000000000000001261253337500147235ustar00rootroot00000000000000msgp-1.0-beta/printer/print.go000066400000000000000000000051431261253337500164110ustar00rootroot00000000000000package printer import ( "bytes" "fmt" "github.com/tinylib/msgp/gen" "github.com/tinylib/msgp/parse" "github.com/ttacon/chalk" "golang.org/x/tools/imports" "io" "io/ioutil" "strings" ) func infof(s string, v ...interface{}) { fmt.Printf(chalk.Magenta.Color(s), v...) } // PrintFile prints the methods for the provided list // of elements to the given file name and canonical // package path. func PrintFile(file string, f *parse.FileSet, mode gen.Method) error { out, tests, err := generate(f, mode) if err != nil { return err } // we'll run goimports on the main file // in another goroutine, and run it here // for the test file. empirically, this // takes about the same amount of time as // doing them in serial when GOMAXPROCS=1, // and faster otherwise. res := goformat(file, out.Bytes()) if tests != nil { testfile := strings.TrimSuffix(file, ".go") + "_test.go" err = format(testfile, tests.Bytes()) if err != nil { return err } infof(">>> Wrote and formatted \"%s\"\n", testfile) } err = <-res if err != nil { return err } return nil } func format(file string, data []byte) error { out, err := imports.Process(file, data, nil) if err != nil { return err } return ioutil.WriteFile(file, out, 0600) } func goformat(file string, data []byte) <-chan error { out := make(chan error, 1) go func(file string, data []byte, end chan error) { end <- format(file, data) infof(">>> Wrote and formatted \"%s\"\n", file) }(file, data, out) return out } func generate(f *parse.FileSet, mode gen.Method) (*bytes.Buffer, *bytes.Buffer, error) { outbuf := bytes.NewBuffer(make([]byte, 0, 4096)) writePkgHeader(outbuf, f.Package) writeImportHeader(outbuf, "github.com/tinylib/msgp/msgp") var testbuf *bytes.Buffer var testwr io.Writer if mode&gen.Test == gen.Test { testbuf = bytes.NewBuffer(make([]byte, 0, 4096)) writePkgHeader(testbuf, f.Package) if mode&(gen.Encode|gen.Decode) != 0 { writeImportHeader(testbuf, "bytes", "github.com/tinylib/msgp/msgp", "testing") } else { writeImportHeader(testbuf, "github.com/tinylib/msgp/msgp", "testing") } testwr = testbuf } return outbuf, testbuf, f.PrintTo(gen.NewPrinter(mode, outbuf, testwr)) } func writePkgHeader(b *bytes.Buffer, name string) { b.WriteString("package ") b.WriteString(name) b.WriteByte('\n') b.WriteString("// NOTE: THIS FILE WAS PRODUCED BY THE\n// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)\n// DO NOT EDIT\n\n") } func writeImportHeader(b *bytes.Buffer, imports ...string) { b.WriteString("import (\n") for _, im := range imports { fmt.Fprintf(b, "\t%q\n", im) } b.WriteString(")\n\n") }