pax_global_header00006660000000000000000000000064135015011470014507gustar00rootroot0000000000000052 comment=87c632ad4fbe701d6f49afc9778ce8b2bbafeabd varint-1.0.0/000077500000000000000000000000001350150114700130105ustar00rootroot00000000000000varint-1.0.0/.gitignore000066400000000000000000000000111350150114700147700ustar00rootroot00000000000000*.o *.txtvarint-1.0.0/.travis.yml000066400000000000000000000000651350150114700151220ustar00rootroot00000000000000language: go go: - 1.12.x env: - GO111MODULE=onvarint-1.0.0/LICENSE000066400000000000000000000020561350150114700140200ustar00rootroot00000000000000MIT License Copyright (c) 2019 Denys Smirnov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. varint-1.0.0/README.md000066400000000000000000000042571350150114700142770ustar00rootroot00000000000000# varint This package provides an optimized implementation of protobuf's varint encoding/decoding. It has no dependencies. Benchmarks comparing to a `binary.Uvarint`: ``` benchmark old ns/op new ns/op delta BenchmarkUvarint/1-8 4.13 2.85 -30.99% BenchmarkUvarint/1_large-8 4.01 2.28 -43.14% BenchmarkUvarint/2-8 6.23 2.87 -53.93% BenchmarkUvarint/2_large-8 5.60 2.86 -48.93% BenchmarkUvarint/3-8 6.55 3.44 -47.48% BenchmarkUvarint/3_large-8 6.54 2.86 -56.27% BenchmarkUvarint/4-8 7.30 3.71 -49.18% BenchmarkUvarint/4_large-8 7.46 3.10 -58.45% BenchmarkUvarint/5-8 8.31 4.12 -50.42% BenchmarkUvarint/5_large-8 8.56 3.48 -59.35% BenchmarkUvarint/6-8 9.42 4.66 -50.53% BenchmarkUvarint/6_large-8 9.91 4.07 -58.93% BenchmarkUvarint/7-8 10.6 5.28 -50.19% BenchmarkUvarint/7_large-8 11.0 4.70 -57.27% BenchmarkUvarint/8-8 11.7 6.02 -48.55% BenchmarkUvarint/8_large-8 12.1 5.19 -57.11% BenchmarkUvarint/9-8 12.9 6.83 -47.05% BenchmarkUvarint/9_large-8 13.1 5.71 -56.41% ``` It also provides additional functionality like `UvarintSize` (similar to `sov*` in `gogo/protobuf`): ``` benchmark old ns/op new ns/op delta BenchmarkUvarintSize/1-8 1.71 0.43 -74.85% BenchmarkUvarintSize/2-8 2.56 0.57 -77.73% BenchmarkUvarintSize/3-8 3.22 0.72 -77.64% BenchmarkUvarintSize/4-8 3.74 0.72 -80.75% BenchmarkUvarintSize/5-8 4.29 0.57 -86.71% BenchmarkUvarintSize/6-8 4.85 0.58 -88.04% BenchmarkUvarintSize/7-8 5.43 0.71 -86.92% BenchmarkUvarintSize/8-8 6.01 0.86 -85.69% BenchmarkUvarintSize/9-8 6.64 1.00 -84.94% ``` # License MITvarint-1.0.0/go.mod000066400000000000000000000000511350150114700141120ustar00rootroot00000000000000module github.com/dennwc/varint go 1.12 varint-1.0.0/proto.go000066400000000000000000000067041350150114700145110ustar00rootroot00000000000000package varint // ProtoTag decodes a protobuf's field number and wire type pair // from buf and returns that value and the number of bytes read (> 0). // If an error occurred, n = 0 is returned. func ProtoTag(buf []byte) (num int, typ byte, n int) { // Same unrolled implementation as in Uvarint. // // But this time we can check if the wire type and field num // are valid when reading the first byte. // // Also, the swifts are now different, because first 3 bits // are for the wire type. // // The implementation will stop at 9 bytes, returning an error. sz := len(buf) if sz == 0 { return 0, 0, 0 } const ( bit = 1 << 7 mask = bit - 1 step = 7 // protobuf typBits = 3 typMask = 1<<3 - 1 ) if sz >= 9 { // no bound checks // i == 0 b := buf[0] if b == 0 { return 0, 0, 0 } typ = b & typMask if typ > 5 { return 0, 0, 0 } if b < bit { num = int(b >> typBits) if num == 0 { return 0, 0, 0 } n = 1 return } num = int((b & mask) >> typBits) var s uint = step - typBits // i == 1 b = buf[1] if b < bit { num |= int(b) << s n = 2 return } num |= int(b&mask) << s s += step // i == 2 b = buf[2] if b < bit { num |= int(b) << s n = 3 return } num |= int(b&mask) << s s += step // i == 3 b = buf[3] if b < bit { num |= int(b) << s n = 4 return } num |= int(b&mask) << s s += step // i == 4 b = buf[4] if b < bit { num |= int(b) << s n = 5 return } num |= int(b&mask) << s s += step // i == 5 b = buf[5] if b < bit { num |= int(b) << s n = 6 return } num |= int(b&mask) << s s += step // i == 6 b = buf[6] if b < bit { num |= int(b) << s n = 7 return } num |= int(b&mask) << s s += step // i == 7 b = buf[7] if b < bit { num |= int(b) << s n = 8 return } num |= int(b&mask) << s s += step // i == 8 b = buf[8] if b < bit { num |= int(b) << s n = 9 return } return 0, 0, 0 // too much } // i == 0 b := buf[0] if b == 0 { return 0, 0, 0 } typ = b & typMask if typ > 5 { return 0, 0, 0 } if b < bit { num = int(b >> typBits) if num == 0 { return 0, 0, 0 } n = 1 return } else if sz == 1 { return 0, 0, 0 } num = int((b & mask) >> typBits) var s uint = step - typBits // i == 1 b = buf[1] if b < bit { num |= int(b) << s n = 2 return } else if sz == 2 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 2 b = buf[2] if b < bit { num |= int(b) << s n = 3 return } else if sz == 3 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 3 b = buf[3] if b < bit { num |= int(b) << s n = 4 return } else if sz == 4 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 4 b = buf[4] if b < bit { num |= int(b) << s n = 5 return } else if sz == 5 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 5 b = buf[5] if b < bit { num |= int(b) << s n = 6 return } else if sz == 6 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 6 b = buf[6] if b < bit { num |= int(b) << s n = 7 return } else if sz == 7 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 7 b = buf[7] if b < bit { num |= int(b) << s n = 8 return } else if sz == 8 { return 0, 0, 0 } num |= int(b&mask) << s s += step // i == 8 b = buf[8] if b < bit { num |= int(b) << s n = 9 return } return 0, 0, 0 // too much } varint-1.0.0/proto_test.go000066400000000000000000000114761350150114700155520ustar00rootroot00000000000000package varint import ( "encoding/binary" "math" "strconv" "testing" ) func protoTag(buf []byte) (num int, typ byte, n int) { var tag uint64 tag, n = Uvarint(buf) if n == 0 || tag == 0 { return 0, 0, 0 } typ, num = byte(tag&0x07), int(tag>>3) if num <= 0 || typ > 5 { return 0, 0, n } return } var casesProtoTag = []int{ int(MaxVal1 >> 3), int(MaxVal2 >> 3), int(MaxVal3 >> 3), int(MaxVal4 >> 3), int(MaxVal5 >> 3), int(MaxVal6 >> 3), int(MaxVal7 >> 3), int(MaxVal8 >> 3), int(MaxVal9 >> 3), } func putProtoTag(b []byte, num int) int { // wire type 1, for test only return binary.PutUvarint(b[:], uint64(num)<<3|1) } func TestProtoTag(t *testing.T) { const wireType = 1 for i, v := range casesProtoTag { sz := i + 1 t.Run(strconv.Itoa(sz), func(t *testing.T) { var b [MaxLen64]byte n := putProtoTag(b[:], v) p := b[:n] if num, typ, n2 := protoTag(p); num != v || typ != wireType || n2 != n { t.Fatalf("%d,%d vs %d,%d", num, typ, v, wireType) } if num, typ, n2 := ProtoTag(p); num != v || typ != wireType || n2 != n { t.Fatalf("%d,%d vs %d,%d", num, typ, v, wireType) } v-- n = putProtoTag(b[:], v) p = b[:n] if num, typ, n2 := protoTag(p); num != v || typ != wireType || n2 != n { t.Fatalf("%d,%d vs %d,%d", num, typ, v, wireType) } if num, typ, n2 := ProtoTag(p); num != v || typ != wireType || n2 != n { t.Fatalf("%d,%d vs %d,%d", num, typ, v, wireType) } t.Run("short", func(t *testing.T) { n := putProtoTag(b[:], v) p := b[:n-1] if _, _, n2 := protoTag(p); n2 != 0 { t.Fatalf("unexpected error: %d", n2) } if _, _, n2 := ProtoTag(p); n2 != 0 { t.Fatalf("unexpected error: %d", n2) } }) t.Run("large", func(t *testing.T) { var b [16]byte n = putProtoTag(b[:], v) p := b[:] if num, typ, n2 := protoTag(p); num != v || typ != wireType || n2 != n { t.Fatalf("%d,%d vs %d,%d", num, typ, v, wireType) } if num, typ, n2 := ProtoTag(p); num != v || typ != wireType || n2 != n { t.Fatalf("%d,%d vs %d,%d", num, typ, v, wireType) } }) }) } t.Run("overflow", func(t *testing.T) { var b [MaxLen64 + 2]byte b[0] = 0xff b[1] = 0xff n := putProtoTag(b[2:], math.MaxInt64) p := b[:n+2] if _, _, n2 := protoTag(p); n2 != 0 { t.Fatalf("unexpected error: %d", n2) } if _, _, n2 := ProtoTag(p); n2 != 0 { t.Fatalf("unexpected error: %d", n2) } }) t.Run("overflow_short", func(t *testing.T) { var b [MaxLen64 + 2]byte b[0] = 0xff b[1] = 0xff n := putProtoTag(b[2:], math.MaxInt64) p := b[:n+2] p[len(p)-1] = 0xff if _, _, n2 := protoTag(p); n2 != 0 { t.Fatalf("unexpected error: %d", n2) } if _, _, n2 := ProtoTag(p); n2 != 0 { t.Fatalf("unexpected error: %d", n2) } }) } func BenchmarkProtoTagSimple(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesProtoTag[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var buf [MaxLen64]byte _ = putProtoTag(buf[:], v) p := buf[:sz] b.ResetTimer() var ( num int typ byte n int ) for i := 0; i < b.N; i++ { num, typ, n = protoTag(p) } if num != v || typ != 1 { b.Fatal(num) } if n != sz { b.Fatal(n) } }) b.Run(strconv.Itoa(sz)+"_large", func(b *testing.B) { var buf [16]byte _ = putProtoTag(buf[:], v) p := buf[:sz] b.ResetTimer() var ( num int typ byte n int ) for i := 0; i < b.N; i++ { num, typ, n = protoTag(p) } if num != v || typ != 1 { b.Fatal(num) } if n != sz { b.Fatal(n) } }) } b.Run("zeros", func(b *testing.B) { var buf [16]byte p := buf[:] b.ResetTimer() var ( num int typ byte n int ) for i := 0; i < b.N; i++ { num, typ, n = protoTag(p) } _, _ = num, typ if n != 0 { b.Fatal(n) } }) } func BenchmarkProtoTag(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesProtoTag[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var buf [MaxLen64]byte _ = putProtoTag(buf[:], v) p := buf[:sz] b.ResetTimer() var ( num int typ byte n int ) for i := 0; i < b.N; i++ { num, typ, n = ProtoTag(p) } if num != v || typ != 1 { b.Fatal(num) } if n != sz { b.Fatal(n) } }) b.Run(strconv.Itoa(sz)+"_large", func(b *testing.B) { var buf [16]byte _ = putProtoTag(buf[:], v) p := buf[:sz] b.ResetTimer() var ( num int typ byte n int ) for i := 0; i < b.N; i++ { num, typ, n = ProtoTag(p) } if num != v || typ != 1 { b.Fatal(num) } if n != sz { b.Fatal(n) } }) } b.Run("zeros", func(b *testing.B) { var buf [16]byte p := buf[:] b.ResetTimer() var ( num int typ byte n int ) for i := 0; i < b.N; i++ { num, typ, n = ProtoTag(p) } _, _ = num, typ if n != 0 { b.Fatal(n) } }) } varint-1.0.0/varint.go000066400000000000000000000102361350150114700146440ustar00rootroot00000000000000package varint const maxUint64 = uint64(1<<64 - 1) // MaxLenN is the maximum length of a varint-encoded N-bit integer. const ( MaxLen8 = 2 MaxLen16 = 3 MaxLen32 = 5 MaxLen64 = 10 ) // MaxValN is the maximum varint-encoded integer that fits in N bytes. const ( MaxVal9 = maxUint64 >> (1 + iota*7) MaxVal8 MaxVal7 MaxVal6 MaxVal5 MaxVal4 MaxVal3 MaxVal2 MaxVal1 ) // UvarintSize returns the number of bytes necessary to encode a given uint. func UvarintSize(x uint64) int { if x <= MaxVal4 { if x <= MaxVal1 { return 1 } else if x <= MaxVal2 { return 2 } else if x <= MaxVal3 { return 3 } return 4 } if x <= MaxVal5 { return 5 } else if x <= MaxVal6 { return 6 } else if x <= MaxVal7 { return 7 } else if x <= MaxVal8 { return 8 } else if x <= MaxVal9 { return 9 } return 10 } // Uvarint decodes a uint64 from buf and returns that value and the // number of bytes read (> 0). If an error occurred, the value is 0 // and the number of bytes n is <= 0 meaning: // // n == 0: buf too small // n < 0: value larger than 64 bits (overflow) // and -n is the number of bytes read // func Uvarint(buf []byte) (uint64, int) { // Fully unrolled implementation of binary.Uvarint. // // It will also eliminate bound checks for buffers larger than 9 bytes. sz := len(buf) if sz == 0 { return 0, 0 } const ( step = 7 bit = 1 << 7 mask = bit - 1 ) if sz >= 10 { // no bound checks // i == 0 b := buf[0] if b < bit { return uint64(b), 1 } x := uint64(b & mask) var s uint = step // i == 1 b = buf[1] if b < bit { return x | uint64(b)< 1 { return 0, -10 // overflow } return x | uint64(b)< 1 { return 0, -10 // overflow } return x | uint64(b)<>= 7 if x == 0 { break } } return n } func uvarintSizeFlat(x uint64) int { if x <= MaxVal1 { return 1 } else if x <= MaxVal2 { return 2 } else if x <= MaxVal3 { return 3 } else if x <= MaxVal4 { return 4 } else if x <= MaxVal5 { return 5 } else if x <= MaxVal6 { return 6 } else if x <= MaxVal7 { return 7 } else if x <= MaxVal8 { return 8 } else if x <= MaxVal9 { return 9 } return 10 } func BenchmarkUvarintSizeLoop(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesUvarintSize[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var n int for i := 0; i < b.N; i++ { n = uvarintSizeLoop(v) } _ = n }) } } func BenchmarkUvarintSizeFlat(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesUvarintSize[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var n int for i := 0; i < b.N; i++ { n = uvarintSizeFlat(v) } _ = n }) } } func BenchmarkUvarintSize(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesUvarintSize[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var n int for i := 0; i < b.N; i++ { n = UvarintSize(v) } _ = n }) } } func BenchmarkUvarintStdlib(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesUvarintSize[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var buf [MaxLen64]byte _ = binary.PutUvarint(buf[:], v) p := buf[:sz] b.ResetTimer() var ( val uint64 n int ) for i := 0; i < b.N; i++ { val, n = binary.Uvarint(p) } if val != v { b.Fatal(val) } if n != sz { b.Fatal(n) } }) b.Run(strconv.Itoa(sz)+"_large", func(b *testing.B) { var buf [16]byte _ = binary.PutUvarint(buf[:], v) p := buf[:] b.ResetTimer() var ( val uint64 n int ) for i := 0; i < b.N; i++ { val, n = binary.Uvarint(p) } if val != v { b.Fatal(val) } if n != sz { b.Fatal(n) } }) } } func BenchmarkUvarint(b *testing.B) { b.StopTimer() for sz := 1; sz <= 9; sz++ { v := casesUvarintSize[sz-1] b.Run(strconv.Itoa(sz), func(b *testing.B) { var buf [MaxLen64]byte _ = binary.PutUvarint(buf[:], v) p := buf[:sz] b.ResetTimer() var ( val uint64 n int ) for i := 0; i < b.N; i++ { val, n = Uvarint(p) } if val != v { b.Fatal(val) } if n != sz { b.Fatal(n) } }) b.Run(strconv.Itoa(sz)+"_large", func(b *testing.B) { var buf [16]byte _ = binary.PutUvarint(buf[:], v) p := buf[:] b.ResetTimer() var ( val uint64 n int ) for i := 0; i < b.N; i++ { val, n = Uvarint(p) } if val != v { b.Fatal(val) } if n != sz { b.Fatal(n) } }) } }