pax_global_header00006660000000000000000000000064127551250240014515gustar00rootroot0000000000000052 comment=e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7 golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/000077500000000000000000000000001275512502400235375ustar00rootroot00000000000000golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/.travis.yml000066400000000000000000000004021275512502400256440ustar00rootroot00000000000000language: go go: - 1.6 script: # build test for supported platforms - GOOS=linux go build - GOOS=darwin go build - GOOS=freebsd go build - GOOS=windows go build - GOARCH=386 go build # run tests on a standard platform - go test -v ./... golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/LICENSE000066400000000000000000000021141275512502400245420ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2016 Aliaksandr Valialkin, VertaMedia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/README.md000066400000000000000000000016421275512502400250210ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/valyala/bytebufferpool.svg)](https://travis-ci.org/valyala/bytebufferpool) [![GoDoc](https://godoc.org/github.com/valyala/bytebufferpool?status.svg)](http://godoc.org/github.com/valyala/bytebufferpool) [![Go Report](http://goreportcard.com/badge/valyala/bytebufferpool)](http://goreportcard.com/report/valyala/bytebufferpool) # bytebufferpool An implementation of a pool of byte buffers with anti-memory-waste protection. The pool may waste limited amount of memory due to fragmentation. This amount equals to the maximum total size of the byte buffers in concurrent use. # Benchmark results Currently bytebufferpool is fastest and most effective buffer pool written in Go. You can find results [here](https://omgnull.github.io/go-benchmark/buffer/). # bytebufferpool users * [fasthttp](https://github.com/valyala/fasthttp) * [quicktemplate](https://github.com/valyala/quicktemplate) golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/bytebuffer.go000066400000000000000000000045211275512502400262250ustar00rootroot00000000000000package bytebufferpool import "io" // ByteBuffer provides byte buffer, which can be used for minimizing // memory allocations. // // ByteBuffer may be used with functions appending data to the given []byte // slice. See example code for details. // // Use Get for obtaining an empty byte buffer. type ByteBuffer struct { // B is a byte buffer to use in append-like workloads. // See example code for details. B []byte } // Len returns the size of the byte buffer. func (b *ByteBuffer) Len() int { return len(b.B) } // ReadFrom implements io.ReaderFrom. // // The function appends all the data read from r to b. func (b *ByteBuffer) ReadFrom(r io.Reader) (int64, error) { p := b.B nStart := int64(len(p)) nMax := int64(cap(p)) n := nStart if nMax == 0 { nMax = 64 p = make([]byte, nMax) } else { p = p[:nMax] } for { if n == nMax { nMax *= 2 bNew := make([]byte, nMax) copy(bNew, p) p = bNew } nn, err := r.Read(p[n:]) n += int64(nn) if err != nil { b.B = p[:n] n -= nStart if err == io.EOF { return n, nil } return n, err } } } // WriteTo implements io.WriterTo. func (b *ByteBuffer) WriteTo(w io.Writer) (int64, error) { n, err := w.Write(b.B) return int64(n), err } // Bytes returns b.B, i.e. all the bytes accumulated in the buffer. // // The purpose of this function is bytes.Buffer compatibility. func (b *ByteBuffer) Bytes() []byte { return b.B } // Write implements io.Writer - it appends p to ByteBuffer.B func (b *ByteBuffer) Write(p []byte) (int, error) { b.B = append(b.B, p...) return len(p), nil } // WriteByte appends the byte c to the buffer. // // The purpose of this function is bytes.Buffer compatibility. // // The function always returns nil. func (b *ByteBuffer) WriteByte(c byte) error { b.B = append(b.B, c) return nil } // WriteString appends s to ByteBuffer.B. func (b *ByteBuffer) WriteString(s string) (int, error) { b.B = append(b.B, s...) return len(s), nil } // Set sets ByteBuffer.B to p. func (b *ByteBuffer) Set(p []byte) { b.B = append(b.B[:0], p...) } // SetString sets ByteBuffer.B to s. func (b *ByteBuffer) SetString(s string) { b.B = append(b.B[:0], s...) } // String returns string representation of ByteBuffer.B. func (b *ByteBuffer) String() string { return string(b.B) } // Reset makes ByteBuffer.B empty. func (b *ByteBuffer) Reset() { b.B = b.B[:0] } golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/bytebuffer_example_test.go000066400000000000000000000006141275512502400307760ustar00rootroot00000000000000package bytebufferpool_test import ( "fmt" "github.com/valyala/bytebufferpool" ) func ExampleByteBuffer() { bb := bytebufferpool.Get() bb.WriteString("first line\n") bb.Write([]byte("second line\n")) bb.B = append(bb.B, "third line\n"...) fmt.Printf("bytebuffer contents=%q", bb.B) // It is safe to release byte buffer now, since it is // no longer used. bytebufferpool.Put(bb) } golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/bytebuffer_test.go000066400000000000000000000057211275512502400272670ustar00rootroot00000000000000package bytebufferpool import ( "bytes" "fmt" "io" "testing" "time" ) func TestByteBufferReadFrom(t *testing.T) { prefix := "foobar" expectedS := "asadfsdafsadfasdfisdsdfa" prefixLen := int64(len(prefix)) expectedN := int64(len(expectedS)) var bb ByteBuffer bb.WriteString(prefix) rf := (io.ReaderFrom)(&bb) for i := 0; i < 20; i++ { r := bytes.NewBufferString(expectedS) n, err := rf.ReadFrom(r) if n != expectedN { t.Fatalf("unexpected n=%d. Expecting %d. iteration %d", n, expectedN, i) } if err != nil { t.Fatalf("unexpected error: %s", err) } bbLen := int64(bb.Len()) expectedLen := prefixLen + int64(i+1)*expectedN if bbLen != expectedLen { t.Fatalf("unexpected byteBuffer length: %d. Expecting %d", bbLen, expectedLen) } for j := 0; j < i; j++ { start := prefixLen + int64(j)*expectedN b := bb.B[start : start+expectedN] if string(b) != expectedS { t.Fatalf("unexpected byteBuffer contents: %q. Expecting %q", b, expectedS) } } } } func TestByteBufferWriteTo(t *testing.T) { expectedS := "foobarbaz" var bb ByteBuffer bb.WriteString(expectedS[:3]) bb.WriteString(expectedS[3:]) wt := (io.WriterTo)(&bb) var w bytes.Buffer for i := 0; i < 10; i++ { n, err := wt.WriteTo(&w) if n != int64(len(expectedS)) { t.Fatalf("unexpected n returned from WriteTo: %d. Expecting %d", n, len(expectedS)) } if err != nil { t.Fatalf("unexpected error: %s", err) } s := string(w.Bytes()) if s != expectedS { t.Fatalf("unexpected string written %q. Expecting %q", s, expectedS) } w.Reset() } } func TestByteBufferGetPutSerial(t *testing.T) { testByteBufferGetPut(t) } func TestByteBufferGetPutConcurrent(t *testing.T) { concurrency := 10 ch := make(chan struct{}, concurrency) for i := 0; i < concurrency; i++ { go func() { testByteBufferGetPut(t) ch <- struct{}{} }() } for i := 0; i < concurrency; i++ { select { case <-ch: case <-time.After(time.Second): t.Fatalf("timeout!") } } } func testByteBufferGetPut(t *testing.T) { for i := 0; i < 10; i++ { expectedS := fmt.Sprintf("num %d", i) b := Get() b.B = append(b.B, "num "...) b.B = append(b.B, fmt.Sprintf("%d", i)...) if string(b.B) != expectedS { t.Fatalf("unexpected result: %q. Expecting %q", b.B, expectedS) } Put(b) } } func testByteBufferGetString(t *testing.T) { for i := 0; i < 10; i++ { expectedS := fmt.Sprintf("num %d", i) b := Get() b.SetString(expectedS) if b.String() != expectedS { t.Fatalf("unexpected result: %q. Expecting %q", b.B, expectedS) } Put(b) } } func TestByteBufferGetStringSerial(t *testing.T) { testByteBufferGetString(t) } func TestByteBufferGetStringConcurrent(t *testing.T) { concurrency := 10 ch := make(chan struct{}, concurrency) for i := 0; i < concurrency; i++ { go func() { testByteBufferGetString(t) ch <- struct{}{} }() } for i := 0; i < concurrency; i++ { select { case <-ch: case <-time.After(time.Second): t.Fatalf("timeout!") } } } golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/bytebuffer_timing_test.go000066400000000000000000000010001275512502400306200ustar00rootroot00000000000000package bytebufferpool import ( "bytes" "testing" ) func BenchmarkByteBufferWrite(b *testing.B) { s := []byte("foobarbaz") b.RunParallel(func(pb *testing.PB) { var buf ByteBuffer for pb.Next() { for i := 0; i < 100; i++ { buf.Write(s) } buf.Reset() } }) } func BenchmarkBytesBufferWrite(b *testing.B) { s := []byte("foobarbaz") b.RunParallel(func(pb *testing.PB) { var buf bytes.Buffer for pb.Next() { for i := 0; i < 100; i++ { buf.Write(s) } buf.Reset() } }) } golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/doc.go000066400000000000000000000004341275512502400246340ustar00rootroot00000000000000// Package bytebufferpool implements a pool of byte buffers // with anti-fragmentation protection. // // The pool may waste limited amount of memory due to fragmentation. // This amount equals to the maximum total size of the byte buffers // in concurrent use. package bytebufferpool golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/pool.go000066400000000000000000000056711275512502400250500ustar00rootroot00000000000000package bytebufferpool import ( "sort" "sync" "sync/atomic" ) const ( minBitSize = 6 // 2**6=64 is a CPU cache line size steps = 20 minSize = 1 << minBitSize maxSize = 1 << (minBitSize + steps - 1) calibrateCallsThreshold = 42000 maxPercentile = 0.95 ) // Pool represents byte buffer pool. // // Distinct pools may be used for distinct types of byte buffers. // Properly determined byte buffer types with their own pools may help reducing // memory waste. type Pool struct { calls [steps]uint64 calibrating uint64 defaultSize uint64 maxSize uint64 pool sync.Pool } var defaultPool Pool // Get returns an empty byte buffer from the pool. // // Got byte buffer may be returned to the pool via Put call. // This reduces the number of memory allocations required for byte buffer // management. func Get() *ByteBuffer { return defaultPool.Get() } // Get returns new byte buffer with zero length. // // The byte buffer may be returned to the pool via Put after the use // in order to minimize GC overhead. func (p *Pool) Get() *ByteBuffer { v := p.pool.Get() if v != nil { return v.(*ByteBuffer) } return &ByteBuffer{ B: make([]byte, 0, atomic.LoadUint64(&p.defaultSize)), } } // Put returns byte buffer to the pool. // // ByteBuffer.B mustn't be touched after returning it to the pool. // Otherwise data races will occur. func Put(b *ByteBuffer) { defaultPool.Put(b) } // Put releases byte buffer obtained via Get to the pool. // // The buffer mustn't be accessed after returning to the pool. func (p *Pool) Put(b *ByteBuffer) { idx := index(len(b.B)) if atomic.AddUint64(&p.calls[idx], 1) > calibrateCallsThreshold { p.calibrate() } maxSize := int(atomic.LoadUint64(&p.maxSize)) if maxSize == 0 || cap(b.B) <= maxSize { b.Reset() p.pool.Put(b) } } func (p *Pool) calibrate() { if !atomic.CompareAndSwapUint64(&p.calibrating, 0, 1) { return } a := make(callSizes, 0, steps) var callsSum uint64 for i := uint64(0); i < steps; i++ { calls := atomic.SwapUint64(&p.calls[i], 0) callsSum += calls a = append(a, callSize{ calls: calls, size: minSize << i, }) } sort.Sort(a) defaultSize := a[0].size maxSize := defaultSize maxSum := uint64(float64(callsSum) * maxPercentile) callsSum = 0 for i := 0; i < steps; i++ { if callsSum > maxSum { break } callsSum += a[i].calls size := a[i].size if size > maxSize { maxSize = size } } atomic.StoreUint64(&p.defaultSize, defaultSize) atomic.StoreUint64(&p.maxSize, maxSize) atomic.StoreUint64(&p.calibrating, 0) } type callSize struct { calls uint64 size uint64 } type callSizes []callSize func (ci callSizes) Len() int { return len(ci) } func (ci callSizes) Less(i, j int) bool { return ci[i].calls > ci[j].calls } func (ci callSizes) Swap(i, j int) { ci[i], ci[j] = ci[j], ci[i] } func index(n int) int { n-- n >>= minBitSize idx := 0 for n > 0 { n >>= 1 idx++ } if idx >= steps { idx = steps - 1 } return idx } golang-github-valyala-bytebufferpool-0.0~git20160817.0.e746df9/pool_test.go000066400000000000000000000032671275512502400261060ustar00rootroot00000000000000package bytebufferpool import ( "math/rand" "testing" "time" ) func TestIndex(t *testing.T) { testIndex(t, 0, 0) testIndex(t, 1, 0) testIndex(t, minSize-1, 0) testIndex(t, minSize, 0) testIndex(t, minSize+1, 1) testIndex(t, 2*minSize-1, 1) testIndex(t, 2*minSize, 1) testIndex(t, 2*minSize+1, 2) testIndex(t, maxSize-1, steps-1) testIndex(t, maxSize, steps-1) testIndex(t, maxSize+1, steps-1) } func testIndex(t *testing.T, n, expectedIdx int) { idx := index(n) if idx != expectedIdx { t.Fatalf("unexpected idx for n=%d: %d. Expecting %d", n, idx, expectedIdx) } } func TestPoolCalibrate(t *testing.T) { for i := 0; i < steps*calibrateCallsThreshold; i++ { n := 1004 if i%15 == 0 { n = rand.Intn(15234) } testGetPut(t, n) } } func TestPoolVariousSizesSerial(t *testing.T) { testPoolVariousSizes(t) } func TestPoolVariousSizesConcurrent(t *testing.T) { concurrency := 5 ch := make(chan struct{}) for i := 0; i < concurrency; i++ { go func() { testPoolVariousSizes(t) ch <- struct{}{} }() } for i := 0; i < concurrency; i++ { select { case <-ch: case <-time.After(3 * time.Second): t.Fatalf("timeout") } } } func testPoolVariousSizes(t *testing.T) { for i := 0; i < steps+1; i++ { n := (1 << uint32(i)) testGetPut(t, n) testGetPut(t, n+1) testGetPut(t, n-1) for j := 0; j < 10; j++ { testGetPut(t, j+n) } } } func testGetPut(t *testing.T, n int) { bb := Get() if len(bb.B) > 0 { t.Fatalf("non-empty byte buffer returned from acquire") } bb.B = allocNBytes(bb.B, n) Put(bb) } func allocNBytes(dst []byte, n int) []byte { diff := n - cap(dst) if diff <= 0 { return dst[:n] } return append(dst, make([]byte, diff)...) }