pax_global_header00006660000000000000000000000064136216313330014513gustar00rootroot0000000000000052 comment=3c4a99247c6656284dd2be8d74438ca9bbb4f342 chunkreader-2.0.1/000077500000000000000000000000001362163133300140065ustar00rootroot00000000000000chunkreader-2.0.1/.travis.yml000066400000000000000000000001131362163133300161120ustar00rootroot00000000000000language: go go: - 1.x - tip matrix: allow_failures: - go: tip chunkreader-2.0.1/LICENSE000066400000000000000000000020611362163133300150120ustar00rootroot00000000000000Copyright (c) 2019 Jack Christensen MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. chunkreader-2.0.1/README.md000066400000000000000000000006241362163133300152670ustar00rootroot00000000000000[![](https://godoc.org/github.com/jackc/chunkreader?status.svg)](https://godoc.org/github.com/jackc/chunkreader) [![Build Status](https://travis-ci.org/jackc/chunkreader.svg)](https://travis-ci.org/jackc/chunkreader) # chunkreader Package chunkreader provides an io.Reader wrapper that minimizes IO reads and memory allocations. Extracted from original implementation in https://github.com/jackc/pgx. chunkreader-2.0.1/chunkreader.go000066400000000000000000000060571362163133300166400ustar00rootroot00000000000000// Package chunkreader provides an io.Reader wrapper that minimizes IO reads and memory allocations. package chunkreader import ( "io" ) // ChunkReader is a io.Reader wrapper that minimizes IO reads and memory allocations. It allocates memory in chunks and // will read as much as will fit in the current buffer in a single call regardless of how large a read is actually // requested. The memory returned via Next is owned by the caller. This avoids the need for an additional copy. // // The downside of this approach is that a large buffer can be pinned in memory even if only a small slice is // referenced. For example, an entire 4096 byte block could be pinned in memory by even a 1 byte slice. In these rare // cases it would be advantageous to copy the bytes to another slice. type ChunkReader struct { r io.Reader buf []byte rp, wp int // buf read position and write position config Config } // Config contains configuration parameters for ChunkReader. type Config struct { MinBufLen int // Minimum buffer length } // New creates and returns a new ChunkReader for r with default configuration. func New(r io.Reader) *ChunkReader { cr, err := NewConfig(r, Config{}) if err != nil { panic("default config can't be bad") } return cr } // NewConfig creates and a new ChunkReader for r configured by config. func NewConfig(r io.Reader, config Config) (*ChunkReader, error) { if config.MinBufLen == 0 { // By historical reasons Postgres currently has 8KB send buffer inside, // so here we want to have at least the same size buffer. // @see https://github.com/postgres/postgres/blob/249d64999615802752940e017ee5166e726bc7cd/src/backend/libpq/pqcomm.c#L134 // @see https://www.postgresql.org/message-id/0cdc5485-cb3c-5e16-4a46-e3b2f7a41322%40ya.ru config.MinBufLen = 8192 } return &ChunkReader{ r: r, buf: make([]byte, config.MinBufLen), config: config, }, nil } // Next returns buf filled with the next n bytes. The caller gains ownership of buf. It is not necessary to make a copy // of buf. If an error occurs, buf will be nil. func (r *ChunkReader) Next(n int) (buf []byte, err error) { // n bytes already in buf if (r.wp - r.rp) >= n { buf = r.buf[r.rp : r.rp+n] r.rp += n return buf, err } // available space in buf is less than n if len(r.buf) < n { r.copyBufContents(r.newBuf(n)) } // buf is large enough, but need to shift filled area to start to make enough contiguous space minReadCount := n - (r.wp - r.rp) if (len(r.buf) - r.wp) < minReadCount { newBuf := r.newBuf(n) r.copyBufContents(newBuf) } if err := r.appendAtLeast(minReadCount); err != nil { return nil, err } buf = r.buf[r.rp : r.rp+n] r.rp += n return buf, nil } func (r *ChunkReader) appendAtLeast(fillLen int) error { n, err := io.ReadAtLeast(r.r, r.buf[r.wp:], fillLen) r.wp += n return err } func (r *ChunkReader) newBuf(size int) []byte { if size < r.config.MinBufLen { size = r.config.MinBufLen } return make([]byte, size) } func (r *ChunkReader) copyBufContents(dest []byte) { r.wp = copy(dest, r.buf[r.rp:r.wp]) r.rp = 0 r.buf = dest } chunkreader-2.0.1/chunkreader_test.go000066400000000000000000000041311362163133300176660ustar00rootroot00000000000000package chunkreader import ( "bytes" "testing" ) func TestChunkReaderNextDoesNotReadIfAlreadyBuffered(t *testing.T) { server := &bytes.Buffer{} r, err := NewConfig(server, Config{MinBufLen: 4}) if err != nil { t.Fatal(err) } src := []byte{1, 2, 3, 4} server.Write(src) n1, err := r.Next(2) if err != nil { t.Fatal(err) } if bytes.Compare(n1, src[0:2]) != 0 { t.Fatalf("Expected read bytes to be %v, but they were %v", src[0:2], n1) } n2, err := r.Next(2) if err != nil { t.Fatal(err) } if bytes.Compare(n2, src[2:4]) != 0 { t.Fatalf("Expected read bytes to be %v, but they were %v", src[2:4], n2) } if bytes.Compare(r.buf, src) != 0 { t.Fatalf("Expected r.buf to be %v, but it was %v", src, r.buf) } if r.rp != 4 { t.Fatalf("Expected r.rp to be %v, but it was %v", 4, r.rp) } if r.wp != 4 { t.Fatalf("Expected r.wp to be %v, but it was %v", 4, r.wp) } } func TestChunkReaderNextExpandsBufAsNeeded(t *testing.T) { server := &bytes.Buffer{} r, err := NewConfig(server, Config{MinBufLen: 4}) if err != nil { t.Fatal(err) } src := []byte{1, 2, 3, 4, 5, 6, 7, 8} server.Write(src) n1, err := r.Next(5) if err != nil { t.Fatal(err) } if bytes.Compare(n1, src[0:5]) != 0 { t.Fatalf("Expected read bytes to be %v, but they were %v", src[0:5], n1) } if len(r.buf) != 5 { t.Fatalf("Expected len(r.buf) to be %v, but it was %v", 5, len(r.buf)) } } func TestChunkReaderDoesNotReuseBuf(t *testing.T) { server := &bytes.Buffer{} r, err := NewConfig(server, Config{MinBufLen: 4}) if err != nil { t.Fatal(err) } src := []byte{1, 2, 3, 4, 5, 6, 7, 8} server.Write(src) n1, err := r.Next(4) if err != nil { t.Fatal(err) } if bytes.Compare(n1, src[0:4]) != 0 { t.Fatalf("Expected read bytes to be %v, but they were %v", src[0:4], n1) } n2, err := r.Next(4) if err != nil { t.Fatal(err) } if bytes.Compare(n2, src[4:8]) != 0 { t.Fatalf("Expected read bytes to be %v, but they were %v", src[4:8], n2) } if bytes.Compare(n1, src[0:4]) != 0 { t.Fatalf("Expected KeepLast to prevent Next from overwriting buf, expected %v but it was %v", src[0:4], n1) } } chunkreader-2.0.1/go.mod000066400000000000000000000000601362163133300151100ustar00rootroot00000000000000module github.com/jackc/chunkreader/v2 go 1.12