pax_global_header00006660000000000000000000000064134617632700014523gustar00rootroot0000000000000052 comment=ea603e88fd42767b5a068592959d137307c79d83 streamquote-1.0.0/000077500000000000000000000000001346176327000140725ustar00rootroot00000000000000streamquote-1.0.0/.travis.yml000066400000000000000000000003051346176327000162010ustar00rootroot00000000000000language: go go: - master - 1.x - 1.12.x - 1.11.x - 1.10.x - 1.9.x - 1.8.x - 1.7.x - 1.6.x - 1.5.x - 1.4.x script: go test -v -bench . -benchmem ./... streamquote-1.0.0/LICENSE000066400000000000000000000027071346176327000151050ustar00rootroot00000000000000Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. streamquote-1.0.0/README.md000066400000000000000000000014401346176327000153500ustar00rootroot00000000000000## streamquote [![Build Status](https://travis-ci.org/nkovacs/streamquote.svg?branch=master)](https://travis-ci.org/nkovacs/streamquote) [![GoDoc](https://godoc.org/github.com/nkovacs/streamquote?status.svg)](https://godoc.org/github.com/nkovacs/streamquote) This package provides a streaming version of `strconv.Quote`. It allows you to quote the data in an `io.Reader` and write it out to an `io.Writer` without having to store the entire input and the entire output in memory. Its primary use case is [go.rice](https://github.com/GeertJohan/go.rice) and similar tools, which need to convert lots of files, some of them quite large, to go strings. ```go converter := streamquote.New() converter.Convert(inputfile, outfile) ``` Unlike `strconv.Quote`, it does not add quotes around the output. streamquote-1.0.0/go.mod000066400000000000000000000000571346176327000152020ustar00rootroot00000000000000module github.com/nkovacs/streamquote go 1.12 streamquote-1.0.0/streamquote.go000066400000000000000000000077531346176327000170060ustar00rootroot00000000000000// Package streamquote implements a streaming version of strconv.Quote. package streamquote import ( "io" "strconv" "unicode/utf8" ) // Converter converts data by escaping control characters and // non-printable characters using Go escape sequences. type Converter interface { // Convert converts the data in "in", writing it to "out". // It uses Go escape sequences (\t, \n, \xFF, \u0100) for control characters // and non-printable characters as defined by strconv.IsPrint. // It is not safe for concurrent use. Convert(in io.Reader, out io.Writer) (int, error) } const bufSize = 100 * 1024 const lowerhex = "0123456789abcdef" type converter struct { readBuffer [bufSize]byte writeBuffer [10]byte } // New returns a new Converter. func New() Converter { return &converter{} } // Convert converts the data in "in", writing it to "out". // It uses Go escape sequences (\t, \n, \xFF, \u0100) for control characters // and non-printable characters as defined by strconv.IsPrint. // It is not safe for concurrent use. func (c *converter) Convert(in io.Reader, out io.Writer) (int, error) { var err error bufSize := len(c.readBuffer) n := 0 var processed = bufSize var dataLen = 0 for { if processed+utf8.UTFMax > bufSize { // need to read more leftover := bufSize - processed if leftover > 0 { copy(c.readBuffer[:leftover], c.readBuffer[processed:]) } read, peekErr := in.Read(c.readBuffer[leftover:]) if peekErr != nil && peekErr != io.EOF { err = peekErr break } dataLen = leftover + read processed = 0 } if dataLen-processed == 0 { break } maxRune := processed + utf8.UTFMax if maxRune > dataLen { maxRune = dataLen } data := c.readBuffer[processed:maxRune] var discard, n2 int r, width := utf8.DecodeRune(data) if width == 1 && r == utf8.RuneError { c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'x' c.writeBuffer[2] = lowerhex[data[0]>>4] c.writeBuffer[3] = lowerhex[data[0]&0xF] out.Write(c.writeBuffer[0:4]) n2 = 4 discard = 1 } else { discard = width if r == rune('"') || r == '\\' { // always backslashed c.writeBuffer[0] = '\\' c.writeBuffer[1] = byte(r) out.Write(c.writeBuffer[0:2]) n2 = 2 } else if strconv.IsPrint(r) { out.Write(data[:width]) n2 = width } else { switch r { case '\a': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'a' out.Write(c.writeBuffer[0:2]) n2 = 2 case '\b': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'b' out.Write(c.writeBuffer[0:2]) n2 = 2 case '\f': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'f' out.Write(c.writeBuffer[0:2]) n2 = 2 case '\n': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'n' out.Write(c.writeBuffer[0:2]) n2 = 2 case '\r': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'r' out.Write(c.writeBuffer[0:2]) n2 = 2 case '\t': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 't' out.Write(c.writeBuffer[0:2]) n2 = 2 case '\v': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'v' out.Write(c.writeBuffer[0:2]) n2 = 2 default: switch { case r < ' ': c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'x' c.writeBuffer[2] = lowerhex[data[0]>>4] c.writeBuffer[3] = lowerhex[data[0]&0xF] out.Write(c.writeBuffer[0:4]) n2 = 4 case r > utf8.MaxRune: r = 0xFFFD fallthrough case r < 0x10000: c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'u' n2 = 2 i := 2 for s := 12; s >= 0; s -= 4 { c.writeBuffer[i] = lowerhex[r>>uint(s)&0xF] i++ n2++ } out.Write(c.writeBuffer[0:i]) default: c.writeBuffer[0] = '\\' c.writeBuffer[1] = 'U' n2 = 2 i := 2 for s := 28; s >= 0; s -= 4 { c.writeBuffer[i] = lowerhex[r>>uint(s)&0xF] i++ n2++ } out.Write(c.writeBuffer[0:i]) } } } } processed += discard n += n2 } return n, err } streamquote-1.0.0/streamquote_test.go000066400000000000000000000067431346176327000200430ustar00rootroot00000000000000package streamquote import ( "bytes" "io" "io/ioutil" "math/rand" "strconv" "strings" "testing" ) // Taken from stdlib's strconv/quote_test.go type quoteTest struct { in string out string ascii string graphic string } var quotetests = []quoteTest{ {"\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`, `"\a\b\f\r\n\t\v"`, `"\a\b\f\r\n\t\v"`}, {"\\", `"\\"`, `"\\"`, `"\\"`}, {"abc\xffdef", `"abc\xffdef"`, `"abc\xffdef"`, `"abc\xffdef"`}, {"\u263a", `"☺"`, `"\u263a"`, `"☺"`}, {"\U0010ffff", `"\U0010ffff"`, `"\U0010ffff"`, `"\U0010ffff"`}, {"\x04", `"\x04"`, `"\x04"`, `"\x04"`}, // Some non-printable but graphic runes. Final column is double-quoted. {"!\u00a0!\u2000!\u3000!", `"!\u00a0!\u2000!\u3000!"`, `"!\u00a0!\u2000!\u3000!"`, "\"!\u00a0!\u2000!\u3000!\""}, } func TestConverter(t *testing.T) { converter := New() for _, tt := range quotetests { var buffer bytes.Buffer converter.Convert(strings.NewReader(tt.in), &buffer) expected := tt.out[1 : len(tt.out)-1] if out := buffer.String(); out != expected { t.Errorf("Quote(%s) = %s, want %s", tt.in, out, expected) } } } // Size of the large string for benchmarking. const largeSize = 10 * 1024 * 1024 // random seed for large string, to make the benchmark consistent. const randSeed = 47 type randomRuneProvider struct { r *rand.Rand limit int read int } func (r *randomRuneProvider) Read(p []byte) (n int, err error) { n = len(p) available := r.limit - r.read if available < n { n = available err = io.EOF } for i := 0; i < n; i++ { p[i] = byte(r.r.Int63()) } r.read += n return } func generateLargeString() io.Reader { source := rand.NewSource(randSeed) r := rand.New(source) return &randomRuneProvider{ r: r, limit: largeSize, } } // TestLargeString tests that converter and strconv.Quote // return the same result for the large string. func TestLargeString(t *testing.T) { largeStringReader := generateLargeString() b, err := ioutil.ReadAll(largeStringReader) if err != nil { t.Fatalf("Failed to read large string into buffer: %v", err) } expected := strconv.Quote(string(b)) buffer := bytes.NewBufferString("\"") converter := New() largeStringReader = generateLargeString() _, err = converter.Convert(largeStringReader, buffer) if err != nil { t.Fatalf("Converter failed: %v", err) } buffer.WriteRune('"') got := buffer.String() if got != expected { t.Fatalf("Large string does not match") } } func BenchmarkConverterSmall(b *testing.B) { converter := New() r := strings.NewReader("\a\b\f\r\n\t\v\a\b\f\r\n\t\v\a\b\f\r\n\t\v") b.ResetTimer() for i := 0; i < b.N; i++ { converter.Convert(r, ioutil.Discard) r.Seek(0, 0) } } func BenchmarkStrconvQuoteSmall(b *testing.B) { for i := 0; i < b.N; i++ { strconv.Quote("\a\b\f\r\n\t\v\a\b\f\r\n\t\v\a\b\f\r\n\t\v") } } func BenchmarkConverterLarge(b *testing.B) { converter := New() b.ResetTimer() for i := 0; i < b.N; i++ { // Using ioutil.Discard here may not seem fair, // because strconv.Quote has to allocate the result string, // but the point of this package is that you don't have to // allocate the entire result string, you can stream the result. converter.Convert(generateLargeString(), ioutil.Discard) } } func BenchmarkStrconvQuoteLarge(b *testing.B) { largeStringReader := generateLargeString() bs, err := ioutil.ReadAll(largeStringReader) if err != nil { b.Fatalf("Failed to read large string into buffer: %v", err) } s := string(bs) for i := 0; i < b.N; i++ { strconv.Quote(s) } }