pax_global_header00006660000000000000000000000064137726604500014524gustar00rootroot0000000000000052 comment=4d9f9c03d2d5a48f6593e834b36b5e58f2e84d81 golang-modernc-internal-1.0.0/000077500000000000000000000000001377266045000162305ustar00rootroot00000000000000golang-modernc-internal-1.0.0/AUTHORS000066400000000000000000000005321377266045000173000ustar00rootroot00000000000000# This file lists authors for copyright purposes. This file is distinct from # the CONTRIBUTORS files. See the latter for an explanation. # # Names should be added to this file as: # Name or Organization # # The email address is not required for organizations. # # Please keep the list sorted. Jan Mercl <0xjnml@gmail.com> golang-modernc-internal-1.0.0/CONTRIBUTORS000066400000000000000000000004251377266045000201110ustar00rootroot00000000000000# This file lists people who contributed code to this repository. The AUTHORS # file lists the copyright holders; this file lists people. # # Names should be added to this file like so: # Name # # Please keep the list sorted. Jan Mercl <0xjnml@gmail.com> golang-modernc-internal-1.0.0/LICENSE000066400000000000000000000027161377266045000172430ustar00rootroot00000000000000Copyright (c) 2016 The Internal Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the names of the authors nor the names of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang-modernc-internal-1.0.0/README.md000066400000000000000000000006611377266045000175120ustar00rootroot00000000000000# internal Things that cannot be imported by 3rd party packages. Packages in this repository * Package [buffer](https://godoc.org/modernc.org/internal/buffer) implements a pool of pointers to byte slices. * Package [file](https://godoc.org/modernc.org/internal/file) provides an os.File-like interface of a memory mapped file. * Package [slice](https://godoc.org/modernc.org/internal/slice) implements pools of pointers to slices. golang-modernc-internal-1.0.0/buffer/000077500000000000000000000000001377266045000175015ustar00rootroot00000000000000golang-modernc-internal-1.0.0/buffer/Makefile000066400000000000000000000030551377266045000211440ustar00rootroot00000000000000# Copyright 2016 The Internal Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. .PHONY: all clean cover cpu editor internalError later mem nuke todo edit grep=--include=*.go --include=*.l --include=*.y --include=*.yy ngrep='TODOOK\|parser\.go\|scanner\.go\|.*_string\.go' all: editor go vet 2>&1 | grep -v $(ngrep) || true go vet 2>&1 | grep -v $(ngrep) || true golint 2>&1 | grep -v $(ngrep) || true make todo unused . || true misspell *.go gosimple || true codesweep || true maligned || true unconvert -apply clean: go clean rm -f *~ *.test *.out cover: t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t cpu: clean go test -run @ -bench . -cpuprofile cpu.out go tool pprof -lines *.test cpu.out edit: @ 1>/dev/null 2>/dev/null gvim -p Makefile log *.go editor: gofmt -l -s -w *.go go test -i go test 2>&1 | tee log go install internalError: egrep -ho '"internal error.*"' *.go | sort | cat -n later: @grep -n $(grep) LATER * || true @grep -n $(grep) MAYBE * || true mem: clean go test -run @ -bench . -memprofile mem.out -memprofilerate 1 -timeout 24h go tool pprof -lines -web -alloc_space *.test mem.out nuke: clean go clean -i todo: @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * | grep -v $(ngrep) || true @grep -nr $(grep) TODO * | grep -v $(ngrep) || true @grep -nr $(grep) BUG * | grep -v $(ngrep) || true @grep -nr $(grep) [^[:alpha:]]println * | grep -v $(ngrep) || true golang-modernc-internal-1.0.0/buffer/all_test.go000066400000000000000000000070271377266045000216450ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package buffer // import "modernc.org/internal/buffer" import ( "bytes" "fmt" "math/rand" "os" "path" "runtime" "strings" "testing" ) func caller(s string, va ...interface{}) { if s == "" { s = strings.Repeat("%v ", len(va)) } _, fn, fl, _ := runtime.Caller(2) fmt.Fprintf(os.Stderr, "caller: %s:%d: ", path.Base(fn), fl) fmt.Fprintf(os.Stderr, s, va...) fmt.Fprintln(os.Stderr) _, fn, fl, _ = runtime.Caller(1) fmt.Fprintf(os.Stderr, "\tcallee: %s:%d: ", path.Base(fn), fl) fmt.Fprintln(os.Stderr) os.Stderr.Sync() } func dbg(s string, va ...interface{}) { if s == "" { s = strings.Repeat("%v ", len(va)) } _, fn, fl, _ := runtime.Caller(1) fmt.Fprintf(os.Stderr, "dbg %s:%d: ", path.Base(fn), fl) fmt.Fprintf(os.Stderr, s, va...) fmt.Fprintln(os.Stderr) os.Stderr.Sync() } func TODO(...interface{}) string { //TODOOK _, fn, fl, _ := runtime.Caller(1) return fmt.Sprintf("TODO: %s:%d:\n", path.Base(fn), fl) //TODOOK } func use(...interface{}) {} func init() { use(caller, dbg, TODO) //TODOOK } // ============================================================================ func test(t testing.TB, allocs, goroutines int) { ready := make(chan int, goroutines) run := make(chan int) done := make(chan int, goroutines) for i := 0; i < goroutines; i++ { go func() { a := rand.Perm(allocs) ready <- 1 <-run for _, v := range a { p := Get(v) b := *p if g, e := len(b), v; g != e { t.Error(g, e) break } Put(p) } done <- 1 }() } for i := 0; i < goroutines; i++ { <-ready } close(run) for i := 0; i < goroutines; i++ { <-done } } func Test2(t *testing.T) { test(t, 1<<15, 32) } func Benchmark1(b *testing.B) { const ( allocs = 1000 goroutines = 100 ) for i := 0; i < b.N; i++ { test(b, allocs, goroutines) } b.SetBytes(goroutines * (allocs*allocs + allocs) / 2) } func TestBytes(t *testing.T) { const N = 1e6 src := make([]byte, N) for i := range src { src[i] = byte(rand.Int()) } var b Bytes for i, v := range src { b.WriteByte(v) if g, e := b.Len(), i+1; g != e { t.Fatal("Len", g, e) } } if !bytes.Equal(b.Bytes(), src) { t.Fatal("content differs") } b.Close() if g, e := b.Len(), 0; g != e { t.Fatal("Len", g, e) } if g, e := len(b.Bytes()), 0; g != e { t.Fatal("Bytes", g, e) } s2 := src for len(s2) != 0 { n := rand.Intn(127) if n > len(s2) { n = len(s2) } n2, _ := b.Write(s2[:n]) if g, e := n2, n; g != e { t.Fatal("Write", g, e) } s2 = s2[n:] } if !bytes.Equal(b.Bytes(), src) { t.Fatal("content differs") } b.Close() s2 = src for len(s2) != 0 { n := rand.Intn(31) if n > len(s2) { n = len(s2) } n2, _ := b.WriteString(string(s2[:n])) if g, e := n2, n; g != e { t.Fatal("Write", g, e) } s2 = s2[n:] } if !bytes.Equal(b.Bytes(), src) { t.Fatal("content differs") } } func benchmarkWriteByte(b *testing.B, n int) { for i := 0; i < b.N; i++ { var buf Bytes for j := 0; j < n; j++ { buf.WriteByte(0) } buf.Close() } b.SetBytes(int64(n)) } func BenchmarkWriteByte_1e3(b *testing.B) { benchmarkWriteByte(b, 1e3) } func BenchmarkWriteByte_1e4(b *testing.B) { benchmarkWriteByte(b, 1e4) } func BenchmarkWriteByte_1e5(b *testing.B) { benchmarkWriteByte(b, 1e5) } func BenchmarkWriteByte_1e6(b *testing.B) { benchmarkWriteByte(b, 1e6) } func BenchmarkWriteByte_1e7(b *testing.B) { benchmarkWriteByte(b, 1e7) } golang-modernc-internal-1.0.0/buffer/buffer.go000066400000000000000000000062241377266045000213050ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package buffer implements a pool of pointers to byte slices. // // Example usage pattern // // p := buffer.Get(size) // b := *p // Now you can use b in any way you need. // ... // // When b will not be used anymore // buffer.Put(p) // ... // // If b or p are not going out of scope soon, optionally // b = nil // p = nil // // Otherwise the pool cannot release the buffer on garbage collection. // // Do not do // // p := buffer.Get(size) // b := *p // ... // buffer.Put(&b) // // or // // b := *buffer.Get(size) // ... // buffer.Put(&b) package buffer // import "modernc.org/internal/buffer" import ( "modernc.org/internal/slice" "io" ) // CGet returns a pointer to a byte slice of len size. The pointed to byte // slice is zeroed up to its cap. CGet panics for size < 0. // // CGet is safe for concurrent use by multiple goroutines. func CGet(size int) *[]byte { return slice.Bytes.CGet(size).(*[]byte) } // Get returns a pointer to a byte slice of len size. The pointed to byte slice // is not zeroed. Get panics for size < 0. // // Get is safe for concurrent use by multiple goroutines. func Get(size int) *[]byte { return slice.Bytes.Get(size).(*[]byte) } // Put puts a pointer to a byte slice into a pool for possible later reuse by // CGet or Get. // // Put is safe for concurrent use by multiple goroutines. func Put(p *[]byte) { slice.Bytes.Put(p) } // Bytes is similar to bytes.Buffer but may generate less garbage when properly // Closed. Zero value is ready to use. type Bytes struct { p *[]byte } // Bytes return the content of b. The result is R/O. func (b *Bytes) Bytes() []byte { if b.p != nil { return *b.p } return nil } // Close will recycle the underlying storage, if any. After Close, b is again // the zero value. func (b *Bytes) Close() error { if b.p != nil { Put(b.p) b.p = nil } return nil } // Len returns the size of content in b. func (b *Bytes) Len() int { if b.p != nil { return len(*b.p) } return 0 } // Reset discard the content of Bytes while keeping the internal storage, if any. func (b *Bytes) Reset() { if b.p != nil { *b.p = (*b.p)[:0] } } // Write writes p into b and returns (len(p), nil). func (b *Bytes) Write(p []byte) (int, error) { n := b.Len() b.grow(n + len(p)) copy((*b.p)[n:], p) return len(p), nil } // WriteByte writes p into b and returns nil. func (b *Bytes) WriteByte(p byte) error { n := b.Len() b.grow(n + 1) (*b.p)[n] = p return nil } // WriteTo writes b's content to w and returns the number of bytes written to w // and an error, if any. func (b *Bytes) WriteTo(w io.Writer) (int64, error) { n, err := w.Write(b.Bytes()) return int64(n), err } // WriteString writes s to b and returns (len(s), nil). func (b *Bytes) WriteString(s string) (int, error) { n := b.Len() b.grow(n + len(s)) copy((*b.p)[n:], s) return len(s), nil } func (b *Bytes) grow(n int) { if b.p != nil { if n <= cap(*b.p) { *b.p = (*b.p)[:n] return } np := Get(2 * n) *np = (*np)[:n] copy(*np, *b.p) Put(b.p) b.p = np return } b.p = Get(n) } golang-modernc-internal-1.0.0/buffer/nonrace_test.go000066400000000000000000000015271377266045000225210ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build !race package buffer // import "modernc.org/internal/buffer" import ( "testing" ) // Does not work with race detector b/c things get purged from the pool. func Test(t *testing.T) { a := [1 << 10]*[]byte{} m := map[*[]byte]struct{}{} for i := range a { p := CGet(i) if _, ok := m[p]; ok { t.Fatal(i) } a[i] = p m[p] = struct{}{} b := *p for j := range b { b[j] = 123 } } for i := range a { Put(a[i]) } for i := range a { p := CGet(i) if _, ok := m[p]; !ok { t.Fatal(i) } delete(m, p) b := *p if g, e := len(b), i; g != e { t.Fatal(g, e) } for j, v := range b[:cap(b)] { if v != 0 { t.Fatal(i, j, v) } } } } golang-modernc-internal-1.0.0/file/000077500000000000000000000000001377266045000171475ustar00rootroot00000000000000golang-modernc-internal-1.0.0/file/Makefile000066400000000000000000000027331377266045000206140ustar00rootroot00000000000000# Copyright 2016 The Internal Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. .PHONY: all clean cover cpu editor internalError later mem nuke todo edit grep=--include=*.go --include=*.l --include=*.y --include=*.yy ngrep='TODOOK\|parser\.go\|scanner\.go\|.*_string\.go' all: editor go vet 2>&1 | grep -v $(ngrep) || true golint 2>&1 | grep -v $(ngrep) || true make todo unused . || true misspell *.go gosimple || true clean: go clean rm -f *~ *.test *.out cover: t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t cpu: clean go test -run @ -bench . -cpuprofile cpu.out go tool pprof -lines *.test cpu.out edit: @ 1>/dev/null 2>/dev/null gvim -p Makefile log *.go editor: gofmt -l -s -w *.go go test 2>&1 | tee log go build internalError: egrep -ho '"internal error.*"' *.go | sort | cat -n later: @grep -n $(grep) LATER * || true @grep -n $(grep) MAYBE * || true mem: clean go test -run @ -bench BenchmarkReadWrite -benchmem -memprofile mem.out -memprofilerate 1 -timeout 24h go tool pprof -lines -web -alloc_space *.test mem.out nuke: clean go clean -i todo: @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * | grep -v $(ngrep) || true @grep -nr $(grep) TODO * | grep -v $(ngrep) || true @grep -nr $(grep) BUG * | grep -v $(ngrep) || true @grep -nr $(grep) [^[:alpha:]]println * | grep -v $(ngrep) || true golang-modernc-internal-1.0.0/file/all_test.go000066400000000000000000000241061377266045000213100ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package file // import "modernc.org/internal/file" import ( "bytes" "encoding/hex" "fmt" "io" "io/ioutil" "math/rand" "os" "path" "runtime" "strings" "testing" "modernc.org/fileutil" "modernc.org/internal/buffer" ) func caller(s string, va ...interface{}) { if s == "" { s = strings.Repeat("%v ", len(va)) } _, fn, fl, _ := runtime.Caller(2) fmt.Fprintf(os.Stderr, "caller: %s:%d: ", path.Base(fn), fl) fmt.Fprintf(os.Stderr, s, va...) fmt.Fprintln(os.Stderr) _, fn, fl, _ = runtime.Caller(1) fmt.Fprintf(os.Stderr, "\tcallee: %s:%d: ", path.Base(fn), fl) fmt.Fprintln(os.Stderr) os.Stderr.Sync() } func dbg(s string, va ...interface{}) { if s == "" { s = strings.Repeat("%v ", len(va)) } _, fn, fl, _ := runtime.Caller(1) fmt.Fprintf(os.Stderr, "dbg %s:%d: ", path.Base(fn), fl) fmt.Fprintf(os.Stderr, s, va...) fmt.Fprintln(os.Stderr) os.Stderr.Sync() } func TODO(...interface{}) string { //TODOOK _, fn, fl, _ := runtime.Caller(1) return fmt.Sprintf("TODO: %s:%d:\n", path.Base(fn), fl) //TODOOK } func use(...interface{}) {} func init() { use(caller, dbg, TODO) //TODOOK } // ============================================================================ const testSize = 1 << 26 var ( _ Interface = (*osFile)(nil) _ testInterface = (*testFile)(nil) _ testInterface = (*testMem)(nil) _ testInterface = (*testOSFile)(nil) ) type testInterface interface { new() (Interface, error) free() error } type osFile struct { *os.File } func (f *osFile) ReadFrom(r io.Reader) (n int64, err error) { return readFrom(f, r) } func (f *osFile) WriteTo(w io.Writer) (n int64, err error) { return writeTo(f, w) } type testFile struct { f Interface name string } func (f *testFile) new() (Interface, error) { f0, err := ioutil.TempFile("", "dbkit-test") if err != nil { return nil, err } if f.f, err = Open(f0); err != nil { return nil, err } f.name = f0.Name() return f.f, nil } func (f *testFile) free() error { if err := f.f.Close(); err != nil { return err } return os.Remove(f.name) } type testOSFile struct { f *osFile } func (f *testOSFile) new() (Interface, error) { f0, err := ioutil.TempFile("", "dbkit-test") if err != nil { return nil, err } f.f = &osFile{f0} return f.f, nil } func (f *testOSFile) free() error { nm := f.f.Name() if err := f.f.Close(); err != nil { return err } return os.Remove(nm) } type testMem struct { mem Interface } func (m *testMem) new() (Interface, error) { mem, err := OpenMem("") if err != nil { return nil, err } m.mem = mem return mem, nil } func (m *testMem) free() error { if mem := m.mem; mem != nil { mem.Close() } return nil } func TestTuncate(t *testing.T) { testTruncate(t, &testFile{}, 1) testTruncate(t, &testMem{}, 1) testTruncate(t, &testOSFile{}, 1) } func testTruncate0(t testing.TB, f Interface) { // Check Truncate works. sz := int64(1e6) if err := f.Truncate(sz); err != nil { t.Error(err) return } fi, err := f.Stat() if err != nil { t.Error(err) return } if g, e := fi.Size(), sz; g != e { t.Error(g, e) return } sz *= 2 if err := f.Truncate(sz); err != nil { t.Error(err) return } if fi, err = f.Stat(); err != nil { t.Error(err) return } if g, e := fi.Size(), sz; g != e { t.Error(g, e) return } sz = 0 if err := f.Truncate(sz); err != nil { t.Error(err) return } if fi, err = f.Stat(); err != nil { t.Error(err) return } if g, e := fi.Size(), sz; g != e { t.Error(g, e) return } // Check Truncate(-1) doesn't work. sz = -1 if err := f.Truncate(sz); err == nil { t.Error(err) } } func testTruncate(t testing.TB, ti testInterface, n int) { f, err := ti.new() if err != nil { t.Error(err) return } defer func() { if err := ti.free(); err != nil { t.Error(err) } }() for i := 0; i < n; i++ { testTruncate0(t, f) } } func BenchmarkTruncateFile(b *testing.B) { testTruncate(b, &testFile{}, b.N) } func BenchmarkTruncateMem(b *testing.B) { testTruncate(b, &testMem{}, b.N) } func BenchmarkTruncateOSFile(b *testing.B) { testTruncate(b, &testOSFile{}, b.N) } func TestReadWrite(t *testing.T) { testReadWrite(t, &testFile{}, 1) testReadWrite(t, &testMem{}, 1) testReadWrite(t, &testOSFile{}, 1) } func testReadWrite0(t testing.TB, f Interface) { const ( N = 1 << 21 M = 10 ) ps := buffer.Get(N) defer buffer.Put(ps) s := *ps pe := buffer.CGet(N) defer buffer.Put(pe) e := *pe rnd := rand.New(rand.NewSource(42)) for i := range e { s[i] = byte(rnd.Intn(256)) } n2 := 0 for i := 0; i < M; i++ { var from, to int for from == to { from = rnd.Intn(N) to = rnd.Intn(N) } if from > to { from, to = to, from } for i := range s[from:to] { s[from+i] = byte(rnd.Intn(256)) } copy(e[from:to], s[from:to]) if to > n2 { n2 = to } n, err := f.WriteAt(s[from:to], int64(from)) if err != nil { t.Error(err) return } if g, e := n, to-from; g != e { t.Error(g, e) return } } fi, err := f.Stat() if err != nil { t.Error(err) return } if g, e := fi.Size(), int64(n2); g != e { t.Error(g, e) return } pb := buffer.Get(n2) defer buffer.Put(pb) b := *pb for i := 0; i <= M; i++ { from := rnd.Intn(n2) to := rnd.Intn(n2) if from > to { from, to = to, from } if i == M { from, to = 0, n2 } n, err := f.ReadAt(b[from:to], int64(from)) if err != nil && (!fileutil.IsEOF(err) && n != 0) { fi, err = f.Stat() if err != nil { t.Error(err) return } t.Error(fi.Size(), from, to, err) return } if g, e := n, to-from; g != e { t.Error(g, e) return } if g, e := b[from:to], e[from:to]; !bytes.Equal(g, e) { t.Errorf( "i %d from %d to %d len(g) %d len(e) %d\n---- got ----\n%s\n---- exp ----\n%s", i, from, to, len(g), len(e), hex.Dump(g), hex.Dump(e), ) return } } pbuf := buffer.Get(n2) defer buffer.Put(pbuf) buf := bytes.NewBuffer(*pbuf) buf.Reset() if _, err := f.WriteTo(buf); err != nil { t.Error(err) return } if g, e := buf.Bytes(), e[:n2]; !bytes.Equal(g, e) { t.Errorf("\nlen %d\n%s\nlen %d\n%s", len(g), hex.Dump(g), len(e), hex.Dump(e)) return } if err := f.Truncate(0); err != nil { t.Error(err) return } if _, err := f.ReadFrom(buf); err != nil { t.Error(err) return } p := buffer.Get(n2) defer buffer.Put(p) roundTrip := *p if n, err := f.ReadAt(roundTrip, 0); err != nil && n == 0 { t.Error(err) return } if g, e := roundTrip, e[:n2]; !bytes.Equal(g, e) { t.Errorf("\nlen %d\n%s\nlen %d\n%s", len(g), hex.Dump(g), len(e), hex.Dump(e)) return } } func testReadWrite(t testing.TB, ti testInterface, n int) { f, err := ti.new() if err != nil { t.Error(err) return } defer func() { if err := ti.free(); err != nil { t.Error(err) } }() for i := 0; i < n; i++ { testReadWrite0(t, f) } } func BenchmarkReadWriteFile(b *testing.B) { testReadWrite(b, &testFile{}, b.N) } func BenchmarkReadWriteMem(b *testing.B) { testReadWrite(b, &testMem{}, b.N) } func BenchmarkReadWriteOSFile(b *testing.B) { testReadWrite(b, &testOSFile{}, b.N) } func benchmarkWriteAt(b *testing.B, ti testInterface) { f, err := ti.new() if err != nil { b.Error(err) return } defer func() { if err := ti.free(); err != nil { b.Error(err) } }() p := buffer.Get(testSize) defer buffer.Put(p) rand.Read(*p) b.ResetTimer() for i := 0; i < b.N; i++ { if _, err := f.WriteAt(*p, 0); err != nil { b.Error(err) return } } b.SetBytes(testSize) } func BenchmarkWriteAtFile(b *testing.B) { benchmarkWriteAt(b, &testFile{}) } func BenchmarkWriteAtMem(b *testing.B) { benchmarkWriteAt(b, &testMem{}) } func BenchmarkWriteAtOSFile(b *testing.B) { benchmarkWriteAt(b, &testOSFile{}) } func benchmarkReadAt(b *testing.B, ti testInterface) { f, err := ti.new() if err != nil { b.Error(err) return } defer func() { if err := ti.free(); err != nil { b.Error(err) } }() p := buffer.Get(testSize) defer buffer.Put(p) rand.Read(*p) if _, err := f.WriteAt(*p, 0); err != nil { b.Error(err) return } b.ResetTimer() for i := 0; i < b.N; i++ { if _, err := f.ReadAt(*p, 0); err != nil && !fileutil.IsEOF(err) { b.Error(err) return } } b.SetBytes(testSize) } func BenchmarkReadAtFile(b *testing.B) { benchmarkReadAt(b, &testFile{}) } func BenchmarkReadAtMem(b *testing.B) { benchmarkReadAt(b, &testMem{}) } func BenchmarkReadAtOSFile(b *testing.B) { benchmarkReadAt(b, &testOSFile{}) } func benchmarkWriteAtOne(b *testing.B, ti testInterface) { const size = 1 << 17 f, err := ti.new() if err != nil { b.Error(err) return } defer func() { if err := ti.free(); err != nil { b.Error(err) } }() p := buffer.Get(size) defer buffer.Put(p) rb := *p rand.Read(rb) b.ResetTimer() for i := 0; i < b.N; i++ { for off := 0; off < size-1; off++ { if _, err := f.WriteAt(rb[off:off+1], int64(off)); err != nil { b.Error(err) return } } } b.SetBytes(size) } func BenchmarkWriteAtOneFile(b *testing.B) { benchmarkWriteAtOne(b, &testFile{}) } func BenchmarkWriteAtOneMem(b *testing.B) { benchmarkWriteAtOne(b, &testMem{}) } func BenchmarkWriteAtOneOSFile(b *testing.B) { benchmarkWriteAtOne(b, &testOSFile{}) } func benchmarkReadAtOne(b *testing.B, ti testInterface) { const size = 1 << 18 f, err := ti.new() if err != nil { b.Error(err) return } defer func() { if err := ti.free(); err != nil { b.Error(err) } }() p := buffer.Get(size) defer buffer.Put(p) rb := *p rand.Read(rb) if _, err := f.WriteAt(rb, 0); err != nil { b.Error(err) return } b.ResetTimer() for i := 0; i < b.N; i++ { for off := 0; off < size-1; off++ { if _, err := f.ReadAt(rb[off:off+1], int64(off)); err != nil && !fileutil.IsEOF(err) { b.Error(err) return } } } b.SetBytes(size) } func BenchmarkReadAtOneFile(b *testing.B) { benchmarkReadAtOne(b, &testFile{}) } func BenchmarkReadAtOneMem(b *testing.B) { benchmarkReadAtOne(b, &testMem{}) } func BenchmarkReadAtOneOSFile(b *testing.B) { benchmarkReadAtOne(b, &testOSFile{}) } golang-modernc-internal-1.0.0/file/file.go000066400000000000000000000212101377266045000204110ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package file provides an os.File-like interface of a memory mapped file. package file // import "modernc.org/internal/file" import ( "fmt" "io" "os" "time" "modernc.org/fileutil" "modernc.org/internal/buffer" "modernc.org/mathutil" "github.com/edsrzf/mmap-go" ) const copyBufSize = 1 << 20 // 1 MB. var ( _ Interface = (*mem)(nil) _ Interface = (*file)(nil) _ os.FileInfo = stat{} sysPage = os.Getpagesize() ) // Interface is a os.File-like entity. type Interface interface { io.ReaderAt io.ReaderFrom io.WriterAt io.WriterTo Close() error Stat() (os.FileInfo, error) Sync() error Truncate(int64) error } // Open returns a new Interface backed by f, or an error, if any. func Open(f *os.File) (Interface, error) { return newFile(f, 1<<30, 20) } // OpenMem returns a new Interface, or an error, if any. The Interface content // is volatile, it's backed only by process' memory. func OpenMem(name string) (Interface, error) { return newMem(name, 18), nil } type memMap map[int64]*[]byte type mem struct { m memMap modTime time.Time name string pgBits uint pgMask int pgSize int size int64 } func newMem(name string, pgBits uint) *mem { pgSize := 1 << pgBits return &mem{ m: memMap{}, modTime: time.Now(), name: name, pgBits: pgBits, pgMask: pgSize - 1, pgSize: pgSize, } } func (f *mem) IsDir() bool { return false } func (f *mem) Mode() os.FileMode { return os.ModeTemporary + 0600 } func (f *mem) ModTime() time.Time { return f.modTime } func (f *mem) Name() string { return f.name } func (f *mem) ReadFrom(r io.Reader) (n int64, err error) { return readFrom(f, r) } func (f *mem) Size() (n int64) { return f.size } func (f *mem) Stat() (os.FileInfo, error) { return f, nil } func (f *mem) Sync() error { return nil } func (f *mem) Sys() interface{} { return nil } func (f *mem) WriteTo(w io.Writer) (n int64, err error) { return writeTo(f, w) } func (f *mem) Close() error { f.Truncate(0) f.m = nil return nil } func (f *mem) ReadAt(b []byte, off int64) (n int, err error) { avail := f.size - off pi := off >> f.pgBits po := int(off) & f.pgMask rem := len(b) if int64(rem) >= avail { rem = int(avail) err = io.EOF } var zeroPage *[]byte for rem != 0 && avail > 0 { pg := f.m[pi] if pg == nil { if zeroPage == nil { zeroPage = buffer.CGet(f.pgSize) defer buffer.Put(zeroPage) } pg = zeroPage } nc := copy(b[:mathutil.Min(rem, f.pgSize)], (*pg)[po:]) pi++ po = 0 rem -= nc n += nc b = b[nc:] } return n, err } func (f *mem) Truncate(size int64) (err error) { if size < 0 { return fmt.Errorf("invalid truncate size: %d", size) } first := size >> f.pgBits if po := size & int64(f.pgMask); po != 0 { if p := f.m[first]; p != nil { b := (*p)[po:] for i := range b { b[i] = 0 } } first++ } last := f.size >> f.pgBits if po := f.size & int64(f.pgMask); po != 0 { if p := f.m[last]; p != nil { b := (*p)[po:] for i := range b { b[i] = 0 } } last++ } for ; first <= last; first++ { if p := f.m[first]; p != nil { buffer.Put(p) } delete(f.m, first) } f.size = size return nil } func (f *mem) WriteAt(b []byte, off int64) (n int, err error) { if len(b) == 0 { return 0, nil } pi := off >> f.pgBits po := int(off) & f.pgMask n = len(b) rem := n var nc int for rem != 0 { pg := f.m[pi] if pg == nil { pg = buffer.CGet(f.pgSize) f.m[pi] = pg } nc = copy((*pg)[po:], b) pi++ po = 0 rem -= nc b = b[nc:] } f.size = mathutil.MaxInt64(f.size, off+int64(n)) return n, nil } type stat struct { os.FileInfo size int64 } func (s stat) Size() int64 { return s.size } type fileMap map[int64]mmap.MMap type file struct { f *os.File m fileMap maxPages int pgBits uint pgMask int pgSize int size int64 fsize int64 } func newFile(f *os.File, maxSize int64, pgBits uint) (*file, error) { if maxSize < 0 { panic("internal error") } pgSize := 1 << pgBits switch { case sysPage > pgSize: pgBits = uint(mathutil.Log2Uint64(uint64(sysPage))) default: pgBits = uint(mathutil.Log2Uint64(uint64(pgSize / sysPage * sysPage))) } pgSize = 1 << pgBits fi := &file{ f: f, m: fileMap{}, maxPages: int(mathutil.MinInt64( 1024, mathutil.MaxInt64(maxSize/int64(pgSize), 1)), ), pgBits: pgBits, pgMask: pgSize - 1, pgSize: pgSize, } info, err := f.Stat() if err != nil { return nil, err } if err = fi.Truncate(info.Size()); err != nil { return nil, err } return fi, nil } func (f *file) ReadFrom(r io.Reader) (n int64, err error) { return readFrom(f, r) } func (f *file) Sync() (err error) { return f.f.Sync() } func (f *file) WriteTo(w io.Writer) (n int64, err error) { return writeTo(f, w) } func (f *file) Close() (err error) { for _, p := range f.m { if err = p.Unmap(); err != nil { return err } } if err = f.f.Truncate(f.size); err != nil { return err } if err = f.f.Sync(); err != nil { return err } if err = f.f.Close(); err != nil { return err } f.m = nil f.f = nil return nil } func (f *file) page(index int64) (mmap.MMap, error) { if len(f.m) == f.maxPages { for i, p := range f.m { if err := p.Unmap(); err != nil { return nil, err } delete(f.m, i) break } } off := index << f.pgBits fsize := off + int64(f.pgSize) if fsize > f.fsize { if err := f.f.Truncate(fsize); err != nil { return nil, err } f.fsize = fsize } p, err := mmap.MapRegion(f.f, f.pgSize, mmap.RDWR, 0, off) if err != nil { return nil, err } f.m[index] = p return p, nil } func (f *file) ReadAt(b []byte, off int64) (n int, err error) { avail := f.size - off pi := off >> f.pgBits po := int(off) & f.pgMask rem := len(b) if int64(rem) >= avail { rem = int(avail) err = io.EOF } for rem != 0 && avail > 0 { pg := f.m[pi] if pg == nil { if pg, err = f.page(pi); err != nil { return n, err } } nc := copy(b[:mathutil.Min(rem, f.pgSize)], pg[po:]) pi++ po = 0 rem -= nc n += nc b = b[nc:] } return n, err } func (f *file) Stat() (os.FileInfo, error) { fi, err := f.f.Stat() if err != nil { return nil, err } return stat{fi, f.size}, nil } func (f *file) Truncate(size int64) (err error) { if size < 0 { return fmt.Errorf("invalid truncate size: %d", size) } first := size >> f.pgBits if po := size & int64(f.pgMask); po != 0 { if p := f.m[first]; p != nil { b := p[po:] for i := range b { b[i] = 0 } } first++ } last := f.size >> f.pgBits if po := f.size & int64(f.pgMask); po != 0 { if p := f.m[last]; p != nil { b := p[po:] for i := range b { b[i] = 0 } } last++ } for ; first <= last; first++ { if p := f.m[first]; p != nil { if err := p.Unmap(); err != nil { return err } } delete(f.m, first) } f.size = size fsize := (size + int64(f.pgSize) - 1) &^ int64(f.pgMask) if fsize != f.fsize { if err := f.f.Truncate(fsize); err != nil { return err } } f.fsize = fsize return nil } func (f *file) WriteAt(b []byte, off int64) (n int, err error) { if len(b) == 0 { return 0, nil } pi := off >> f.pgBits po := int(off) & f.pgMask n = len(b) rem := n var nc int for rem != 0 { pg := f.m[pi] if pg == nil { pg, err = f.page(pi) if err != nil { return n, err } } nc = copy(pg[po:], b) pi++ po = 0 rem -= nc b = b[nc:] } f.size = mathutil.MaxInt64(f.size, off+int64(n)) return n, nil } // ---------------------------------------------------------------------------- func readFrom(f Interface, r io.Reader) (n int64, err error) { f.Truncate(0) p := buffer.Get(copyBufSize) b := *p defer buffer.Put(p) var off int64 var werr error for { rn, rerr := r.Read(b) if rn != 0 { _, werr = f.WriteAt(b[:rn], off) n += int64(rn) off += int64(rn) } if rerr != nil { if !fileutil.IsEOF(rerr) { err = rerr } break } if werr != nil { err = werr break } } return n, err } func writeTo(f Interface, w io.Writer) (n int64, err error) { p := buffer.Get(copyBufSize) b := *p defer buffer.Put(p) var off int64 var werr error for { rn, rerr := f.ReadAt(b, off) if rn != 0 { _, werr = w.Write(b[:rn]) n += int64(rn) off += int64(rn) } if rerr != nil { if !fileutil.IsEOF(rerr) { err = rerr } break } if werr != nil { err = werr break } } return n, err } golang-modernc-internal-1.0.0/slice/000077500000000000000000000000001377266045000173275ustar00rootroot00000000000000golang-modernc-internal-1.0.0/slice/Makefile000066400000000000000000000027001377266045000207660ustar00rootroot00000000000000# Copyright 2016 The Internal Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. .PHONY: all clean cover cpu editor internalError later mem nuke todo edit grep=--include=*.go --include=*.l --include=*.y --include=*.yy ngrep='TODOOK\|parser\.go\|scanner\.go\|.*_string\.go' all: editor go vet 2>&1 | grep -v $(ngrep) || true golint 2>&1 | grep -v $(ngrep) || true make todo unused . || true misspell *.go gosimple || true clean: go clean rm -f *~ *.test *.out cover: t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t cpu: clean go test -run @ -bench . -cpuprofile cpu.out go tool pprof -lines *.test cpu.out edit: @ 1>/dev/null 2>/dev/null gvim -p Makefile log *.go editor: gofmt -l -s -w *.go go test 2>&1 | tee log go build internalError: egrep -ho '"internal error.*"' *.go | sort | cat -n later: @grep -n $(grep) LATER * || true @grep -n $(grep) MAYBE * || true mem: clean go test -run @ -bench . -memprofile mem.out -memprofilerate 1 -timeout 24h go tool pprof -lines -web -alloc_space *.test mem.out nuke: clean go clean -i todo: @grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * | grep -v $(ngrep) || true @grep -nr $(grep) TODO * | grep -v $(ngrep) || true @grep -nr $(grep) BUG * | grep -v $(ngrep) || true @grep -nr $(grep) [^[:alpha:]]println * | grep -v $(ngrep) || true golang-modernc-internal-1.0.0/slice/all_test.go000066400000000000000000000040751377266045000214730ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package slice // import "modernc.org/internal/slice" import ( "fmt" "math/rand" "os" "path" "runtime" "strings" "testing" ) func caller(s string, va ...interface{}) { if s == "" { s = strings.Repeat("%v ", len(va)) } _, fn, fl, _ := runtime.Caller(2) fmt.Fprintf(os.Stderr, "caller: %s:%d: ", path.Base(fn), fl) fmt.Fprintf(os.Stderr, s, va...) fmt.Fprintln(os.Stderr) _, fn, fl, _ = runtime.Caller(1) fmt.Fprintf(os.Stderr, "\tcallee: %s:%d: ", path.Base(fn), fl) fmt.Fprintln(os.Stderr) os.Stderr.Sync() } func dbg(s string, va ...interface{}) { if s == "" { s = strings.Repeat("%v ", len(va)) } _, fn, fl, _ := runtime.Caller(1) fmt.Fprintf(os.Stderr, "dbg %s:%d: ", path.Base(fn), fl) fmt.Fprintf(os.Stderr, s, va...) fmt.Fprintln(os.Stderr) os.Stderr.Sync() } func TODO(...interface{}) string { //TODOOK _, fn, fl, _ := runtime.Caller(1) return fmt.Sprintf("TODO: %s:%d:\n", path.Base(fn), fl) //TODOOK } func use(...interface{}) {} func init() { use(caller, dbg, TODO) //TODOOK } // ============================================================================ func test(t testing.TB, allocs, goroutines int, pool *Pool) { ready := make(chan int, goroutines) run := make(chan int) done := make(chan int, goroutines) for i := 0; i < goroutines; i++ { go func() { a := rand.Perm(allocs) ready <- 1 <-run for _, v := range a { p := pool.Get(v).(*[]byte) b := *p if g, e := len(b), v; g != e { t.Error(g, e) break } pool.Put(p) } done <- 1 }() } for i := 0; i < goroutines; i++ { <-ready } close(run) for i := 0; i < goroutines; i++ { <-done } } func Test2(t *testing.T) { test(t, 1<<15, 32, newBytes()) } func Benchmark1(b *testing.B) { const ( allocs = 1000 goroutines = 100 ) pool := newBytes() for i := 0; i < b.N; i++ { test(b, allocs, goroutines, pool) } b.SetBytes(goroutines * (allocs*allocs + allocs) / 2) } golang-modernc-internal-1.0.0/slice/pool.go000066400000000000000000000075011377266045000206320ustar00rootroot00000000000000// Copyright 2016 The Internal Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package slice implements pools of pointers to slices. package slice // import "modernc.org/internal/slice" import ( "sync" "modernc.org/mathutil" ) var ( // Bytes is a ready to use *[]byte Pool. Bytes *Pool // Ints is a ready to use *[]int Pool. Ints *Pool ) func init() { Bytes = newBytes() Ints = NewPool( func(size int) interface{} { // create b := make([]int, size) return &b }, func(s interface{}) { // clear b := *s.(*[]int) b = b[:cap(b)] for i := range b { b[i] = 0 } }, func(s interface{}, size int) { // setSize p := s.(*[]int) *p = (*p)[:size] }, func(s interface{}) int { return cap(*s.(*[]int)) }, // cap ) } func newBytes() *Pool { return NewPool( func(size int) interface{} { // create b := make([]byte, size) return &b }, func(s interface{}) { // clear b := *s.(*[]byte) b = b[:cap(b)] for i := range b { b[i] = 0 } }, func(s interface{}, size int) { // setSize p := s.(*[]byte) *p = (*p)[:size] }, func(s interface{}) int { return cap(*s.(*[]byte)) }, // cap ) } // Pool implements a pool of pointers to slices. // // Example usage pattern (assuming pool is, for example, a *[]byte Pool) // // p := pool.Get(size).(*[]byte) // b := *p // Now you can use b in any way you need. // ... // // When b will not be used anymore // pool.Put(p) // ... // // If b or p are not going out of scope soon, optionally // b = nil // p = nil // // Otherwise the pool cannot release the slice on garbage collection. // // Do not do // // p := pool.Get(size).(*[]byte) // b := *p // ... // pool.Put(&b) // // or // // b := *pool.Get(size).(*[]byte) // ... // pool.Put(&b) type Pool struct { cap func(interface{}) int clear func(interface{}) m [63]sync.Pool null interface{} setSize func(interface{}, int) } // NewPool returns a newly created Pool. Assuming the desired slice type is // []T: // // The create function returns a *[]T of len == cap == size. // // The argument of clear is *[]T and the function sets all the slice elements // to the respective zero value. // // The setSize function gets a *[]T and sets its len to size. // // The cap function gets a *[]T and returns its capacity. func NewPool( create func(size int) interface{}, clear func(interface{}), setSize func(p interface{}, size int), cap func(p interface{}) int, ) *Pool { p := &Pool{clear: clear, setSize: setSize, cap: cap, null: create(0)} for i := range p.m { size := 1 << uint(i) p.m[i] = sync.Pool{New: func() interface{} { // 0: 1 - 1 // 1: 10 - 10 // 2: 11 - 100 // 3: 101 - 1000 // 4: 1001 - 10000 // 5: 10001 - 100000 return create(size) }} } return p } // CGet returns a *[]T of len size. The pointed to slice is zeroed up to its // cap. CGet panics for size < 0. // // CGet is safe for concurrent use by multiple goroutines. func (p *Pool) CGet(size int) interface{} { s := p.Get(size) p.clear(s) return s } // Get returns a *[]T of len size. The pointed to slice is not zeroed. Get // panics for size < 0. // // Get is safe for concurrent use by multiple goroutines. func (p *Pool) Get(size int) interface{} { var index int switch { case size < 0: panic("Pool.Get: negative size") case size == 0: return p.null case size > 1: index = mathutil.Log2Uint64(uint64(size-1)) + 1 } s := p.m[index].Get() p.setSize(s, size) return s } // Put puts a *[]T into a pool for possible later reuse by CGet or Get. Put // panics is its argument is not of type *[]T. // // Put is safe for concurrent use by multiple goroutines. func (p *Pool) Put(b interface{}) { size := p.cap(b) if size == 0 { return } p.m[mathutil.Log2Uint64(uint64(size))].Put(b) }