pax_global_header00006660000000000000000000000064144405763360014526gustar00rootroot0000000000000052 comment=35c9d55c11fd82ace2a8923c684ceb8300fdbc33 fastzip-0.1.11/000077500000000000000000000000001444057633600132665ustar00rootroot00000000000000fastzip-0.1.11/.github/000077500000000000000000000000001444057633600146265ustar00rootroot00000000000000fastzip-0.1.11/.github/workflows/000077500000000000000000000000001444057633600166635ustar00rootroot00000000000000fastzip-0.1.11/.github/workflows/go.yml000066400000000000000000000010571444057633600200160ustar00rootroot00000000000000name: Go on: push: branches: [main] pull_request: branches: [main] jobs: build: strategy: matrix: go: [1.18.x, 1.19.x, 1.20.x] os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - name: Setup go uses: actions/setup-go@v2 with: go-version: ${{ matrix.go }} - name: Run tests run: go test --cpu 1,4 -race -coverprofile coverage.txt -covermode atomic ./... - uses: codecov/codecov-action@v2 fastzip-0.1.11/LICENSE000066400000000000000000000020541444057633600142740ustar00rootroot00000000000000MIT License Copyright (c) 2019 Arran Walker Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.fastzip-0.1.11/README.md000066400000000000000000000133761444057633600145570ustar00rootroot00000000000000# fastzip [![godoc](https://godoc.org/github.com/saracen/fastzip?status.svg)](http://godoc.org/github.com/saracen/fastzip) [![Build Status](https://travis-ci.org/saracen/fastzip.svg?branch=master)](https://travis-ci.org/saracen/fastzip) Fastzip is an opinionated Zip archiver and extractor with a focus on speed. - Archiving and extraction of files and directories can only occur within a specified directory. - Permissions, ownership (uid, gid on linux/unix) and modification times are preserved. - Buffers used for copying files are recycled to reduce allocations. - Files are archived and extracted concurrently. - By default, the excellent [`github.com/klauspost/compress/flate`](https://github.com/klauspost/compress) library is used for compression and decompression. ## Example ### Archiver ```go // Create archive file w, err := os.Create("archive.zip") if err != nil { panic(err) } defer w.Close() // Create new Archiver a, err := fastzip.NewArchiver(w, "~/fastzip-archiving") if err != nil { panic(err) } defer a.Close() // Register a non-default level compressor if required // a.RegisterCompressor(zip.Deflate, fastzip.FlateCompressor(1)) // Walk directory, adding the files we want to add files := make(map[string]os.FileInfo) err = filepath.Walk("~/fastzip-archiving", func(pathname string, info os.FileInfo, err error) error { files[pathname] = info return nil }) // Archive if err = a.Archive(context.Background(), files); err != nil { panic(err) } ``` ### Extractor ```go // Create new extractor e, err := fastzip.NewExtractor("archive.zip", "~/fastzip-extraction") if err != nil { panic(err) } defer e.Close() // Extract archive files if err = e.Extract(context.Background()); err != nil { panic(err) } ``` ## Benchmarks Archiving and extracting a Go 1.13 GOROOT directory, 342M, 10308 files. StandardFlate is using `compress/flate`, NonStandardFlate is `klauspost/compress/flate`, both on level 5. This was performed on a server with an SSD and 24-cores. Each test was conducted using the `WithArchiverConcurrency` and `WithExtractorConcurrency` options of 1, 2, 4, 8 and 16. ``` $ go test -bench Benchmark* -archivedir go1.13 -benchtime=30s -timeout=20m goos: linux goarch: amd64 pkg: github.com/saracen/fastzip BenchmarkArchiveStore_1-24 39 788604969 ns/op 421.66 MB/s 9395405 B/op 266271 allocs/op BenchmarkArchiveStandardFlate_1-24 2 16154127468 ns/op 20.58 MB/s 12075824 B/op 257251 allocs/op BenchmarkArchiveStandardFlate_2-24 4 8686391074 ns/op 38.28 MB/s 15898644 B/op 260757 allocs/op BenchmarkArchiveStandardFlate_4-24 7 4391603068 ns/op 75.72 MB/s 19295604 B/op 260871 allocs/op BenchmarkArchiveStandardFlate_8-24 14 2291624196 ns/op 145.10 MB/s 21999205 B/op 260970 allocs/op BenchmarkArchiveStandardFlate_16-24 16 2105056696 ns/op 157.96 MB/s 29237232 B/op 261225 allocs/op BenchmarkArchiveNonStandardFlate_1-24 6 6011250439 ns/op 55.32 MB/s 11070960 B/op 257204 allocs/op BenchmarkArchiveNonStandardFlate_2-24 9 3629347294 ns/op 91.62 MB/s 18870130 B/op 262279 allocs/op BenchmarkArchiveNonStandardFlate_4-24 18 1766182097 ns/op 188.27 MB/s 22976928 B/op 262349 allocs/op BenchmarkArchiveNonStandardFlate_8-24 34 1002516188 ns/op 331.69 MB/s 29860872 B/op 262473 allocs/op BenchmarkArchiveNonStandardFlate_16-24 46 757112363 ns/op 439.20 MB/s 42036132 B/op 262714 allocs/op BenchmarkExtractStore_1-24 20 1625582744 ns/op 202.66 MB/s 22900375 B/op 330528 allocs/op BenchmarkExtractStore_2-24 42 786644031 ns/op 418.80 MB/s 22307976 B/op 329272 allocs/op BenchmarkExtractStore_4-24 92 384075767 ns/op 857.76 MB/s 22247288 B/op 328667 allocs/op BenchmarkExtractStore_8-24 165 215884636 ns/op 1526.02 MB/s 22354996 B/op 328459 allocs/op BenchmarkExtractStore_16-24 226 157087517 ns/op 2097.20 MB/s 22258691 B/op 328393 allocs/op BenchmarkExtractStandardFlate_1-24 6 5501808448 ns/op 23.47 MB/s 86148462 B/op 495586 allocs/op BenchmarkExtractStandardFlate_2-24 13 2748387174 ns/op 46.99 MB/s 84232141 B/op 491343 allocs/op BenchmarkExtractStandardFlate_4-24 21 1511063035 ns/op 85.47 MB/s 84998750 B/op 490124 allocs/op BenchmarkExtractStandardFlate_8-24 32 995911009 ns/op 129.67 MB/s 86188957 B/op 489574 allocs/op BenchmarkExtractStandardFlate_16-24 46 652641882 ns/op 197.88 MB/s 88256113 B/op 489575 allocs/op BenchmarkExtractNonStandardFlate_1-24 7 4989810851 ns/op 25.88 MB/s 64552948 B/op 373541 allocs/op BenchmarkExtractNonStandardFlate_2-24 13 2478287953 ns/op 52.11 MB/s 63413947 B/op 373183 allocs/op BenchmarkExtractNonStandardFlate_4-24 26 1333552250 ns/op 96.84 MB/s 63546389 B/op 373925 allocs/op BenchmarkExtractNonStandardFlate_8-24 37 817039739 ns/op 158.06 MB/s 64354655 B/op 375357 allocs/op BenchmarkExtractNonStandardFlate_16-24 63 566984549 ns/op 227.77 MB/s 65444227 B/op 379664 allocs/op ``` fastzip-0.1.11/archiver.go000066400000000000000000000224441444057633600154260ustar00rootroot00000000000000package fastzip import ( "bufio" "context" "fmt" "io" "os" "path/filepath" "runtime" "sort" "strings" "sync" "sync/atomic" "time" "unicode/utf8" "github.com/klauspost/compress/zip" "github.com/klauspost/compress/zstd" "github.com/saracen/fastzip/internal/filepool" "github.com/saracen/zipextra" "golang.org/x/sync/errgroup" ) const irregularModes = os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeNamedPipe var bufioReaderPool = sync.Pool{ New: func() interface{} { return bufio.NewReaderSize(nil, 32*1024) }, } var ( defaultCompressor = FlateCompressor(-1) defaultZstdCompressor = ZstdCompressor(int(zstd.SpeedDefault)) ) // Archiver is an opinionated Zip archiver. // // Only regular files, symlinks and directories are supported. Only files that // are children of the specified chroot directory will be archived. // // Access permissions, ownership (unix) and modification times are preserved. type Archiver struct { // This 2 fields are accessed via atomic operations // They are at the start of the struct so they are properly 8 byte aligned written, entries int64 zw *zip.Writer options archiverOptions chroot string m sync.Mutex compressors map[uint16]zip.Compressor } // NewArchiver returns a new Archiver. func NewArchiver(w io.Writer, chroot string, opts ...ArchiverOption) (*Archiver, error) { var err error if chroot, err = filepath.Abs(chroot); err != nil { return nil, err } a := &Archiver{ chroot: chroot, compressors: make(map[uint16]zip.Compressor), } a.options.method = zip.Deflate a.options.concurrency = runtime.GOMAXPROCS(0) a.options.stageDir = chroot a.options.bufferSize = -1 for _, o := range opts { err := o(&a.options) if err != nil { return nil, err } } a.zw = zip.NewWriter(w) a.zw.SetOffset(a.options.offset) // register flate compressor a.RegisterCompressor(zip.Deflate, defaultCompressor) a.RegisterCompressor(zstd.ZipMethodWinZip, defaultZstdCompressor) return a, nil } // RegisterCompressor registers custom compressors for a specified method ID. // The common methods Store and Deflate are built in. func (a *Archiver) RegisterCompressor(method uint16, comp zip.Compressor) { a.zw.RegisterCompressor(method, comp) a.compressors[method] = comp } // Close closes the underlying ZipWriter. func (a *Archiver) Close() error { return a.zw.Close() } // Written returns how many bytes and entries have been written to the archive. // Written can be called whilst archiving is in progress. func (a *Archiver) Written() (bytes, entries int64) { return atomic.LoadInt64(&a.written), atomic.LoadInt64(&a.entries) } // Archive archives all files, symlinks and directories. func (a *Archiver) Archive(ctx context.Context, files map[string]os.FileInfo) (err error) { names := make([]string, 0, len(files)) for name := range files { names = append(names, name) } sort.Strings(names) var fp *filepool.FilePool concurrency := a.options.concurrency if len(files) < concurrency { concurrency = len(files) } if concurrency > 1 { fp, err = filepool.New(a.options.stageDir, concurrency, a.options.bufferSize) if err != nil { return err } defer dclose(fp, &err) } wg, ctx := errgroup.WithContext(ctx) defer func() { if werr := wg.Wait(); werr != nil { err = werr } }() hdrs := make([]zip.FileHeader, len(names)) for i, name := range names { fi := files[name] if fi.Mode()&irregularModes != 0 { continue } path, err := filepath.Abs(name) if err != nil { return err } if !strings.HasPrefix(path, a.chroot+string(filepath.Separator)) && path != a.chroot { return fmt.Errorf("%s cannot be archived from outside of chroot (%s)", name, a.chroot) } rel, err := filepath.Rel(a.chroot, path) if err != nil { return err } hdr := &hdrs[i] fileInfoHeader(rel, fi, hdr) if ctx.Err() != nil { return ctx.Err() } switch { case hdr.Mode()&os.ModeSymlink != 0: err = a.createSymlink(path, fi, hdr) case hdr.Mode().IsDir(): err = a.createDirectory(fi, hdr) default: if hdr.UncompressedSize64 > 0 { hdr.Method = a.options.method } if fp == nil { err = a.createFile(ctx, path, fi, hdr, nil) incOnSuccess(&a.entries, err) } else { f := fp.Get() wg.Go(func() error { err := a.createFile(ctx, path, fi, hdr, f) fp.Put(f) incOnSuccess(&a.entries, err) return err }) } } if err != nil { return err } } return wg.Wait() } func fileInfoHeader(name string, fi os.FileInfo, hdr *zip.FileHeader) { hdr.Name = filepath.ToSlash(name) hdr.UncompressedSize64 = uint64(fi.Size()) hdr.Modified = fi.ModTime() hdr.SetMode(fi.Mode()) if hdr.Mode().IsDir() { hdr.Name += "/" } const uint32max = (1 << 32) - 1 if hdr.UncompressedSize64 > uint32max { hdr.UncompressedSize = uint32max } else { hdr.UncompressedSize = uint32(hdr.UncompressedSize64) } } func (a *Archiver) createDirectory(fi os.FileInfo, hdr *zip.FileHeader) error { a.m.Lock() defer a.m.Unlock() _, err := a.createHeader(fi, hdr) incOnSuccess(&a.entries, err) return err } func (a *Archiver) createSymlink(path string, fi os.FileInfo, hdr *zip.FileHeader) error { a.m.Lock() defer a.m.Unlock() w, err := a.createHeader(fi, hdr) if err != nil { return err } link, err := os.Readlink(path) if err != nil { return err } _, err = io.WriteString(w, link) incOnSuccess(&a.entries, err) return err } func (a *Archiver) createFile(ctx context.Context, path string, fi os.FileInfo, hdr *zip.FileHeader, tmp *filepool.File) error { f, err := os.Open(path) if err != nil { return err } defer f.Close() return a.compressFile(ctx, f, fi, hdr, tmp) } // compressFile pre-compresses the file first to a file from the filepool, // making use of zip.CreateRaw. This allows for concurrent files to be // compressed and then added to the zip file when ready. // If no filepool file is available (when using a concurrency of 1) or the // compressed file is larger than the uncompressed version, the file is moved // to the zip file using the conventional zip.CreateHeader. func (a *Archiver) compressFile(ctx context.Context, f *os.File, fi os.FileInfo, hdr *zip.FileHeader, tmp *filepool.File) error { comp, ok := a.compressors[hdr.Method] // if we don't have the registered compressor, it most likely means Store is // being used, so we revert to non-concurrent behaviour if !ok || tmp == nil { return a.compressFileSimple(ctx, f, fi, hdr) } fw, err := comp(tmp) if err != nil { return err } br := bufioReaderPool.Get().(*bufio.Reader) defer bufioReaderPool.Put(br) br.Reset(f) _, err = io.Copy(io.MultiWriter(fw, tmp.Hasher()), br) dclose(fw, &err) if err != nil { return err } hdr.CompressedSize64 = tmp.Written() // if compressed file is larger, use the uncompressed version. if hdr.CompressedSize64 > hdr.UncompressedSize64 { f.Seek(0, io.SeekStart) hdr.Method = zip.Store return a.compressFileSimple(ctx, f, fi, hdr) } hdr.CRC32 = tmp.Checksum() a.m.Lock() defer a.m.Unlock() w, err := a.createHeaderRaw(fi, hdr) if err != nil { return err } br.Reset(tmp) _, err = br.WriteTo(countWriter{w, &a.written, ctx}) return err } // compressFileSimple uses the conventional zip.createHeader. This differs from // compressFile as it locks the zip _whilst_ compressing (if the method is not // Store). func (a *Archiver) compressFileSimple(ctx context.Context, f *os.File, fi os.FileInfo, hdr *zip.FileHeader) error { br := bufioReaderPool.Get().(*bufio.Reader) defer bufioReaderPool.Put(br) br.Reset(f) a.m.Lock() defer a.m.Unlock() w, err := a.createHeader(fi, hdr) if err != nil { return err } _, err = br.WriteTo(countWriter{w, &a.written, ctx}) return err } func (a *Archiver) createHeaderRaw(fi os.FileInfo, fh *zip.FileHeader) (io.Writer, error) { // When the standard Go library's version of CreateRaw was added, rather // than solely focus on custom compression in "raw" mode, it also removed // the convenience of setting up common zip flags and timestamp logic. This // here replicates what CreateHeader() does: // https://github.com/golang/go/blob/go1.17/src/archive/zip/writer.go#L271 const zipVersion20 = 20 utf8Valid1, utf8Require1 := detectUTF8(fh.Name) utf8Valid2, utf8Require2 := detectUTF8(fh.Comment) switch { case fh.NonUTF8: fh.Flags &^= 0x800 case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2): fh.Flags |= 0x800 } fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 fh.ReaderVersion = zipVersion20 if !fh.Modified.IsZero() { fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified) fh.Extra = append(fh.Extra, zipextra.NewExtendedTimestamp(fh.Modified).Encode()...) } fh.Flags |= 0x8 return a.createRaw(fi, fh) } // https://github.com/golang/go/blob/go1.17.7/src/archive/zip/writer.go#L229 func detectUTF8(s string) (valid, require bool) { for i := 0; i < len(s); { r, size := utf8.DecodeRuneInString(s[i:]) i += size if r < 0x20 || r > 0x7d || r == 0x5c { if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) { return false, false } require = true } } return true, require } // https://github.com/golang/go/blob/go1.17.7/src/archive/zip/struct.go#L242 func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) { fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9) fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11) return } fastzip-0.1.11/archiver_options.go000066400000000000000000000036161444057633600172010ustar00rootroot00000000000000package fastzip import ( "errors" ) var ( ErrMinConcurrency = errors.New("concurrency must be at least 1") ) // ArchiverOption is an option used when creating an archiver. type ArchiverOption func(*archiverOptions) error type archiverOptions struct { method uint16 concurrency int bufferSize int stageDir string offset int64 } // WithArchiverMethod sets the zip method to be used for compressible files. func WithArchiverMethod(method uint16) ArchiverOption { return func(o *archiverOptions) error { o.method = method return nil } } // WithArchiverConcurrency will set the maximum number of files to be // compressed concurrently. The default is set to GOMAXPROCS. func WithArchiverConcurrency(n int) ArchiverOption { return func(o *archiverOptions) error { if n <= 0 { return ErrMinConcurrency } o.concurrency = n return nil } } // WithArchiverBufferSize sets the buffer size for each file to be compressed // concurrently. If a compressed file's data exceeds the buffer size, a // temporary file is written (to the stage directory) to hold the additional // data. The default is 2 mebibytes, so if concurrency is 16, 32 mebibytes of // memory will be allocated. func WithArchiverBufferSize(n int) ArchiverOption { return func(o *archiverOptions) error { if n < 0 { n = 0 } o.bufferSize = n return nil } } // WithStageDirectory sets the directory to be used to stage compressed files // before they're written to the archive. The default is the directory to be // archived. func WithStageDirectory(dir string) ArchiverOption { return func(o *archiverOptions) error { o.stageDir = dir return nil } } // WithArchiverOffset sets the offset of the beginning of the zip data. This // should be used when zip data is appended to an existing file. func WithArchiverOffset(n int64) ArchiverOption { return func(o *archiverOptions) error { o.offset = n return nil } } fastzip-0.1.11/archiver_test.go000066400000000000000000000334401444057633600164630ustar00rootroot00000000000000package fastzip import ( "context" "flag" "fmt" "io" "io/ioutil" "os" "path/filepath" "runtime" "sort" "strings" "testing" "time" "github.com/klauspost/compress/zip" "github.com/klauspost/compress/zstd" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var fixedModTime = time.Date(2020, time.February, 1, 6, 0, 0, 0, time.UTC) type testFile struct { mode os.FileMode contents string } func testCreateFiles(t *testing.T, files map[string]testFile) (map[string]os.FileInfo, string) { dir := t.TempDir() filenames := make([]string, 0, len(files)) for path := range files { filenames = append(filenames, path) } sort.Strings(filenames) var err error for _, path := range filenames { tf := files[path] path = filepath.Join(dir, path) switch { case tf.mode&os.ModeSymlink != 0 && tf.mode&os.ModeDir != 0: err = os.Symlink(tf.contents, path) case tf.mode&os.ModeDir != 0: err = os.Mkdir(path, tf.mode) case tf.mode&os.ModeSymlink != 0: err = os.Symlink(tf.contents, path) default: err = os.WriteFile(path, []byte(tf.contents), tf.mode) } require.NoError(t, err) require.NoError(t, lchmod(path, tf.mode)) require.NoError(t, lchtimes(path, tf.mode, fixedModTime, fixedModTime)) } archiveFiles := make(map[string]os.FileInfo) err = filepath.Walk(dir, func(pathname string, fi os.FileInfo, err error) error { archiveFiles[pathname] = fi return nil }) require.NoError(t, err) return archiveFiles, dir } func testCreateArchive(t *testing.T, dir string, files map[string]os.FileInfo, fn func(filename, chroot string), opts ...ArchiverOption) { f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, dir, opts...) require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) _, entries := a.Written() require.EqualValues(t, len(files), entries) fn(f.Name(), dir) } func TestArchive(t *testing.T) { symMode := os.FileMode(0777) if runtime.GOOS == "windows" { symMode = 0666 } testFiles := map[string]testFile{ "foo": {mode: os.ModeDir | 0777}, "foo/foo.go": {mode: 0666}, "bar": {mode: os.ModeDir | 0777}, "bar/bar.go": {mode: 0666}, "bar/foo": {mode: os.ModeDir | 0777}, "bar/foo/bar": {mode: os.ModeDir | 0777}, "bar/foo/bar/foo": {mode: os.ModeDir | 0777}, "bar/foo/bar/foo/bar": {mode: 0666}, "bar/symlink": {mode: os.ModeSymlink | symMode, contents: "bar/foo/bar/foo"}, "bar/symlink.go": {mode: os.ModeSymlink | symMode, contents: "foo/foo.go"}, "bar/compressible": {mode: 0666, contents: "11111111111111111111111111111111111111111111111111"}, "bar/uncompressible": {mode: 0666, contents: "A3#bez&OqCusPr)d&D]Vot9Eo0z^5O*VZm3:sO3HptL.H-4cOv"}, "empty_dir": {mode: os.ModeDir | 0777}, "large_file": {mode: 0666, contents: strings.Repeat("abcdefzmkdldjsdfkjsdfsdfiqwpsdfa", 65536)}, } tests := map[string][]ArchiverOption{ "default options": nil, "no buffer": {WithArchiverBufferSize(0)}, "with store": {WithArchiverMethod(zip.Store)}, "with concurrency 2": {WithArchiverConcurrency(2)}, } for tn, opts := range tests { t.Run(tn, func(t *testing.T) { files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) testCreateArchive(t, dir, files, func(filename, chroot string) { for pathname, fi := range testExtract(t, filename, testFiles) { if fi.IsDir() { continue } if runtime.GOOS == "windows" && fi.Mode()&os.ModeSymlink != 0 { continue } assert.Equal(t, fixedModTime.Unix(), fi.ModTime().Unix(), "file %v mod time not equal", pathname) } }, opts...) }) } } func TestArchiveCancelContext(t *testing.T) { twoMB := strings.Repeat("1", 2*1024*1024) testFiles := map[string]testFile{} for i := 0; i < 100; i++ { testFiles[fmt.Sprintf("file_%d", i)] = testFile{mode: 0666, contents: twoMB} } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, dir, WithArchiverConcurrency(1)) a.RegisterCompressor(zip.Deflate, FlateCompressor(1)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan struct{}) go func() { defer func() { done <- struct{}{} }() require.EqualError(t, a.Archive(ctx, files), "context canceled") }() defer func() { require.NoError(t, a.Close()) }() for { select { case <-done: return default: // cancel as soon as any data is written if bytes, _ := a.Written(); bytes > 0 { cancel() } } } } func TestArchiveWithCompressor(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, dir) a.RegisterCompressor(zip.Deflate, FlateCompressor(1)) require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) bytes, entries := a.Written() require.EqualValues(t, 0, bytes) require.EqualValues(t, 3, entries) testExtract(t, f.Name(), testFiles) } func TestArchiveWithMethod(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, dir, WithArchiverMethod(zip.Store)) require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) bytes, entries := a.Written() require.EqualValues(t, 0, bytes) require.EqualValues(t, 3, entries) testExtract(t, f.Name(), testFiles) } func TestArchiveWithStageDirectory(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, chroot := testCreateFiles(t, testFiles) defer os.RemoveAll(chroot) dir := t.TempDir() f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, chroot, WithStageDirectory(dir)) require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) bytes, entries := a.Written() require.EqualValues(t, 0, bytes) require.EqualValues(t, 3, entries) stageFiles, err := os.ReadDir(dir) require.NoError(t, err) require.Zero(t, len(stageFiles)) testExtract(t, f.Name(), testFiles) } func TestArchiveWithConcurrency(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } concurrencyTests := []struct { concurrency int pass bool }{ {-1, false}, {0, false}, {1, true}, {30, true}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) for _, test := range concurrencyTests { func() { f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, dir, WithArchiverConcurrency(test.concurrency)) if !test.pass { require.Error(t, err) return } require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) bytes, entries := a.Written() require.EqualValues(t, 0, bytes) require.EqualValues(t, 3, entries) testExtract(t, f.Name(), testFiles) }() } } func TestArchiveWithBufferSize(t *testing.T) { testFiles := map[string]testFile{ "foobar.go": {mode: 0666}, "compressible": {mode: 0666, contents: "11111111111111111111111111111111111111111111111111"}, "uncompressible": {mode: 0666, contents: "A3#bez&OqCusPr)d&D]Vot9Eo0z^5O*VZm3:sO3HptL.H-4cOv"}, "empty_dir": {mode: os.ModeDir | 0777}, "large_file": {mode: 0666, contents: strings.Repeat("abcdefzmkdldjsdfkjsdfsdfiqwpsdfa", 65536)}, } tests := []struct { buffersize int zero bool }{ {-100, false}, {-2, false}, {-1, false}, {0, true}, {32 * 1024, true}, {64 * 1024, true}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) for _, test := range tests { func() { f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() a, err := NewArchiver(f, dir, WithArchiverBufferSize(test.buffersize)) require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) if !test.zero { require.Equal(t, 0, a.options.bufferSize) } else { require.Equal(t, test.buffersize, a.options.bufferSize) } _, entries := a.Written() require.EqualValues(t, 6, entries) testExtract(t, f.Name(), testFiles) }() } } func TestArchiveChroot(t *testing.T) { dir := t.TempDir() f, err := os.Create(filepath.Join(dir, "archive.zip")) require.NoError(t, err) defer f.Close() require.NoError(t, os.MkdirAll(filepath.Join(dir, "chroot"), 0777)) a, err := NewArchiver(f, filepath.Join(dir, "chroot")) require.NoError(t, err) tests := []struct { paths []string good bool }{ {[]string{"chroot/good"}, true}, {[]string{"chroot/good", "bad"}, false}, {[]string{"bad"}, false}, {[]string{"chroot/../bad"}, false}, {[]string{"chroot/../chroot/good"}, true}, } for _, test := range tests { files := make(map[string]os.FileInfo) for _, filename := range test.paths { w, err := os.Create(filepath.Join(dir, filename)) require.NoError(t, err) stat, err := w.Stat() require.NoError(t, err) require.NoError(t, w.Close()) files[w.Name()] = stat } err = a.Archive(context.Background(), files) if test.good { assert.NoError(t, err) } else { assert.Error(t, err) } } } func TestArchiveWithOffset(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) f, err := ioutil.TempFile("", "fastzip-test") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() f.Seek(1000, io.SeekStart) a, err := NewArchiver(f, dir, WithArchiverOffset(1000)) require.NoError(t, err) require.NoError(t, a.Archive(context.Background(), files)) require.NoError(t, a.Close()) bytes, entries := a.Written() require.EqualValues(t, 0, bytes) require.EqualValues(t, 3, entries) testExtract(t, f.Name(), testFiles) } var archiveDir = flag.String("archivedir", runtime.GOROOT(), "The directory to use for archive benchmarks") func benchmarkArchiveOptions(b *testing.B, stdDeflate bool, options ...ArchiverOption) { files := make(map[string]os.FileInfo) size := int64(0) filepath.Walk(*archiveDir, func(filename string, fi os.FileInfo, err error) error { files[filename] = fi size += fi.Size() return nil }) dir := b.TempDir() options = append(options, WithStageDirectory(dir)) b.ReportAllocs() b.SetBytes(size) b.ResetTimer() for n := 0; n < b.N; n++ { f, err := os.Create(filepath.Join(dir, "fastzip-benchmark.zip")) require.NoError(b, err) a, err := NewArchiver(f, *archiveDir, options...) if stdDeflate { a.RegisterCompressor(zip.Deflate, StdFlateCompressor(-1)) } else { a.RegisterCompressor(zip.Deflate, FlateCompressor(-1)) } require.NoError(b, err) err = a.Archive(context.Background(), files) require.NoError(b, err) require.NoError(b, a.Close()) require.NoError(b, f.Close()) require.NoError(b, os.Remove(f.Name())) } } func BenchmarkArchiveStore_1(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(1), WithArchiverMethod(zip.Store)) } func BenchmarkArchiveStandardFlate_1(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(1)) } func BenchmarkArchiveStandardFlate_2(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(2)) } func BenchmarkArchiveStandardFlate_4(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(4)) } func BenchmarkArchiveStandardFlate_8(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(8)) } func BenchmarkArchiveStandardFlate_16(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(16)) } func BenchmarkArchiveNonStandardFlate_1(b *testing.B) { benchmarkArchiveOptions(b, false, WithArchiverConcurrency(1)) } func BenchmarkArchiveNonStandardFlate_2(b *testing.B) { benchmarkArchiveOptions(b, false, WithArchiverConcurrency(2)) } func BenchmarkArchiveNonStandardFlate_4(b *testing.B) { benchmarkArchiveOptions(b, false, WithArchiverConcurrency(4)) } func BenchmarkArchiveNonStandardFlate_8(b *testing.B) { benchmarkArchiveOptions(b, false, WithArchiverConcurrency(8)) } func BenchmarkArchiveNonStandardFlate_16(b *testing.B) { benchmarkArchiveOptions(b, false, WithArchiverConcurrency(16), WithArchiverMethod(zstd.ZipMethodWinZip)) } func BenchmarkArchiveZstd_1(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(1), WithArchiverMethod(zstd.ZipMethodWinZip)) } func BenchmarkArchiveZstd_2(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(2), WithArchiverMethod(zstd.ZipMethodWinZip)) } func BenchmarkArchiveZstd_4(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(4), WithArchiverMethod(zstd.ZipMethodWinZip)) } func BenchmarkArchiveZstd_8(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(8), WithArchiverMethod(zstd.ZipMethodWinZip)) } func BenchmarkArchiveZstd_16(b *testing.B) { benchmarkArchiveOptions(b, true, WithArchiverConcurrency(16), WithArchiverMethod(zstd.ZipMethodWinZip)) } fastzip-0.1.11/archiver_unix.go000066400000000000000000000014141444057633600164630ustar00rootroot00000000000000//go:build !windows // +build !windows package fastzip import ( "io" "math/big" "os" "syscall" "github.com/klauspost/compress/zip" "github.com/saracen/zipextra" ) func (a *Archiver) createHeader(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) { stat, ok := fi.Sys().(*syscall.Stat_t) if ok { hdr.Extra = append(hdr.Extra, zipextra.NewInfoZIPNewUnix(big.NewInt(int64(stat.Uid)), big.NewInt(int64(stat.Gid))).Encode()...) } return a.zw.CreateHeader(hdr) } func (a *Archiver) createRaw(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) { stat, ok := fi.Sys().(*syscall.Stat_t) if ok { hdr.Extra = append(hdr.Extra, zipextra.NewInfoZIPNewUnix(big.NewInt(int64(stat.Uid)), big.NewInt(int64(stat.Gid))).Encode()...) } return a.zw.CreateRaw(hdr) } fastzip-0.1.11/archiver_windows.go000066400000000000000000000005461444057633600171770ustar00rootroot00000000000000//go:build windows // +build windows package fastzip import ( "io" "os" "github.com/klauspost/compress/zip" ) func (a *Archiver) createHeader(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) { return a.zw.CreateHeader(hdr) } func (a *Archiver) createRaw(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) { return a.zw.CreateRaw(hdr) } fastzip-0.1.11/extractor.go000066400000000000000000000160731444057633600156370ustar00rootroot00000000000000package fastzip import ( "bufio" "context" "fmt" "io" "os" "path/filepath" "runtime" "strings" "sync" "sync/atomic" "time" "github.com/klauspost/compress/zip" "github.com/klauspost/compress/zstd" "github.com/saracen/zipextra" "golang.org/x/sync/errgroup" ) var bufioWriterPool = sync.Pool{ New: func() interface{} { return bufio.NewWriterSize(nil, 32*1024) }, } var ( defaultDecompressor = FlateDecompressor() defaultZstdDecompressor = ZstdDecompressor() ) // Extractor is an opinionated Zip file extractor. // // Files are extracted in parallel. Only regular files, symlinks and directories // are supported. Files can only be extracted to the specified chroot directory. // // Access permissions, ownership (unix) and modification times are preserved. type Extractor struct { // This 2 fields are accessed via atomic operations // They are at the start of the struct so they are properly 8 byte aligned written, entries int64 zr *zip.Reader closer io.Closer m sync.Mutex options extractorOptions chroot string } // NewExtractor opens a zip file and returns a new extractor. // // Close() should be called to close the extractor's underlying zip.Reader // when done. func NewExtractor(filename, chroot string, opts ...ExtractorOption) (*Extractor, error) { zr, err := zip.OpenReader(filename) if err != nil { return nil, err } return newExtractor(&zr.Reader, zr, chroot, opts) } // NewExtractor returns a new extractor, reading from the reader provided. // // The size of the archive should be provided. // // Unlike with NewExtractor(), calling Close() on the extractor is unnecessary. func NewExtractorFromReader(r io.ReaderAt, size int64, chroot string, opts ...ExtractorOption) (*Extractor, error) { zr, err := zip.NewReader(r, size) if err != nil { return nil, err } return newExtractor(zr, nil, chroot, opts) } func newExtractor(r *zip.Reader, c io.Closer, chroot string, opts []ExtractorOption) (*Extractor, error) { var err error if chroot, err = filepath.Abs(chroot); err != nil { return nil, err } e := &Extractor{ chroot: chroot, zr: r, closer: c, } e.options.concurrency = runtime.GOMAXPROCS(0) for _, o := range opts { err := o(&e.options) if err != nil { return nil, err } } e.RegisterDecompressor(zip.Deflate, defaultDecompressor) e.RegisterDecompressor(zstd.ZipMethodWinZip, defaultZstdDecompressor) return e, nil } // RegisterDecompressor allows custom decompressors for a specified method ID. // The common methods Store and Deflate are built in. func (e *Extractor) RegisterDecompressor(method uint16, dcomp zip.Decompressor) { e.zr.RegisterDecompressor(method, dcomp) } // Files returns the file within the archive. func (e *Extractor) Files() []*zip.File { return e.zr.File } // Close closes the underlying ZipReader. func (e *Extractor) Close() error { if e.closer == nil { return nil } return e.closer.Close() } // Written returns how many bytes and entries have been written to disk. // Written can be called whilst extraction is in progress. func (e *Extractor) Written() (bytes, entries int64) { return atomic.LoadInt64(&e.written), atomic.LoadInt64(&e.entries) } // Extract extracts files, creates symlinks and directories from the // archive. func (e *Extractor) Extract(ctx context.Context) (err error) { limiter := make(chan struct{}, e.options.concurrency) wg, ctx := errgroup.WithContext(ctx) defer func() { if werr := wg.Wait(); werr != nil { err = werr } }() for i, file := range e.zr.File { if file.Mode()&irregularModes != 0 { continue } var path string path, err = filepath.Abs(filepath.Join(e.chroot, file.Name)) if err != nil { return err } if !strings.HasPrefix(path, e.chroot+string(filepath.Separator)) && path != e.chroot { return fmt.Errorf("%s cannot be extracted outside of chroot (%s)", path, e.chroot) } if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { return err } if ctx.Err() != nil { return ctx.Err() } switch { case file.Mode()&os.ModeSymlink != 0: // defer the creation of symlinks // this is to prevent a traversal vulnerability where a symlink is // first created and then files are additional extracted into it continue case file.Mode().IsDir(): err = e.createDirectory(path, file) default: limiter <- struct{}{} gf := e.zr.File[i] wg.Go(func() error { defer func() { <-limiter }() err := e.createFile(ctx, path, gf) if err == nil { err = e.updateFileMetadata(path, gf) } return err }) } if err != nil { return err } } if err := wg.Wait(); err != nil { return err } // handle deferred symlink creation and update directory metadata // (otherwise modification dates are incorrect) for _, file := range e.zr.File { if file.Mode()&os.ModeSymlink == 0 && !file.Mode().IsDir() { continue } path, err := filepath.Abs(filepath.Join(e.chroot, file.Name)) if err != nil { return err } if file.Mode()&os.ModeSymlink != 0 { if err := e.createSymlink(path, file); err != nil { return err } continue } err = e.updateFileMetadata(path, file) if err != nil { return err } } return nil } func (e *Extractor) createDirectory(path string, file *zip.File) error { err := os.Mkdir(path, 0777) if os.IsExist(err) { err = nil } incOnSuccess(&e.entries, err) return err } func (e *Extractor) createSymlink(path string, file *zip.File) error { if err := os.Remove(path); err != nil && !os.IsNotExist(err) { return err } r, err := file.Open() if err != nil { return err } defer r.Close() name, err := io.ReadAll(r) if err != nil { return err } if err := os.Symlink(string(name), path); err != nil { return err } err = e.updateFileMetadata(path, file) incOnSuccess(&e.entries, err) return err } func (e *Extractor) createFile(ctx context.Context, path string, file *zip.File) (err error) { if err := os.Remove(path); err != nil && !os.IsNotExist(err) { return err } r, err := file.Open() if err != nil { return err } defer dclose(r, &err) f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return err } defer dclose(f, &err) bw := bufioWriterPool.Get().(*bufio.Writer) defer bufioWriterPool.Put(bw) bw.Reset(countWriter{f, &e.written, ctx}) if _, err = bw.ReadFrom(r); err != nil { return err } err = bw.Flush() incOnSuccess(&e.entries, err) return err } func (e *Extractor) updateFileMetadata(path string, file *zip.File) error { fields, err := zipextra.Parse(file.Extra) if err != nil { return err } if err := lchtimes(path, file.Mode(), time.Now(), file.Modified); err != nil { return err } if err := lchmod(path, file.Mode()); err != nil { return err } unixfield, ok := fields[zipextra.ExtraFieldUnixN] if !ok { return nil } unix, err := unixfield.InfoZIPNewUnix() if err != nil { return err } err = lchown(path, int(unix.Uid.Int64()), int(unix.Gid.Int64())) if err == nil { return nil } if e.options.chownErrorHandler == nil { return nil } e.m.Lock() defer e.m.Unlock() return e.options.chownErrorHandler(file.Name, err) } fastzip-0.1.11/extractor_options.go000066400000000000000000000017341444057633600174100ustar00rootroot00000000000000package fastzip // ExtractorOption is an option used when creating an extractor. type ExtractorOption func(*extractorOptions) error type extractorOptions struct { concurrency int chownErrorHandler func(name string, err error) error } // WithExtractorConcurrency will set the maximum number of files being // extracted concurrently. The default is set to GOMAXPROCS. func WithExtractorConcurrency(n int) ExtractorOption { return func(o *extractorOptions) error { if n <= 0 { return ErrMinConcurrency } o.concurrency = n return nil } } // WithExtractorChownErrorHandler sets an error handler to be called if errors are // encountered when trying to preserve ownership of extracted files. Returning // nil will continue extraction, returning any error will cause Extract() to // error. func WithExtractorChownErrorHandler(fn func(name string, err error) error) ExtractorOption { return func(o *extractorOptions) error { o.chownErrorHandler = fn return nil } } fastzip-0.1.11/extractor_test.go000066400000000000000000000221371444057633600166740ustar00rootroot00000000000000package fastzip import ( "context" "fmt" "os" "path/filepath" "strings" "testing" "github.com/klauspost/compress/zip" "github.com/klauspost/compress/zstd" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func testExtract(t *testing.T, filename string, files map[string]testFile) map[string]os.FileInfo { dir := t.TempDir() e, err := NewExtractor(filename, dir) require.NoError(t, err) defer e.Close() for _, f := range e.Files() { assert.Equal(t, filepath.ToSlash(f.Name), f.Name, "zip file path separator not /") } require.NoError(t, e.Extract(context.Background())) result := make(map[string]os.FileInfo) err = filepath.Walk(dir, func(pathname string, fi os.FileInfo, err error) error { if err != nil { return err } rel, err := filepath.Rel(dir, pathname) if err != nil { return err } if rel == "." { return nil } rel = filepath.ToSlash(rel) require.Contains(t, files, rel) result[pathname] = fi mode := files[rel].mode assert.Equal(t, mode.Perm(), fi.Mode().Perm(), "file %v perm not equal", rel) assert.Equal(t, mode.IsDir(), fi.IsDir(), "file %v is_dir not equal", rel) assert.Equal(t, mode&os.ModeSymlink, fi.Mode()&os.ModeSymlink, "file %v mode symlink not equal", rel) if fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 { return nil } contents, err := os.ReadFile(pathname) require.NoError(t, err) assert.Equal(t, string(files[rel].contents), string(contents)) return nil }) require.NoError(t, err) return result } func TestExtractCancelContext(t *testing.T) { twoMB := strings.Repeat("1", 2*1024*1024) testFiles := map[string]testFile{} for i := 0; i < 100; i++ { testFiles[fmt.Sprintf("file_%d", i)] = testFile{mode: 0666, contents: twoMB} } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) testCreateArchive(t, dir, files, func(filename, chroot string) { e, err := NewExtractor(filename, dir, WithExtractorConcurrency(1)) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan struct{}) go func() { defer func() { done <- struct{}{} }() require.EqualError(t, e.Extract(ctx), "context canceled") }() for { select { case <-done: return default: // cancel as soon as any data is written if bytes, _ := e.Written(); bytes > 0 { cancel() } } } }) } func TestExtractorWithDecompressor(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) testCreateArchive(t, dir, files, func(filename, chroot string) { e, err := NewExtractor(filename, dir) require.NoError(t, err) e.RegisterDecompressor(zip.Deflate, StdFlateDecompressor()) defer e.Close() require.NoError(t, e.Extract(context.Background())) }) } func TestExtractorWithConcurrency(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } concurrencyTests := []struct { concurrency int pass bool }{ {-1, false}, {0, false}, {1, true}, {30, true}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) testCreateArchive(t, dir, files, func(filename, chroot string) { for _, test := range concurrencyTests { e, err := NewExtractor(filename, dir, WithExtractorConcurrency(test.concurrency)) if test.pass { assert.NoError(t, err) require.NoError(t, e.Close()) } else { assert.Error(t, err) } } }) } func TestExtractorWithChownErrorHandler(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) testCreateArchive(t, dir, files, func(filename, chroot string) { e, err := NewExtractor(filename, dir, WithExtractorChownErrorHandler(func(name string, err error) error { assert.Fail(t, "should have no error") return nil })) assert.NoError(t, err) assert.NoError(t, e.Extract(context.Background())) require.NoError(t, e.Close()) }) } func TestExtractorFromReader(t *testing.T) { testFiles := map[string]testFile{ "foo.go": {mode: 0666}, "bar.go": {mode: 0666}, } files, dir := testCreateFiles(t, testFiles) defer os.RemoveAll(dir) testCreateArchive(t, dir, files, func(filename, chroot string) { f, err := os.Open(filename) require.NoError(t, err) fi, err := f.Stat() require.NoError(t, err) e, err := NewExtractorFromReader(f, fi.Size(), chroot) require.NoError(t, err) require.NoError(t, e.Extract(context.Background())) require.NoError(t, e.Close()) }) } func TestExtractorDetectSymlinkTraversal(t *testing.T) { dir := t.TempDir() archivePath := filepath.Join(dir, "vuln.zip") f, err := os.Create(archivePath) require.NoError(t, err) zw := zip.NewWriter(f) // create symlink symlink := &zip.FileHeader{Name: "root/inner"} symlink.SetMode(os.ModeSymlink) w, err := zw.CreateHeader(symlink) require.NoError(t, err) _, err = w.Write([]byte("../")) require.NoError(t, err) // create file within symlink _, err = zw.Create("root/inner/vuln") require.NoError(t, err) zw.Close() f.Close() e, err := NewExtractor(archivePath, dir) require.NoError(t, err) defer e.Close() require.Error(t, e.Extract(context.Background())) } func aopts(options ...ArchiverOption) []ArchiverOption { return options } func benchmarkExtractOptions(b *testing.B, stdDeflate bool, ao []ArchiverOption, eo ...ExtractorOption) { files := make(map[string]os.FileInfo) filepath.Walk(*archiveDir, func(filename string, fi os.FileInfo, err error) error { files[filename] = fi return nil }) dir := b.TempDir() archiveName := filepath.Join(dir, "fastzip-benchmark-extract.zip") f, err := os.Create(archiveName) require.NoError(b, err) defer os.Remove(f.Name()) ao = append(ao, WithStageDirectory(dir)) a, err := NewArchiver(f, *archiveDir, ao...) require.NoError(b, err) err = a.Archive(context.Background(), files) require.NoError(b, err) require.NoError(b, a.Close()) require.NoError(b, f.Close()) b.ReportAllocs() b.ResetTimer() fi, _ := os.Stat(archiveName) b.SetBytes(fi.Size()) for n := 0; n < b.N; n++ { e, err := NewExtractor(archiveName, dir, eo...) if stdDeflate { e.RegisterDecompressor(zip.Deflate, StdFlateDecompressor()) } require.NoError(b, err) require.NoError(b, e.Extract(context.Background())) } } func BenchmarkExtractStore_1(b *testing.B) { benchmarkExtractOptions(b, true, aopts(WithArchiverMethod(zip.Store)), WithExtractorConcurrency(1)) } func BenchmarkExtractStore_2(b *testing.B) { benchmarkExtractOptions(b, true, aopts(WithArchiverMethod(zip.Store)), WithExtractorConcurrency(2)) } func BenchmarkExtractStore_4(b *testing.B) { benchmarkExtractOptions(b, true, aopts(WithArchiverMethod(zip.Store)), WithExtractorConcurrency(4)) } func BenchmarkExtractStore_8(b *testing.B) { benchmarkExtractOptions(b, true, aopts(WithArchiverMethod(zip.Store)), WithExtractorConcurrency(8)) } func BenchmarkExtractStore_16(b *testing.B) { benchmarkExtractOptions(b, true, aopts(WithArchiverMethod(zip.Store)), WithExtractorConcurrency(16)) } func BenchmarkExtractStandardFlate_1(b *testing.B) { benchmarkExtractOptions(b, true, nil, WithExtractorConcurrency(1)) } func BenchmarkExtractStandardFlate_2(b *testing.B) { benchmarkExtractOptions(b, true, nil, WithExtractorConcurrency(2)) } func BenchmarkExtractStandardFlate_4(b *testing.B) { benchmarkExtractOptions(b, true, nil, WithExtractorConcurrency(4)) } func BenchmarkExtractStandardFlate_8(b *testing.B) { benchmarkExtractOptions(b, true, nil, WithExtractorConcurrency(8)) } func BenchmarkExtractStandardFlate_16(b *testing.B) { benchmarkExtractOptions(b, true, nil, WithExtractorConcurrency(16)) } func BenchmarkExtractNonStandardFlate_1(b *testing.B) { benchmarkExtractOptions(b, false, nil, WithExtractorConcurrency(1)) } func BenchmarkExtractNonStandardFlate_2(b *testing.B) { benchmarkExtractOptions(b, false, nil, WithExtractorConcurrency(2)) } func BenchmarkExtractNonStandardFlate_4(b *testing.B) { benchmarkExtractOptions(b, false, nil, WithExtractorConcurrency(4)) } func BenchmarkExtractNonStandardFlate_8(b *testing.B) { benchmarkExtractOptions(b, false, nil, WithExtractorConcurrency(8)) } func BenchmarkExtractNonStandardFlate_16(b *testing.B) { benchmarkExtractOptions(b, false, nil, WithExtractorConcurrency(16)) } func BenchmarkExtractZstd_1(b *testing.B) { benchmarkExtractOptions(b, false, aopts(WithArchiverMethod(zstd.ZipMethodWinZip)), WithExtractorConcurrency(1)) } func BenchmarkExtractZstd_2(b *testing.B) { benchmarkExtractOptions(b, false, aopts(WithArchiverMethod(zstd.ZipMethodWinZip)), WithExtractorConcurrency(2)) } func BenchmarkExtractZstd_4(b *testing.B) { benchmarkExtractOptions(b, false, aopts(WithArchiverMethod(zstd.ZipMethodWinZip)), WithExtractorConcurrency(4)) } func BenchmarkExtractZstd_8(b *testing.B) { benchmarkExtractOptions(b, false, aopts(WithArchiverMethod(zstd.ZipMethodWinZip)), WithExtractorConcurrency(8)) } func BenchmarkExtractZstd_16(b *testing.B) { benchmarkExtractOptions(b, false, aopts(WithArchiverMethod(zstd.ZipMethodWinZip)), WithExtractorConcurrency(16)) } fastzip-0.1.11/extractor_unix.go000066400000000000000000000015411444057633600166740ustar00rootroot00000000000000// +build !windows package fastzip import ( "os" "runtime" "time" "golang.org/x/sys/unix" ) func lchmod(name string, mode os.FileMode) error { var flags int if runtime.GOOS == "linux" { if mode&os.ModeSymlink != 0 { return nil } } else { flags = unix.AT_SYMLINK_NOFOLLOW } err := unix.Fchmodat(unix.AT_FDCWD, name, uint32(mode), flags) if err != nil { return &os.PathError{Op: "lchmod", Path: name, Err: err} } return nil } func lchtimes(name string, mode os.FileMode, atime, mtime time.Time) error { at := unix.NsecToTimeval(atime.UnixNano()) mt := unix.NsecToTimeval(mtime.UnixNano()) tv := [2]unix.Timeval{at, mt} err := unix.Lutimes(name, tv[:]) if err != nil { return &os.PathError{Op: "lchtimes", Path: name, Err: err} } return nil } func lchown(name string, uid, gid int) error { return os.Lchown(name, uid, gid) } fastzip-0.1.11/extractor_windows.go000066400000000000000000000006451444057633600174070ustar00rootroot00000000000000// +build windows package fastzip import ( "os" "time" ) func lchmod(name string, mode os.FileMode) error { if mode&os.ModeSymlink != 0 { return nil } return os.Chmod(name, mode) } func lchtimes(name string, mode os.FileMode, atime, mtime time.Time) error { if mode&os.ModeSymlink != 0 { return nil } return os.Chtimes(name, atime, mtime) } func lchown(name string, uid, gid int) error { return nil } fastzip-0.1.11/go.mod000066400000000000000000000006111444057633600143720ustar00rootroot00000000000000module github.com/saracen/fastzip go 1.18 require ( github.com/klauspost/compress v1.16.5 github.com/saracen/zipextra v0.0.0-20220303013732-0187cb0159ea github.com/stretchr/testify v1.8.3 golang.org/x/sync v0.2.0 golang.org/x/sys v0.8.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) fastzip-0.1.11/go.sum000066400000000000000000000030711444057633600144220ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saracen/zipextra v0.0.0-20220303013732-0187cb0159ea h1:8czYLkvzZRE+AElIQeDffQdgR+CC3wKEFILYU/1PeX4= github.com/saracen/zipextra v0.0.0-20220303013732-0187cb0159ea/go.mod h1:hnzuad9d2wdd3z8fC6UouHQK5qZxqv3F/E6MMzXc7q0= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= fastzip-0.1.11/internal/000077500000000000000000000000001444057633600151025ustar00rootroot00000000000000fastzip-0.1.11/internal/filepool/000077500000000000000000000000001444057633600167135ustar00rootroot00000000000000fastzip-0.1.11/internal/filepool/filepool.go000066400000000000000000000064131444057633600210570ustar00rootroot00000000000000package filepool import ( "errors" "fmt" "hash" "hash/crc32" "io" "os" "path/filepath" "strings" ) var ErrPoolSizeLessThanZero = errors.New("pool size must be greater than zero") const defaultBufferSize = 2 * 1024 * 1024 type filePoolCloseError []error func (e filePoolCloseError) Len() int { return len(e) } func (e filePoolCloseError) Error() string { if len(e) == 1 { return e[0].Error() } var sb strings.Builder for _, err := range e { sb.WriteString(err.Error() + "\n") } return sb.String() } func (e filePoolCloseError) Unwrap() error { if len(e) > 1 { return e[1:] } return nil } // FilePool represents a pool of files that can be used as buffers. type FilePool struct { files []*File limiter chan int } // New returns a new FilePool. func New(dir string, poolSize int, bufferSize int) (*FilePool, error) { if poolSize <= 0 { return nil, ErrPoolSizeLessThanZero } fp := &FilePool{} fp.files = make([]*File, poolSize) fp.limiter = make(chan int, poolSize) if bufferSize < 0 { bufferSize = defaultBufferSize } for i := range fp.files { fp.files[i] = newFile(dir, i, bufferSize) fp.limiter <- i } return fp, nil } // Get gets a file from the pool. func (fp *FilePool) Get() *File { idx := <-fp.limiter return fp.files[idx] } // Put puts a file back into the pool. func (fp *FilePool) Put(f *File) { f.reset() fp.limiter <- f.idx } // Close closes and removes all files in the pool. func (fp *FilePool) Close() error { var err filePoolCloseError for _, f := range fp.files { if f == nil || f.f == nil { continue } if cerr := f.f.Close(); cerr != nil { err = append(err, cerr) } if rerr := os.Remove(f.f.Name()); rerr != nil && !os.IsNotExist(rerr) { err = append(err, rerr) } } fp.files = nil if err.Len() > 0 { return err } return nil } // File is a file backed buffer. type File struct { dir string idx int w int64 r int64 crc hash.Hash32 f *os.File buf []byte size int } func newFile(dir string, idx, size int) *File { return &File{ dir: dir, idx: idx, size: size, crc: crc32.NewIEEE(), } } func (f *File) Write(p []byte) (n int, err error) { if f.buf == nil && f.size > 0 { f.buf = make([]byte, f.size) } if f.w < int64(len(f.buf)) { n = copy(f.buf[f.w:], p) p = p[n:] f.w += int64(n) } if len(p) > 0 { if f.f == nil { f.f, err = os.Create(filepath.Join(f.dir, fmt.Sprintf("fastzip_%02d", f.idx))) if err != nil { return n, err } } bn := n n, err = f.f.WriteAt(p, f.w-int64(len(f.buf))) f.w += int64(n) n += bn if err != nil { return n, err } } return n, err } func (f *File) Read(p []byte) (n int, err error) { remaining := f.w - f.r if remaining <= 0 { return 0, io.EOF } if int64(len(p)) > remaining { p = p[:remaining] } if f.r < int64(len(f.buf)) { n = copy(p, f.buf[f.r:]) f.r += int64(n) p = p[n:] } if len(p) > 0 && f.r >= int64(len(f.buf)) { bn := n n, err = f.f.ReadAt(p, f.r-int64(len(f.buf))) f.r += int64(n) n += bn } return n, err } func (f *File) Written() uint64 { return uint64(f.w) } func (f *File) Hasher() io.Writer { return f.crc } func (f *File) Checksum() uint32 { return f.crc.Sum32() } func (f *File) reset() { f.w = 0 f.r = 0 f.crc.Reset() if f.f != nil { f.f.Truncate(0) } } fastzip-0.1.11/internal/filepool/filepool_test.go000066400000000000000000000100741444057633600221140ustar00rootroot00000000000000package filepool import ( "bytes" "errors" "fmt" "io" "os" "path/filepath" "runtime" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestFilePoolSizes(t *testing.T) { tests := []struct { size int err error }{ {-1, ErrPoolSizeLessThanZero}, {0, ErrPoolSizeLessThanZero}, {4, nil}, {8, nil}, } for _, tc := range tests { t.Run(fmt.Sprintf("size %d", tc.size), func(t *testing.T) { dir := t.TempDir() fp, err := New(dir, tc.size, 0) require.Equal(t, tc.err, err) if tc.err != nil { return } // writing should produce the temporary file for i := 0; i < tc.size; i++ { f := fp.Get() _, err = f.Write([]byte("foobar")) assert.NoError(t, err) fp.Put(f) _, err = os.Lstat(filepath.Join(dir, fmt.Sprintf("fastzip_%02d", i))) assert.NoError(t, err, fmt.Sprintf("fastzip_%02d should exist", i)) } // closing should cleanup temporary files assert.NoError(t, fp.Close()) for i := 0; i < tc.size; i++ { _, err = os.Lstat(filepath.Join(dir, fmt.Sprintf("fastzip_%02d", i))) assert.Error(t, err, fmt.Sprintf("fastzip_%02d shouldn't exist", i)) } }) } } func TestFilePoolReset(t *testing.T) { dir := t.TempDir() fp, err := New(dir, 16, 0) require.NoError(t, err) for i := range fp.files { file := fp.Get() _, err = file.Write(bytes.Repeat([]byte("0"), i)) assert.NoError(t, err) b, err := io.ReadAll(file) assert.NoError(t, err) assert.Len(t, b, i) assert.Equal(t, uint64(i), file.Written()) _, err = file.Hasher().Write([]byte("hello")) assert.NoError(t, err) assert.Equal(t, uint32(0x3610a686), file.Checksum()) fp.Put(file) } for range fp.files { file := fp.Get() b, err := io.ReadAll(file) assert.NoError(t, err) assert.Len(t, b, 0) assert.Equal(t, uint64(0), file.Written()) assert.Equal(t, uint32(0), file.Checksum()) fp.Put(file) } assert.NoError(t, fp.Close()) } func TestFilePoolCloseError(t *testing.T) { dir := t.TempDir() fp, err := New(dir, 16, 0) require.NoError(t, err) for _, file := range fp.files { f := fp.Get() _, err := f.Write([]byte("foobar")) assert.NoError(t, err) fp.Put(f) require.NoError(t, file.f.Close()) } err = fp.Close() require.Error(t, err, "expected already closed error") assert.Contains(t, err.Error(), "file already closed\n") count := 0 for { count++ if err = errors.Unwrap(err); err == nil { break } } assert.Equal(t, 16, count) } func TestFilePoolNoErrorOnAlreadyDeleted(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Skipping test on windows (cannot delete in-use file)") } dir := t.TempDir() fp, err := New(dir, 16, 0) require.NoError(t, err) for range fp.files { f := fp.Get() _, err := f.Write([]byte("foobar")) assert.NoError(t, err) fp.Put(f) } err = os.RemoveAll(dir) require.NoError(t, err) assert.NoError(t, fp.Close()) } func TestFilePoolFileBuffer(t *testing.T) { dir := t.TempDir() tests := map[string]struct { data []byte fileExists bool }{ "below buffer length": { data: []byte("123456789"), fileExists: false, }, "equal to buffer length": { data: []byte("1234567890"), fileExists: false, }, "above buffer length": { data: []byte("1234567890x"), fileExists: true, }, } for tn, tc := range tests { t.Run(tn, func(t *testing.T) { fp, err := New(dir, 1, 10) require.NoError(t, err) defer fp.Close() require.Len(t, fp.files, 1) f := fp.files[0] n, err := f.Write(tc.data) assert.NoError(t, err) assert.Equal(t, len(tc.data), n) _, err = os.Lstat(filepath.Join(dir, "fastzip_00")) if tc.fileExists { assert.NoError(t, err, "fastzip_00 should exist") } else { assert.Error(t, err, "fastzip_00 should not exist") } // split reads to ensure read/write indexes track correctly buf := make([]byte, 20) size := 0 { n, err := f.Read(buf[:5]) assert.NoError(t, err) size += n } { n, err := f.Read(buf[5:]) assert.NoError(t, err) size += n } assert.Equal(t, tc.data, buf[:size]) }) } } fastzip-0.1.11/register.go000066400000000000000000000074471444057633600154550ustar00rootroot00000000000000package fastzip import ( "bufio" "io" "sync" stdflate "compress/flate" "github.com/klauspost/compress/flate" "github.com/klauspost/compress/zstd" ) type flater interface { Close() error Flush() error Reset(dst io.Writer) Write(data []byte) (n int, err error) } func newFlateReaderPool(newReaderFn func(w io.Reader) io.ReadCloser) *sync.Pool { pool := &sync.Pool{} pool.New = func() interface{} { return &flateReader{pool, bufio.NewReaderSize(nil, 32*1024), newReaderFn(nil)} } return pool } type flateReader struct { pool *sync.Pool buf *bufio.Reader io.ReadCloser } func (fr *flateReader) Reset(r io.Reader) { fr.buf.Reset(r) fr.ReadCloser.(flate.Resetter).Reset(fr.buf, nil) } func (fr *flateReader) Close() error { err := fr.ReadCloser.Close() fr.pool.Put(fr) return err } // FlateDecompressor returns a pooled performant zip.Decompressor. func FlateDecompressor() func(r io.Reader) io.ReadCloser { pool := newFlateReaderPool(flate.NewReader) return func(r io.Reader) io.ReadCloser { fr := pool.Get().(*flateReader) fr.Reset(r) return fr } } // StdFlateDecompressor returns a pooled standard library zip.Decompressor. func StdFlateDecompressor() func(r io.Reader) io.ReadCloser { pool := newFlateReaderPool(stdflate.NewReader) return func(r io.Reader) io.ReadCloser { fr := pool.Get().(*flateReader) fr.Reset(r) return fr } } type zstdReader struct { pool *sync.Pool buf *bufio.Reader *zstd.Decoder } func (zr *zstdReader) Close() error { err := zr.Decoder.Reset(nil) zr.pool.Put(zr) return err } // ZstdDecompressor returns a pooled zstd decoder. func ZstdDecompressor() func(r io.Reader) io.ReadCloser { pool := &sync.Pool{} pool.New = func() interface{} { r, _ := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderMaxWindow(128<<20), zstd.WithDecoderConcurrency(1)) return &zstdReader{pool, bufio.NewReaderSize(nil, 32*1024), r} } return func(r io.Reader) io.ReadCloser { fr := pool.Get().(*zstdReader) fr.Decoder.Reset(r) return fr } } func newFlateWriterPool(level int, newWriterFn func(w io.Writer, level int) (flater, error)) *sync.Pool { pool := &sync.Pool{} pool.New = func() interface{} { fw, err := newWriterFn(nil, level) if err != nil { panic(err) } return &flateWriter{pool, fw} } return pool } type flateWriter struct { pool *sync.Pool flater } func (fw *flateWriter) Reset(w io.Writer) { fw.flater.Reset(w) } func (fw *flateWriter) Close() error { err := fw.flater.Close() fw.pool.Put(fw) return err } // FlateCompressor returns a pooled performant zip.Compressor configured to a // specified compression level. Invalid flate levels will panic. func FlateCompressor(level int) func(w io.Writer) (io.WriteCloser, error) { pool := newFlateWriterPool(level, func(w io.Writer, level int) (flater, error) { return flate.NewWriter(w, level) }) return func(w io.Writer) (io.WriteCloser, error) { fw := pool.Get().(*flateWriter) fw.Reset(w) return fw, nil } } // StdFlateCompressor returns a pooled standard library zip.Compressor // configured to a specified compression level. Invalid flate levels will // panic. func StdFlateCompressor(level int) func(w io.Writer) (io.WriteCloser, error) { pool := newFlateWriterPool(level, func(w io.Writer, level int) (flater, error) { return stdflate.NewWriter(w, level) }) return func(w io.Writer) (io.WriteCloser, error) { fw := pool.Get().(*flateWriter) fw.Reset(w) return fw, nil } } func ZstdCompressor(level int) func(w io.Writer) (io.WriteCloser, error) { pool := newFlateWriterPool(level, func(w io.Writer, level int) (flater, error) { return zstd.NewWriter(w, zstd.WithEncoderCRC(false), zstd.WithEncoderLevel(zstd.EncoderLevel(level))) }) return func(w io.Writer) (io.WriteCloser, error) { fw := pool.Get().(*flateWriter) fw.Reset(w) return fw, nil } } fastzip-0.1.11/util.go000066400000000000000000000010231444057633600145660ustar00rootroot00000000000000package fastzip import ( "context" "io" "sync/atomic" ) func dclose(c io.Closer, err *error) { if cerr := c.Close(); cerr != nil && *err == nil { *err = cerr } } func incOnSuccess(inc *int64, err error) { if err == nil { atomic.AddInt64(inc, 1) } } type countWriter struct { w io.Writer written *int64 ctx context.Context } func (w countWriter) Write(p []byte) (n int, err error) { if err = w.ctx.Err(); err == nil { n, err = w.w.Write(p) atomic.AddInt64(w.written, int64(n)) } return n, err }