pax_global_header00006660000000000000000000000064144475765430014535gustar00rootroot0000000000000052 comment=84acd0aec6c1b253e1c50fdec5e1c9d514c0c58a reflink-1.0.1/000077500000000000000000000000001444757654300131665ustar00rootroot00000000000000reflink-1.0.1/.gitignore000066400000000000000000000000071444757654300151530ustar00rootroot00000000000000.*.swp reflink-1.0.1/LICENSE000066400000000000000000000020631444757654300141740ustar00rootroot00000000000000MIT License Copyright (c) 2020 Karpelès Lab Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. reflink-1.0.1/Makefile000066400000000000000000000004411444757654300146250ustar00rootroot00000000000000#!/bin/make GOROOT:=$(shell PATH="/pkg/main/dev-lang.go/bin:$$PATH" go env GOROOT) GO=$(GOROOT)/bin/go GOPATH:=$(shell $(GO) env GOPATH) .PHONY: test deps all: $(GOPATH)/bin/goimports -w -l . $(GO) build -v ./... deps: $(GO) get -v -t ./... test: $(GO) test -v ./... -test.count 1 reflink-1.0.1/README.md000066400000000000000000000024321444757654300144460ustar00rootroot00000000000000[![GoDoc](https://godoc.org/github.com/KarpelesLab/reflink?status.svg)](https://godoc.org/github.com/KarpelesLab/reflink) # reflink Perform reflink operation on compatible filesystems (btrfs or xfs). ## What is a reflink? There are a number of type of links existing on Linux: * symlinks * hardlinks * reflinks Reflinks are a new kind of links found in btrfs and xfs which act similar to hard links, except modifying one of the two files will not change the other, and typically only the changed data will take space on the disk (copy-on-write). ## Can I use reflinks? A machine needs to have a compatible OS and filesystem to perform reflinks. Known to work are: * btrfs on Linux * xfs on Linux Other OSes have similar features, to be implemented in the future. * Windows has `DUPLICATE_EXTENTS_TO_FILE` * Solaris has `reflink` * MacOS has `clonefile` ## Usage ```golang err := reflink.Always("original_file.bin", "snapshot-001.bin") // or err := reflink.Auto("source_img.png", "modified_img.png") ``` `reflink.Always` will fail if reflink is not supported or didn't work for any reason, while `reflink.Auto` will fallback to a regular file copy. # Notes * The arguments have been put in the same order as `os.Link` or `os.Rename` rather than `io.Copy` as we are dealing with filenames. reflink-1.0.1/api.go000066400000000000000000000072731444757654300142770ustar00rootroot00000000000000package reflink import ( "fmt" "io" "io/fs" "io/ioutil" "os" "path/filepath" ) // Always will perform a reflink operation and fail on error. // // This is equivalent to command cp --reflink=always func Always(src, dst string) error { return reflinkFile(src, dst, false) } // Auto will attempt to perform a reflink operation and fallback to normal data // copy if reflink is not supported. // // This is equivalent to cp --reflink=auto func Auto(src, dst string) error { return reflinkFile(src, dst, true) } // reflinkFile perform the reflink operation in order to copy src into dst using // the underlying filesystem's copy-on-write reflink system. If this fails (for // example the filesystem does not support reflink) and fallback is true, then // copy_file_range will be used, and if that fails too io.Copy will be used to // copy the data. func reflinkFile(src, dst string, fallback bool) error { s, err := os.Open(src) if err != nil { return err } defer s.Close() // generate temporary file for output tmp, err := ioutil.TempFile(filepath.Dir(dst), "") if err != nil { return err } // copy to temp file err = reflinkInternal(tmp, s) // if reflink failed but we allow fallback, first attempt using copyFileRange (will actually clone bytes on some filesystems) if (err != nil) && fallback { var st fs.FileInfo st, err = s.Stat() if err == nil { _, err = copyFileRange(tmp, s, 0, 0, st.Size()) } } // if everything failed and we fallback, attempt io.Copy if (err != nil) && fallback { // reflink failed but fallback enabled, perform a normal copy instead _, err = io.Copy(tmp, s) } tmp.Close() // we're not writing to this anymore // if an error happened, remove temp file and signal error if err != nil { os.Remove(tmp.Name()) return err } // keep src file mode if possible if st, err := s.Stat(); err == nil { tmp.Chmod(st.Mode()) } // replace dst file err = os.Rename(tmp.Name(), dst) if err != nil { // failed to rename (dst is not writable?) os.Remove(tmp.Name()) return err } return nil } // Reflink performs the reflink operation on the passed files, replacing // dst's contents with src. If fallback is true and reflink fails, // copy_file_range will be used first, and if that fails too io.Copy will // be used to copy the data. func Reflink(dst, src *os.File, fallback bool) error { err := reflinkInternal(dst, src) if (err != nil) && fallback { // reflink failed, but we can fallback, but first we need to know the file's size var st fs.FileInfo st, err = src.Stat() if err != nil { // couldn't stat source, this can't be helped return fmt.Errorf("failed to stat source: %w", err) } _, err = copyFileRange(dst, src, 0, 0, st.Size()) if err != nil { // copyFileRange failed too, switch to simple io copy reader := io.NewSectionReader(src, 0, st.Size()) writer := §ionWriter{w: dst} dst.Truncate(0) // assuming any error in trucate will result in copy error _, err = io.Copy(writer, reader) } } return err } // Partial performs a range reflink operation on the passed files, replacing // part of dst's contents with data from src. If fallback is true and reflink // fails, copy_file_range will be used first, and if that fails too io.CopyN // will be used to copy the data. func Partial(dst, src *os.File, dstOffset, srcOffset, n int64, fallback bool) error { err := reflinkRangeInternal(dst, src, dstOffset, srcOffset, n) if (err != nil) && fallback { _, err = copyFileRange(dst, src, dstOffset, srcOffset, n) } if (err != nil) && fallback { // seek both src & dst reader := io.NewSectionReader(src, srcOffset, n) writer := §ionWriter{w: dst, base: dstOffset} _, err = io.CopyN(writer, reader, n) } return err } reflink-1.0.1/errors.go000066400000000000000000000005471444757654300150370ustar00rootroot00000000000000package reflink import "errors" // ErrReflinkUnsupported is returned by Always() if the operation is not // supported on the current operating system. Auto() will never return this // error. var ( ErrReflinkUnsupported = errors.New("reflink is not supported on this OS") ErrReflinkFailed = errors.New("reflink is not supported on this OS or file") ) reflink-1.0.1/go.mod000066400000000000000000000001341444757654300142720ustar00rootroot00000000000000module github.com/KarpelesLab/reflink go 1.19 require golang.org/x/sys v0.9.0 // indirect reflink-1.0.1/go.sum000066400000000000000000000002271444757654300143220ustar00rootroot00000000000000golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= reflink-1.0.1/reflink.go000066400000000000000000000005511444757654300151500ustar00rootroot00000000000000//go:build !linux package reflink import "os" func reflinkInternal(d, s *os.File) error { return ErrReflinkUnsupported } func reflinkRangeInternal(dst, src *os.File, dstOffset, srcOffset, n int64) error { return ErrReflinkUnsupported } func copyFileRange(dst, src *os.File, dstOffset, srcOffset, n int64) (int64, error) { return ErrReflinkUnsupported } reflink-1.0.1/reflink_linux.go000066400000000000000000000043451444757654300163740ustar00rootroot00000000000000//go:build linux package reflink import ( "errors" "os" "golang.org/x/sys/unix" ) // reflinkInternal performs the actual reflink action without worrying about fallback func reflinkInternal(d, s *os.File) error { ss, err := s.SyscallConn() if err != nil { return err } sd, err := d.SyscallConn() if err != nil { return err } var err2, err3 error err = sd.Control(func(dfd uintptr) { err2 = ss.Control(func(sfd uintptr) { // int ioctl(int dest_fd, FICLONE, int src_fd); err3 = unix.IoctlFileClone(int(dfd), int(sfd)) }) }) if err != nil { // sd.Control failed return err } if err2 != nil { // ss.Control failed return err2 } if err3 != nil && errors.Is(err3, unix.ENOTSUP) { return ErrReflinkFailed } // err3 is ioctl() response return err3 } func reflinkRangeInternal(dst, src *os.File, dstOffset, srcOffset, n int64) error { ss, err := src.SyscallConn() if err != nil { return err } sd, err := dst.SyscallConn() if err != nil { return err } var err2, err3 error err = sd.Control(func(dfd uintptr) { err2 = ss.Control(func(sfd uintptr) { req := &unix.FileCloneRange{ Src_fd: int64(sfd), Src_offset: uint64(srcOffset), Src_length: uint64(n), Dest_offset: uint64(dstOffset), } // int ioctl(int dest_fd, FICLONE, int src_fd); err3 = unix.IoctlFileCloneRange(int(dfd), req) }) }) if err != nil { // sd.Control failed return err } if err2 != nil { // ss.Control failed return err2 } if err3 != nil && errors.Is(err3, unix.ENOTSUP) { return ErrReflinkFailed } // err3 is ioctl() response return err3 } func copyFileRange(dst, src *os.File, dstOffset, srcOffset, n int64) (int64, error) { ss, err := src.SyscallConn() if err != nil { return 0, err } sd, err := dst.SyscallConn() if err != nil { return 0, err } var resN int var err2, err3 error err = sd.Control(func(dfd uintptr) { err2 = ss.Control(func(sfd uintptr) { // call syscall resN, err3 = unix.CopyFileRange(int(sfd), &srcOffset, int(dfd), &dstOffset, int(n), 0) }) }) if err != nil { // sd.Control failed return int64(resN), err } if err2 != nil { // ss.Control failed return int64(resN), err2 } // err3 is ioctl() response return int64(resN), err3 } reflink-1.0.1/reflink_test.go000066400000000000000000000052331444757654300162110ustar00rootroot00000000000000package reflink_test import ( "bytes" "crypto/rand" "errors" "io" "os" "path/filepath" "testing" "github.com/KarpelesLab/reflink" ) func TestReflink(t *testing.T) { d, err := os.MkdirTemp("", "reflinktest*") if err != nil { t.Fatalf("failed to create temporary directory: %s", err) return } defer os.RemoveAll(d) buf := make([]byte, 1024*1024) // 1MB _, err = io.ReadFull(rand.Reader, buf) if err != nil { t.Errorf("failed to fill test buffer with random bytes: %s", err) } err = os.WriteFile(filepath.Join(d, "src.bin"), buf, 0666) if err != nil { t.Fatalf("failed to create initial test file: %s", err) return } // perform reflink err = reflink.Always(filepath.Join(d, "src.bin"), filepath.Join(d, "test1.bin")) if err != nil { if errors.Is(err, reflink.ErrReflinkUnsupported) { t.Logf("cannot test reflink on this OS: %s", err) } else if errors.Is(err, reflink.ErrReflinkFailed) { t.Logf("cannot test reflink on this configuration: %s", err) } else { t.Errorf("failed to reflink.Always: %s", err) } } // perform reflink auto err = reflink.Auto(filepath.Join(d, "src.bin"), filepath.Join(d, "test2.bin")) if err != nil { t.Errorf("failed to reflink.Auto: %s", err) } err = testFile(filepath.Join(d, "test2.bin"), buf) if err != nil { t.Errorf("bad output file for reflink.Auto: %s", err) } in, err := os.Open(filepath.Join(d, "src.bin")) if err != nil { t.Fatalf("failed to open source file for reading: %s", err) return } defer in.Close() out, err := os.Create(filepath.Join(d, "test3.bin")) if err != nil { t.Fatalf("failed to create target file for writing: %s", err) return } defer out.Close() err = reflink.Reflink(out, in, true) if err != nil { t.Errorf("reflink on file failed: %s", err) } err = testOsFile(out, buf) if err != nil { t.Errorf("reflink target file content fails: %s", err) } out.Truncate(0) err = reflink.Partial(out, in, 0, 512*1024, 256*1024, true) if err != nil { t.Errorf("failed to reflink.Partial(fallback=true): %s", err) } err = testOsFile(out, buf[512*1024:(512+256)*1024]) if err != nil { t.Errorf("reflink target file content fails: %s", err) } } func testFile(fn string, target []byte) error { data, err := os.ReadFile(fn) if err != nil { return err } if !bytes.Equal(data, target) { return errors.New("file content does not match") } return nil } func testOsFile(f *os.File, target []byte) error { st, err := f.Stat() if err != nil { return err } r := io.NewSectionReader(f, 0, st.Size()) buf, err := io.ReadAll(r) if err != nil { return err } if !bytes.Equal(buf, target) { return errors.New("file content does not match") } return nil } reflink-1.0.1/writer.go000066400000000000000000000015271444757654300150360ustar00rootroot00000000000000package reflink import ( "errors" "io" ) // sectionWriter is a helper used when we need to fallback into copying data manually type sectionWriter struct { w io.WriterAt // target file base int64 // base position in file off int64 // current relative offset } // Write writes & updates offset func (s *sectionWriter) Write(p []byte) (int, error) { n, err := s.w.WriteAt(p, s.base+s.off) s.off += int64(n) return n, err } func (s *sectionWriter) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: // nothing needed case io.SeekCurrent: offset += s.off case io.SeekEnd: // we don't support io.SeekEnd fallthrough default: return s.off, errors.New("Seek: invalid whence") } if offset < 0 { return s.off, errors.New("Seek: invalid offset") } s.off = offset return offset, nil }