pax_global_header00006660000000000000000000000064136204410610014507gustar00rootroot0000000000000052 comment=39625986c7abc45f3f89218a90db2f1f3e8b1850 golang-github-karrick-godirwalk-1.15.3/000077500000000000000000000000001362044106100177325ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/.gitignore000066400000000000000000000006421362044106100217240ustar00rootroot00000000000000# Binaries for programs and plugins *.exe *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ examples/remove-empty-directories/remove-empty-directories examples/sizes/sizes examples/walk-fast/walk-fast examples/walk-stdlib/walk-stdlib golang-github-karrick-godirwalk-1.15.3/LICENSE000066400000000000000000000024521362044106100207420ustar00rootroot00000000000000BSD 2-Clause License Copyright (c) 2017, Karrick McDermott All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. golang-github-karrick-godirwalk-1.15.3/README.md000066400000000000000000000251661362044106100212230ustar00rootroot00000000000000# godirwalk `godirwalk` is a library for traversing a directory tree on a file system. [![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk) [![Build Status](https://dev.azure.com/microsoft0235/microsoft/_apis/build/status/karrick.godirwalk?branchName=master)](https://dev.azure.com/microsoft0235/microsoft/_build/latest?definitionId=1&branchName=master) In short, why do I use this library? 1. It's faster than `filepath.Walk`. 1. It's more correct on Windows than `filepath.Walk`. 1. It's more easy to use than `filepath.Walk`. 1. It's more flexible than `filepath.Walk`. ## Usage Example Additional examples are provided in the `examples/` subdirectory. This library will normalize the provided top level directory name based on the os-specific path separator by calling `filepath.Clean` on its first argument. However it always provides the pathname created by using the correct os-specific path separator when invoking the provided callback function. ```Go dirname := "some/directory/root" err := godirwalk.Walk(dirname, &godirwalk.Options{ Callback: func(osPathname string, de *godirwalk.Dirent) error { fmt.Printf("%s %s\n", de.ModeType(), osPathname) return nil }, Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc) }) ``` This library not only provides functions for traversing a file system directory tree, but also for obtaining a list of immediate descendants of a particular directory, typically much more quickly than using `os.ReadDir` or `os.ReadDirnames`. ## Description Here's why I use `godirwalk` in preference to `filepath.Walk`, `os.ReadDir`, and `os.ReadDirnames`. ### It's faster than `filepath.Walk` When compared against `filepath.Walk` in benchmarks, it has been observed to run between five and ten times the speed on darwin, at speeds comparable to the that of the unix `find` utility; about twice the speed on linux; and about four times the speed on Windows. How does it obtain this performance boost? It does less work to give you nearly the same output. This library calls the same `syscall` functions to do the work, but it makes fewer calls, does not throw away information that it might need, and creates less memory churn along the way by reusing the same scratch buffer for reading from a directory rather than reallocating a new buffer every time it reads file system entry data from the operating system. While traversing a file system directory tree, `filepath.Walk` obtains the list of immediate descendants of a directory, and throws away the file system node type information provided by the operating system that comes with the node's name. Then, immediately prior to invoking the callback function, `filepath.Walk` invokes `os.Stat` for each node, and passes the returned `os.FileInfo` information to the callback. While the `os.FileInfo` information provided by `os.Stat` is extremely helpful--and even includes the `os.FileMode` data--providing it requires an additional system call for each node. Because most callbacks only care about what the node type is, this library does not throw the type information away, but rather provides that information to the callback function in the form of a `os.FileMode` value. Note that the provided `os.FileMode` value that this library provides only has the node type information, and does not have the permission bits, sticky bits, or other information from the file's mode. If the callback does care about a particular node's entire `os.FileInfo` data structure, the callback can easiy invoke `os.Stat` when needed, and only when needed. #### Benchmarks ##### macOS ```Bash $ go test -bench=. -benchmem goos: darwin goarch: amd64 pkg: github.com/karrick/godirwalk BenchmarkReadDirnamesStandardLibrary-12 50000 26250 ns/op 10360 B/op 16 allocs/op BenchmarkReadDirnamesThisLibrary-12 50000 24372 ns/op 5064 B/op 20 allocs/op BenchmarkFilepathWalk-12 1 1099524875 ns/op 228415912 B/op 416952 allocs/op BenchmarkGodirwalk-12 2 526754589 ns/op 103110464 B/op 451442 allocs/op BenchmarkGodirwalkUnsorted-12 3 509219296 ns/op 100751400 B/op 378800 allocs/op BenchmarkFlameGraphFilepathWalk-12 1 7478618820 ns/op 2284138176 B/op 4169453 allocs/op BenchmarkFlameGraphGodirwalk-12 1 4977264058 ns/op 1031105328 B/op 4514423 allocs/op PASS ok github.com/karrick/godirwalk 21.219s ``` ##### Linux ```Bash $ go test -bench=. -benchmem goos: linux goarch: amd64 pkg: github.com/karrick/godirwalk BenchmarkReadDirnamesStandardLibrary-12 100000 15458 ns/op 10360 B/op 16 allocs/op BenchmarkReadDirnamesThisLibrary-12 100000 14646 ns/op 5064 B/op 20 allocs/op BenchmarkFilepathWalk-12 2 631034745 ns/op 228210216 B/op 416939 allocs/op BenchmarkGodirwalk-12 3 358714883 ns/op 102988664 B/op 451437 allocs/op BenchmarkGodirwalkUnsorted-12 3 355363915 ns/op 100629234 B/op 378796 allocs/op BenchmarkFlameGraphFilepathWalk-12 1 6086913991 ns/op 2282104720 B/op 4169417 allocs/op BenchmarkFlameGraphGodirwalk-12 1 3456398824 ns/op 1029886400 B/op 4514373 allocs/op PASS ok github.com/karrick/godirwalk 19.179s ``` ### It's more correct on Windows than `filepath.Walk` I did not previously care about this either, but humor me. We all love how we can write once and run everywhere. It is essential for the language's adoption, growth, and success, that the software we create can run unmodified on all architectures and operating systems supported by Go. When the traversed file system has a logical loop caused by symbolic links to directories, on unix `filepath.Walk` ignores symbolic links and traverses the entire directory tree without error. On Windows however, `filepath.Walk` will continue following directory symbolic links, even though it is not supposed to, eventually causing `filepath.Walk` to terminate early and return an error when the pathname gets too long from concatenating endless loops of symbolic links onto the pathname. This error comes from Windows, passes through `filepath.Walk`, and to the upstream client running `filepath.Walk`. The takeaway is that behavior is different based on which platform `filepath.Walk` is running. While this is clearly not intentional, until it is fixed in the standard library, it presents a compatibility problem. This library correctly identifies symbolic links that point to directories and will only follow them when `FollowSymbolicLinks` is set to true. Behavior on Windows and other operating systems is identical. ### It's more easy to use than `filepath.Walk` Since this library does not invoke `os.Stat` on every file system node it encounters, there is no possible error event for the callback function to filter on. The third argument in the `filepath.WalkFunc` function signature to pass the error from `os.Stat` to the callback function is no longer necessary, and thus eliminated from signature of the callback function from this library. Also, `filepath.Walk` invokes the callback function with a solidus delimited pathname regardless of the os-specific path separator. This library invokes the callback function with the os-specific pathname separator, obviating a call to `filepath.Clean` in the callback function for each node prior to actually using the provided pathname. In other words, even on Windows, `filepath.Walk` will invoke the callback with `some/path/to/foo.txt`, requiring well written clients to perform pathname normalization for every file prior to working with the specified file. In truth, many clients developed on unix and not tested on Windows neglect this subtlety, and will result in software bugs when running on Windows. This library would invoke the callback function with `some\path\to\foo.txt` for the same file when running on Windows, eliminating the need to normalize the pathname by the client, and lessen the likelyhood that a client will work on unix but not on Windows. ### It's more flexible than `filepath.Walk` #### Configurable Handling of Symbolic Links The default behavior of this library is to ignore symbolic links to directories when walking a directory tree, just like `filepath.Walk` does. However, it does invoke the callback function with each node it finds, including symbolic links. If a particular use case exists to follow symbolic links when traversing a directory tree, this library can be invoked in manner to do so, by setting the `FollowSymbolicLinks` parameter to true. #### Configurable Sorting of Directory Children The default behavior of this library is to always sort the immediate descendants of a directory prior to visiting each node, just like `filepath.Walk` does. This is usually the desired behavior. However, this does come at slight performance and memory penalties required to sort the names when a directory node has many entries. Additionally if caller specifies `Unsorted` enumeration, reading directories is lazily performed as the caller consumes entries. If a particular use case exists that does not require sorting the directory's immediate descendants prior to visiting its nodes, this library will skip the sorting step when the `Unsorted` parameter is set to true. Here's an interesting read of the potential hazzards of traversing a file system hierarchy in a non-deterministic order. If you know the problem you are solving is not affected by the order files are visited, then I encourage you to use `Unsorted`. Otherwise skip setting this option. [Researchers find bug in Python script may have affected hundreds of studies](https://arstechnica.com/information-technology/2019/10/chemists-discover-cross-platform-python-scripts-not-so-cross-platform/) #### Configurable Post Children Callback This library provides upstream code with the ability to specify a callback to be invoked for each directory after its children are processed. This has been used to recursively delete empty directories after traversing the file system in a more efficient manner. See the `examples/clean-empties` directory for an example of this usage. #### Configurable Error Callback This library provides upstream code with the ability to specify a callback to be invoked for errors that the operating system returns, allowing the upstream code to determine the next course of action to take, whether to halt walking the hierarchy, as it would do were no error callback provided, or skip the node that caused the error. See the `examples/walk-fast` directory for an example of this usage. golang-github-karrick-godirwalk-1.15.3/azure-pipelines.yml000066400000000000000000000022031362044106100235660ustar00rootroot00000000000000# Go # Build your Go project. # Add steps that test, save build artifacts, deploy, and more: # https://docs.microsoft.com/azure/devops/pipelines/languages/go trigger: - master variables: GOVERSION: 1.13 jobs: - job: Linux pool: vmImage: 'ubuntu-latest' steps: - task: GoTool@0 displayName: 'Use Go $(GOVERSION)' inputs: version: $(GOVERSION) - task: Go@0 inputs: command: test arguments: -race -v ./... displayName: 'Execute Tests' - job: Mac pool: vmImage: 'macos-latest' steps: - task: GoTool@0 displayName: 'Use Go $(GOVERSION)' inputs: version: $(GOVERSION) - task: Go@0 inputs: command: test arguments: -race -v ./... displayName: 'Execute Tests' - job: Windows pool: vmImage: 'windows-latest' steps: - task: GoTool@0 displayName: 'Use Go $(GOVERSION)' inputs: version: $(GOVERSION) - task: Go@0 inputs: command: test arguments: -race -v ./... displayName: 'Execute Tests' golang-github-karrick-godirwalk-1.15.3/debug_development.go000066400000000000000000000004341362044106100237520ustar00rootroot00000000000000// +build godirwalk_debug package godirwalk import ( "fmt" "os" ) // debug formats and prints arguments to stderr for development builds func debug(f string, a ...interface{}) { // fmt.Fprintf(os.Stderr, f, a...) os.Stderr.Write([]byte("godirwalk: " + fmt.Sprintf(f, a...))) } golang-github-karrick-godirwalk-1.15.3/debug_release.go000066400000000000000000000002001362044106100230370ustar00rootroot00000000000000// +build !godirwalk_debug package godirwalk // debug is a no-op for release builds func debug(_ string, _ ...interface{}) {} golang-github-karrick-godirwalk-1.15.3/dirent.go000066400000000000000000000100061362044106100215430ustar00rootroot00000000000000package godirwalk import ( "os" "path/filepath" ) // Dirent stores the name and file system mode type of discovered file system // entries. type Dirent struct { name string // base name of the file system entry. path string // path name of the file system entry. modeType os.FileMode // modeType is the type of file system entry. } // NewDirent returns a newly initialized Dirent structure, or an error. This // function does not follow symbolic links. // // This function is rarely used, as Dirent structures are provided by other // functions in this library that read and walk directories, but is provided, // however, for the occasion when a program needs to create a Dirent. func NewDirent(osPathname string) (*Dirent, error) { modeType, err := modeType(osPathname) if err != nil { return nil, err } return &Dirent{ name: filepath.Base(osPathname), path: filepath.Dir(osPathname), modeType: modeType, }, nil } // IsDir returns true if and only if the Dirent represents a file system // directory. Note that on some operating systems, more than one file mode bit // may be set for a node. For instance, on Windows, a symbolic link that points // to a directory will have both the directory and the symbolic link bits set. func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 } // IsDirOrSymlinkToDir returns true if and only if the Dirent represents a file // system directory, or a symbolic link to a directory. Note that if the Dirent // is not a directory but is a symbolic link, this method will resolve by // sending a request to the operating system to follow the symbolic link. func (de Dirent) IsDirOrSymlinkToDir() (bool, error) { if de.IsDir() { return true, nil } if !de.IsSymlink() { return false, nil } // Does this symlink point to a directory? info, err := os.Stat(filepath.Join(de.path, de.name)) if err != nil { return false, err } return info.IsDir(), nil } // IsRegular returns true if and only if the Dirent represents a regular file. // That is, it ensures that no mode type bits are set. func (de Dirent) IsRegular() bool { return de.modeType&os.ModeType == 0 } // IsSymlink returns true if and only if the Dirent represents a file system // symbolic link. Note that on some operating systems, more than one file mode // bit may be set for a node. For instance, on Windows, a symbolic link that // points to a directory will have both the directory and the symbolic link bits // set. func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 } // IsDevice returns true if and only if the Dirent represents a device file. func (de Dirent) IsDevice() bool { return de.modeType&os.ModeDevice != 0 } // ModeType returns the mode bits that specify the file system node type. We // could make our own enum-like data type for encoding the file type, but Go's // runtime already gives us architecture independent file modes, as discussed in // `os/types.go`: // // Go's runtime FileMode type has same definition on all systems, so that // information about files can be moved from one system to another portably. func (de Dirent) ModeType() os.FileMode { return de.modeType } // Name returns the base name of the file system entry. func (de Dirent) Name() string { return de.name } // reset releases memory held by entry err and name, and resets mode type to 0. func (de *Dirent) reset() { de.name = "" de.path = "" de.modeType = 0 } // Dirents represents a slice of Dirent pointers, which are sortable by base // name. This type satisfies the `sort.Interface` interface. type Dirents []*Dirent // Len returns the count of Dirent structures in the slice. func (l Dirents) Len() int { return len(l) } // Less returns true if and only if the base name of the element specified by // the first index is lexicographically less than that of the second index. func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name } // Swap exchanges the two Dirent entries specified by the two provided indexes. func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] } golang-github-karrick-godirwalk-1.15.3/dirent_test.go000066400000000000000000000066511362044106100226150ustar00rootroot00000000000000package godirwalk import ( "os" "path/filepath" "testing" ) func TestDirent(t *testing.T) { // TODO: IsDevice() should be tested, but that would require updating // scaffolding to create a device. t.Run("file", func(t *testing.T) { de, err := NewDirent(filepath.Join(scaffolingRoot, "d0", "f1")) ensureError(t, err) if got, want := de.Name(), "f1"; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.ModeType(), os.FileMode(0); got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsDir(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } got, err := de.IsDirOrSymlinkToDir() ensureError(t, err) if want := false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsRegular(), true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsSymlink(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) t.Run("directory", func(t *testing.T) { de, err := NewDirent(filepath.Join(scaffolingRoot, "d0")) ensureError(t, err) if got, want := de.Name(), "d0"; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.ModeType(), os.ModeDir; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsDir(), true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } got, err := de.IsDirOrSymlinkToDir() ensureError(t, err) if want := true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsRegular(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsSymlink(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) t.Run("symlink", func(t *testing.T) { t.Run("to file", func(t *testing.T) { de, err := NewDirent(filepath.Join(scaffolingRoot, "d0", "symlinks", "toF1")) ensureError(t, err) if got, want := de.Name(), "toF1"; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.ModeType(), os.ModeSymlink; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsDir(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } got, err := de.IsDirOrSymlinkToDir() ensureError(t, err) if want := false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsRegular(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsSymlink(), true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) t.Run("to directory", func(t *testing.T) { de, err := NewDirent(filepath.Join(scaffolingRoot, "d0", "symlinks", "toD1")) ensureError(t, err) if got, want := de.Name(), "toD1"; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.ModeType(), os.ModeSymlink; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsDir(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } got, err := de.IsDirOrSymlinkToDir() ensureError(t, err) if want := true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsRegular(), false; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := de.IsSymlink(), true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) }) } golang-github-karrick-godirwalk-1.15.3/doc.go000066400000000000000000000025471362044106100210360ustar00rootroot00000000000000/* Package godirwalk provides functions to read and traverse directory trees. In short, why do I use this library? * It's faster than `filepath.Walk`. * It's more correct on Windows than `filepath.Walk`. * It's more easy to use than `filepath.Walk`. * It's more flexible than `filepath.Walk`. USAGE This library will normalize the provided top level directory name based on the os-specific path separator by calling `filepath.Clean` on its first argument. However it always provides the pathname created by using the correct os-specific path separator when invoking the provided callback function. dirname := "some/directory/root" err := godirwalk.Walk(dirname, &godirwalk.Options{ Callback: func(osPathname string, de *godirwalk.Dirent) error { fmt.Printf("%s %s\n", de.ModeType(), osPathname) return nil }, }) This library not only provides functions for traversing a file system directory tree, but also for obtaining a list of immediate descendants of a particular directory, typically much more quickly than using `os.ReadDir` or `os.ReadDirnames`. scratchBuffer := make([]byte, godirwalk.MinimumScratchBufferSize) names, err := godirwalk.ReadDirnames("some/directory", scratchBuffer) // ... entries, err := godirwalk.ReadDirents("another/directory", scratchBuffer) // ... */ package godirwalk golang-github-karrick-godirwalk-1.15.3/ensure_test.go000066400000000000000000000043751362044106100226320ustar00rootroot00000000000000package godirwalk import ( "fmt" "path/filepath" "sort" "strings" "testing" ) func ensureError(tb testing.TB, err error, contains ...string) { tb.Helper() if len(contains) == 0 || (len(contains) == 1 && contains[0] == "") { if err != nil { tb.Fatalf("GOT: %v; WANT: %v", err, contains) } } else if err == nil { tb.Errorf("GOT: %v; WANT: %v", err, contains) } else { for _, stub := range contains { if stub != "" && !strings.Contains(err.Error(), stub) { tb.Errorf("GOT: %v; WANT: %q", err, stub) } } } } func ensureStringSlicesMatch(tb testing.TB, actual, expected []string) { tb.Helper() results := make(map[string]int) for _, s := range actual { results[s] = -1 } for _, s := range expected { results[s]++ } keys := make([]string, 0, len(results)) for k := range results { keys = append(keys, k) } sort.Strings(keys) for _, s := range keys { v, ok := results[s] if !ok { panic(fmt.Errorf("cannot find key: %s", s)) // panic because this function is broken } switch v { case -1: tb.Errorf("GOT: %q (extra)", s) case 0: // both slices have this key case 1: tb.Errorf("WANT: %q (missing)", s) default: panic(fmt.Errorf("key has invalid value: %s: %d", s, v)) // panic because this function is broken } } } func ensureDirentsMatch(tb testing.TB, actual, expected Dirents) { tb.Helper() sort.Sort(actual) sort.Sort(expected) al := len(actual) el := len(expected) var ai, ei int for ai < al || ei < el { if ai == al { tb.Errorf("GOT: %s %s (extra)", expected[ei].Name(), expected[ei].ModeType()) ei++ } else if ei == el { tb.Errorf("WANT: %s %s (missing)", actual[ai].Name(), actual[ai].ModeType()) ai++ } else { epn := filepath.Join(expected[ei].path, expected[ei].Name()) apn := filepath.Join(actual[ai].path, actual[ai].Name()) if apn < epn { tb.Errorf("GOT: %s %s (extra)", apn, actual[ai].ModeType()) ai++ } else if epn < apn { tb.Errorf("WANT: %s %s (missing)", epn, expected[ei].ModeType()) ei++ } else { // names match; check mode types if got, want := actual[ai].ModeType(), expected[ei].ModeType(); got != want { tb.Errorf("GOT: %v; WANT: %v", actual[ai].ModeType(), expected[ei].ModeType()) } ai++ ei++ } } } } golang-github-karrick-godirwalk-1.15.3/examples/000077500000000000000000000000001362044106100215505ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/find-fast/000077500000000000000000000000001362044106100234235ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/find-fast/go.mod000066400000000000000000000004501362044106100245300ustar00rootroot00000000000000module github.com/karrick/godirwalk/examples/find-fast replace github.com/karrick/godirwalk => ../../ require ( github.com/karrick/godirwalk v1.13.5 github.com/karrick/golf v1.4.0 github.com/mattn/go-isatty v0.0.11 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect ) go 1.13 golang-github-karrick-godirwalk-1.15.3/examples/find-fast/go.sum000066400000000000000000000012141362044106100245540ustar00rootroot00000000000000github.com/karrick/golf v1.4.0 h1:9i9HnUh7uCyUFJhIqg311HBibw4f2pbGldi0ZM2FhaQ= github.com/karrick/golf v1.4.0/go.mod h1:qGN0IhcEL+IEgCXp00RvH32UP59vtwc8w5YcIdArNRk= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang-github-karrick-godirwalk-1.15.3/examples/find-fast/main.go000066400000000000000000000054721362044106100247060ustar00rootroot00000000000000/* * find-fast * * Walks a file system hierarchy using this library. */ package main import ( "fmt" "os" "path/filepath" "regexp" "github.com/karrick/godirwalk" "github.com/karrick/golf" "github.com/mattn/go-isatty" ) var NoColor = os.Getenv("TERM") == "dumb" || !(isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())) func main() { optRegex := golf.String("regex", "", "Do not print unless full path matches regex.") optQuiet := golf.Bool("quiet", false, "Do not print intermediate errors to stderr.") golf.Parse() programName, err := os.Executable() if err != nil { programName = os.Args[0] } programName = filepath.Base(programName) var nameRE *regexp.Regexp if *optRegex != "" { nameRE, err = regexp.Compile(*optRegex) if err != nil { fmt.Fprintf(os.Stderr, "%s: invalid regex pattern: %s\n", programName, err) os.Exit(2) } } var buf []byte // only used when color output options := &godirwalk.Options{ ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { if !*optQuiet { fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) } return godirwalk.SkipNode }, Unsorted: true, } switch { case nameRE == nil: // When no name pattern provided, print everything. options.Callback = func(osPathname string, _ *godirwalk.Dirent) error { _, err := fmt.Println(osPathname) return err } case NoColor: // Name pattern was provided, but color not permitted. options.Callback = func(osPathname string, _ *godirwalk.Dirent) error { var err error if nameRE.FindString(osPathname) != "" { _, err = fmt.Println(osPathname) } return err } default: // Name pattern provided, and color is permitted. buf = append(buf, "\033[22m"...) // very first print should set normal intensity options.Callback = func(osPathname string, _ *godirwalk.Dirent) error { matches := nameRE.FindAllStringSubmatchIndex(osPathname, -1) if len(matches) == 0 { return nil // entry does not match pattern } var prev int for _, tuple := range matches { buf = append(buf, osPathname[prev:tuple[0]]...) // print text before match buf = append(buf, "\033[1m"...) // bold intensity buf = append(buf, osPathname[tuple[0]:tuple[1]]...) // print match buf = append(buf, "\033[22m"...) // normal intensity prev = tuple[1] } buf = append(buf, osPathname[prev:]...) // print remaining text after final match _, err := os.Stdout.Write(append(buf, '\n')) // don't forget newline buf = buf[:0] // reset buffer for next string return err } } dirname := "." if golf.NArg() > 0 { dirname = golf.Arg(0) } if err = godirwalk.Walk(dirname, options); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) os.Exit(1) } } golang-github-karrick-godirwalk-1.15.3/examples/remove-empty-directories/000077500000000000000000000000001362044106100265135ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/remove-empty-directories/main.go000066400000000000000000000032571362044106100277750ustar00rootroot00000000000000/* * remove-empty-directories * * Walks a file system hierarchy and removes all directories with no children. */ package main import ( "fmt" "os" "path/filepath" "github.com/karrick/godirwalk" ) func main() { if len(os.Args) < 2 { fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", filepath.Base(os.Args[0])) os.Exit(2) } var count, total int var err error for _, arg := range os.Args[1:] { count, err = pruneEmptyDirectories(arg) total += count if err != nil { break } } fmt.Fprintf(os.Stderr, "Removed %d empty directories\n", total) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) os.Exit(1) } } func pruneEmptyDirectories(osDirname string) (int, error) { var count int err := godirwalk.Walk(osDirname, &godirwalk.Options{ Unsorted: true, Callback: func(_ string, _ *godirwalk.Dirent) error { // no-op while diving in; all the fun happens in PostChildrenCallback return nil }, PostChildrenCallback: func(osPathname string, _ *godirwalk.Dirent) error { s, err := godirwalk.NewScanner(osPathname) if err != nil { return err } // Attempt to read only the first directory entry. Remember that // Scan skips both "." and ".." entries. hasAtLeastOneChild := s.Scan() // If error reading from directory, wrap up and return. if err := s.Err(); err != nil { return err } if hasAtLeastOneChild { return nil // do not remove directory with at least one child } if osPathname == osDirname { return nil // do not remove directory that was provided top-level directory } err = os.Remove(osPathname) if err == nil { count++ } return err }, }) return count, err } golang-github-karrick-godirwalk-1.15.3/examples/scanner/000077500000000000000000000000001362044106100232015ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/scanner/main.go000066400000000000000000000023251362044106100244560ustar00rootroot00000000000000package main import ( "flag" "fmt" "os" "path/filepath" "github.com/karrick/godirwalk" ) func main() { dirname := "." if flag.NArg() > 0 { dirname = flag.Arg(0) } scanner, err := godirwalk.NewScanner(dirname) if err != nil { fatal("cannot scan directory: %s", err) } for scanner.Scan() { dirent, err := scanner.Dirent() if err != nil { warning("cannot get dirent: %s", err) continue } name := dirent.Name() if name == "break" { break } if name == "continue" { continue } fmt.Printf("%v %v\n", dirent.ModeType(), name) } if err := scanner.Err(); err != nil { fatal("cannot scan directory: %s", err) } } var ( optQuiet = flag.Bool("quiet", false, "Elide printing of non-critical error messages.") programName string ) func init() { var err error if programName, err = os.Executable(); err != nil { programName = os.Args[0] } programName = filepath.Base(programName) flag.Parse() } func stderr(f string, args ...interface{}) { fmt.Fprintf(os.Stderr, programName+": "+fmt.Sprintf(f, args...)+"\n") } func fatal(f string, args ...interface{}) { stderr(f, args...) os.Exit(1) } func warning(f string, args ...interface{}) { if !*optQuiet { stderr(f, args...) } } golang-github-karrick-godirwalk-1.15.3/examples/sizes/000077500000000000000000000000001362044106100227055ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/sizes/main.go000066400000000000000000000044351362044106100241660ustar00rootroot00000000000000/* * sizes * * Walks a file system hierarchy and prints sizes of file system objects, * recursively printing sizes of directories. */ package main import ( "fmt" "os" "path/filepath" "github.com/karrick/godirwalk" ) var progname = filepath.Base(os.Args[0]) func main() { if len(os.Args) < 2 { fmt.Fprintf(os.Stderr, "usage: %s dir1 [dir2 [dir3...]]\n", progname) os.Exit(2) } for _, arg := range os.Args[1:] { if err := sizes(arg); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", progname, err) } } } func sizes(osDirname string) error { sizes := newSizesStack() return godirwalk.Walk(osDirname, &godirwalk.Options{ Callback: func(osPathname string, de *godirwalk.Dirent) error { if de.IsDir() { sizes.EnterDirectory() return nil } st, err := os.Stat(osPathname) if err != nil { return err } size := st.Size() sizes.Accumulate(size) _, err = fmt.Printf("%s % 12d %s\n", st.Mode(), size, osPathname) return err }, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { fmt.Fprintf(os.Stderr, "%s: %s\n", progname, err) return godirwalk.SkipNode }, PostChildrenCallback: func(osPathname string, de *godirwalk.Dirent) error { size := sizes.LeaveDirectory() sizes.Accumulate(size) // add this directory's size to parent directory. st, err := os.Stat(osPathname) switch err { case nil: _, err = fmt.Printf("%s % 12d %s\n", st.Mode(), size, osPathname) default: // ignore the error and just show the mode type _, err = fmt.Printf("%s % 12d %s\n", de.ModeType(), size, osPathname) } return err }, }) } // sizesStack encapsulates operations on stack of directory sizes, with similar // but slightly modified LIFO semantics to push and pop on a regular stack. type sizesStack struct { sizes []int64 // stack of sizes top int // index of top of stack } func newSizesStack() *sizesStack { // Initialize with dummy value at top of stack to eliminate special cases. return &sizesStack{sizes: make([]int64, 1, 32)} } func (s *sizesStack) EnterDirectory() { s.sizes = append(s.sizes, 0) s.top++ } func (s *sizesStack) LeaveDirectory() (i int64) { i, s.sizes = s.sizes[s.top], s.sizes[:s.top] s.top-- return i } func (s *sizesStack) Accumulate(i int64) { s.sizes[s.top] += i } golang-github-karrick-godirwalk-1.15.3/examples/walk-fast/000077500000000000000000000000001362044106100234415ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/walk-fast/main.go000066400000000000000000000020061362044106100247120ustar00rootroot00000000000000/* * walk-fast * * Walks a file system hierarchy using this library. */ package main import ( "flag" "fmt" "os" "github.com/karrick/godirwalk" ) func main() { optVerbose := flag.Bool("verbose", false, "Print file system entries.") flag.Parse() dirname := "." if flag.NArg() > 0 { dirname = flag.Arg(0) } err := godirwalk.Walk(dirname, &godirwalk.Options{ Callback: func(osPathname string, de *godirwalk.Dirent) error { if *optVerbose { fmt.Printf("%s %s\n", de.ModeType(), osPathname) } return nil }, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { if *optVerbose { fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) } // For the purposes of this example, a simple SkipNode will suffice, // although in reality perhaps additional logic might be called for. return godirwalk.SkipNode }, Unsorted: true, // set true for faster yet non-deterministic enumeration (see godoc) }) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(1) } } golang-github-karrick-godirwalk-1.15.3/examples/walk-stdlib/000077500000000000000000000000001362044106100237655ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/examples/walk-stdlib/main.go000066400000000000000000000011371362044106100252420ustar00rootroot00000000000000/* * walk-fast * * Walks a file system hierarchy using the standard library. */ package main import ( "flag" "fmt" "os" "path/filepath" ) func main() { optVerbose := flag.Bool("verbose", false, "Print file system entries.") flag.Parse() dirname := "." if flag.NArg() > 0 { dirname = flag.Arg(0) } err := filepath.Walk(dirname, func(osPathname string, info os.FileInfo, err error) error { if err != nil { return err } if *optVerbose { fmt.Printf("%s %s\n", info.Mode(), osPathname) } return nil }) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(1) } } golang-github-karrick-godirwalk-1.15.3/go.mod000066400000000000000000000000551362044106100210400ustar00rootroot00000000000000module github.com/karrick/godirwalk go 1.13 golang-github-karrick-godirwalk-1.15.3/go.sum000066400000000000000000000000001362044106100210530ustar00rootroot00000000000000golang-github-karrick-godirwalk-1.15.3/inoWithFileno.go000066400000000000000000000002351362044106100230370ustar00rootroot00000000000000// +build dragonfly freebsd openbsd netbsd package godirwalk import "syscall" func inoFromDirent(de *syscall.Dirent) uint64 { return uint64(de.Fileno) } golang-github-karrick-godirwalk-1.15.3/inoWithIno.go000066400000000000000000000002271362044106100223510ustar00rootroot00000000000000// +build aix darwin linux nacl solaris package godirwalk import "syscall" func inoFromDirent(de *syscall.Dirent) uint64 { return uint64(de.Ino) } golang-github-karrick-godirwalk-1.15.3/modeType.go000066400000000000000000000012651362044106100220530ustar00rootroot00000000000000package godirwalk import ( "os" ) // modeType returns the mode type of the file system entry identified by // osPathname by calling os.LStat function, to intentionally not follow symbolic // links. // // Even though os.LStat provides all file mode bits, we want to ensure same // values returned to caller regardless of whether we obtained file mode bits // from syscall or stat call. Therefore mask out the additional file mode bits // that are provided by stat but not by the syscall, so users can rely on their // values. func modeType(osPathname string) (os.FileMode, error) { fi, err := os.Lstat(osPathname) if err == nil { return fi.Mode() & os.ModeType, nil } return 0, err } golang-github-karrick-godirwalk-1.15.3/modeTypeWithType.go000066400000000000000000000020431362044106100235440ustar00rootroot00000000000000// +build darwin dragonfly freebsd linux netbsd openbsd package godirwalk import ( "os" "path/filepath" "syscall" ) // modeTypeFromDirent converts a syscall defined constant, which is in purview // of OS, to a constant defined by Go, assumed by this project to be stable. // // When the syscall constant is not recognized, this function falls back to a // Stat on the file system. func modeTypeFromDirent(de *syscall.Dirent, osDirname, osBasename string) (os.FileMode, error) { switch de.Type { case syscall.DT_REG: return 0, nil case syscall.DT_DIR: return os.ModeDir, nil case syscall.DT_LNK: return os.ModeSymlink, nil case syscall.DT_CHR: return os.ModeDevice | os.ModeCharDevice, nil case syscall.DT_BLK: return os.ModeDevice, nil case syscall.DT_FIFO: return os.ModeNamedPipe, nil case syscall.DT_SOCK: return os.ModeSocket, nil default: // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), then // resolve actual mode by reading file information. return modeType(filepath.Join(osDirname, osBasename)) } } golang-github-karrick-godirwalk-1.15.3/modeTypeWithoutType.go000066400000000000000000000010311362044106100242700ustar00rootroot00000000000000// +build aix js nacl solaris package godirwalk import ( "os" "path/filepath" "syscall" ) // modeTypeFromDirent converts a syscall defined constant, which is in purview // of OS, to a constant defined by Go, assumed by this project to be stable. // // Because some operating system syscall.Dirent structures do not include a Type // field, fall back on Stat of the file system. func modeTypeFromDirent(_ *syscall.Dirent, osDirname, osBasename string) (os.FileMode, error) { return modeType(filepath.Join(osDirname, osBasename)) } golang-github-karrick-godirwalk-1.15.3/nameWithNamlen.go000066400000000000000000000014731362044106100231750ustar00rootroot00000000000000// +build aix darwin dragonfly freebsd netbsd openbsd package godirwalk import ( "reflect" "syscall" "unsafe" ) func nameFromDirent(de *syscall.Dirent) []byte { // Because this GOOS' syscall.Dirent provides a Namlen field that says how // long the name is, this function does not need to search for the NULL // byte. ml := int(de.Namlen) // Convert syscall.Dirent.Name, which is array of int8, to []byte, by // overwriting Cap, Len, and Data slice header fields to values from // syscall.Dirent fields. Setting the Cap, Len, and Data field values for // the slice header modifies what the slice header points to, and in this // case, the name buffer. var name []byte sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) sh.Cap = ml sh.Len = ml sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) return name } golang-github-karrick-godirwalk-1.15.3/nameWithoutNamlen.go000066400000000000000000000023071362044106100237220ustar00rootroot00000000000000// +build nacl linux js solaris package godirwalk import ( "bytes" "reflect" "syscall" "unsafe" ) // nameOffset is a compile time constant const nameOffset = int(unsafe.Offsetof(syscall.Dirent{}.Name)) func nameFromDirent(de *syscall.Dirent) (name []byte) { // Because this GOOS' syscall.Dirent does not provide a field that specifies // the name length, this function must first calculate the max possible name // length, and then search for the NULL byte. ml := int(de.Reclen) - nameOffset // Convert syscall.Dirent.Name, which is array of int8, to []byte, by // overwriting Cap, Len, and Data slice header fields to the max possible // name length computed above, and finding the terminating NULL byte. sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) sh.Cap = ml sh.Len = ml sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) if index := bytes.IndexByte(name, 0); index >= 0 { // Found NULL byte; set slice's cap and len accordingly. sh.Cap = index sh.Len = index return } // NOTE: This branch is not expected, but included for defensive // programming, and provides a hard stop on the name based on the structure // field array size. sh.Cap = len(de.Name) sh.Len = sh.Cap return } golang-github-karrick-godirwalk-1.15.3/readdir.go000066400000000000000000000046111362044106100216750ustar00rootroot00000000000000package godirwalk // ReadDirents returns a sortable slice of pointers to Dirent structures, each // representing the file system name and mode type for one of the immediate // descendant of the specified directory. If the specified directory is a // symbolic link, it will be resolved. // // If an optional scratch buffer is provided that is at least one page of // memory, it will be used when reading directory entries from the file // system. If you plan on calling this function in a loop, you will have // significantly better performance if you allocate a scratch buffer and use it // each time you call this function. // // children, err := godirwalk.ReadDirents(osDirname, nil) // if err != nil { // return nil, errors.Wrap(err, "cannot get list of directory children") // } // sort.Sort(children) // for _, child := range children { // fmt.Printf("%s %s\n", child.ModeType, child.Name) // } func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) { return readDirents(osDirname, scratchBuffer) } // ReadDirnames returns a slice of strings, representing the immediate // descendants of the specified directory. If the specified directory is a // symbolic link, it will be resolved. // // If an optional scratch buffer is provided that is at least one page of // memory, it will be used when reading directory entries from the file // system. If you plan on calling this function in a loop, you will have // significantly better performance if you allocate a scratch buffer and use it // each time you call this function. // // Note that this function, depending on operating system, may or may not invoke // the ReadDirents function, in order to prepare the list of immediate // descendants. Therefore, if your program needs both the names and the file // system mode types of descendants, it will always be faster to invoke // ReadDirents directly, rather than calling this function, then looping over // the results and calling os.Stat or os.LStat for each entry. // // children, err := godirwalk.ReadDirnames(osDirname, nil) // if err != nil { // return nil, errors.Wrap(err, "cannot get list of directory children") // } // sort.Strings(children) // for _, child := range children { // fmt.Printf("%s\n", child) // } func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { return readDirnames(osDirname, scratchBuffer) } golang-github-karrick-godirwalk-1.15.3/readdir_test.go000066400000000000000000000050341362044106100227340ustar00rootroot00000000000000package godirwalk import ( "os" "path/filepath" "testing" ) func TestReadDirents(t *testing.T) { t.Run("without symlinks", func(t *testing.T) { testroot := filepath.Join(scaffolingRoot, "d0") actual, err := ReadDirents(testroot, nil) ensureError(t, err) expected := Dirents{ &Dirent{ name: maxName, path: testroot, modeType: os.FileMode(0), }, &Dirent{ name: "d1", path: testroot, modeType: os.ModeDir, }, &Dirent{ name: "f1", path: testroot, modeType: os.FileMode(0), }, &Dirent{ name: "skips", path: testroot, modeType: os.ModeDir, }, &Dirent{ name: "symlinks", path: testroot, modeType: os.ModeDir, }, } ensureDirentsMatch(t, actual, expected) }) t.Run("with symlinks", func(t *testing.T) { testroot := filepath.Join(scaffolingRoot, "d0/symlinks") actual, err := ReadDirents(testroot, nil) ensureError(t, err) // Because some platforms set multiple mode type bits, when we create // the expected slice, we need to ensure the mode types are set // appropriately for this platform. We have another test function to // ensure NewDirent does this correctly, so let's call NewDirent for // each of the expected children entries. var expected Dirents for _, child := range []string{"nothing", "toAbs", "toD1", "toF1", "d4"} { de, err := NewDirent(filepath.Join(testroot, child)) if err != nil { t.Fatal(err) } expected = append(expected, de) } ensureDirentsMatch(t, actual, expected) }) } func TestReadDirnames(t *testing.T) { actual, err := ReadDirnames(filepath.Join(scaffolingRoot, "d0"), nil) ensureError(t, err) expected := []string{maxName, "d1", "f1", "skips", "symlinks"} ensureStringSlicesMatch(t, actual, expected) } func BenchmarkReadDirnamesStandardLibrary(b *testing.B) { if testing.Short() { b.Skip("Skipping benchmark using user's Go source directory") } f := func(osDirname string) ([]string, error) { dh, err := os.Open(osDirname) if err != nil { return nil, err } return dh.Readdirnames(-1) } var count int for i := 0; i < b.N; i++ { actual, err := f(goPrefix) if err != nil { b.Fatal(err) } count = len(actual) } _ = count } func BenchmarkReadDirnamesGodirwalk(b *testing.B) { if testing.Short() { b.Skip("Skipping benchmark using user's Go source directory") } var count int for i := 0; i < b.N; i++ { actual, err := ReadDirnames(goPrefix, nil) if err != nil { b.Fatal(err) } count = len(actual) } _ = count } golang-github-karrick-godirwalk-1.15.3/readdir_unix.go000066400000000000000000000065421362044106100227450ustar00rootroot00000000000000// +build !windows package godirwalk import ( "os" "syscall" "unsafe" ) // MinimumScratchBufferSize specifies the minimum size of the scratch buffer // that ReadDirents, ReadDirnames, Scanner, and Walk will use when reading file // entries from the operating system. During program startup it is initialized // to the result from calling `os.Getpagesize()` for non Windows environments, // and 0 for Windows. var MinimumScratchBufferSize = os.Getpagesize() func newScratchBuffer() []byte { return make([]byte, MinimumScratchBufferSize) } func readDirents(osDirname string, scratchBuffer []byte) ([]*Dirent, error) { var entries []*Dirent var workBuffer []byte dh, err := os.Open(osDirname) if err != nil { return nil, err } fd := int(dh.Fd()) if len(scratchBuffer) < MinimumScratchBufferSize { scratchBuffer = newScratchBuffer() } for { if len(workBuffer) == 0 { n, err := syscall.ReadDirent(fd, scratchBuffer) // n, err := unix.ReadDirent(fd, scratchBuffer) if err != nil { _ = dh.Close() return nil, err } if n <= 0 { // end of directory: normal exit if err = dh.Close(); err != nil { return nil, err } return entries, nil } workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read } sde := (*syscall.Dirent)(unsafe.Pointer(&workBuffer[0])) // point entry to first syscall.Dirent in buffer workBuffer = workBuffer[reclen(sde):] // advance buffer for next iteration through loop if inoFromDirent(sde) == 0 { continue // inode set to 0 indicates an entry that was marked as deleted } nameSlice := nameFromDirent(sde) nameLength := len(nameSlice) if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { continue } childName := string(nameSlice) mt, err := modeTypeFromDirent(sde, osDirname, childName) if err != nil { _ = dh.Close() return nil, err } entries = append(entries, &Dirent{name: childName, path: osDirname, modeType: mt}) } } func readDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { var entries []string var workBuffer []byte var sde *syscall.Dirent dh, err := os.Open(osDirname) if err != nil { return nil, err } fd := int(dh.Fd()) if len(scratchBuffer) < MinimumScratchBufferSize { scratchBuffer = newScratchBuffer() } for { if len(workBuffer) == 0 { n, err := syscall.ReadDirent(fd, scratchBuffer) // n, err := unix.ReadDirent(fd, scratchBuffer) if err != nil { _ = dh.Close() return nil, err } if n <= 0 { // end of directory: normal exit if err = dh.Close(); err != nil { return nil, err } return entries, nil } workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read } // Handle first entry in the work buffer. sde = (*syscall.Dirent)(unsafe.Pointer(&workBuffer[0])) // point entry to first syscall.Dirent in buffer workBuffer = workBuffer[reclen(sde):] // advance buffer for next iteration through loop if inoFromDirent(sde) == 0 { continue // inode set to 0 indicates an entry that was marked as deleted } nameSlice := nameFromDirent(sde) nameLength := len(nameSlice) if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { continue } entries = append(entries, string(nameSlice)) } } golang-github-karrick-godirwalk-1.15.3/readdir_windows.go000066400000000000000000000025441362044106100234520ustar00rootroot00000000000000// +build windows package godirwalk import "os" // MinimumScratchBufferSize specifies the minimum size of the scratch buffer // that ReadDirents, ReadDirnames, Scanner, and Walk will use when reading file // entries from the operating system. During program startup it is initialized // to the result from calling `os.Getpagesize()` for non Windows environments, // and 0 for Windows. var MinimumScratchBufferSize = 0 func newScratchBuffer() []byte { return nil } func readDirents(osDirname string, _ []byte) ([]*Dirent, error) { dh, err := os.Open(osDirname) if err != nil { return nil, err } fileinfos, err := dh.Readdir(-1) if err != nil { _ = dh.Close() return nil, err } entries := make([]*Dirent, len(fileinfos)) for i, fi := range fileinfos { entries[i] = &Dirent{ name: fi.Name(), path: osDirname, modeType: fi.Mode() & os.ModeType, } } if err = dh.Close(); err != nil { return nil, err } return entries, nil } func readDirnames(osDirname string, _ []byte) ([]string, error) { dh, err := os.Open(osDirname) if err != nil { return nil, err } fileinfos, err := dh.Readdir(-1) if err != nil { _ = dh.Close() return nil, err } entries := make([]string, len(fileinfos)) for i, fi := range fileinfos { entries[i] = fi.Name() } if err = dh.Close(); err != nil { return nil, err } return entries, nil } golang-github-karrick-godirwalk-1.15.3/reclenFromNamlen.go000066400000000000000000000002231362044106100235050ustar00rootroot00000000000000// +build dragonfly package godirwalk import "syscall" func reclen(de *syscall.Dirent) uint64 { return (16 + uint64(de.Namlen) + 1 + 7) &^ 7 } golang-github-karrick-godirwalk-1.15.3/reclenFromReclen.go000066400000000000000000000002551362044106100235100ustar00rootroot00000000000000// +build nacl linux js solaris aix darwin freebsd netbsd openbsd package godirwalk import "syscall" func reclen(de *syscall.Dirent) uint64 { return uint64(de.Reclen) } golang-github-karrick-godirwalk-1.15.3/scaffoling_test.go000066400000000000000000000125731362044106100234430ustar00rootroot00000000000000package godirwalk import ( "flag" "fmt" "io/ioutil" "os" "path/filepath" "testing" ) // maxName is the tested maximum length of a filename this library will // handle. Previous attempts to set it to one less than the size of // syscall.Dirent.Name array resulted in runtime errors trying to create // a test scaffolding file whose size exceeded 255 bytes. This filename // is 255 characters long. const maxName = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // scaffolingRoot is the temporary directory root for scaffold directory. var scaffolingRoot string func TestMain(m *testing.M) { flag.Parse() var code int // program exit code // All tests use the same directory test scaffolding. Create the directory // hierarchy, run the tests, then remove the root directory of the test // scaffolding. defer func() { if err := teardown(); err != nil { fmt.Fprintf(os.Stderr, "godirwalk teardown: %s\n", err) code = 1 } os.Exit(code) }() // When cannot complete setup, dump the directory so we see what we have, // then bail. if err := setup(); err != nil { fmt.Fprintf(os.Stderr, "godirwalk setup: %s\n", err) dumpDirectory() code = 1 return } code = m.Run() // When any test was a failure, then use standard library to walk test // scaffolding directory and print its contents. if code != 0 { dumpDirectory() } } func setup() error { var err error scaffolingRoot, err = ioutil.TempDir(os.TempDir(), "godirwalk-") if err != nil { return err } entries := []Creater{ file{"d0/" + maxName}, file{"d0/f0"}, // will be deleted after symlink for it created file{"d0/f1"}, // file{"d0/d1/f2"}, // file{"d0/skips/d2/f3"}, // node precedes skip file{"d0/skips/d2/skip"}, // skip is non-directory file{"d0/skips/d2/z1"}, // node follows skip non-directory: should never be visited file{"d0/skips/d3/f4"}, // node precedes skip file{"d0/skips/d3/skip/f5"}, // skip is directory: this node should never be visited file{"d0/skips/d3/z2"}, // node follows skip directory: should be visited link{"d0/symlinks/nothing", "../f0"}, // referent will be deleted link{"d0/symlinks/toF1", "../f1"}, // link{"d0/symlinks/toD1", "../d1"}, // link{"d0/symlinks/d4/toSD1", "../toD1"}, // chained symbolic links link{"d0/symlinks/d4/toSF1", "../toF1"}, // chained symbolic links } for _, entry := range entries { if err := entry.Create(); err != nil { return fmt.Errorf("cannot create scaffolding entry: %s", err) } } oldname, err := filepath.Abs(filepath.Join(scaffolingRoot, "d0/f1")) if err != nil { return fmt.Errorf("cannot create scaffolding entry: %s", err) } if err := (link{"d0/symlinks/toAbs", oldname}).Create(); err != nil { return fmt.Errorf("cannot create scaffolding entry: %s", err) } if err := os.Remove(filepath.Join(scaffolingRoot, "d0/f0")); err != nil { return fmt.Errorf("cannot remove file from test scaffolding: %s", err) } return nil } func teardown() error { if scaffolingRoot == "" { return nil // if we do not even have a test root directory then exit } if err := os.RemoveAll(scaffolingRoot); err != nil { return err } return nil } func dumpDirectory() { trim := len(scaffolingRoot) // trim rootDir from prefix of strings err := filepath.Walk(scaffolingRoot, func(osPathname string, info os.FileInfo, err error) error { if err != nil { // we have no info, so get it info, err2 := os.Lstat(osPathname) if err2 != nil { fmt.Fprintf(os.Stderr, "?--------- %s: %s\n", osPathname[trim:], err2) } else { fmt.Fprintf(os.Stderr, "%s %s: %s\n", info.Mode(), osPathname[trim:], err) } return nil } var suffix string if info.Mode()&os.ModeSymlink != 0 { referent, err := os.Readlink(osPathname) if err != nil { suffix = fmt.Sprintf(": cannot read symlink: %s", err) err = nil } else { suffix = fmt.Sprintf(" -> %s", referent) } } fmt.Fprintf(os.Stderr, "%s %s%s\n", info.Mode(), osPathname[trim:], suffix) return nil }) if err != nil { fmt.Fprintf(os.Stderr, "cannot walk test directory: %s\n", err) } } //////////////////////////////////////// // helpers to create file system entries for test scaffolding type Creater interface { Create() error } type file struct { name string } func (f file) Create() error { newname := filepath.Join(scaffolingRoot, filepath.FromSlash(f.name)) if err := os.MkdirAll(filepath.Dir(newname), os.ModePerm); err != nil { return fmt.Errorf("cannot create directory for test scaffolding: %s", err) } if err := ioutil.WriteFile(newname, []byte(newname+"\n"), os.ModePerm); err != nil { return fmt.Errorf("cannot create file for test scaffolding: %s", err) } return nil } type link struct { name, referent string } func (s link) Create() error { newname := filepath.Join(scaffolingRoot, filepath.FromSlash(s.name)) if err := os.MkdirAll(filepath.Dir(newname), os.ModePerm); err != nil { return fmt.Errorf("cannot create directory for test scaffolding: %s", err) } oldname := filepath.FromSlash(s.referent) if err := os.Symlink(oldname, newname); err != nil { return fmt.Errorf("cannot create symbolic link for test scaffolding: %s", err) } return nil } golang-github-karrick-godirwalk-1.15.3/scandir_test.go000066400000000000000000000036321362044106100227470ustar00rootroot00000000000000package godirwalk import ( "os" "path/filepath" "testing" ) func TestScanner(t *testing.T) { t.Run("collect names", func(t *testing.T) { var actual []string scanner, err := NewScanner(filepath.Join(scaffolingRoot, "d0")) ensureError(t, err) for scanner.Scan() { actual = append(actual, scanner.Name()) } ensureError(t, scanner.Err()) expected := []string{maxName, "d1", "f1", "skips", "symlinks"} ensureStringSlicesMatch(t, actual, expected) }) t.Run("collect dirents", func(t *testing.T) { var actual []*Dirent testroot := filepath.Join(scaffolingRoot, "d0") scanner, err := NewScanner(testroot) ensureError(t, err) for scanner.Scan() { dirent, err := scanner.Dirent() ensureError(t, err) actual = append(actual, dirent) } ensureError(t, scanner.Err()) expected := Dirents{ &Dirent{ name: maxName, path: testroot, modeType: os.FileMode(0), }, &Dirent{ name: "d1", path: testroot, modeType: os.ModeDir, }, &Dirent{ name: "f1", path: testroot, modeType: os.FileMode(0), }, &Dirent{ name: "skips", path: testroot, modeType: os.ModeDir, }, &Dirent{ name: "symlinks", path: testroot, modeType: os.ModeDir, }, } ensureDirentsMatch(t, actual, expected) }) t.Run("symlink to directory", func(t *testing.T) { scanner, err := NewScanner(filepath.Join(scaffolingRoot, "d0/symlinks")) ensureError(t, err) var found bool for scanner.Scan() { if scanner.Name() != "toD1" { continue } found = true de, err := scanner.Dirent() ensureError(t, err) got, err := de.IsDirOrSymlinkToDir() ensureError(t, err) if want := true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } } ensureError(t, scanner.Err()) if got, want := found, true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) } golang-github-karrick-godirwalk-1.15.3/scandir_unix.go000066400000000000000000000115441362044106100227540ustar00rootroot00000000000000// +build !windows package godirwalk import ( "os" "syscall" "unsafe" ) // Scanner is an iterator to enumerate the contents of a directory. type Scanner struct { scratchBuffer []byte // read directory bytes from file system into this buffer workBuffer []byte // points into scratchBuffer, from which we chunk out directory entries osDirname string childName string err error // err is the error associated with scanning directory statErr error // statErr is any error return while attempting to stat an entry dh *os.File // used to close directory after done reading de *Dirent // most recently decoded directory entry sde *syscall.Dirent fd int // file descriptor used to read entries from directory } // NewScanner returns a new directory Scanner that lazily enumerates the // contents of a single directory. // // scanner, err := godirwalk.NewScanner(dirname) // if err != nil { // fatal("cannot scan directory: %s", err) // } // // for scanner.Scan() { // dirent, err := scanner.Dirent() // if err != nil { // warning("cannot get dirent: %s", err) // continue // } // name := dirent.Name() // if name == "break" { // break // } // if name == "continue" { // continue // } // fmt.Printf("%v %v\n", dirent.ModeType(), dirent.Name()) // } // if err := scanner.Err(); err != nil { // fatal("cannot scan directory: %s", err) // } func NewScanner(osDirname string) (*Scanner, error) { return NewScannerWithScratchBuffer(osDirname, nil) } // NewScannerWithScratchBuffer returns a new directory Scanner that lazily // enumerates the contents of a single directory. On platforms other than // Windows it uses the provided scratch buffer to read from the file system. On // Windows the scratch buffer is ignored. func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) { dh, err := os.Open(osDirname) if err != nil { return nil, err } if len(scratchBuffer) < MinimumScratchBufferSize { scratchBuffer = newScratchBuffer() } scanner := &Scanner{ scratchBuffer: scratchBuffer, osDirname: osDirname, dh: dh, fd: int(dh.Fd()), } return scanner, nil } // Dirent returns the current directory entry while scanning a directory. func (s *Scanner) Dirent() (*Dirent, error) { if s.de == nil { s.de = &Dirent{name: s.childName, path: s.osDirname} s.de.modeType, s.statErr = modeTypeFromDirent(s.sde, s.osDirname, s.childName) } return s.de, s.statErr } // done is called when directory scanner unable to continue, with either the // triggering error, or nil when there are simply no more entries to read from // the directory. func (s *Scanner) done(err error) { if s.dh == nil { return } if cerr := s.dh.Close(); err == nil { s.err = cerr } s.osDirname, s.childName = "", "" s.scratchBuffer, s.workBuffer = nil, nil s.dh, s.de, s.sde, s.statErr = nil, nil, nil, nil s.fd = 0 } // Err returns any error associated with scanning a directory. It is normal to // call Err after Scan returns false, even though they both ensure Scanner // resources are released. Do not call until done scanning a directory. func (s *Scanner) Err() error { s.done(nil) return s.err } // Name returns the base name of the current directory entry while scanning a // directory. func (s *Scanner) Name() string { return s.childName } // Scan potentially reads and then decodes the next directory entry from the // file system. // // When it returns false, this releases resources used by the Scanner then // returns any error associated with closing the file system directory resource. func (s *Scanner) Scan() bool { if s.dh == nil { return false } s.de = nil for { // When the work buffer has nothing remaining to decode, we need to load // more data from disk. if len(s.workBuffer) == 0 { n, err := syscall.ReadDirent(s.fd, s.scratchBuffer) // n, err := unix.ReadDirent(s.fd, s.scratchBuffer) if err != nil { s.done(err) return false } if n <= 0 { // end of directory: normal exit s.done(nil) return false } s.workBuffer = s.scratchBuffer[:n] // trim work buffer to number of bytes read } s.sde = (*syscall.Dirent)(unsafe.Pointer(&s.workBuffer[0])) // point entry to first syscall.Dirent in buffer s.workBuffer = s.workBuffer[reclen(s.sde):] // advance buffer for next iteration through loop if inoFromDirent(s.sde) == 0 { continue // inode set to 0 indicates an entry that was marked as deleted } nameSlice := nameFromDirent(s.sde) nameLength := len(nameSlice) if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { continue } s.childName = string(nameSlice) return true } } golang-github-karrick-godirwalk-1.15.3/scandir_windows.go000066400000000000000000000066111362044106100234620ustar00rootroot00000000000000// +build windows package godirwalk import ( "fmt" "os" ) // Scanner is an iterator to enumerate the contents of a directory. type Scanner struct { osDirname string childName string dh *os.File // dh is handle to open directory de *Dirent err error // err is the error associated with scanning directory childMode os.FileMode } // NewScanner returns a new directory Scanner that lazily enumerates the // contents of a single directory. // // scanner, err := godirwalk.NewScanner(dirname) // if err != nil { // fatal("cannot scan directory: %s", err) // } // // for scanner.Scan() { // dirent, err := scanner.Dirent() // if err != nil { // warning("cannot get dirent: %s", err) // continue // } // name := dirent.Name() // if name == "break" { // break // } // if name == "continue" { // continue // } // fmt.Printf("%v %v\n", dirent.ModeType(), dirent.Name()) // } // if err := scanner.Err(); err != nil { // fatal("cannot scan directory: %s", err) // } func NewScanner(osDirname string) (*Scanner, error) { dh, err := os.Open(osDirname) if err != nil { return nil, err } scanner := &Scanner{ osDirname: osDirname, dh: dh, } return scanner, nil } // NewScannerWithScratchBuffer returns a new directory Scanner that lazily // enumerates the contents of a single directory. On platforms other than // Windows it uses the provided scratch buffer to read from the file system. On // Windows the scratch buffer parameter is ignored. func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) { return NewScanner(osDirname) } // Dirent returns the current directory entry while scanning a directory. func (s *Scanner) Dirent() (*Dirent, error) { if s.de == nil { s.de = &Dirent{ name: s.childName, path: s.osDirname, modeType: s.childMode, } } return s.de, nil } // done is called when directory scanner unable to continue, with either the // triggering error, or nil when there are simply no more entries to read from // the directory. func (s *Scanner) done(err error) { if s.dh == nil { return } if cerr := s.dh.Close(); err == nil { s.err = cerr } s.childName, s.osDirname = "", "" s.de, s.dh = nil, nil } // Err returns any error associated with scanning a directory. It is normal to // call Err after Scan returns false, even though they both ensure Scanner // resources are released. Do not call until done scanning a directory. func (s *Scanner) Err() error { s.done(nil) return s.err } // Name returns the base name of the current directory entry while scanning a // directory. func (s *Scanner) Name() string { return s.childName } // Scan potentially reads and then decodes the next directory entry from the // file system. // // When it returns false, this releases resources used by the Scanner then // returns any error associated with closing the file system directory resource. func (s *Scanner) Scan() bool { if s.dh == nil { return false } s.de = nil fileinfos, err := s.dh.Readdir(1) if err != nil { s.done(err) return false } if l := len(fileinfos); l != 1 { s.done(fmt.Errorf("expected a single entry rather than %d", l)) return false } fi := fileinfos[0] s.childMode = fi.Mode() & os.ModeType s.childName = fi.Name() return true } golang-github-karrick-godirwalk-1.15.3/scanner.go000066400000000000000000000016711362044106100217170ustar00rootroot00000000000000package godirwalk import "sort" type scanner interface { Dirent() (*Dirent, error) Err() error Name() string Scan() bool } // sortedScanner enumerates through a directory's contents after reading the // entire directory and sorting the entries by name. Used by walk to simplify // its implementation. type sortedScanner struct { dd []*Dirent de *Dirent } func newSortedScanner(osPathname string, scratchBuffer []byte) (*sortedScanner, error) { deChildren, err := ReadDirents(osPathname, scratchBuffer) if err != nil { return nil, err } sort.Sort(deChildren) return &sortedScanner{dd: deChildren}, nil } func (d *sortedScanner) Err() error { d.dd, d.de = nil, nil return nil } func (d *sortedScanner) Dirent() (*Dirent, error) { return d.de, nil } func (d *sortedScanner) Name() string { return d.de.name } func (d *sortedScanner) Scan() bool { if len(d.dd) > 0 { d.de, d.dd = d.dd[0], d.dd[1:] return true } return false } golang-github-karrick-godirwalk-1.15.3/walk.go000066400000000000000000000276251362044106100212330ustar00rootroot00000000000000package godirwalk import ( "errors" "fmt" "os" "path/filepath" ) // Options provide parameters for how the Walk function operates. type Options struct { // ErrorCallback specifies a function to be invoked in the case of an error // that could potentially be ignored while walking a file system // hierarchy. When set to nil or left as its zero-value, any error condition // causes Walk to immediately return the error describing what took // place. When non-nil, this user supplied function is invoked with the OS // pathname of the file system object that caused the error along with the // error that took place. The return value of the supplied ErrorCallback // function determines whether the error will cause Walk to halt immediately // as it would were no ErrorCallback value provided, or skip this file // system node yet continue on with the remaining nodes in the file system // hierarchy. // // ErrorCallback is invoked both for errors that are returned by the // runtime, and for errors returned by other user supplied callback // functions. ErrorCallback func(string, error) ErrorAction // FollowSymbolicLinks specifies whether Walk will follow symbolic links // that refer to directories. When set to false or left as its zero-value, // Walk will still invoke the callback function with symbolic link nodes, // but if the symbolic link refers to a directory, it will not recurse on // that directory. When set to true, Walk will recurse on symbolic links // that refer to a directory. FollowSymbolicLinks bool // Unsorted controls whether or not Walk will sort the immediate descendants // of a directory by their relative names prior to visiting each of those // entries. // // When set to false or left at its zero-value, Walk will get the list of // immediate descendants of a particular directory, sort that list by // lexical order of their names, and then visit each node in the list in // sorted order. This will cause Walk to always traverse the same directory // tree in the same order, however may be inefficient for directories with // many immediate descendants. // // When set to true, Walk skips sorting the list of immediate descendants // for a directory, and simply visits each node in the order the operating // system enumerated them. This will be more fast, but with the side effect // that the traversal order may be different from one invocation to the // next. Unsorted bool // Callback is a required function that Walk will invoke for every file // system node it encounters. Callback WalkFunc // PostChildrenCallback is an option function that Walk will invoke for // every file system directory it encounters after its children have been // processed. PostChildrenCallback WalkFunc // ScratchBuffer is an optional byte slice to use as a scratch buffer for // Walk to use when reading directory entries, to reduce amount of garbage // generation. Not all architectures take advantage of the scratch // buffer. If omitted or the provided buffer has fewer bytes than // MinimumScratchBufferSize, then a buffer with MinimumScratchBufferSize // bytes will be created and used once per Walk invocation. ScratchBuffer []byte // AllowNonDirectory causes Walk to bypass the check that ensures it is // being called on a directory node, or when FollowSymbolicLinks is true, a // symbolic link that points to a directory. Leave this value false to have // Walk return an error when called on a non-directory. Set this true to // have Walk run even when called on a non-directory node. AllowNonDirectory bool } // ErrorAction defines a set of actions the Walk function could take based on // the occurrence of an error while walking the file system. See the // documentation for the ErrorCallback field of the Options structure for more // information. type ErrorAction int const ( // Halt is the ErrorAction return value when the upstream code wants to halt // the walk process when a runtime error takes place. It matches the default // action the Walk function would take were no ErrorCallback provided. Halt ErrorAction = iota // SkipNode is the ErrorAction return value when the upstream code wants to // ignore the runtime error for the current file system node, skip // processing of the node that caused the error, and continue walking the // file system hierarchy with the remaining nodes. SkipNode ) // WalkFunc is the type of the function called for each file system node visited // by Walk. The pathname argument will contain the argument to Walk as a prefix; // that is, if Walk is called with "dir", which is a directory containing the // file "a", the provided WalkFunc will be invoked with the argument "dir/a", // using the correct os.PathSeparator for the Go Operating System architecture, // GOOS. The directory entry argument is a pointer to a Dirent for the node, // providing access to both the basename and the mode type of the file system // node. // // If an error is returned by the Callback or PostChildrenCallback functions, // and no ErrorCallback function is provided, processing stops. If an // ErrorCallback function is provided, then it is invoked with the OS pathname // of the node that caused the error along along with the error. The return // value of the ErrorCallback function determines whether to halt processing, or // skip this node and continue processing remaining file system nodes. // // The exception is when the function returns the special value // filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a // directory, Walk skips the directory's contents entirely. If the function // returns filepath.SkipDir when invoked on a non-directory file system node, // Walk skips the remaining files in the containing directory. Note that any // supplied ErrorCallback function is not invoked with filepath.SkipDir when the // Callback or PostChildrenCallback functions return that special value. type WalkFunc func(osPathname string, directoryEntry *Dirent) error // Walk walks the file tree rooted at the specified directory, calling the // specified callback function for each file system node in the tree, including // root, symbolic links, and other node types. // // This function is often much faster than filepath.Walk because it does not // invoke os.Stat for every node it encounters, but rather obtains the file // system node type when it reads the parent directory. // // If a runtime error occurs, either from the operating system or from the // upstream Callback or PostChildrenCallback functions, processing typically // halts. However, when an ErrorCallback function is provided in the provided // Options structure, that function is invoked with the error along with the OS // pathname of the file system node that caused the error. The ErrorCallback // function's return value determines the action that Walk will then take. // // func main() { // dirname := "." // if len(os.Args) > 1 { // dirname = os.Args[1] // } // err := godirwalk.Walk(dirname, &godirwalk.Options{ // Callback: func(osPathname string, de *godirwalk.Dirent) error { // fmt.Printf("%s %s\n", de.ModeType(), osPathname) // return nil // }, // ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { // // Your program may want to log the error somehow. // fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) // // // For the purposes of this example, a simple SkipNode will suffice, // // although in reality perhaps additional logic might be called for. // return godirwalk.SkipNode // }, // }) // if err != nil { // fmt.Fprintf(os.Stderr, "%s\n", err) // os.Exit(1) // } // } func Walk(pathname string, options *Options) error { if options == nil || options.Callback == nil { return errors.New("cannot walk without non-nil options and Callback function") } pathname = filepath.Clean(pathname) var fi os.FileInfo var err error if options.FollowSymbolicLinks { fi, err = os.Stat(pathname) } else { fi, err = os.Lstat(pathname) } if err != nil { return err } mode := fi.Mode() if !options.AllowNonDirectory && mode&os.ModeDir == 0 { return fmt.Errorf("cannot Walk non-directory: %s", pathname) } dirent := &Dirent{ name: filepath.Base(pathname), path: filepath.Dir(pathname), modeType: mode & os.ModeType, } if len(options.ScratchBuffer) < MinimumScratchBufferSize { options.ScratchBuffer = newScratchBuffer() } // If ErrorCallback is nil, set to a default value that halts the walk // process on all operating system errors. This is done to allow error // handling to be more succinct in the walk code. if options.ErrorCallback == nil { options.ErrorCallback = defaultErrorCallback } if err = walk(pathname, dirent, options); err != filepath.SkipDir { return err } return nil // silence SkipDir for top level } // defaultErrorCallback always returns Halt because if the upstream code did not // provide an ErrorCallback function, walking the file system hierarchy ought to // halt upon any operating system error. func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt } // walk recursively traverses the file system node specified by pathname and the // Dirent. func walk(osPathname string, dirent *Dirent, options *Options) error { err := options.Callback(osPathname, dirent) if err != nil { if err == filepath.SkipDir { return err } if action := options.ErrorCallback(osPathname, err); action == SkipNode { return nil } return err } if dirent.IsSymlink() { if !options.FollowSymbolicLinks { return nil } // Does this symlink point to a directory? info, err := os.Stat(osPathname) if err != nil { if action := options.ErrorCallback(osPathname, err); action == SkipNode { return nil } return err } if !info.IsDir() { return nil } } else if !dirent.IsDir() { return nil } // If get here, then specified pathname refers to a directory or a // symbolic link to a directory. var ds scanner if options.Unsorted { // When upstream does not request a sorted iteration, it's more memory // efficient to read a single child at a time from the file system. ds, err = NewScanner(osPathname) } else { // When upstream wants a sorted iteration, we must read the entire // directory and sort through the child names, and then iterate on each // child. ds, err = newSortedScanner(osPathname, options.ScratchBuffer) } if err != nil { if action := options.ErrorCallback(osPathname, err); action == SkipNode { return nil } return err } for ds.Scan() { deChild, err := ds.Dirent() osChildname := filepath.Join(osPathname, deChild.name) if err != nil { if action := options.ErrorCallback(osChildname, err); action == SkipNode { return nil } return err } err = walk(osChildname, deChild, options) debug("osChildname: %q; error: %v\n", osChildname, err) if err == nil { continue } if err != filepath.SkipDir { return err } // When received SkipDir on a directory or a symbolic link to a // directory, stop processing that directory but continue processing // siblings. When received on a non-directory, stop processing // remaining siblings. isDir, err := deChild.IsDirOrSymlinkToDir() if err != nil { if action := options.ErrorCallback(osChildname, err); action == SkipNode { continue // ignore and continue with next sibling } return err // caller does not approve of this error } if !isDir { break // stop processing remaining siblings, but allow post children callback } // continue processing remaining siblings } if err = ds.Err(); err != nil { return err } if options.PostChildrenCallback == nil { return nil } err = options.PostChildrenCallback(osPathname, dirent) if err == nil || err == filepath.SkipDir { return err } if action := options.ErrorCallback(osPathname, err); action == SkipNode { return nil } return err } golang-github-karrick-godirwalk-1.15.3/walk_test.go000066400000000000000000000224371362044106100222660ustar00rootroot00000000000000package godirwalk import ( "os" "path/filepath" "sort" "testing" ) func filepathWalk(tb testing.TB, osDirname string) []string { tb.Helper() var entries []string err := filepath.Walk(osDirname, func(osPathname string, info os.FileInfo, err error) error { if err != nil { return err } if info.Name() == "skip" { return filepath.SkipDir } entries = append(entries, filepath.FromSlash(osPathname)) return nil }) ensureError(tb, err) return entries } func godirwalkWalk(tb testing.TB, osDirname string) []string { tb.Helper() var entries []string err := Walk(osDirname, &Options{ Callback: func(osPathname string, dirent *Dirent) error { if dirent.Name() == "skip" { return filepath.SkipDir } entries = append(entries, filepath.FromSlash(osPathname)) return nil }, }) ensureError(tb, err) return entries } func godirwalkWalkUnsorted(tb testing.TB, osDirname string) []string { tb.Helper() var entries []string err := Walk(osDirname, &Options{ Callback: func(osPathname string, dirent *Dirent) error { if dirent.Name() == "skip" { return filepath.SkipDir } entries = append(entries, filepath.FromSlash(osPathname)) return nil }, Unsorted: true, }) ensureError(tb, err) return entries } // Ensure the results from calling this library's Walk function exactly match // those returned by filepath.Walk func ensureSameAsStandardLibrary(tb testing.TB, osDirname string) { tb.Helper() osDirname = filepath.Join(scaffolingRoot, osDirname) actual := godirwalkWalk(tb, osDirname) sort.Strings(actual) expected := filepathWalk(tb, osDirname) ensureStringSlicesMatch(tb, actual, expected) } // Test the entire test root hierarchy with all of its artifacts. This library // advertises itself as visiting the same file system entries as the standard // library, and responding to discovered errors the same way, including // responding to filepath.SkipDir exactly like the standard library does. This // test ensures that behavior is correct by enumerating the contents of the test // root directory. func TestWalkCompatibleWithFilepathWalk(t *testing.T) { t.Run("test root", func(t *testing.T) { ensureSameAsStandardLibrary(t, "d0") }) t.Run("ignore skips", func(t *testing.T) { // When filepath.SkipDir is returned, the remainder of the children in a // directory are not visited. This causes results to be different when // visiting in lexicographical order or natural order. For this test, we // want to ensure godirwalk can optimize traversals when unsorted using // the Scanner, but recognize that we cannot test against standard // library when we skip any nodes within it. osDirname := filepath.Join(scaffolingRoot, "d0/d1") actual := godirwalkWalkUnsorted(t, osDirname) sort.Strings(actual) expected := filepathWalk(t, osDirname) ensureStringSlicesMatch(t, actual, expected) }) } // Test cases for encountering the filepath.SkipDir error at different // relative positions from the invocation argument. func TestWalkSkipDir(t *testing.T) { t.Run("skip file at root", func(t *testing.T) { ensureSameAsStandardLibrary(t, "d0/skips/d2") }) t.Run("skip dir at root", func(t *testing.T) { ensureSameAsStandardLibrary(t, "d0/skips/d3") }) t.Run("skip nodes under root", func(t *testing.T) { ensureSameAsStandardLibrary(t, "d0/skips") }) t.Run("SkipDirOnSymlink", func(t *testing.T) { var actual []string err := Walk(filepath.Join(scaffolingRoot, "d0/skips"), &Options{ Callback: func(osPathname string, dirent *Dirent) error { if dirent.Name() == "skip" { return filepath.SkipDir } actual = append(actual, filepath.FromSlash(osPathname)) return nil }, FollowSymbolicLinks: true, }) ensureError(t, err) expected := []string{ filepath.Join(scaffolingRoot, "d0/skips"), filepath.Join(scaffolingRoot, "d0/skips/d2"), filepath.Join(scaffolingRoot, "d0/skips/d2/f3"), filepath.Join(scaffolingRoot, "d0/skips/d3"), filepath.Join(scaffolingRoot, "d0/skips/d3/f4"), filepath.Join(scaffolingRoot, "d0/skips/d3/z2"), } ensureStringSlicesMatch(t, actual, expected) }) } func TestWalkFollowSymbolicLinks(t *testing.T) { var actual []string var errorCallbackVisited bool err := Walk(filepath.Join(scaffolingRoot, "d0/symlinks"), &Options{ Callback: func(osPathname string, _ *Dirent) error { actual = append(actual, filepath.FromSlash(osPathname)) return nil }, ErrorCallback: func(osPathname string, err error) ErrorAction { if filepath.Base(osPathname) == "nothing" { errorCallbackVisited = true return SkipNode } return Halt }, FollowSymbolicLinks: true, }) ensureError(t, err) if got, want := errorCallbackVisited, true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } expected := []string{ filepath.Join(scaffolingRoot, "d0/symlinks"), filepath.Join(scaffolingRoot, "d0/symlinks/d4"), filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSD1"), // chained symbolic link filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSD1/f2"), // chained symbolic link filepath.Join(scaffolingRoot, "d0/symlinks/d4/toSF1"), // chained symbolic link filepath.Join(scaffolingRoot, "d0/symlinks/nothing"), filepath.Join(scaffolingRoot, "d0/symlinks/toAbs"), filepath.Join(scaffolingRoot, "d0/symlinks/toD1"), filepath.Join(scaffolingRoot, "d0/symlinks/toD1/f2"), filepath.Join(scaffolingRoot, "d0/symlinks/toF1"), } ensureStringSlicesMatch(t, actual, expected) } // While filepath.Walk will deliver the no access error to the regular callback, // godirwalk should deliver it first to the ErrorCallback handler, then take // action based on the return value of that callback function. func TestErrorCallback(t *testing.T) { t.Run("halt", func(t *testing.T) { var callbackVisited, errorCallbackVisited bool err := Walk(filepath.Join(scaffolingRoot, "d0/symlinks"), &Options{ Callback: func(osPathname string, dirent *Dirent) error { switch dirent.Name() { case "nothing": callbackVisited = true } return nil }, ErrorCallback: func(osPathname string, err error) ErrorAction { switch filepath.Base(osPathname) { case "nothing": errorCallbackVisited = true return Halt // Direct Walk to propagate error to caller } t.Fatalf("unexpected error callback for %s: %s", osPathname, err) return SkipNode }, FollowSymbolicLinks: true, }) ensureError(t, err, "nothing") // Ensure caller receives propagated access error if got, want := callbackVisited, true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := errorCallbackVisited, true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) t.Run("skipnode", func(t *testing.T) { var callbackVisited, errorCallbackVisited bool err := Walk(filepath.Join(scaffolingRoot, "d0/symlinks"), &Options{ Callback: func(osPathname string, dirent *Dirent) error { switch dirent.Name() { case "nothing": callbackVisited = true } return nil }, ErrorCallback: func(osPathname string, err error) ErrorAction { switch filepath.Base(osPathname) { case "nothing": errorCallbackVisited = true return SkipNode // Direct Walk to ignore this error } t.Fatalf("unexpected error callback for %s: %s", osPathname, err) return Halt }, FollowSymbolicLinks: true, }) ensureError(t, err) // Ensure caller receives no access error if got, want := callbackVisited, true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } if got, want := errorCallbackVisited, true; got != want { t.Errorf("GOT: %v; WANT: %v", got, want) } }) } // Invokes PostChildrenCallback for all directories and nothing else. func TestPostChildrenCallback(t *testing.T) { var actual []string err := Walk(filepath.Join(scaffolingRoot, "d0"), &Options{ Callback: func(_ string, _ *Dirent) error { return nil }, PostChildrenCallback: func(osPathname string, _ *Dirent) error { actual = append(actual, osPathname) return nil }, }) ensureError(t, err) expected := []string{ filepath.Join(scaffolingRoot, "d0"), filepath.Join(scaffolingRoot, "d0/d1"), filepath.Join(scaffolingRoot, "d0/skips"), filepath.Join(scaffolingRoot, "d0/skips/d2"), filepath.Join(scaffolingRoot, "d0/skips/d3"), filepath.Join(scaffolingRoot, "d0/skips/d3/skip"), filepath.Join(scaffolingRoot, "d0/symlinks"), filepath.Join(scaffolingRoot, "d0/symlinks/d4"), } ensureStringSlicesMatch(t, actual, expected) } const flameIterations = 10 var goPrefix = filepath.Join(os.Getenv("GOPATH"), "src") func BenchmarkFilepath(b *testing.B) { if testing.Short() { b.Skip("Skipping benchmark using user's Go source directory") } for i := 0; i < b.N; i++ { _ = filepathWalk(b, goPrefix) } } func BenchmarkGodirwalk(b *testing.B) { if testing.Short() { b.Skip("Skipping benchmark using user's Go source directory") } for i := 0; i < b.N; i++ { _ = godirwalkWalk(b, goPrefix) } } func BenchmarkGodirwalkUnsorted(b *testing.B) { if testing.Short() { b.Skip("Skipping benchmark using user's Go source directory") } for i := 0; i < b.N; i++ { _ = godirwalkWalkUnsorted(b, goPrefix) } } func BenchmarkFlameGraphFilepath(b *testing.B) { for i := 0; i < flameIterations; i++ { _ = filepathWalk(b, goPrefix) } } func BenchmarkFlameGraphGodirwalk(b *testing.B) { for i := 0; i < flameIterations; i++ { _ = godirwalkWalk(b, goPrefix) } }