pax_global_header00006660000000000000000000000064135350072010014506gustar00rootroot0000000000000052 comment=837825176105fa678a25d9fc161f8a3cffccd7ba panicparse-1.3.0/000077500000000000000000000000001353500720100136345ustar00rootroot00000000000000panicparse-1.3.0/.travis.yml000066400000000000000000000042361353500720100157520ustar00rootroot00000000000000# Copyright 2014 Marc-Antoine Ruel. All rights reserved. # Use of this source code is governed under the Apache License, Version 2.0 # that can be found in the LICENSE file. sudo: false language: go dist: xenial matrix: include: - go: 1.12.x env: GO111MODULE=on cache: directories: # go1.10+ 'go test' cache on linux (macOS and Windows are # different). - $HOME/.cache/go-build # go1.11+ with GO111MODULE=on - $GOPATH/pkg/mod # Cache tools sources. Manually verified that both misspell and ineffassign # only depend on the stdlib. - $GOPATH/src/github\.com/client9 - $GOPATH/src/github\.com/gordonklaus # For shadow. - $GOPATH/src/golang\.org - go: 1.8.7 before_script: - echo $TRAVIS_GO_VERSION - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then go get -u -v github.com/client9/misspell/cmd/misspell github.com/gordonklaus/ineffassign golang.org/x/lint/golint golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow; fi script: # Checks run on the latest version. - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then echo 'Check Code is well formatted'; ! gofmt -s -d . | read; fi - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then go list ./... | grep -v /vendor/ | xargs -L1 golint -set_exit_status; fi - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then go vet ./...; fi - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then ineffassign .; fi - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then go vet -vettool=$GOPATH/bin/shadow; fi - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then bash -c 'set -e; echo "" > coverage.txt; for d in $(go list ./...); do go test -covermode=count -coverprofile=p.out $d; if [ -f p.out ]; then cat p.out >> coverage.txt; rm p.out; fi; done'; fi - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then go test -race ./...; fi # Check run on older versions. - if [[ $TRAVIS_GO_VERSION == 1.8.7 ]]; then go test ./...; fi - if [[ $TRAVIS_GO_VERSION == 1.8.7 ]]; then if find . -path ./.git -prune -o -type f -executable -print | grep -e . ; then echo 'Do not commit executables'; false; fi; fi after_success: - if [[ $TRAVIS_GO_VERSION != 1.8.7 ]]; then bash <(curl -s https://codecov.io/bash); fi panicparse-1.3.0/LICENSE000066400000000000000000000261231353500720100146450ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2015 Marc-Antoine Ruel Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. panicparse-1.3.0/README.md000066400000000000000000000116111353500720100151130ustar00rootroot00000000000000# panicparse Parses panic stack traces, densifies and deduplicates goroutines with similar stack traces. Helps debugging crashes and deadlocks in heavily parallelized process. [![GoDoc](https://godoc.org/github.com/maruel/panicparse/stack?status.svg)](https://godoc.org/github.com/maruel/panicparse/stack) [![Go Report Card](https://goreportcard.com/badge/github.com/maruel/panicparse)](https://goreportcard.com/report/github.com/maruel/panicparse) [![Coverage Status](https://codecov.io/gh/maruel/panicparse/graph/badge.svg)](https://codecov.io/gh/maruel/panicparse) [![Build Status](https://travis-ci.org/maruel/panicparse.svg)](https://travis-ci.org/maruel/panicparse) panicparse helps make sense of Go crash dumps: ![Screencast](https://raw.githubusercontent.com/wiki/maruel/panicparse/parse.gif "Screencast") ## Features * >50% more compact output than original stack dump yet more readable. * Exported symbols are bold, private symbols are darker. * Stdlib is green, main is yellow, rest is red. * Deduplicates redundant goroutine stacks. Useful for large server crashes. * Arguments as pointer IDs instead of raw pointer values. * Pushes stdlib-only stacks at the bottom to help focus on important code. * Usable as a library! [![GoDoc](https://godoc.org/github.com/maruel/panicparse/stack?status.svg)](https://godoc.org/github.com/maruel/panicparse/stack) * Warning: please pin the major version (i.e. vendor it via [dep](https://github.com/golang/dep)) as breaking changes happen on [major version update](https://semver.org/). * Parses the source files if available to augment the output. * Works on Windows. ## Authors `panicparse` was created with ❤️️ and passion by [Marc-Antoine Ruel](https://github.com/maruel) and [friends](https://github.com/maruel/panicparse/graphs/contributors). ## Installation go get github.com/maruel/panicparse/cmd/pp ## Usage ### Piping a stack trace from another process #### TL;DR * Ubuntu (bash v4 or zsh): `|&` * OSX, [install bash 4+](README.md#updating-bash-on-osx), then: `|&` * Windows _or_ OSX with stock bash v3: `2>&1 |` * [Fish](http://fishshell.com/) shell: `^|` #### Longer version `pp` streams its stdin to stdout as long as it doesn't detect any panic. `panic()` and Go's native deadlock detector [print to stderr](https://golang.org/src/runtime/panic1.go) via the native [`print()` function](https://golang.org/pkg/builtin/#print). **Bash v4** or **zsh**: `|&` tells the shell to redirect stderr to stdout, it's an alias for `2>&1 |` ([bash v4](https://www.gnu.org/software/bash/manual/bash.html#Pipelines), [zsh](http://zsh.sourceforge.net/Doc/Release/Shell-Grammar.html#Simple-Commands-_0026-Pipelines)): go test -v |&pp **Windows or OSX native bash** [(which is 3.2.57)](http://meta.ath0.com/2012/02/05/apples-great-gpl-purge/): They don't have this shortcut, so use the long form: go test -v 2>&1 | pp **Fish**: It uses [^ for stderr redirection](http://fishshell.com/docs/current/tutorial.html#tut_pipes_and_redirections) so the shortcut is `^|`: go test -v ^|pp **PowerShell**: [It has broken `2>&1` redirection](https://connect.microsoft.com/PowerShell/feedback/details/765551/in-powershell-v3-you-cant-redirect-stderr-to-stdout-without-generating-error-records). The workaround is to shell out to cmd.exe. :( ### Investigate deadlock On POSIX, use `Ctrl-\` to send SIGQUIT to your process, `pp` will ignore the signal and will parse the stack trace. ### Parsing from a file To dump to a file then parse, pass the file path of a stack trace go test 2> stack.txt pp stack.txt ## Tips ### Disable inlining Starting with go1.11, the toolchain starts to inline more often. This causes traces to be less informative. You can use the following to help diagnosing issues: go install -gcflags '-N -l' path/to/foo foo |& pp or go test -gcflags '-N -l' ./... |& pp ### GOTRACEBACK Starting with Go 1.6, [`GOTRACEBACK`](https://golang.org/pkg/runtime/) defaults to `single` instead of `all` / `1` that was used in 1.5 and before. To get all goroutines trace and not just the crashing one, set the environment variable: export GOTRACEBACK=all or `set GOTRACEBACK=all` on Windows. Probably worth to put it in your `.bashrc`. ### Updating bash on OSX Install bash v4+ on OSX via [homebrew](http://brew.sh) or [macports](https://www.macports.org/). Your future self will appreciate having done that. ### If you have `/usr/bin/pp` installed If you try `pp` for the first time and you get: Creating tables and indexes... Done. and/or /usr/bin/pp5.18: No input files specified you may be running the _Perl PAR Packager_ instead of panicparse. You have two choices, either you put `$GOPATH/bin` at the begining of `$PATH` or use long name `panicparse` with: go get github.com/maruel/panicparse then using `panicparse` instead of `pp`: go test 2> panicparse panicparse-1.3.0/cmd/000077500000000000000000000000001353500720100143775ustar00rootroot00000000000000panicparse-1.3.0/cmd/panic/000077500000000000000000000000001353500720100154715ustar00rootroot00000000000000panicparse-1.3.0/cmd/panic/internal/000077500000000000000000000000001353500720100173055ustar00rootroot00000000000000panicparse-1.3.0/cmd/panic/internal/internal.go000066400000000000000000000006071353500720100214530ustar00rootroot00000000000000// Copyright 2017 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // Package internal is for use for panic. package internal // Callback calls back a function through an external then internal function. func Callback(f func()) { callback(f) } func callback(f func()) { f() } panicparse-1.3.0/cmd/panic/main.go000066400000000000000000002323001353500720100167440ustar00rootroot00000000000000// Copyright 2017 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // panic crashes in various ways. // // It is a tool to help test pp. package main // To install, run: // go install github.com/maruel/panicparse/cmd/panic // panic -help // panic str |& pp // // Some panics require the race detector with -race: // go install -race github.com/maruel/panicparse/cmd/panic // panic race |& pp // // To add a new panic stack signature, add it to types type below, keeping the // list ordered by name. If you need utility functions, add it in the section // below. That's it! import ( "fmt" "io" "os" "os/exec" "runtime" "sort" "strings" "sync" "syscall" "time" "github.com/maruel/panicparse/cmd/panic/internal" ) // Mocked in test. var stdErr io.Writer = os.Stderr // Utility functions. func panicint(i int) { panic(i) } func panicstr(a string) { panic(a) } func panicslicestr(a []string) { panic(a) } func panicArgsElided(a, b, c, d, e, f, g, h, i, j, k int) { panic(a) } func recurse(i int) { if i > 0 { recurse(i - 1) return } panic(42) } func panicRaceDisabled() { help := "'panic race' can only be used when built with the race detector.\n" + "To build, use:\n" + " go install -race github.com/maruel/panicparse/cmd/panic\n" io.WriteString(stdErr, help) } func rerunWithFastCrash() { if os.Getenv("GORACE") != "log_path=stderr halt_on_error=1" { os.Setenv("GORACE", "log_path=stderr halt_on_error=1") c := exec.Command(os.Args[0], os.Args[1:]...) c.Stderr = os.Stderr if err, ok := c.Run().(*exec.ExitError); ok { if status, ok := err.Sys().(syscall.WaitStatus); ok { os.Exit(status.ExitStatus()) } os.Exit(1) } os.Exit(0) } } func panicRaceEnabled() { rerunWithFastCrash() i := 0 for j := 0; j < 2; j++ { go func() { for { i++ } }() } time.Sleep(time.Minute) } func panicRace() { if raceEnabled { panicRaceEnabled() } else { panicRaceDisabled() } } // // types is all the supported types of panics. // // Keep the list sorted. // // TODO(maruel): Figure out a way to reliably trigger "(scan)" output: // - disable automatic GC with runtime.SetGCPercent(-1) // - a goroutine with a large number of items in the stack // - large heap to make the scanning process slow enough // - trigger a manual GC with go runtime.GC() // - panic in the meantime // This would still not be deterministic. // // TODO(maruel): Figure out a way to reliably trigger sleep output. var types = map[string]struct { desc string f func() }{ "args_elided": { "too many args in stack line, causing the call arguments to be elided", func() { panicArgsElided(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) }, }, "chan_receive": { "goroutine blocked on <-c", func() { c := make(chan bool) go func() { <-c <-c }() c <- true panic(42) }, }, "chan_send": { "goroutine blocked on c<-", func() { c := make(chan bool) go func() { c <- true c <- true }() <-c panic(42) }, }, "goroutine_1": { "panic in one goroutine", func() { go func() { panicint(42) }() time.Sleep(time.Minute) }, }, "goroutine_100": { "start 100 goroutines before panicking", func() { var wg sync.WaitGroup for i := 0; i < 100; i++ { wg.Add(1) go func() { wg.Done() time.Sleep(time.Minute) }() } wg.Wait() panicint(42) }, }, "goroutine_dedupe_pointers": { "start 100 goroutines with different pointers before panicking", func() { var wg sync.WaitGroup for i := 0; i < 100; i++ { wg.Add(1) go func(b *int) { wg.Done() time.Sleep(time.Minute) }(new(int)) } wg.Wait() panicint(42) }, }, "int": { "panic(42)", func() { panicint(42) }, }, "locked": { "thread locked goroutine via runtime.LockOSThread()", func() { runtime.LockOSThread() panic(42) }, }, "other": { "panics with other package in the call stack, with both exported and unexpected functions", func() { internal.Callback(func() { panic("allo") }) }, }, "asleep": { "panics with 'all goroutines are asleep - deadlock'", func() { // When built with the race detector, this hangs. I suspect this is due // because the race detector starts a separate goroutine which causes // checkdead() to not trigger. See checkdead() in src/runtime/proc.go. // https://github.com/golang/go/issues/20588 // // Repro: // go install -race github.com/maruel/panicparse/cmd/panic; panic asleep var mu sync.Mutex mu.Lock() mu.Lock() }, }, "race": { "cause a crash by race detector", panicRace, }, "stack_cut_off": { "recursive calls with too many call lines in traceback, causing higher up calls to missing", func() { // Observed limit is 99. // src/runtime/runtime2.go:const _TracebackMaxFrames = 100 recurse(100) }, }, "stack_cut_off_named": { "named calls with too many call lines in traceback, causing higher up calls to missing", func() { // As of go1.12.5, up to recurse1215() is printed. recurse2000() }, }, "simple": { // This is not used for real, here for documentation. "skip the map for a shorter stack trace", func() {}, }, "slice_str": { "panic([]string{\"allo\"}) with cap=2", func() { a := make([]string, 1, 2) a[0] = "allo" panicslicestr(a) }, }, "stdlib": { "panics with stdlib in the call stack, with both exported and unexpected functions", func() { strings.FieldsFunc("a", func(rune) bool { panic("allo") }) }, }, "stdlib_and_other": { "panics with both other and stdlib packages in the call stack", func() { strings.FieldsFunc("a", func(rune) bool { internal.Callback(func() { panic("allo") }) return false }) }, }, "str": { "panic(\"allo\")", func() { panicstr("allo") }, }, } // func main() { if len(os.Args) == 2 { n := os.Args[1] if f, ok := types[n]; ok { fmt.Printf("GOTRACEBACK=%s\n", os.Getenv("GOTRACEBACK")) if n == "simple" { // Since the map lookup creates another call stack entry, add a one-off // "simple" panic style to test the very minimal case. // types["simple"].f is never called. panic("simple") } f.f() os.Exit(3) } fmt.Fprintf(stdErr, "unknown panic style %q\n", n) os.Exit(1) } usage() } func usage() { t := `usage: panic This tool is meant to be used with pp to test different parsing scenarios and ensure output on different version of the Go toolchain can be successfully parsed. Set GOTRACEBACK before running this tool to see how it affects the panic output. Select the way to panic: ` io.WriteString(stdErr, t) names := make([]string, 0, len(types)) m := 0 for n := range types { names = append(names, n) if i := len(n); i > m { m = i } } sort.Strings(names) for _, n := range names { fmt.Fprintf(stdErr, "- %-*s %s\n", m, n, types[n].desc) } os.Exit(2) } // func recurse00() { panic("the end") } func recurse01() { recurse00() } func recurse02() { recurse01() } func recurse03() { recurse02() } func recurse04() { recurse03() } func recurse05() { recurse04() } func recurse06() { recurse05() } func recurse07() { recurse06() } func recurse08() { recurse07() } func recurse09() { recurse08() } func recurse10() { recurse09() } func recurse11() { recurse10() } func recurse12() { recurse11() } func recurse13() { recurse12() } func recurse14() { recurse13() } func recurse15() { recurse14() } func recurse16() { recurse15() } func recurse17() { recurse16() } func recurse18() { recurse17() } func recurse19() { recurse18() } func recurse20() { recurse19() } func recurse21() { recurse20() } func recurse22() { recurse21() } func recurse23() { recurse22() } func recurse24() { recurse23() } func recurse25() { recurse24() } func recurse26() { recurse25() } func recurse27() { recurse26() } func recurse28() { recurse27() } func recurse29() { recurse28() } func recurse30() { recurse29() } func recurse31() { recurse30() } func recurse32() { recurse31() } func recurse33() { recurse32() } func recurse34() { recurse33() } func recurse35() { recurse34() } func recurse36() { recurse35() } func recurse37() { recurse36() } func recurse38() { recurse37() } func recurse39() { recurse38() } func recurse40() { recurse39() } func recurse41() { recurse40() } func recurse42() { recurse41() } func recurse43() { recurse42() } func recurse44() { recurse43() } func recurse45() { recurse44() } func recurse46() { recurse45() } func recurse47() { recurse46() } func recurse48() { recurse47() } func recurse49() { recurse48() } func recurse50() { recurse49() } func recurse51() { recurse50() } func recurse52() { recurse51() } func recurse53() { recurse52() } func recurse54() { recurse53() } func recurse55() { recurse54() } func recurse56() { recurse55() } func recurse57() { recurse56() } func recurse58() { recurse57() } func recurse59() { recurse58() } func recurse60() { recurse59() } func recurse61() { recurse60() } func recurse62() { recurse61() } func recurse63() { recurse62() } func recurse64() { recurse63() } func recurse65() { recurse64() } func recurse66() { recurse65() } func recurse67() { recurse66() } func recurse68() { recurse67() } func recurse69() { recurse68() } func recurse70() { recurse69() } func recurse71() { recurse70() } func recurse72() { recurse71() } func recurse73() { recurse72() } func recurse74() { recurse73() } func recurse75() { recurse74() } func recurse76() { recurse75() } func recurse77() { recurse76() } func recurse78() { recurse77() } func recurse79() { recurse78() } func recurse80() { recurse79() } func recurse81() { recurse80() } func recurse82() { recurse81() } func recurse83() { recurse82() } func recurse84() { recurse83() } func recurse85() { recurse84() } func recurse86() { recurse85() } func recurse87() { recurse86() } func recurse88() { recurse87() } func recurse89() { recurse88() } func recurse90() { recurse89() } func recurse91() { recurse90() } func recurse92() { recurse91() } func recurse93() { recurse92() } func recurse94() { recurse93() } func recurse95() { recurse94() } func recurse96() { recurse95() } func recurse97() { recurse96() } func recurse98() { recurse97() } func recurse99() { recurse98() } func recurse100() { recurse99() } func recurse101() { recurse100() } func recurse102() { recurse101() } func recurse103() { recurse102() } func recurse104() { recurse103() } func recurse105() { recurse104() } func recurse106() { recurse105() } func recurse107() { recurse106() } func recurse108() { recurse107() } func recurse109() { recurse108() } func recurse110() { recurse109() } func recurse111() { recurse110() } func recurse112() { recurse111() } func recurse113() { recurse112() } func recurse114() { recurse113() } func recurse115() { recurse114() } func recurse116() { recurse115() } func recurse117() { recurse116() } func recurse118() { recurse117() } func recurse119() { recurse118() } func recurse120() { recurse119() } func recurse121() { recurse120() } func recurse122() { recurse121() } func recurse123() { recurse122() } func recurse124() { recurse123() } func recurse125() { recurse124() } func recurse126() { recurse125() } func recurse127() { recurse126() } func recurse128() { recurse127() } func recurse129() { recurse128() } func recurse130() { recurse129() } func recurse131() { recurse130() } func recurse132() { recurse131() } func recurse133() { recurse132() } func recurse134() { recurse133() } func recurse135() { recurse134() } func recurse136() { recurse135() } func recurse137() { recurse136() } func recurse138() { recurse137() } func recurse139() { recurse138() } func recurse140() { recurse139() } func recurse141() { recurse140() } func recurse142() { recurse141() } func recurse143() { recurse142() } func recurse144() { recurse143() } func recurse145() { recurse144() } func recurse146() { recurse145() } func recurse147() { recurse146() } func recurse148() { recurse147() } func recurse149() { recurse148() } func recurse150() { recurse149() } func recurse151() { recurse150() } func recurse152() { recurse151() } func recurse153() { recurse152() } func recurse154() { recurse153() } func recurse155() { recurse154() } func recurse156() { recurse155() } func recurse157() { recurse156() } func recurse158() { recurse157() } func recurse159() { recurse158() } func recurse160() { recurse159() } func recurse161() { recurse160() } func recurse162() { recurse161() } func recurse163() { recurse162() } func recurse164() { recurse163() } func recurse165() { recurse164() } func recurse166() { recurse165() } func recurse167() { recurse166() } func recurse168() { recurse167() } func recurse169() { recurse168() } func recurse170() { recurse169() } func recurse171() { recurse170() } func recurse172() { recurse171() } func recurse173() { recurse172() } func recurse174() { recurse173() } func recurse175() { recurse174() } func recurse176() { recurse175() } func recurse177() { recurse176() } func recurse178() { recurse177() } func recurse179() { recurse178() } func recurse180() { recurse179() } func recurse181() { recurse180() } func recurse182() { recurse181() } func recurse183() { recurse182() } func recurse184() { recurse183() } func recurse185() { recurse184() } func recurse186() { recurse185() } func recurse187() { recurse186() } func recurse188() { recurse187() } func recurse189() { recurse188() } func recurse190() { recurse189() } func recurse191() { recurse190() } func recurse192() { recurse191() } func recurse193() { recurse192() } func recurse194() { recurse193() } func recurse195() { recurse194() } func recurse196() { recurse195() } func recurse197() { recurse196() } func recurse198() { recurse197() } func recurse199() { recurse198() } func recurse200() { recurse199() } func recurse201() { recurse200() } func recurse202() { recurse201() } func recurse203() { recurse202() } func recurse204() { recurse203() } func recurse205() { recurse204() } func recurse206() { recurse205() } func recurse207() { recurse206() } func recurse208() { recurse207() } func recurse209() { recurse208() } func recurse210() { recurse209() } func recurse211() { recurse210() } func recurse212() { recurse211() } func recurse213() { recurse212() } func recurse214() { recurse213() } func recurse215() { recurse214() } func recurse216() { recurse215() } func recurse217() { recurse216() } func recurse218() { recurse217() } func recurse219() { recurse218() } func recurse220() { recurse219() } func recurse221() { recurse220() } func recurse222() { recurse221() } func recurse223() { recurse222() } func recurse224() { recurse223() } func recurse225() { recurse224() } func recurse226() { recurse225() } func recurse227() { recurse226() } func recurse228() { recurse227() } func recurse229() { recurse228() } func recurse230() { recurse229() } func recurse231() { recurse230() } func recurse232() { recurse231() } func recurse233() { recurse232() } func recurse234() { recurse233() } func recurse235() { recurse234() } func recurse236() { recurse235() } func recurse237() { recurse236() } func recurse238() { recurse237() } func recurse239() { recurse238() } func recurse240() { recurse239() } func recurse241() { recurse240() } func recurse242() { recurse241() } func recurse243() { recurse242() } func recurse244() { recurse243() } func recurse245() { recurse244() } func recurse246() { recurse245() } func recurse247() { recurse246() } func recurse248() { recurse247() } func recurse249() { recurse248() } func recurse250() { recurse249() } func recurse251() { recurse250() } func recurse252() { recurse251() } func recurse253() { recurse252() } func recurse254() { recurse253() } func recurse255() { recurse254() } func recurse256() { recurse255() } func recurse257() { recurse256() } func recurse258() { recurse257() } func recurse259() { recurse258() } func recurse260() { recurse259() } func recurse261() { recurse260() } func recurse262() { recurse261() } func recurse263() { recurse262() } func recurse264() { recurse263() } func recurse265() { recurse264() } func recurse266() { recurse265() } func recurse267() { recurse266() } func recurse268() { recurse267() } func recurse269() { recurse268() } func recurse270() { recurse269() } func recurse271() { recurse270() } func recurse272() { recurse271() } func recurse273() { recurse272() } func recurse274() { recurse273() } func recurse275() { recurse274() } func recurse276() { recurse275() } func recurse277() { recurse276() } func recurse278() { recurse277() } func recurse279() { recurse278() } func recurse280() { recurse279() } func recurse281() { recurse280() } func recurse282() { recurse281() } func recurse283() { recurse282() } func recurse284() { recurse283() } func recurse285() { recurse284() } func recurse286() { recurse285() } func recurse287() { recurse286() } func recurse288() { recurse287() } func recurse289() { recurse288() } func recurse290() { recurse289() } func recurse291() { recurse290() } func recurse292() { recurse291() } func recurse293() { recurse292() } func recurse294() { recurse293() } func recurse295() { recurse294() } func recurse296() { recurse295() } func recurse297() { recurse296() } func recurse298() { recurse297() } func recurse299() { recurse298() } func recurse300() { recurse299() } func recurse301() { recurse300() } func recurse302() { recurse301() } func recurse303() { recurse302() } func recurse304() { recurse303() } func recurse305() { recurse304() } func recurse306() { recurse305() } func recurse307() { recurse306() } func recurse308() { recurse307() } func recurse309() { recurse308() } func recurse310() { recurse309() } func recurse311() { recurse310() } func recurse312() { recurse311() } func recurse313() { recurse312() } func recurse314() { recurse313() } func recurse315() { recurse314() } func recurse316() { recurse315() } func recurse317() { recurse316() } func recurse318() { recurse317() } func recurse319() { recurse318() } func recurse320() { recurse319() } func recurse321() { recurse320() } func recurse322() { recurse321() } func recurse323() { recurse322() } func recurse324() { recurse323() } func recurse325() { recurse324() } func recurse326() { recurse325() } func recurse327() { recurse326() } func recurse328() { recurse327() } func recurse329() { recurse328() } func recurse330() { recurse329() } func recurse331() { recurse330() } func recurse332() { recurse331() } func recurse333() { recurse332() } func recurse334() { recurse333() } func recurse335() { recurse334() } func recurse336() { recurse335() } func recurse337() { recurse336() } func recurse338() { recurse337() } func recurse339() { recurse338() } func recurse340() { recurse339() } func recurse341() { recurse340() } func recurse342() { recurse341() } func recurse343() { recurse342() } func recurse344() { recurse343() } func recurse345() { recurse344() } func recurse346() { recurse345() } func recurse347() { recurse346() } func recurse348() { recurse347() } func recurse349() { recurse348() } func recurse350() { recurse349() } func recurse351() { recurse350() } func recurse352() { recurse351() } func recurse353() { recurse352() } func recurse354() { recurse353() } func recurse355() { recurse354() } func recurse356() { recurse355() } func recurse357() { recurse356() } func recurse358() { recurse357() } func recurse359() { recurse358() } func recurse360() { recurse359() } func recurse361() { recurse360() } func recurse362() { recurse361() } func recurse363() { recurse362() } func recurse364() { recurse363() } func recurse365() { recurse364() } func recurse366() { recurse365() } func recurse367() { recurse366() } func recurse368() { recurse367() } func recurse369() { recurse368() } func recurse370() { recurse369() } func recurse371() { recurse370() } func recurse372() { recurse371() } func recurse373() { recurse372() } func recurse374() { recurse373() } func recurse375() { recurse374() } func recurse376() { recurse375() } func recurse377() { recurse376() } func recurse378() { recurse377() } func recurse379() { recurse378() } func recurse380() { recurse379() } func recurse381() { recurse380() } func recurse382() { recurse381() } func recurse383() { recurse382() } func recurse384() { recurse383() } func recurse385() { recurse384() } func recurse386() { recurse385() } func recurse387() { recurse386() } func recurse388() { recurse387() } func recurse389() { recurse388() } func recurse390() { recurse389() } func recurse391() { recurse390() } func recurse392() { recurse391() } func recurse393() { recurse392() } func recurse394() { recurse393() } func recurse395() { recurse394() } func recurse396() { recurse395() } func recurse397() { recurse396() } func recurse398() { recurse397() } func recurse399() { recurse398() } func recurse400() { recurse399() } func recurse401() { recurse400() } func recurse402() { recurse401() } func recurse403() { recurse402() } func recurse404() { recurse403() } func recurse405() { recurse404() } func recurse406() { recurse405() } func recurse407() { recurse406() } func recurse408() { recurse407() } func recurse409() { recurse408() } func recurse410() { recurse409() } func recurse411() { recurse410() } func recurse412() { recurse411() } func recurse413() { recurse412() } func recurse414() { recurse413() } func recurse415() { recurse414() } func recurse416() { recurse415() } func recurse417() { recurse416() } func recurse418() { recurse417() } func recurse419() { recurse418() } func recurse420() { recurse419() } func recurse421() { recurse420() } func recurse422() { recurse421() } func recurse423() { recurse422() } func recurse424() { recurse423() } func recurse425() { recurse424() } func recurse426() { recurse425() } func recurse427() { recurse426() } func recurse428() { recurse427() } func recurse429() { recurse428() } func recurse430() { recurse429() } func recurse431() { recurse430() } func recurse432() { recurse431() } func recurse433() { recurse432() } func recurse434() { recurse433() } func recurse435() { recurse434() } func recurse436() { recurse435() } func recurse437() { recurse436() } func recurse438() { recurse437() } func recurse439() { recurse438() } func recurse440() { recurse439() } func recurse441() { recurse440() } func recurse442() { recurse441() } func recurse443() { recurse442() } func recurse444() { recurse443() } func recurse445() { recurse444() } func recurse446() { recurse445() } func recurse447() { recurse446() } func recurse448() { recurse447() } func recurse449() { recurse448() } func recurse450() { recurse449() } func recurse451() { recurse450() } func recurse452() { recurse451() } func recurse453() { recurse452() } func recurse454() { recurse453() } func recurse455() { recurse454() } func recurse456() { recurse455() } func recurse457() { recurse456() } func recurse458() { recurse457() } func recurse459() { recurse458() } func recurse460() { recurse459() } func recurse461() { recurse460() } func recurse462() { recurse461() } func recurse463() { recurse462() } func recurse464() { recurse463() } func recurse465() { recurse464() } func recurse466() { recurse465() } func recurse467() { recurse466() } func recurse468() { recurse467() } func recurse469() { recurse468() } func recurse470() { recurse469() } func recurse471() { recurse470() } func recurse472() { recurse471() } func recurse473() { recurse472() } func recurse474() { recurse473() } func recurse475() { recurse474() } func recurse476() { recurse475() } func recurse477() { recurse476() } func recurse478() { recurse477() } func recurse479() { recurse478() } func recurse480() { recurse479() } func recurse481() { recurse480() } func recurse482() { recurse481() } func recurse483() { recurse482() } func recurse484() { recurse483() } func recurse485() { recurse484() } func recurse486() { recurse485() } func recurse487() { recurse486() } func recurse488() { recurse487() } func recurse489() { recurse488() } func recurse490() { recurse489() } func recurse491() { recurse490() } func recurse492() { recurse491() } func recurse493() { recurse492() } func recurse494() { recurse493() } func recurse495() { recurse494() } func recurse496() { recurse495() } func recurse497() { recurse496() } func recurse498() { recurse497() } func recurse499() { recurse498() } func recurse500() { recurse499() } func recurse501() { recurse500() } func recurse502() { recurse501() } func recurse503() { recurse502() } func recurse504() { recurse503() } func recurse505() { recurse504() } func recurse506() { recurse505() } func recurse507() { recurse506() } func recurse508() { recurse507() } func recurse509() { recurse508() } func recurse510() { recurse509() } func recurse511() { recurse510() } func recurse512() { recurse511() } func recurse513() { recurse512() } func recurse514() { recurse513() } func recurse515() { recurse514() } func recurse516() { recurse515() } func recurse517() { recurse516() } func recurse518() { recurse517() } func recurse519() { recurse518() } func recurse520() { recurse519() } func recurse521() { recurse520() } func recurse522() { recurse521() } func recurse523() { recurse522() } func recurse524() { recurse523() } func recurse525() { recurse524() } func recurse526() { recurse525() } func recurse527() { recurse526() } func recurse528() { recurse527() } func recurse529() { recurse528() } func recurse530() { recurse529() } func recurse531() { recurse530() } func recurse532() { recurse531() } func recurse533() { recurse532() } func recurse534() { recurse533() } func recurse535() { recurse534() } func recurse536() { recurse535() } func recurse537() { recurse536() } func recurse538() { recurse537() } func recurse539() { recurse538() } func recurse540() { recurse539() } func recurse541() { recurse540() } func recurse542() { recurse541() } func recurse543() { recurse542() } func recurse544() { recurse543() } func recurse545() { recurse544() } func recurse546() { recurse545() } func recurse547() { recurse546() } func recurse548() { recurse547() } func recurse549() { recurse548() } func recurse550() { recurse549() } func recurse551() { recurse550() } func recurse552() { recurse551() } func recurse553() { recurse552() } func recurse554() { recurse553() } func recurse555() { recurse554() } func recurse556() { recurse555() } func recurse557() { recurse556() } func recurse558() { recurse557() } func recurse559() { recurse558() } func recurse560() { recurse559() } func recurse561() { recurse560() } func recurse562() { recurse561() } func recurse563() { recurse562() } func recurse564() { recurse563() } func recurse565() { recurse564() } func recurse566() { recurse565() } func recurse567() { recurse566() } func recurse568() { recurse567() } func recurse569() { recurse568() } func recurse570() { recurse569() } func recurse571() { recurse570() } func recurse572() { recurse571() } func recurse573() { recurse572() } func recurse574() { recurse573() } func recurse575() { recurse574() } func recurse576() { recurse575() } func recurse577() { recurse576() } func recurse578() { recurse577() } func recurse579() { recurse578() } func recurse580() { recurse579() } func recurse581() { recurse580() } func recurse582() { recurse581() } func recurse583() { recurse582() } func recurse584() { recurse583() } func recurse585() { recurse584() } func recurse586() { recurse585() } func recurse587() { recurse586() } func recurse588() { recurse587() } func recurse589() { recurse588() } func recurse590() { recurse589() } func recurse591() { recurse590() } func recurse592() { recurse591() } func recurse593() { recurse592() } func recurse594() { recurse593() } func recurse595() { recurse594() } func recurse596() { recurse595() } func recurse597() { recurse596() } func recurse598() { recurse597() } func recurse599() { recurse598() } func recurse600() { recurse599() } func recurse601() { recurse600() } func recurse602() { recurse601() } func recurse603() { recurse602() } func recurse604() { recurse603() } func recurse605() { recurse604() } func recurse606() { recurse605() } func recurse607() { recurse606() } func recurse608() { recurse607() } func recurse609() { recurse608() } func recurse610() { recurse609() } func recurse611() { recurse610() } func recurse612() { recurse611() } func recurse613() { recurse612() } func recurse614() { recurse613() } func recurse615() { recurse614() } func recurse616() { recurse615() } func recurse617() { recurse616() } func recurse618() { recurse617() } func recurse619() { recurse618() } func recurse620() { recurse619() } func recurse621() { recurse620() } func recurse622() { recurse621() } func recurse623() { recurse622() } func recurse624() { recurse623() } func recurse625() { recurse624() } func recurse626() { recurse625() } func recurse627() { recurse626() } func recurse628() { recurse627() } func recurse629() { recurse628() } func recurse630() { recurse629() } func recurse631() { recurse630() } func recurse632() { recurse631() } func recurse633() { recurse632() } func recurse634() { recurse633() } func recurse635() { recurse634() } func recurse636() { recurse635() } func recurse637() { recurse636() } func recurse638() { recurse637() } func recurse639() { recurse638() } func recurse640() { recurse639() } func recurse641() { recurse640() } func recurse642() { recurse641() } func recurse643() { recurse642() } func recurse644() { recurse643() } func recurse645() { recurse644() } func recurse646() { recurse645() } func recurse647() { recurse646() } func recurse648() { recurse647() } func recurse649() { recurse648() } func recurse650() { recurse649() } func recurse651() { recurse650() } func recurse652() { recurse651() } func recurse653() { recurse652() } func recurse654() { recurse653() } func recurse655() { recurse654() } func recurse656() { recurse655() } func recurse657() { recurse656() } func recurse658() { recurse657() } func recurse659() { recurse658() } func recurse660() { recurse659() } func recurse661() { recurse660() } func recurse662() { recurse661() } func recurse663() { recurse662() } func recurse664() { recurse663() } func recurse665() { recurse664() } func recurse666() { recurse665() } func recurse667() { recurse666() } func recurse668() { recurse667() } func recurse669() { recurse668() } func recurse670() { recurse669() } func recurse671() { recurse670() } func recurse672() { recurse671() } func recurse673() { recurse672() } func recurse674() { recurse673() } func recurse675() { recurse674() } func recurse676() { recurse675() } func recurse677() { recurse676() } func recurse678() { recurse677() } func recurse679() { recurse678() } func recurse680() { recurse679() } func recurse681() { recurse680() } func recurse682() { recurse681() } func recurse683() { recurse682() } func recurse684() { recurse683() } func recurse685() { recurse684() } func recurse686() { recurse685() } func recurse687() { recurse686() } func recurse688() { recurse687() } func recurse689() { recurse688() } func recurse690() { recurse689() } func recurse691() { recurse690() } func recurse692() { recurse691() } func recurse693() { recurse692() } func recurse694() { recurse693() } func recurse695() { recurse694() } func recurse696() { recurse695() } func recurse697() { recurse696() } func recurse698() { recurse697() } func recurse699() { recurse698() } func recurse700() { recurse699() } func recurse701() { recurse700() } func recurse702() { recurse701() } func recurse703() { recurse702() } func recurse704() { recurse703() } func recurse705() { recurse704() } func recurse706() { recurse705() } func recurse707() { recurse706() } func recurse708() { recurse707() } func recurse709() { recurse708() } func recurse710() { recurse709() } func recurse711() { recurse710() } func recurse712() { recurse711() } func recurse713() { recurse712() } func recurse714() { recurse713() } func recurse715() { recurse714() } func recurse716() { recurse715() } func recurse717() { recurse716() } func recurse718() { recurse717() } func recurse719() { recurse718() } func recurse720() { recurse719() } func recurse721() { recurse720() } func recurse722() { recurse721() } func recurse723() { recurse722() } func recurse724() { recurse723() } func recurse725() { recurse724() } func recurse726() { recurse725() } func recurse727() { recurse726() } func recurse728() { recurse727() } func recurse729() { recurse728() } func recurse730() { recurse729() } func recurse731() { recurse730() } func recurse732() { recurse731() } func recurse733() { recurse732() } func recurse734() { recurse733() } func recurse735() { recurse734() } func recurse736() { recurse735() } func recurse737() { recurse736() } func recurse738() { recurse737() } func recurse739() { recurse738() } func recurse740() { recurse739() } func recurse741() { recurse740() } func recurse742() { recurse741() } func recurse743() { recurse742() } func recurse744() { recurse743() } func recurse745() { recurse744() } func recurse746() { recurse745() } func recurse747() { recurse746() } func recurse748() { recurse747() } func recurse749() { recurse748() } func recurse750() { recurse749() } func recurse751() { recurse750() } func recurse752() { recurse751() } func recurse753() { recurse752() } func recurse754() { recurse753() } func recurse755() { recurse754() } func recurse756() { recurse755() } func recurse757() { recurse756() } func recurse758() { recurse757() } func recurse759() { recurse758() } func recurse760() { recurse759() } func recurse761() { recurse760() } func recurse762() { recurse761() } func recurse763() { recurse762() } func recurse764() { recurse763() } func recurse765() { recurse764() } func recurse766() { recurse765() } func recurse767() { recurse766() } func recurse768() { recurse767() } func recurse769() { recurse768() } func recurse770() { recurse769() } func recurse771() { recurse770() } func recurse772() { recurse771() } func recurse773() { recurse772() } func recurse774() { recurse773() } func recurse775() { recurse774() } func recurse776() { recurse775() } func recurse777() { recurse776() } func recurse778() { recurse777() } func recurse779() { recurse778() } func recurse780() { recurse779() } func recurse781() { recurse780() } func recurse782() { recurse781() } func recurse783() { recurse782() } func recurse784() { recurse783() } func recurse785() { recurse784() } func recurse786() { recurse785() } func recurse787() { recurse786() } func recurse788() { recurse787() } func recurse789() { recurse788() } func recurse790() { recurse789() } func recurse791() { recurse790() } func recurse792() { recurse791() } func recurse793() { recurse792() } func recurse794() { recurse793() } func recurse795() { recurse794() } func recurse796() { recurse795() } func recurse797() { recurse796() } func recurse798() { recurse797() } func recurse799() { recurse798() } func recurse800() { recurse799() } func recurse801() { recurse800() } func recurse802() { recurse801() } func recurse803() { recurse802() } func recurse804() { recurse803() } func recurse805() { recurse804() } func recurse806() { recurse805() } func recurse807() { recurse806() } func recurse808() { recurse807() } func recurse809() { recurse808() } func recurse810() { recurse809() } func recurse811() { recurse810() } func recurse812() { recurse811() } func recurse813() { recurse812() } func recurse814() { recurse813() } func recurse815() { recurse814() } func recurse816() { recurse815() } func recurse817() { recurse816() } func recurse818() { recurse817() } func recurse819() { recurse818() } func recurse820() { recurse819() } func recurse821() { recurse820() } func recurse822() { recurse821() } func recurse823() { recurse822() } func recurse824() { recurse823() } func recurse825() { recurse824() } func recurse826() { recurse825() } func recurse827() { recurse826() } func recurse828() { recurse827() } func recurse829() { recurse828() } func recurse830() { recurse829() } func recurse831() { recurse830() } func recurse832() { recurse831() } func recurse833() { recurse832() } func recurse834() { recurse833() } func recurse835() { recurse834() } func recurse836() { recurse835() } func recurse837() { recurse836() } func recurse838() { recurse837() } func recurse839() { recurse838() } func recurse840() { recurse839() } func recurse841() { recurse840() } func recurse842() { recurse841() } func recurse843() { recurse842() } func recurse844() { recurse843() } func recurse845() { recurse844() } func recurse846() { recurse845() } func recurse847() { recurse846() } func recurse848() { recurse847() } func recurse849() { recurse848() } func recurse850() { recurse849() } func recurse851() { recurse850() } func recurse852() { recurse851() } func recurse853() { recurse852() } func recurse854() { recurse853() } func recurse855() { recurse854() } func recurse856() { recurse855() } func recurse857() { recurse856() } func recurse858() { recurse857() } func recurse859() { recurse858() } func recurse860() { recurse859() } func recurse861() { recurse860() } func recurse862() { recurse861() } func recurse863() { recurse862() } func recurse864() { recurse863() } func recurse865() { recurse864() } func recurse866() { recurse865() } func recurse867() { recurse866() } func recurse868() { recurse867() } func recurse869() { recurse868() } func recurse870() { recurse869() } func recurse871() { recurse870() } func recurse872() { recurse871() } func recurse873() { recurse872() } func recurse874() { recurse873() } func recurse875() { recurse874() } func recurse876() { recurse875() } func recurse877() { recurse876() } func recurse878() { recurse877() } func recurse879() { recurse878() } func recurse880() { recurse879() } func recurse881() { recurse880() } func recurse882() { recurse881() } func recurse883() { recurse882() } func recurse884() { recurse883() } func recurse885() { recurse884() } func recurse886() { recurse885() } func recurse887() { recurse886() } func recurse888() { recurse887() } func recurse889() { recurse888() } func recurse890() { recurse889() } func recurse891() { recurse890() } func recurse892() { recurse891() } func recurse893() { recurse892() } func recurse894() { recurse893() } func recurse895() { recurse894() } func recurse896() { recurse895() } func recurse897() { recurse896() } func recurse898() { recurse897() } func recurse899() { recurse898() } func recurse900() { recurse899() } func recurse901() { recurse900() } func recurse902() { recurse901() } func recurse903() { recurse902() } func recurse904() { recurse903() } func recurse905() { recurse904() } func recurse906() { recurse905() } func recurse907() { recurse906() } func recurse908() { recurse907() } func recurse909() { recurse908() } func recurse910() { recurse909() } func recurse911() { recurse910() } func recurse912() { recurse911() } func recurse913() { recurse912() } func recurse914() { recurse913() } func recurse915() { recurse914() } func recurse916() { recurse915() } func recurse917() { recurse916() } func recurse918() { recurse917() } func recurse919() { recurse918() } func recurse920() { recurse919() } func recurse921() { recurse920() } func recurse922() { recurse921() } func recurse923() { recurse922() } func recurse924() { recurse923() } func recurse925() { recurse924() } func recurse926() { recurse925() } func recurse927() { recurse926() } func recurse928() { recurse927() } func recurse929() { recurse928() } func recurse930() { recurse929() } func recurse931() { recurse930() } func recurse932() { recurse931() } func recurse933() { recurse932() } func recurse934() { recurse933() } func recurse935() { recurse934() } func recurse936() { recurse935() } func recurse937() { recurse936() } func recurse938() { recurse937() } func recurse939() { recurse938() } func recurse940() { recurse939() } func recurse941() { recurse940() } func recurse942() { recurse941() } func recurse943() { recurse942() } func recurse944() { recurse943() } func recurse945() { recurse944() } func recurse946() { recurse945() } func recurse947() { recurse946() } func recurse948() { recurse947() } func recurse949() { recurse948() } func recurse950() { recurse949() } func recurse951() { recurse950() } func recurse952() { recurse951() } func recurse953() { recurse952() } func recurse954() { recurse953() } func recurse955() { recurse954() } func recurse956() { recurse955() } func recurse957() { recurse956() } func recurse958() { recurse957() } func recurse959() { recurse958() } func recurse960() { recurse959() } func recurse961() { recurse960() } func recurse962() { recurse961() } func recurse963() { recurse962() } func recurse964() { recurse963() } func recurse965() { recurse964() } func recurse966() { recurse965() } func recurse967() { recurse966() } func recurse968() { recurse967() } func recurse969() { recurse968() } func recurse970() { recurse969() } func recurse971() { recurse970() } func recurse972() { recurse971() } func recurse973() { recurse972() } func recurse974() { recurse973() } func recurse975() { recurse974() } func recurse976() { recurse975() } func recurse977() { recurse976() } func recurse978() { recurse977() } func recurse979() { recurse978() } func recurse980() { recurse979() } func recurse981() { recurse980() } func recurse982() { recurse981() } func recurse983() { recurse982() } func recurse984() { recurse983() } func recurse985() { recurse984() } func recurse986() { recurse985() } func recurse987() { recurse986() } func recurse988() { recurse987() } func recurse989() { recurse988() } func recurse990() { recurse989() } func recurse991() { recurse990() } func recurse992() { recurse991() } func recurse993() { recurse992() } func recurse994() { recurse993() } func recurse995() { recurse994() } func recurse996() { recurse995() } func recurse997() { recurse996() } func recurse998() { recurse997() } func recurse999() { recurse998() } func recurse1000() { recurse999() } func recurse1001() { recurse1000() } func recurse1002() { recurse1001() } func recurse1003() { recurse1002() } func recurse1004() { recurse1003() } func recurse1005() { recurse1004() } func recurse1006() { recurse1005() } func recurse1007() { recurse1006() } func recurse1008() { recurse1007() } func recurse1009() { recurse1008() } func recurse1010() { recurse1009() } func recurse1011() { recurse1010() } func recurse1012() { recurse1011() } func recurse1013() { recurse1012() } func recurse1014() { recurse1013() } func recurse1015() { recurse1014() } func recurse1016() { recurse1015() } func recurse1017() { recurse1016() } func recurse1018() { recurse1017() } func recurse1019() { recurse1018() } func recurse1020() { recurse1019() } func recurse1021() { recurse1020() } func recurse1022() { recurse1021() } func recurse1023() { recurse1022() } func recurse1024() { recurse1023() } func recurse1025() { recurse1024() } func recurse1026() { recurse1025() } func recurse1027() { recurse1026() } func recurse1028() { recurse1027() } func recurse1029() { recurse1028() } func recurse1030() { recurse1029() } func recurse1031() { recurse1030() } func recurse1032() { recurse1031() } func recurse1033() { recurse1032() } func recurse1034() { recurse1033() } func recurse1035() { recurse1034() } func recurse1036() { recurse1035() } func recurse1037() { recurse1036() } func recurse1038() { recurse1037() } func recurse1039() { recurse1038() } func recurse1040() { recurse1039() } func recurse1041() { recurse1040() } func recurse1042() { recurse1041() } func recurse1043() { recurse1042() } func recurse1044() { recurse1043() } func recurse1045() { recurse1044() } func recurse1046() { recurse1045() } func recurse1047() { recurse1046() } func recurse1048() { recurse1047() } func recurse1049() { recurse1048() } func recurse1050() { recurse1049() } func recurse1051() { recurse1050() } func recurse1052() { recurse1051() } func recurse1053() { recurse1052() } func recurse1054() { recurse1053() } func recurse1055() { recurse1054() } func recurse1056() { recurse1055() } func recurse1057() { recurse1056() } func recurse1058() { recurse1057() } func recurse1059() { recurse1058() } func recurse1060() { recurse1059() } func recurse1061() { recurse1060() } func recurse1062() { recurse1061() } func recurse1063() { recurse1062() } func recurse1064() { recurse1063() } func recurse1065() { recurse1064() } func recurse1066() { recurse1065() } func recurse1067() { recurse1066() } func recurse1068() { recurse1067() } func recurse1069() { recurse1068() } func recurse1070() { recurse1069() } func recurse1071() { recurse1070() } func recurse1072() { recurse1071() } func recurse1073() { recurse1072() } func recurse1074() { recurse1073() } func recurse1075() { recurse1074() } func recurse1076() { recurse1075() } func recurse1077() { recurse1076() } func recurse1078() { recurse1077() } func recurse1079() { recurse1078() } func recurse1080() { recurse1079() } func recurse1081() { recurse1080() } func recurse1082() { recurse1081() } func recurse1083() { recurse1082() } func recurse1084() { recurse1083() } func recurse1085() { recurse1084() } func recurse1086() { recurse1085() } func recurse1087() { recurse1086() } func recurse1088() { recurse1087() } func recurse1089() { recurse1088() } func recurse1090() { recurse1089() } func recurse1091() { recurse1090() } func recurse1092() { recurse1091() } func recurse1093() { recurse1092() } func recurse1094() { recurse1093() } func recurse1095() { recurse1094() } func recurse1096() { recurse1095() } func recurse1097() { recurse1096() } func recurse1098() { recurse1097() } func recurse1099() { recurse1098() } func recurse1100() { recurse1099() } func recurse1101() { recurse1100() } func recurse1102() { recurse1101() } func recurse1103() { recurse1102() } func recurse1104() { recurse1103() } func recurse1105() { recurse1104() } func recurse1106() { recurse1105() } func recurse1107() { recurse1106() } func recurse1108() { recurse1107() } func recurse1109() { recurse1108() } func recurse1110() { recurse1109() } func recurse1111() { recurse1110() } func recurse1112() { recurse1111() } func recurse1113() { recurse1112() } func recurse1114() { recurse1113() } func recurse1115() { recurse1114() } func recurse1116() { recurse1115() } func recurse1117() { recurse1116() } func recurse1118() { recurse1117() } func recurse1119() { recurse1118() } func recurse1120() { recurse1119() } func recurse1121() { recurse1120() } func recurse1122() { recurse1121() } func recurse1123() { recurse1122() } func recurse1124() { recurse1123() } func recurse1125() { recurse1124() } func recurse1126() { recurse1125() } func recurse1127() { recurse1126() } func recurse1128() { recurse1127() } func recurse1129() { recurse1128() } func recurse1130() { recurse1129() } func recurse1131() { recurse1130() } func recurse1132() { recurse1131() } func recurse1133() { recurse1132() } func recurse1134() { recurse1133() } func recurse1135() { recurse1134() } func recurse1136() { recurse1135() } func recurse1137() { recurse1136() } func recurse1138() { recurse1137() } func recurse1139() { recurse1138() } func recurse1140() { recurse1139() } func recurse1141() { recurse1140() } func recurse1142() { recurse1141() } func recurse1143() { recurse1142() } func recurse1144() { recurse1143() } func recurse1145() { recurse1144() } func recurse1146() { recurse1145() } func recurse1147() { recurse1146() } func recurse1148() { recurse1147() } func recurse1149() { recurse1148() } func recurse1150() { recurse1149() } func recurse1151() { recurse1150() } func recurse1152() { recurse1151() } func recurse1153() { recurse1152() } func recurse1154() { recurse1153() } func recurse1155() { recurse1154() } func recurse1156() { recurse1155() } func recurse1157() { recurse1156() } func recurse1158() { recurse1157() } func recurse1159() { recurse1158() } func recurse1160() { recurse1159() } func recurse1161() { recurse1160() } func recurse1162() { recurse1161() } func recurse1163() { recurse1162() } func recurse1164() { recurse1163() } func recurse1165() { recurse1164() } func recurse1166() { recurse1165() } func recurse1167() { recurse1166() } func recurse1168() { recurse1167() } func recurse1169() { recurse1168() } func recurse1170() { recurse1169() } func recurse1171() { recurse1170() } func recurse1172() { recurse1171() } func recurse1173() { recurse1172() } func recurse1174() { recurse1173() } func recurse1175() { recurse1174() } func recurse1176() { recurse1175() } func recurse1177() { recurse1176() } func recurse1178() { recurse1177() } func recurse1179() { recurse1178() } func recurse1180() { recurse1179() } func recurse1181() { recurse1180() } func recurse1182() { recurse1181() } func recurse1183() { recurse1182() } func recurse1184() { recurse1183() } func recurse1185() { recurse1184() } func recurse1186() { recurse1185() } func recurse1187() { recurse1186() } func recurse1188() { recurse1187() } func recurse1189() { recurse1188() } func recurse1190() { recurse1189() } func recurse1191() { recurse1190() } func recurse1192() { recurse1191() } func recurse1193() { recurse1192() } func recurse1194() { recurse1193() } func recurse1195() { recurse1194() } func recurse1196() { recurse1195() } func recurse1197() { recurse1196() } func recurse1198() { recurse1197() } func recurse1199() { recurse1198() } func recurse1200() { recurse1199() } func recurse1201() { recurse1200() } func recurse1202() { recurse1201() } func recurse1203() { recurse1202() } func recurse1204() { recurse1203() } func recurse1205() { recurse1204() } func recurse1206() { recurse1205() } func recurse1207() { recurse1206() } func recurse1208() { recurse1207() } func recurse1209() { recurse1208() } func recurse1210() { recurse1209() } func recurse1211() { recurse1210() } func recurse1212() { recurse1211() } func recurse1213() { recurse1212() } func recurse1214() { recurse1213() } func recurse1215() { recurse1214() } func recurse1216() { recurse1215() } func recurse1217() { recurse1216() } func recurse1218() { recurse1217() } func recurse1219() { recurse1218() } func recurse1220() { recurse1219() } func recurse1221() { recurse1220() } func recurse1222() { recurse1221() } func recurse1223() { recurse1222() } func recurse1224() { recurse1223() } func recurse1225() { recurse1224() } func recurse1226() { recurse1225() } func recurse1227() { recurse1226() } func recurse1228() { recurse1227() } func recurse1229() { recurse1228() } func recurse1230() { recurse1229() } func recurse1231() { recurse1230() } func recurse1232() { recurse1231() } func recurse1233() { recurse1232() } func recurse1234() { recurse1233() } func recurse1235() { recurse1234() } func recurse1236() { recurse1235() } func recurse1237() { recurse1236() } func recurse1238() { recurse1237() } func recurse1239() { recurse1238() } func recurse1240() { recurse1239() } func recurse1241() { recurse1240() } func recurse1242() { recurse1241() } func recurse1243() { recurse1242() } func recurse1244() { recurse1243() } func recurse1245() { recurse1244() } func recurse1246() { recurse1245() } func recurse1247() { recurse1246() } func recurse1248() { recurse1247() } func recurse1249() { recurse1248() } func recurse1250() { recurse1249() } func recurse1251() { recurse1250() } func recurse1252() { recurse1251() } func recurse1253() { recurse1252() } func recurse1254() { recurse1253() } func recurse1255() { recurse1254() } func recurse1256() { recurse1255() } func recurse1257() { recurse1256() } func recurse1258() { recurse1257() } func recurse1259() { recurse1258() } func recurse1260() { recurse1259() } func recurse1261() { recurse1260() } func recurse1262() { recurse1261() } func recurse1263() { recurse1262() } func recurse1264() { recurse1263() } func recurse1265() { recurse1264() } func recurse1266() { recurse1265() } func recurse1267() { recurse1266() } func recurse1268() { recurse1267() } func recurse1269() { recurse1268() } func recurse1270() { recurse1269() } func recurse1271() { recurse1270() } func recurse1272() { recurse1271() } func recurse1273() { recurse1272() } func recurse1274() { recurse1273() } func recurse1275() { recurse1274() } func recurse1276() { recurse1275() } func recurse1277() { recurse1276() } func recurse1278() { recurse1277() } func recurse1279() { recurse1278() } func recurse1280() { recurse1279() } func recurse1281() { recurse1280() } func recurse1282() { recurse1281() } func recurse1283() { recurse1282() } func recurse1284() { recurse1283() } func recurse1285() { recurse1284() } func recurse1286() { recurse1285() } func recurse1287() { recurse1286() } func recurse1288() { recurse1287() } func recurse1289() { recurse1288() } func recurse1290() { recurse1289() } func recurse1291() { recurse1290() } func recurse1292() { recurse1291() } func recurse1293() { recurse1292() } func recurse1294() { recurse1293() } func recurse1295() { recurse1294() } func recurse1296() { recurse1295() } func recurse1297() { recurse1296() } func recurse1298() { recurse1297() } func recurse1299() { recurse1298() } func recurse1300() { recurse1299() } func recurse1301() { recurse1300() } func recurse1302() { recurse1301() } func recurse1303() { recurse1302() } func recurse1304() { recurse1303() } func recurse1305() { recurse1304() } func recurse1306() { recurse1305() } func recurse1307() { recurse1306() } func recurse1308() { recurse1307() } func recurse1309() { recurse1308() } func recurse1310() { recurse1309() } func recurse1311() { recurse1310() } func recurse1312() { recurse1311() } func recurse1313() { recurse1312() } func recurse1314() { recurse1313() } func recurse1315() { recurse1314() } func recurse1316() { recurse1315() } func recurse1317() { recurse1316() } func recurse1318() { recurse1317() } func recurse1319() { recurse1318() } func recurse1320() { recurse1319() } func recurse1321() { recurse1320() } func recurse1322() { recurse1321() } func recurse1323() { recurse1322() } func recurse1324() { recurse1323() } func recurse1325() { recurse1324() } func recurse1326() { recurse1325() } func recurse1327() { recurse1326() } func recurse1328() { recurse1327() } func recurse1329() { recurse1328() } func recurse1330() { recurse1329() } func recurse1331() { recurse1330() } func recurse1332() { recurse1331() } func recurse1333() { recurse1332() } func recurse1334() { recurse1333() } func recurse1335() { recurse1334() } func recurse1336() { recurse1335() } func recurse1337() { recurse1336() } func recurse1338() { recurse1337() } func recurse1339() { recurse1338() } func recurse1340() { recurse1339() } func recurse1341() { recurse1340() } func recurse1342() { recurse1341() } func recurse1343() { recurse1342() } func recurse1344() { recurse1343() } func recurse1345() { recurse1344() } func recurse1346() { recurse1345() } func recurse1347() { recurse1346() } func recurse1348() { recurse1347() } func recurse1349() { recurse1348() } func recurse1350() { recurse1349() } func recurse1351() { recurse1350() } func recurse1352() { recurse1351() } func recurse1353() { recurse1352() } func recurse1354() { recurse1353() } func recurse1355() { recurse1354() } func recurse1356() { recurse1355() } func recurse1357() { recurse1356() } func recurse1358() { recurse1357() } func recurse1359() { recurse1358() } func recurse1360() { recurse1359() } func recurse1361() { recurse1360() } func recurse1362() { recurse1361() } func recurse1363() { recurse1362() } func recurse1364() { recurse1363() } func recurse1365() { recurse1364() } func recurse1366() { recurse1365() } func recurse1367() { recurse1366() } func recurse1368() { recurse1367() } func recurse1369() { recurse1368() } func recurse1370() { recurse1369() } func recurse1371() { recurse1370() } func recurse1372() { recurse1371() } func recurse1373() { recurse1372() } func recurse1374() { recurse1373() } func recurse1375() { recurse1374() } func recurse1376() { recurse1375() } func recurse1377() { recurse1376() } func recurse1378() { recurse1377() } func recurse1379() { recurse1378() } func recurse1380() { recurse1379() } func recurse1381() { recurse1380() } func recurse1382() { recurse1381() } func recurse1383() { recurse1382() } func recurse1384() { recurse1383() } func recurse1385() { recurse1384() } func recurse1386() { recurse1385() } func recurse1387() { recurse1386() } func recurse1388() { recurse1387() } func recurse1389() { recurse1388() } func recurse1390() { recurse1389() } func recurse1391() { recurse1390() } func recurse1392() { recurse1391() } func recurse1393() { recurse1392() } func recurse1394() { recurse1393() } func recurse1395() { recurse1394() } func recurse1396() { recurse1395() } func recurse1397() { recurse1396() } func recurse1398() { recurse1397() } func recurse1399() { recurse1398() } func recurse1400() { recurse1399() } func recurse1401() { recurse1400() } func recurse1402() { recurse1401() } func recurse1403() { recurse1402() } func recurse1404() { recurse1403() } func recurse1405() { recurse1404() } func recurse1406() { recurse1405() } func recurse1407() { recurse1406() } func recurse1408() { recurse1407() } func recurse1409() { recurse1408() } func recurse1410() { recurse1409() } func recurse1411() { recurse1410() } func recurse1412() { recurse1411() } func recurse1413() { recurse1412() } func recurse1414() { recurse1413() } func recurse1415() { recurse1414() } func recurse1416() { recurse1415() } func recurse1417() { recurse1416() } func recurse1418() { recurse1417() } func recurse1419() { recurse1418() } func recurse1420() { recurse1419() } func recurse1421() { recurse1420() } func recurse1422() { recurse1421() } func recurse1423() { recurse1422() } func recurse1424() { recurse1423() } func recurse1425() { recurse1424() } func recurse1426() { recurse1425() } func recurse1427() { recurse1426() } func recurse1428() { recurse1427() } func recurse1429() { recurse1428() } func recurse1430() { recurse1429() } func recurse1431() { recurse1430() } func recurse1432() { recurse1431() } func recurse1433() { recurse1432() } func recurse1434() { recurse1433() } func recurse1435() { recurse1434() } func recurse1436() { recurse1435() } func recurse1437() { recurse1436() } func recurse1438() { recurse1437() } func recurse1439() { recurse1438() } func recurse1440() { recurse1439() } func recurse1441() { recurse1440() } func recurse1442() { recurse1441() } func recurse1443() { recurse1442() } func recurse1444() { recurse1443() } func recurse1445() { recurse1444() } func recurse1446() { recurse1445() } func recurse1447() { recurse1446() } func recurse1448() { recurse1447() } func recurse1449() { recurse1448() } func recurse1450() { recurse1449() } func recurse1451() { recurse1450() } func recurse1452() { recurse1451() } func recurse1453() { recurse1452() } func recurse1454() { recurse1453() } func recurse1455() { recurse1454() } func recurse1456() { recurse1455() } func recurse1457() { recurse1456() } func recurse1458() { recurse1457() } func recurse1459() { recurse1458() } func recurse1460() { recurse1459() } func recurse1461() { recurse1460() } func recurse1462() { recurse1461() } func recurse1463() { recurse1462() } func recurse1464() { recurse1463() } func recurse1465() { recurse1464() } func recurse1466() { recurse1465() } func recurse1467() { recurse1466() } func recurse1468() { recurse1467() } func recurse1469() { recurse1468() } func recurse1470() { recurse1469() } func recurse1471() { recurse1470() } func recurse1472() { recurse1471() } func recurse1473() { recurse1472() } func recurse1474() { recurse1473() } func recurse1475() { recurse1474() } func recurse1476() { recurse1475() } func recurse1477() { recurse1476() } func recurse1478() { recurse1477() } func recurse1479() { recurse1478() } func recurse1480() { recurse1479() } func recurse1481() { recurse1480() } func recurse1482() { recurse1481() } func recurse1483() { recurse1482() } func recurse1484() { recurse1483() } func recurse1485() { recurse1484() } func recurse1486() { recurse1485() } func recurse1487() { recurse1486() } func recurse1488() { recurse1487() } func recurse1489() { recurse1488() } func recurse1490() { recurse1489() } func recurse1491() { recurse1490() } func recurse1492() { recurse1491() } func recurse1493() { recurse1492() } func recurse1494() { recurse1493() } func recurse1495() { recurse1494() } func recurse1496() { recurse1495() } func recurse1497() { recurse1496() } func recurse1498() { recurse1497() } func recurse1499() { recurse1498() } func recurse1500() { recurse1499() } func recurse1501() { recurse1500() } func recurse1502() { recurse1501() } func recurse1503() { recurse1502() } func recurse1504() { recurse1503() } func recurse1505() { recurse1504() } func recurse1506() { recurse1505() } func recurse1507() { recurse1506() } func recurse1508() { recurse1507() } func recurse1509() { recurse1508() } func recurse1510() { recurse1509() } func recurse1511() { recurse1510() } func recurse1512() { recurse1511() } func recurse1513() { recurse1512() } func recurse1514() { recurse1513() } func recurse1515() { recurse1514() } func recurse1516() { recurse1515() } func recurse1517() { recurse1516() } func recurse1518() { recurse1517() } func recurse1519() { recurse1518() } func recurse1520() { recurse1519() } func recurse1521() { recurse1520() } func recurse1522() { recurse1521() } func recurse1523() { recurse1522() } func recurse1524() { recurse1523() } func recurse1525() { recurse1524() } func recurse1526() { recurse1525() } func recurse1527() { recurse1526() } func recurse1528() { recurse1527() } func recurse1529() { recurse1528() } func recurse1530() { recurse1529() } func recurse1531() { recurse1530() } func recurse1532() { recurse1531() } func recurse1533() { recurse1532() } func recurse1534() { recurse1533() } func recurse1535() { recurse1534() } func recurse1536() { recurse1535() } func recurse1537() { recurse1536() } func recurse1538() { recurse1537() } func recurse1539() { recurse1538() } func recurse1540() { recurse1539() } func recurse1541() { recurse1540() } func recurse1542() { recurse1541() } func recurse1543() { recurse1542() } func recurse1544() { recurse1543() } func recurse1545() { recurse1544() } func recurse1546() { recurse1545() } func recurse1547() { recurse1546() } func recurse1548() { recurse1547() } func recurse1549() { recurse1548() } func recurse1550() { recurse1549() } func recurse1551() { recurse1550() } func recurse1552() { recurse1551() } func recurse1553() { recurse1552() } func recurse1554() { recurse1553() } func recurse1555() { recurse1554() } func recurse1556() { recurse1555() } func recurse1557() { recurse1556() } func recurse1558() { recurse1557() } func recurse1559() { recurse1558() } func recurse1560() { recurse1559() } func recurse1561() { recurse1560() } func recurse1562() { recurse1561() } func recurse1563() { recurse1562() } func recurse1564() { recurse1563() } func recurse1565() { recurse1564() } func recurse1566() { recurse1565() } func recurse1567() { recurse1566() } func recurse1568() { recurse1567() } func recurse1569() { recurse1568() } func recurse1570() { recurse1569() } func recurse1571() { recurse1570() } func recurse1572() { recurse1571() } func recurse1573() { recurse1572() } func recurse1574() { recurse1573() } func recurse1575() { recurse1574() } func recurse1576() { recurse1575() } func recurse1577() { recurse1576() } func recurse1578() { recurse1577() } func recurse1579() { recurse1578() } func recurse1580() { recurse1579() } func recurse1581() { recurse1580() } func recurse1582() { recurse1581() } func recurse1583() { recurse1582() } func recurse1584() { recurse1583() } func recurse1585() { recurse1584() } func recurse1586() { recurse1585() } func recurse1587() { recurse1586() } func recurse1588() { recurse1587() } func recurse1589() { recurse1588() } func recurse1590() { recurse1589() } func recurse1591() { recurse1590() } func recurse1592() { recurse1591() } func recurse1593() { recurse1592() } func recurse1594() { recurse1593() } func recurse1595() { recurse1594() } func recurse1596() { recurse1595() } func recurse1597() { recurse1596() } func recurse1598() { recurse1597() } func recurse1599() { recurse1598() } func recurse1600() { recurse1599() } func recurse1601() { recurse1600() } func recurse1602() { recurse1601() } func recurse1603() { recurse1602() } func recurse1604() { recurse1603() } func recurse1605() { recurse1604() } func recurse1606() { recurse1605() } func recurse1607() { recurse1606() } func recurse1608() { recurse1607() } func recurse1609() { recurse1608() } func recurse1610() { recurse1609() } func recurse1611() { recurse1610() } func recurse1612() { recurse1611() } func recurse1613() { recurse1612() } func recurse1614() { recurse1613() } func recurse1615() { recurse1614() } func recurse1616() { recurse1615() } func recurse1617() { recurse1616() } func recurse1618() { recurse1617() } func recurse1619() { recurse1618() } func recurse1620() { recurse1619() } func recurse1621() { recurse1620() } func recurse1622() { recurse1621() } func recurse1623() { recurse1622() } func recurse1624() { recurse1623() } func recurse1625() { recurse1624() } func recurse1626() { recurse1625() } func recurse1627() { recurse1626() } func recurse1628() { recurse1627() } func recurse1629() { recurse1628() } func recurse1630() { recurse1629() } func recurse1631() { recurse1630() } func recurse1632() { recurse1631() } func recurse1633() { recurse1632() } func recurse1634() { recurse1633() } func recurse1635() { recurse1634() } func recurse1636() { recurse1635() } func recurse1637() { recurse1636() } func recurse1638() { recurse1637() } func recurse1639() { recurse1638() } func recurse1640() { recurse1639() } func recurse1641() { recurse1640() } func recurse1642() { recurse1641() } func recurse1643() { recurse1642() } func recurse1644() { recurse1643() } func recurse1645() { recurse1644() } func recurse1646() { recurse1645() } func recurse1647() { recurse1646() } func recurse1648() { recurse1647() } func recurse1649() { recurse1648() } func recurse1650() { recurse1649() } func recurse1651() { recurse1650() } func recurse1652() { recurse1651() } func recurse1653() { recurse1652() } func recurse1654() { recurse1653() } func recurse1655() { recurse1654() } func recurse1656() { recurse1655() } func recurse1657() { recurse1656() } func recurse1658() { recurse1657() } func recurse1659() { recurse1658() } func recurse1660() { recurse1659() } func recurse1661() { recurse1660() } func recurse1662() { recurse1661() } func recurse1663() { recurse1662() } func recurse1664() { recurse1663() } func recurse1665() { recurse1664() } func recurse1666() { recurse1665() } func recurse1667() { recurse1666() } func recurse1668() { recurse1667() } func recurse1669() { recurse1668() } func recurse1670() { recurse1669() } func recurse1671() { recurse1670() } func recurse1672() { recurse1671() } func recurse1673() { recurse1672() } func recurse1674() { recurse1673() } func recurse1675() { recurse1674() } func recurse1676() { recurse1675() } func recurse1677() { recurse1676() } func recurse1678() { recurse1677() } func recurse1679() { recurse1678() } func recurse1680() { recurse1679() } func recurse1681() { recurse1680() } func recurse1682() { recurse1681() } func recurse1683() { recurse1682() } func recurse1684() { recurse1683() } func recurse1685() { recurse1684() } func recurse1686() { recurse1685() } func recurse1687() { recurse1686() } func recurse1688() { recurse1687() } func recurse1689() { recurse1688() } func recurse1690() { recurse1689() } func recurse1691() { recurse1690() } func recurse1692() { recurse1691() } func recurse1693() { recurse1692() } func recurse1694() { recurse1693() } func recurse1695() { recurse1694() } func recurse1696() { recurse1695() } func recurse1697() { recurse1696() } func recurse1698() { recurse1697() } func recurse1699() { recurse1698() } func recurse1700() { recurse1699() } func recurse1701() { recurse1700() } func recurse1702() { recurse1701() } func recurse1703() { recurse1702() } func recurse1704() { recurse1703() } func recurse1705() { recurse1704() } func recurse1706() { recurse1705() } func recurse1707() { recurse1706() } func recurse1708() { recurse1707() } func recurse1709() { recurse1708() } func recurse1710() { recurse1709() } func recurse1711() { recurse1710() } func recurse1712() { recurse1711() } func recurse1713() { recurse1712() } func recurse1714() { recurse1713() } func recurse1715() { recurse1714() } func recurse1716() { recurse1715() } func recurse1717() { recurse1716() } func recurse1718() { recurse1717() } func recurse1719() { recurse1718() } func recurse1720() { recurse1719() } func recurse1721() { recurse1720() } func recurse1722() { recurse1721() } func recurse1723() { recurse1722() } func recurse1724() { recurse1723() } func recurse1725() { recurse1724() } func recurse1726() { recurse1725() } func recurse1727() { recurse1726() } func recurse1728() { recurse1727() } func recurse1729() { recurse1728() } func recurse1730() { recurse1729() } func recurse1731() { recurse1730() } func recurse1732() { recurse1731() } func recurse1733() { recurse1732() } func recurse1734() { recurse1733() } func recurse1735() { recurse1734() } func recurse1736() { recurse1735() } func recurse1737() { recurse1736() } func recurse1738() { recurse1737() } func recurse1739() { recurse1738() } func recurse1740() { recurse1739() } func recurse1741() { recurse1740() } func recurse1742() { recurse1741() } func recurse1743() { recurse1742() } func recurse1744() { recurse1743() } func recurse1745() { recurse1744() } func recurse1746() { recurse1745() } func recurse1747() { recurse1746() } func recurse1748() { recurse1747() } func recurse1749() { recurse1748() } func recurse1750() { recurse1749() } func recurse1751() { recurse1750() } func recurse1752() { recurse1751() } func recurse1753() { recurse1752() } func recurse1754() { recurse1753() } func recurse1755() { recurse1754() } func recurse1756() { recurse1755() } func recurse1757() { recurse1756() } func recurse1758() { recurse1757() } func recurse1759() { recurse1758() } func recurse1760() { recurse1759() } func recurse1761() { recurse1760() } func recurse1762() { recurse1761() } func recurse1763() { recurse1762() } func recurse1764() { recurse1763() } func recurse1765() { recurse1764() } func recurse1766() { recurse1765() } func recurse1767() { recurse1766() } func recurse1768() { recurse1767() } func recurse1769() { recurse1768() } func recurse1770() { recurse1769() } func recurse1771() { recurse1770() } func recurse1772() { recurse1771() } func recurse1773() { recurse1772() } func recurse1774() { recurse1773() } func recurse1775() { recurse1774() } func recurse1776() { recurse1775() } func recurse1777() { recurse1776() } func recurse1778() { recurse1777() } func recurse1779() { recurse1778() } func recurse1780() { recurse1779() } func recurse1781() { recurse1780() } func recurse1782() { recurse1781() } func recurse1783() { recurse1782() } func recurse1784() { recurse1783() } func recurse1785() { recurse1784() } func recurse1786() { recurse1785() } func recurse1787() { recurse1786() } func recurse1788() { recurse1787() } func recurse1789() { recurse1788() } func recurse1790() { recurse1789() } func recurse1791() { recurse1790() } func recurse1792() { recurse1791() } func recurse1793() { recurse1792() } func recurse1794() { recurse1793() } func recurse1795() { recurse1794() } func recurse1796() { recurse1795() } func recurse1797() { recurse1796() } func recurse1798() { recurse1797() } func recurse1799() { recurse1798() } func recurse1800() { recurse1799() } func recurse1801() { recurse1800() } func recurse1802() { recurse1801() } func recurse1803() { recurse1802() } func recurse1804() { recurse1803() } func recurse1805() { recurse1804() } func recurse1806() { recurse1805() } func recurse1807() { recurse1806() } func recurse1808() { recurse1807() } func recurse1809() { recurse1808() } func recurse1810() { recurse1809() } func recurse1811() { recurse1810() } func recurse1812() { recurse1811() } func recurse1813() { recurse1812() } func recurse1814() { recurse1813() } func recurse1815() { recurse1814() } func recurse1816() { recurse1815() } func recurse1817() { recurse1816() } func recurse1818() { recurse1817() } func recurse1819() { recurse1818() } func recurse1820() { recurse1819() } func recurse1821() { recurse1820() } func recurse1822() { recurse1821() } func recurse1823() { recurse1822() } func recurse1824() { recurse1823() } func recurse1825() { recurse1824() } func recurse1826() { recurse1825() } func recurse1827() { recurse1826() } func recurse1828() { recurse1827() } func recurse1829() { recurse1828() } func recurse1830() { recurse1829() } func recurse1831() { recurse1830() } func recurse1832() { recurse1831() } func recurse1833() { recurse1832() } func recurse1834() { recurse1833() } func recurse1835() { recurse1834() } func recurse1836() { recurse1835() } func recurse1837() { recurse1836() } func recurse1838() { recurse1837() } func recurse1839() { recurse1838() } func recurse1840() { recurse1839() } func recurse1841() { recurse1840() } func recurse1842() { recurse1841() } func recurse1843() { recurse1842() } func recurse1844() { recurse1843() } func recurse1845() { recurse1844() } func recurse1846() { recurse1845() } func recurse1847() { recurse1846() } func recurse1848() { recurse1847() } func recurse1849() { recurse1848() } func recurse1850() { recurse1849() } func recurse1851() { recurse1850() } func recurse1852() { recurse1851() } func recurse1853() { recurse1852() } func recurse1854() { recurse1853() } func recurse1855() { recurse1854() } func recurse1856() { recurse1855() } func recurse1857() { recurse1856() } func recurse1858() { recurse1857() } func recurse1859() { recurse1858() } func recurse1860() { recurse1859() } func recurse1861() { recurse1860() } func recurse1862() { recurse1861() } func recurse1863() { recurse1862() } func recurse1864() { recurse1863() } func recurse1865() { recurse1864() } func recurse1866() { recurse1865() } func recurse1867() { recurse1866() } func recurse1868() { recurse1867() } func recurse1869() { recurse1868() } func recurse1870() { recurse1869() } func recurse1871() { recurse1870() } func recurse1872() { recurse1871() } func recurse1873() { recurse1872() } func recurse1874() { recurse1873() } func recurse1875() { recurse1874() } func recurse1876() { recurse1875() } func recurse1877() { recurse1876() } func recurse1878() { recurse1877() } func recurse1879() { recurse1878() } func recurse1880() { recurse1879() } func recurse1881() { recurse1880() } func recurse1882() { recurse1881() } func recurse1883() { recurse1882() } func recurse1884() { recurse1883() } func recurse1885() { recurse1884() } func recurse1886() { recurse1885() } func recurse1887() { recurse1886() } func recurse1888() { recurse1887() } func recurse1889() { recurse1888() } func recurse1890() { recurse1889() } func recurse1891() { recurse1890() } func recurse1892() { recurse1891() } func recurse1893() { recurse1892() } func recurse1894() { recurse1893() } func recurse1895() { recurse1894() } func recurse1896() { recurse1895() } func recurse1897() { recurse1896() } func recurse1898() { recurse1897() } func recurse1899() { recurse1898() } func recurse1900() { recurse1899() } func recurse1901() { recurse1900() } func recurse1902() { recurse1901() } func recurse1903() { recurse1902() } func recurse1904() { recurse1903() } func recurse1905() { recurse1904() } func recurse1906() { recurse1905() } func recurse1907() { recurse1906() } func recurse1908() { recurse1907() } func recurse1909() { recurse1908() } func recurse1910() { recurse1909() } func recurse1911() { recurse1910() } func recurse1912() { recurse1911() } func recurse1913() { recurse1912() } func recurse1914() { recurse1913() } func recurse1915() { recurse1914() } func recurse1916() { recurse1915() } func recurse1917() { recurse1916() } func recurse1918() { recurse1917() } func recurse1919() { recurse1918() } func recurse1920() { recurse1919() } func recurse1921() { recurse1920() } func recurse1922() { recurse1921() } func recurse1923() { recurse1922() } func recurse1924() { recurse1923() } func recurse1925() { recurse1924() } func recurse1926() { recurse1925() } func recurse1927() { recurse1926() } func recurse1928() { recurse1927() } func recurse1929() { recurse1928() } func recurse1930() { recurse1929() } func recurse1931() { recurse1930() } func recurse1932() { recurse1931() } func recurse1933() { recurse1932() } func recurse1934() { recurse1933() } func recurse1935() { recurse1934() } func recurse1936() { recurse1935() } func recurse1937() { recurse1936() } func recurse1938() { recurse1937() } func recurse1939() { recurse1938() } func recurse1940() { recurse1939() } func recurse1941() { recurse1940() } func recurse1942() { recurse1941() } func recurse1943() { recurse1942() } func recurse1944() { recurse1943() } func recurse1945() { recurse1944() } func recurse1946() { recurse1945() } func recurse1947() { recurse1946() } func recurse1948() { recurse1947() } func recurse1949() { recurse1948() } func recurse1950() { recurse1949() } func recurse1951() { recurse1950() } func recurse1952() { recurse1951() } func recurse1953() { recurse1952() } func recurse1954() { recurse1953() } func recurse1955() { recurse1954() } func recurse1956() { recurse1955() } func recurse1957() { recurse1956() } func recurse1958() { recurse1957() } func recurse1959() { recurse1958() } func recurse1960() { recurse1959() } func recurse1961() { recurse1960() } func recurse1962() { recurse1961() } func recurse1963() { recurse1962() } func recurse1964() { recurse1963() } func recurse1965() { recurse1964() } func recurse1966() { recurse1965() } func recurse1967() { recurse1966() } func recurse1968() { recurse1967() } func recurse1969() { recurse1968() } func recurse1970() { recurse1969() } func recurse1971() { recurse1970() } func recurse1972() { recurse1971() } func recurse1973() { recurse1972() } func recurse1974() { recurse1973() } func recurse1975() { recurse1974() } func recurse1976() { recurse1975() } func recurse1977() { recurse1976() } func recurse1978() { recurse1977() } func recurse1979() { recurse1978() } func recurse1980() { recurse1979() } func recurse1981() { recurse1980() } func recurse1982() { recurse1981() } func recurse1983() { recurse1982() } func recurse1984() { recurse1983() } func recurse1985() { recurse1984() } func recurse1986() { recurse1985() } func recurse1987() { recurse1986() } func recurse1988() { recurse1987() } func recurse1989() { recurse1988() } func recurse1990() { recurse1989() } func recurse1991() { recurse1990() } func recurse1992() { recurse1991() } func recurse1993() { recurse1992() } func recurse1994() { recurse1993() } func recurse1995() { recurse1994() } func recurse1996() { recurse1995() } func recurse1997() { recurse1996() } func recurse1998() { recurse1997() } func recurse1999() { recurse1998() } func recurse2000() { recurse1999() } panicparse-1.3.0/cmd/panic/main_no_race.go000066400000000000000000000003531353500720100204330ustar00rootroot00000000000000// Copyright 2017 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // +build !race package main const raceEnabled = false panicparse-1.3.0/cmd/panic/main_race.go000066400000000000000000000003511353500720100177350ustar00rootroot00000000000000// Copyright 2017 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // +build race package main const raceEnabled = true panicparse-1.3.0/cmd/panic/main_test.go000066400000000000000000000017461353500720100200130ustar00rootroot00000000000000// Copyright 2017 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package main import ( "io/ioutil" "os" "testing" ) func TestMain(t *testing.T) { if !testing.Verbose() { stdErr = ioutil.Discard defer func() { stdErr = os.Stderr }() } for name, l := range types { if name == "simple" { // It's safe. l.f() continue } if name == "goroutine_1" || name == "asleep" { // goroutine_1 panics in a separate goroutine, so it's tricky to catch. // asleep will just hang because there are other goroutine. continue } if name == "race" { if raceEnabled { // It's not safe, it'll crash the program in a way that cannot be trapped. continue } // It's safe. l.f() continue } t.Run(name, func(t *testing.T) { defer func() { if err := recover(); err == nil { t.Fatal("expected error") } }() l.f() }) } } panicparse-1.3.0/cmd/pp/000077500000000000000000000000001353500720100150165ustar00rootroot00000000000000panicparse-1.3.0/cmd/pp/main.go000066400000000000000000000014251353500720100162730ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // panicparse: analyzes stack dump of Go processes and simplifies it. // // It is mostly useful on servers will large number of identical goroutines, // making the crash dump harder to read than strictly necessary. // // Colors: // - Magenta: first goroutine to be listed. // - Yellow: main package. // - Green: standard library. // - Red: other packages. // // Bright colors are used for exported symbols. package main import ( "fmt" "os" "github.com/maruel/panicparse/internal" ) func main() { if err := internal.Main(); err != nil { fmt.Fprintf(os.Stderr, "Failed: %s\n", err) os.Exit(1) } } panicparse-1.3.0/go.mod000066400000000000000000000002751353500720100147460ustar00rootroot00000000000000module github.com/maruel/panicparse go 1.11 require ( github.com/mattn/go-colorable v0.1.1 github.com/mattn/go-isatty v0.0.7 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b ) panicparse-1.3.0/go.sum000066400000000000000000000015351353500720100147730ustar00rootroot00000000000000github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= panicparse-1.3.0/internal/000077500000000000000000000000001353500720100154505ustar00rootroot00000000000000panicparse-1.3.0/internal/goversion1.6.go000066400000000000000000000003751353500720100202440ustar00rootroot00000000000000// Copyright 2016 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // +build go1.6 package internal const ( showGOTRACEBACKBanner = true ) panicparse-1.3.0/internal/goversion1.go000066400000000000000000000004171353500720100200750ustar00rootroot00000000000000// Copyright 2016 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // +build go1.1 // +build !go1.6 package internal const ( showGOTRACEBACKBanner = false ) panicparse-1.3.0/internal/html.go000066400000000000000000000167151353500720100167550ustar00rootroot00000000000000// Copyright 2017 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package internal import ( "html/template" "os" "time" "github.com/maruel/panicparse/stack" ) func writeToHTML(html string, buckets []*stack.Bucket, needsEnv bool) error { m := template.FuncMap{ "funcClass": funcClass, "notoColorEmoji1F4A3": notoColorEmoji1F4A3, } if len(buckets) > 1 { m["routineClass"] = routineClass } else { m["routineClass"] = func(bucket *stack.Bucket) template.HTML { return "Routine" } } t, err := template.New("htmlTpl").Funcs(m).Parse(htmlTpl) if err != nil { return err } data := struct { Buckets []*stack.Bucket Now time.Time NeedsEnv bool }{buckets, time.Now().Truncate(time.Second), needsEnv} f, err := os.Create(html) if err != nil { return err } err1 := t.Execute(f, data) err2 := f.Close() if err1 != nil { return err1 } return err2 } func funcClass(line *stack.Call) template.HTML { if line.IsStdlib { if line.Func.IsExported() { return "FuncStdLibExported" } return "FuncStdLib" } else if line.IsPkgMain() { return "FuncMain" } else if line.Func.IsExported() { return "FuncOtherExported" } return "FuncOther" } func routineClass(bucket *stack.Bucket) template.HTML { if bucket.First { return "RoutineFirst" } return "Routine" } const htmlTpl = ` {{- define "RenderCall" -}} {{.SrcLine}} {{.Func.Name}}({{.Args}}) {{- end -}} PanicParse
Generated on {{.Now.String}}. {{if .NeedsEnv}}
To see all goroutines, visit github.com/maruel/panicparse.
{{end}}
{{range .Buckets}}

{{if .First}}Panicking {{end}}Routine

{{len .IDs}}: {{.State}} {{if .SleepMax -}} {{- if ne .SleepMin .SleepMax}} [{{.SleepMin}}~{{.SleepMax}} minutes] {{- else}} [{{.SleepMax}} minutes] {{- end -}} {{- end}} {{if .Locked}} [locked] {{- end -}} {{- if .CreatedBy.SrcPath}} [Created by {{template "RenderCall" .CreatedBy}}] {{- end -}}

Stack

{{range .Signature.Stack.Calls}} - {{template "RenderCall" .}}
{{- end}} {{if .Stack.Elided}}(...)
{{end}} {{end}}
` // notoColorEmoji1F4A3 is the bomb emoji U+1F4A3 in Noto Emoji as a PNG. // // Source: https://www.google.com/get/noto/help/emoji/smileys-people.html // License: http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=OFL // // Created with: // python -c "import base64;a=base64.b64encode(open('emoji_u1f4a3.png','rb').read()); print '\n'.join(a[i:i+70] for i in range(0,len(a),70))" func notoColorEmoji1F4A3() template.HTML { return "" + "iVBORw0KGgoAAAANSUhEUgAAAIgAAACACAMAAADnN9ENAAAA5FBMVEVMaXFvTjQhISEhIS" + "EhISEhISEhISEhISEhISEhISEhISEhISHRbBTRbBTRbBQhISHRbBTRbBTRbBTRbBQ6OjrR" + "bBTZcBTsehbzfhf1fxfidRVOTk75oiL7uCr90zL3kB3/6zv2hhlHMR5OTk5KSkpKSkpRUV" + "FKSkpKSkpMTExBQUE2NjYpKSlOTk5EREQ8PDw4ODgtLS1RUVFQUFAmJiYkJCQ0NDRLS0tT" + "U1MoKCgyMjJHR0dTU1NdXV0rKytXV1dVVVUvLy9eXl5aWlpcXFxcXFxeXl5fX186OjphYW" + "E/Pz9KSkrdB5CTAAAATHRSTlMAEUZggJjf/6/vcL86e1nP2//vmSC7//////9E////////" + "/2WPpYHp////////////////////z/////8w6P////+p/8f///////+/QBb3BQAACWJJRE" + "FUeAHslgWC6yAURUOk1N3d3d29+9/SzzxCStvQEfp9zgZycu8DnvTNN78YJCuqqsry77WQ" + "NRum2BX0u7JQiYWJw/l7NBz4AderQkFut8frdn+kFBu2wodeYeHxBwjBkFd6StiFOYiboF" + "CAxf9MRXHc9KGpqurDBpqghzcYuCOCeMp21oKelTBVCQt5kDiisXhCJ5aMQkFu6+lg4tDY" + "r2rikRCPKFgQYlGeiYpN7OHbpMj8OgQ8PAGdWIIlnnwbFKYdtgDAJj+MDgZSX/Zwsx4mby" + "YR6RbntRZVegQDX7/tI1YexMTNObQ+y992EUWRQJIJQjqTzWRyRjtRiMTqKtWQJCTCD4TM" + "aS6bBzLGxLKRKNer1MELX0wEmYHk8pSMGUmIlMI+LC7uTeEQmhEvnZBC/kqGTolfkmSn72" + "NPbFhoWOHswmczeYYC7YaV4MfBHl+BEYmCSJYVSUM3ukgRM5C7g4edqAqLMBq0m1sRh8oe" + "Fl4zzp8swtFApXKlmkIkECD8+mpYEZ8iWZGq1YFKKazg582IDiu8bk7Ob5YaOnVCs9UWu+" + "C99D7LWR5fVeY/YpVOp9Po9rp1g25/AIGIXmhp0yMLgSTIhcYDVYaj0ag1Hk/a0yZ1qVVK" + "6KsmfnMVSbMepBkv32M2nw87rfZioatMxsveirqUBLoxHr1CJpvNZmBQSSB+icd6M9dFpt" + "tt21CZ4EHfKKny9Ug4awA/kNRmt993loPBAFSICcbtFpRUeuViFIPFiENpp9NYHg6HexW8" + "1Suaiaysscc8gsg6jdTxpFNf6oALUaEmBz2SsMBKEkjeL88Bt8VFWj1fCKvWdDolKmYoYD" + "L5ejcSApNoMsZqBH+0byfbiStNEICbFZt/uIN3WtpASWIUwgOiZWgMyPL7v8+tCuIkBdlw" + "W4WWHS/AdyKzRCEfXy7Iw+PHntlti+k0TZ1FKCJJgteV0wHGhr/1Lvp4/HGQvGdJVVVTZy" + "HFl6TGDEIM3FiUIvnrv53zkXyH4PNzm5mknrhUNo7CUjAeSIbGmNW3Vih/gHGKY1jE53uc" + "1BJYSOF4KCmM6YcqaPmvy/86l88MKPbzcXIKLKSwFJFUPMDtpvPDKT63cZKMJSCRglJE4k" + "7x0hjTaZHAOpzjYBIKCpsThpQLSc4D3GaOdWQ0+CEFEo7HSkpIxjzAraXz4RxbURhJQQtK" + "gYSdYE2mPMBtZYWxjMggIRYLKSiFks0Gw9nExjy16XBnxYBxNHghRSTcE65JEXM4rTl2qI" + "OKkRdnIcWXcDgzXktam8s76zBQzOcZ4q6IsIBCSVVhYTmckpJ2HWBA8XoMMEri1oQnJ89L" + "x++3cF6U46hYI8CQ4ks4HBzhYdHK0+TDc5ABxDsCCygi4ZpIJYvF0EIGqzsdffvtskschA" + "4wLGHrcrRoCYfDSnBVe+nc5Yis4zAWRwYHFQwpFxKBuEq6Uyt5untBCs8hjB1DCiVnlfDg" + "5PkCdzUT3TmYDINxezqH46jY7+3FxF0Vd75EVcLZdPPCmEH4cFb202RBfIdT2K4ONo5CyQ" + "iSE+T5NJvu8q4z/LE/fBYWwsHY/Xgn45NxGEr8SvyDc4R06zt+W0S7wyGrk1MhdJAhkr2T" + "rEXy09ng/toLL2SfcENkMHRoiYVkrERBnKQKriSynzmqORlAlMObjgyHs+G5kSXp5sGV/N" + "jZQmQyKMQOxnNIpBI1m40sCSoJOjgPux0LKW4XQgkOjpqNBwm9wPYtJFGToeNaJaPjbPS2" + "utRh98bvu/1rLZAMEFWISAjxl0RBNkE//Fb2U4sTBI5bkJ2/JBqCFCHfOH27mBMHKXzI4R" + "pk/1PI8zmkCpnNx3aXaYh1NICkF5BZwKOkYz+1aBUS+OYmsp9aa8i+wWjUjuDc9BqvyPa9" + "AkSW9b5Tg0yNeWm8ItusmkwuniP6wUrHBSTREFmSpk+R7dZMq4l6sh6uP1kdBLc09WQVyL" + "D5Rc1eCGtAkiPk5pLIZDKuiH7EM40hD4R426oqufmlNwHEOs4hSdN7WmQhqYOoLxslEYd8" + "1ejTK5C6MWTtIN6SuHPD4fgSOvxCrnznBZxfQtbPrOTi7kyJdkghNyCpMV+NIP31+4gQVs" + "Itubg9yzVerqxyePWKhEHWDiLnhpVQAgoCBh08u7qQuyCv69FSKrmQgLLDm3jHEIdXiJ5M" + "IOTxdZ0B4h0cSvzfnFswzh1SiJ5MACSyn7h89iuhJIMEFCoc49KhJxN2aghxlSiJvJdg1n" + "DMnUMV4k0m9Dmysp/3zErOJKDAwrxahnKcCuFkAp6sjP20qauEwzmTgIJAAYbvuCjEhzT/" + "0htkr/VmyX0VCSnOwoABR0EHBnOlkLw55Ct7HTvImUQoFsN4L1rVS0VVSMh9pD/P4pmTYD" + "giUW98Y4/BPvRgJGnzG1pk698oiX4HblwK5ZC3rHSE31kf7FJOZxtfQgotEmGIQw1GEvLr" + "dzCaJ6WthJKKElAQGs4Y1x0Bv2uYp9E8Ls8kpIhFEI5Bx5SO44nxIcG/9CL7GF2WM5FgPK" + "DAwlBBBs4LHbqQgN++yCAeJcMzCSiwiCahAgyM5YZjFvZn4Kd4FJdOgo0VCizEwAAG69AO" + "P5Ow9yOrOB6lw2FZniSWIhbJBAphYE+VI+yNEfMVxyZ/o8SjwMKIAgzUoR1MFfpH4EcTx8" + "9HiVBgAeaUqTDoGMKhCgl/0TowcZFbCRcFFFokQJwx4FCM0PesrMTE6QISjwKLRBSWIWOh" + "o4VCmBdjTL7IheIsxEioEMYVB9/FByYyxtQLSkiBBaFAFML4mWN235+OesaYzcKnwEIMEV" + "CAcd2x4N9rQtMZGFPkXVJoIQYBAgrFUIOJghkcTtJ1ElBgAcZLSYVmSFK1qUHDqbqk0AIM" + "AwQUYNChF4SDCU/HnZxNlxRY3qBhiPAUOsOC33Z3ZTWgxLOAg8AAhWJI2vhLONeEElqoEY" + "JSKAdP7r15FIlgJBqhHVzUtiTLblBKOlqUVIsAx9LQ0aYkGTZlLCYtO3h2iobjKceG56XF" + "PLwYm7pBKYupsRlE39rOk3GZLn51Owpj89VpmyH/lVCkv0LZYCoDjqXtdPoGqf5lQHlaGN" + "Tx0L6BeegZJFnmV1djg6OitqPtRKSYZDrTMyrTOuC/BIJbGRhmXKfLktmkk8QwphfQRkA6" + "jz1zIy+PAbsRbInQi8qgF6C4H9PvfQ2E8NXrR7z9tJvf+Z1/ANt+S+GBXoDpAAAAAElFTk" + "SuQmCC" } panicparse-1.3.0/internal/html_test.go000066400000000000000000000020521353500720100200010ustar00rootroot00000000000000// Copyright 2018 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package internal import ( "io/ioutil" "os" "testing" "github.com/maruel/panicparse/stack" ) func TestWriteToHTML(t *testing.T) { f, err := ioutil.TempFile("", "panicparse") if err != nil { t.Fatal(err) } n := f.Name() f.Close() defer func() { if err := os.Remove(n); err != nil { t.Fatal(err) } }() buckets := []*stack.Bucket{ { Signature: stack.Signature{ State: "chan receive", Stack: stack.Stack{ Calls: []stack.Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 72, Func: stack.Func{Raw: "main.func·001"}, Args: stack.Args{Values: []stack.Arg{{Value: 0x11000000, Name: ""}, {Value: 2}}}, }, }, }, }, IDs: []int{1, 2}, First: true, }, { IDs: []int{3}, }, } if err := writeToHTML(n, buckets, true); err != nil { t.Fatal(err) } } panicparse-1.3.0/internal/main.go000066400000000000000000000130241353500720100167230ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // Package internal implements panicparse // // It is mostly useful on servers will large number of identical goroutines, // making the crash dump harder to read than strictly necessary. // // Colors: // - Magenta: first goroutine to be listed. // - Yellow: main package. // - Green: standard library. // - Red: other packages. // // Bright colors are used for exported symbols. package internal import ( "errors" "flag" "fmt" "io" "io/ioutil" "log" "os" "os/signal" "regexp" "syscall" "github.com/maruel/panicparse/stack" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/mgutz/ansi" ) // resetFG is similar to ansi.Reset except that it doesn't reset the // background color, only the foreground color and the style. // // That much for the "ansi" abstraction layer... const resetFG = ansi.DefaultFG + "\033[m" // defaultPalette is the default recommended palette. var defaultPalette = Palette{ EOLReset: resetFG, RoutineFirst: ansi.ColorCode("magenta+b"), CreatedBy: ansi.LightBlack, Package: ansi.ColorCode("default+b"), SrcFile: resetFG, FuncStdLib: ansi.Green, FuncStdLibExported: ansi.ColorCode("green+b"), FuncMain: ansi.ColorCode("yellow+b"), FuncOther: ansi.Red, FuncOtherExported: ansi.ColorCode("red+b"), Arguments: resetFG, } func writeToConsole(out io.Writer, p *Palette, buckets []*stack.Bucket, fullPath, needsEnv bool, filter, match *regexp.Regexp) error { if needsEnv { _, _ = io.WriteString(out, "\nTo see all goroutines, visit https://github.com/maruel/panicparse#gotraceback\n\n") } srcLen, pkgLen := CalcLengths(buckets, fullPath) for _, bucket := range buckets { header := p.BucketHeader(bucket, fullPath, len(buckets) > 1) if filter != nil && filter.MatchString(header) { continue } if match != nil && !match.MatchString(header) { continue } _, _ = io.WriteString(out, header) _, _ = io.WriteString(out, p.StackLines(&bucket.Signature, srcLen, pkgLen, fullPath)) } return nil } // process copies stdin to stdout and processes any "panic: " line found. // // If html is used, a stack trace is written to this file instead. func process(in io.Reader, out io.Writer, p *Palette, s stack.Similarity, fullPath, parse, rebase bool, html string, filter, match *regexp.Regexp) error { c, err := stack.ParseDump(in, out, rebase) if c == nil || err != nil { return err } if rebase { log.Printf("GOROOT=%s", c.GOROOT) log.Printf("GOPATH=%s", c.GOPATHs) } needsEnv := len(c.Goroutines) == 1 && showBanner() if parse { stack.Augment(c.Goroutines) } buckets := stack.Aggregate(c.Goroutines, s) if html == "" { return writeToConsole(out, p, buckets, fullPath, needsEnv, filter, match) } return writeToHTML(html, buckets, needsEnv) } func showBanner() bool { if !showGOTRACEBACKBanner { return false } gtb := os.Getenv("GOTRACEBACK") return gtb == "" || gtb == "single" } // Main is implemented here so both 'pp' and 'panicparse' executables can be // compiled. This is to work around the Perl Package manager 'pp' that is // preinstalled on some OSes. func Main() error { aggressive := flag.Bool("aggressive", false, "Aggressive deduplication including non pointers") parse := flag.Bool("parse", true, "Parses source files to deduct types; use -parse=false to work around bugs in source parser") rebase := flag.Bool("rebase", true, "Guess GOROOT and GOPATH") verboseFlag := flag.Bool("v", false, "Enables verbose logging output") filterFlag := flag.String("f", "", "Regexp to filter out headers that match, ex: -f 'IO wait|syscall'") matchFlag := flag.String("m", "", "Regexp to filter by only headers that match, ex: -m 'semacquire'") // Console only. fullPath := flag.Bool("full-path", false, "Print full sources path") noColor := flag.Bool("no-color", !isatty.IsTerminal(os.Stdout.Fd()) || os.Getenv("TERM") == "dumb", "Disable coloring") forceColor := flag.Bool("force-color", false, "Forcibly enable coloring when with stdout is redirected") // HTML only. html := flag.String("html", "", "Output an HTML file") flag.Parse() log.SetFlags(log.Lmicroseconds) if !*verboseFlag { log.SetOutput(ioutil.Discard) } var err error var filter *regexp.Regexp if *filterFlag != "" { if filter, err = regexp.Compile(*filterFlag); err != nil { return err } } var match *regexp.Regexp if *matchFlag != "" { if match, err = regexp.Compile(*matchFlag); err != nil { return err } } s := stack.AnyPointer if *aggressive { s = stack.AnyValue } var out io.Writer = os.Stdout p := &defaultPalette if *html == "" { if *noColor && !*forceColor { p = &Palette{} } else { out = colorable.NewColorableStdout() } } var in *os.File switch flag.NArg() { case 0: in = os.Stdin // Explicitly silence SIGQUIT, as it is useful to gather the stack dump // from the piped command.. signals := make(chan os.Signal) go func() { for { <-signals } }() signal.Notify(signals, os.Interrupt, syscall.SIGQUIT) case 1: // Do not handle SIGQUIT when passed a file to process. name := flag.Arg(0) if in, err = os.Open(name); err != nil { return fmt.Errorf("did you mean to specify a valid stack dump file name? %s", err) } defer in.Close() default: return errors.New("pipe from stdin or specify a single file") } return process(in, out, p, s, *fullPath, *parse, *rebase, *html, filter, match) } panicparse-1.3.0/internal/main_test.go000066400000000000000000000176401353500720100177720ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package internal import ( "bytes" "regexp" "strings" "testing" "github.com/maruel/panicparse/stack" ) var data = []string{ "panic: runtime error: index out of range", "", "goroutine 11 [running, 5 minutes, locked to thread]:", "github.com/luci/luci-go/client/archiver.(*archiver).PushFile(0xc208032410, 0xc20968a3c0, 0x5b, 0xc20988c280, 0x7d, 0x0, 0x0)", " /gopath/path/to/archiver.go:325 +0x2c4", "github.com/luci/luci-go/client/isolate.archive(0x7fbdab7a5218, 0xc208032410, 0xc20803b0b0, 0x22, 0xc208046370, 0xc20804666a, 0x17, 0x0, 0x0, 0x0, ...)", " /gopath/path/to/isolate.go:148 +0x12d2", "github.com/luci/luci-go/client/isolate.Archive(0x7fbdab7a5218, 0xc208032410, 0xc20803b0b0, 0x22, 0xc208046370, 0x0, 0x0)", " /gopath/path/to/isolate.go:102 +0xc9", "main.func·004(0x7fffc3b8f13a, 0x2c)", " /gopath/path/to/batch_archive.go:166 +0x7cd", "created by main.(*batchArchiveRun).main", " /gopath/path/to/batch_archive.go:167 +0x42c", "", "goroutine 1 [running]:", "gopkg.in/yaml%2ev2.handleErr(0xc208033b20)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "reflect.Value.assignTo(0x570860, 0xc20803f3e0, 0x15)", " c:/go/src/reflect/value.go:2125 +0x368", "main.main()", " /gopath/src/github.com/maruel/pre-commit-go/main.go:428 +0x27", "", "goroutine 2 [running, 1 minutes]:", "gopkg.in/yaml%2ev2.handleErr(0xc208033b20)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "reflect.Value.assignTo(0x570860, 0xc20803f3e0, 0x15)", " c:/go/src/reflect/value.go:2125 +0x368", "main.main()", " /gopath/src/github.com/maruel/pre-commit-go/main.go:428 +0x27", "", } func TestProcess(t *testing.T) { out := &bytes.Buffer{} if err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &defaultPalette, stack.AnyPointer, false, false, true, "", nil, nil); err != nil { t.Fatal(err) } expected := []string{ "panic: runtime error: index out of range", "", "\x1b[1;35m1: running [5 minutes] [locked]\x1b[90m [Created by main.(*batchArchiveRun).main @ batch_archive.go:167]\x1b[39m\x1b[m", " \x1b[1;39marchiver \x1b[39m\x1b[marchiver.go:325 \x1b[1;31m(*archiver).PushFile\x1b[39m\x1b[m(#1, 0xc20968a3c0, 0x5b, 0xc20988c280, 0x7d, 0, 0)\x1b[39m\x1b[m", " \x1b[1;39misolate \x1b[39m\x1b[misolate.go:148 \x1b[31marchive\x1b[39m\x1b[m(#4, #1, #2, 0x22, #3, 0xc20804666a, 0x17, 0, 0, 0, ...)\x1b[39m\x1b[m", " \x1b[1;39misolate \x1b[39m\x1b[misolate.go:102 \x1b[1;31mArchive\x1b[39m\x1b[m(#4, #1, #2, 0x22, #3, 0, 0)\x1b[39m\x1b[m", " \x1b[1;39mmain \x1b[39m\x1b[mbatch_archive.go:166 \x1b[1;33mfunc·004\x1b[39m\x1b[m(0x7fffc3b8f13a, 0x2c)\x1b[39m\x1b[m", "2: running [0~1 minutes]\x1b[39m\x1b[m", " \x1b[1;39myaml.v2 \x1b[39m\x1b[myaml.go:153 \x1b[31mhandleErr\x1b[39m\x1b[m(#5)\x1b[39m\x1b[m", " \x1b[1;39mreflect \x1b[39m\x1b[mvalue.go:2125 \x1b[32mValue.assignTo\x1b[39m\x1b[m(0x570860, #6, 0x15)\x1b[39m\x1b[m", " \x1b[1;39mmain \x1b[39m\x1b[mmain.go:428 \x1b[1;33mmain\x1b[39m\x1b[m()\x1b[39m\x1b[m", "", } actual := strings.Split(out.String(), "\n") compareLines(t, expected, actual) } func TestProcessFullPath(t *testing.T) { out := &bytes.Buffer{} if err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &defaultPalette, stack.AnyValue, true, false, true, "", nil, nil); err != nil { t.Fatal(err) } expected := []string{ "panic: runtime error: index out of range", "", "\x1b[1;35m1: running [5 minutes] [locked]\x1b[90m [Created by main.(*batchArchiveRun).main @ /gopath/path/to/batch_archive.go:167]\x1b[39m\x1b[m", " \x1b[1;39marchiver \x1b[39m\x1b[m/gopath/path/to/archiver.go:325 \x1b[1;31m(*archiver).PushFile\x1b[39m\x1b[m(#1, 0xc20968a3c0, 0x5b, 0xc20988c280, 0x7d, 0, 0)\x1b[39m\x1b[m", " \x1b[1;39misolate \x1b[39m\x1b[m/gopath/path/to/isolate.go:148 \x1b[31marchive\x1b[39m\x1b[m(#4, #1, #2, 0x22, #3, 0xc20804666a, 0x17, 0, 0, 0, ...)\x1b[39m\x1b[m", " \x1b[1;39misolate \x1b[39m\x1b[m/gopath/path/to/isolate.go:102 \x1b[1;31mArchive\x1b[39m\x1b[m(#4, #1, #2, 0x22, #3, 0, 0)\x1b[39m\x1b[m", " \x1b[1;39mmain \x1b[39m\x1b[m/gopath/path/to/batch_archive.go:166 \x1b[1;33mfunc·004\x1b[39m\x1b[m(0x7fffc3b8f13a, 0x2c)\x1b[39m\x1b[m", "2: running [0~1 minutes]\x1b[39m\x1b[m", " \x1b[1;39myaml.v2 \x1b[39m\x1b[m/gopath/src/gopkg.in/yaml.v2/yaml.go:153 \x1b[31mhandleErr\x1b[39m\x1b[m(#5)\x1b[39m\x1b[m", " \x1b[1;39mreflect \x1b[39m\x1b[mc:/go/src/reflect/value.go:2125 \x1b[32mValue.assignTo\x1b[39m\x1b[m(0x570860, #6, 0x15)\x1b[39m\x1b[m", " \x1b[1;39mmain \x1b[39m\x1b[m/gopath/src/github.com/maruel/pre-commit-go/main.go:428 \x1b[1;33mmain\x1b[39m\x1b[m()\x1b[39m\x1b[m", "", } actual := strings.Split(out.String(), "\n") compareLines(t, expected, actual) } func TestProcessNoColor(t *testing.T) { out := &bytes.Buffer{} if err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &Palette{}, stack.AnyPointer, false, false, true, "", nil, nil); err != nil { t.Fatal(err) } expected := []string{ "panic: runtime error: index out of range", "", "1: running [5 minutes] [locked] [Created by main.(*batchArchiveRun).main @ batch_archive.go:167]", " archiver archiver.go:325 (*archiver).PushFile(#1, 0xc20968a3c0, 0x5b, 0xc20988c280, 0x7d, 0, 0)", " isolate isolate.go:148 archive(#4, #1, #2, 0x22, #3, 0xc20804666a, 0x17, 0, 0, 0, ...)", " isolate isolate.go:102 Archive(#4, #1, #2, 0x22, #3, 0, 0)", " main batch_archive.go:166 func·004(0x7fffc3b8f13a, 0x2c)", "2: running [0~1 minutes]", " yaml.v2 yaml.go:153 handleErr(#5)", " reflect value.go:2125 Value.assignTo(0x570860, #6, 0x15)", " main main.go:428 main()", "", } actual := strings.Split(out.String(), "\n") compareLines(t, expected, actual) } func compareLines(t *testing.T, expected, actual []string) { for i := 0; i < len(actual) && i < len(expected); i++ { if expected[i] != actual[i] { t.Fatalf("Different lines #%d:\n- %q\n- %q", i, expected[i], actual[i]) } } if len(expected) != len(actual) { t.Fatalf("different length %d != %d", len(expected), len(actual)) } } func TestProcessMatch(t *testing.T) { out := &bytes.Buffer{} err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &Palette{}, stack.AnyPointer, false, false, true, "", nil, regexp.MustCompile(`batchArchiveRun`)) if err != nil { t.Fatal(err) } expected := []string{ "panic: runtime error: index out of range", "", "1: running [5 minutes] [locked] [Created by main.(*batchArchiveRun).main @ batch_archive.go:167]", " archiver archiver.go:325 (*archiver).PushFile(#1, 0xc20968a3c0, 0x5b, 0xc20988c280, 0x7d, 0, 0)", " isolate isolate.go:148 archive(#4, #1, #2, 0x22, #3, 0xc20804666a, 0x17, 0, 0, 0, ...)", " isolate isolate.go:102 Archive(#4, #1, #2, 0x22, #3, 0, 0)", " main batch_archive.go:166 func·004(0x7fffc3b8f13a, 0x2c)", "", } actual := strings.Split(out.String(), "\n") compareLines(t, expected, actual) } func TestProcessFilter(t *testing.T) { out := &bytes.Buffer{} err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &Palette{}, stack.AnyPointer, false, false, true, "", regexp.MustCompile(`batchArchiveRun`), nil) if err != nil { t.Fatal(err) } expected := []string{ "panic: runtime error: index out of range", "", "2: running [0~1 minutes]", " yaml.v2 yaml.go:153 handleErr(#5)", " reflect value.go:2125 Value.assignTo(0x570860, #6, 0x15)", " main main.go:428 main()", "", } actual := strings.Split(out.String(), "\n") compareLines(t, expected, actual) } panicparse-1.3.0/internal/ui.go000066400000000000000000000064371353500720100164260ustar00rootroot00000000000000// Copyright 2016 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package internal import ( "fmt" "strings" "github.com/maruel/panicparse/stack" ) // Palette defines the color used. // // An empty object Palette{} can be used to disable coloring. type Palette struct { EOLReset string // Routine header. RoutineFirst string // The first routine printed. Routine string // Following routines. CreatedBy string // Call line. Package string SrcFile string FuncStdLib string FuncStdLibExported string FuncMain string FuncOther string FuncOtherExported string Arguments string } // CalcLengths returns the maximum length of the source lines and package names. func CalcLengths(buckets []*stack.Bucket, fullPath bool) (int, int) { srcLen := 0 pkgLen := 0 for _, bucket := range buckets { for _, line := range bucket.Signature.Stack.Calls { l := 0 if fullPath { l = len(line.FullSrcLine()) } else { l = len(line.SrcLine()) } if l > srcLen { srcLen = l } l = len(line.Func.PkgName()) if l > pkgLen { pkgLen = l } } } return srcLen, pkgLen } // functionColor returns the color to be used for the function name based on // the type of package the function is in. func (p *Palette) functionColor(line *stack.Call) string { if line.IsStdlib { if line.Func.IsExported() { return p.FuncStdLibExported } return p.FuncStdLib } else if line.IsPkgMain() { return p.FuncMain } else if line.Func.IsExported() { return p.FuncOtherExported } return p.FuncOther } // routineColor returns the color for the header of the goroutines bucket. func (p *Palette) routineColor(bucket *stack.Bucket, multipleBuckets bool) string { if bucket.First && multipleBuckets { return p.RoutineFirst } return p.Routine } // BucketHeader prints the header of a goroutine signature. func (p *Palette) BucketHeader(bucket *stack.Bucket, fullPath, multipleBuckets bool) string { extra := "" if s := bucket.SleepString(); s != "" { extra += " [" + s + "]" } if bucket.Locked { extra += " [locked]" } if c := bucket.CreatedByString(fullPath); c != "" { extra += p.CreatedBy + " [Created by " + c + "]" } return fmt.Sprintf( "%s%d: %s%s%s\n", p.routineColor(bucket, multipleBuckets), len(bucket.IDs), bucket.State, extra, p.EOLReset) } // callLine prints one stack line. func (p *Palette) callLine(line *stack.Call, srcLen, pkgLen int, fullPath bool) string { src := "" if fullPath { src = line.FullSrcLine() } else { src = line.SrcLine() } return fmt.Sprintf( " %s%-*s %s%-*s %s%s%s(%s)%s", p.Package, pkgLen, line.Func.PkgName(), p.SrcFile, srcLen, src, p.functionColor(line), line.Func.Name(), p.Arguments, &line.Args, p.EOLReset) } // StackLines prints one complete stack trace, without the header. func (p *Palette) StackLines(signature *stack.Signature, srcLen, pkgLen int, fullPath bool) string { out := make([]string, len(signature.Stack.Calls)) for i := range signature.Stack.Calls { out[i] = p.callLine(&signature.Stack.Calls[i], srcLen, pkgLen, fullPath) } if signature.Stack.Elided { out = append(out, " (...)") } return strings.Join(out, "\n") + "\n" } panicparse-1.3.0/internal/ui_test.go000066400000000000000000000132401353500720100174530ustar00rootroot00000000000000// Copyright 2016 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package internal import ( "flag" "io/ioutil" "log" "os" "testing" "github.com/maruel/panicparse/stack" ) var testPalette = &Palette{ EOLReset: "A", RoutineFirst: "B", Routine: "C", CreatedBy: "D", Package: "E", SrcFile: "F", FuncStdLib: "G", FuncStdLibExported: "H", FuncMain: "I", FuncOther: "J", FuncOtherExported: "K", Arguments: "L", } func TestCalcLengths(t *testing.T) { b := []*stack.Bucket{ { Signature: stack.Signature{ Stack: stack.Stack{ Calls: []stack.Call{ { SrcPath: "/gopath/foo/baz.go", Line: 123, Func: stack.Func{Raw: "main.func·001"}, }, }, }, }, IDs: []int{}, First: true, }, } srcLen, pkgLen := CalcLengths(b, true) // When printing, it prints the remote path, not the transposed local path. compareString(t, "/gopath/foo/baz.go:123", b[0].Signature.Stack.Calls[0].FullSrcLine()) compareInt(t, len("/gopath/foo/baz.go:123"), srcLen) compareString(t, "main", b[0].Signature.Stack.Calls[0].Func.PkgName()) compareInt(t, len("main"), pkgLen) srcLen, pkgLen = CalcLengths(b, false) compareString(t, "baz.go:123", b[0].Signature.Stack.Calls[0].SrcLine()) compareInt(t, len("baz.go:123"), srcLen) compareString(t, "main", b[0].Signature.Stack.Calls[0].Func.PkgName()) compareInt(t, len("main"), pkgLen) } func TestBucketHeader(t *testing.T) { b := &stack.Bucket{ Signature: stack.Signature{ State: "chan receive", CreatedBy: stack.Call{ SrcPath: "/gopath/src/github.com/foo/bar/baz.go", Line: 74, Func: stack.Func{Raw: "main.mainImpl"}, }, SleepMax: 6, SleepMin: 2, }, IDs: []int{1, 2}, First: true, } // When printing, it prints the remote path, not the transposed local path. compareString(t, "B2: chan receive [2~6 minutes]D [Created by main.mainImpl @ /gopath/src/github.com/foo/bar/baz.go:74]A\n", testPalette.BucketHeader(b, true, true)) compareString(t, "C2: chan receive [2~6 minutes]D [Created by main.mainImpl @ /gopath/src/github.com/foo/bar/baz.go:74]A\n", testPalette.BucketHeader(b, true, false)) compareString(t, "B2: chan receive [2~6 minutes]D [Created by main.mainImpl @ baz.go:74]A\n", testPalette.BucketHeader(b, false, true)) compareString(t, "C2: chan receive [2~6 minutes]D [Created by main.mainImpl @ baz.go:74]A\n", testPalette.BucketHeader(b, false, false)) b = &stack.Bucket{ Signature: stack.Signature{ State: "b0rked", SleepMax: 6, SleepMin: 6, Locked: true, }, IDs: []int{}, First: true, } compareString(t, "C0: b0rked [6 minutes] [locked]A\n", testPalette.BucketHeader(b, false, false)) } func TestStackLines(t *testing.T) { s := &stack.Signature{ State: "idle", Stack: stack.Stack{ Calls: []stack.Call{ { SrcPath: "/goroot/src/runtime/sys_linux_amd64.s", Line: 400, Func: stack.Func{Raw: "runtime.Epollwait"}, Args: stack.Args{ Values: []stack.Arg{ {Value: 0x4}, {Value: 0x7fff671c7118}, {Value: 0xffffffff00000080}, {}, {Value: 0xffffffff0028c1be}, {}, {}, {}, {}, {}, }, Elided: true, }, IsStdlib: true, }, { SrcPath: "/goroot/src/runtime/netpoll_epoll.go", Line: 68, Func: stack.Func{Raw: "runtime.netpoll"}, Args: stack.Args{Values: []stack.Arg{{Value: 0x901b01}, {}}}, IsStdlib: true, }, { SrcPath: "/gopath/src/main.go", Line: 1472, Func: stack.Func{Raw: "main.Main"}, Args: stack.Args{Values: []stack.Arg{{Value: 0xc208012000}}}, }, { SrcPath: "/gopath/src/foo/bar.go", Line: 1575, Func: stack.Func{Raw: "foo.OtherExported"}, }, { SrcPath: "/gopath/src/foo/bar.go", Line: 10, Func: stack.Func{Raw: "foo.otherPrivate"}, }, }, Elided: true, }, } // When printing, it prints the remote path, not the transposed local path. expected := "" + " Eruntime F/goroot/src/runtime/sys_linux_amd64.s:400 HEpollwaitL(0x4, 0x7fff671c7118, 0xffffffff00000080, 0, 0xffffffff0028c1be, 0, 0, 0, 0, 0, ...)A\n" + " Eruntime F/goroot/src/runtime/netpoll_epoll.go:68 GnetpollL(0x901b01, 0)A\n" + " Emain F/gopath/src/main.go:1472 IMainL(0xc208012000)A\n" + " Efoo F/gopath/src/foo/bar.go:1575 KOtherExportedL()A\n" + " Efoo F/gopath/src/foo/bar.go:10 JotherPrivateL()A\n" + " (...)\n" compareString(t, expected, testPalette.StackLines(s, 10, 10, true)) expected = "" + " Eruntime Fsys_linux_amd64.s:400 HEpollwaitL(0x4, 0x7fff671c7118, 0xffffffff00000080, 0, 0xffffffff0028c1be, 0, 0, 0, 0, 0, ...)A\n" + " Eruntime Fnetpoll_epoll.go:68 GnetpollL(0x901b01, 0)A\n" + " Emain Fmain.go:1472 IMainL(0xc208012000)A\n" + " Efoo Fbar.go:1575 KOtherExportedL()A\n" + " Efoo Fbar.go:10 JotherPrivateL()A\n" + " (...)\n" compareString(t, expected, testPalette.StackLines(s, 10, 10, false)) } func compareString(t *testing.T, expected, actual string) { if expected != actual { i := 0 for i < len(expected) && i < len(actual) && expected[i] == actual[i] { i++ } t.Fatalf("Delta at offset %d:\n- %q\n- %q", i, expected, actual) } } func compareInt(t *testing.T, expected, actual int) { if expected != actual { t.Fatalf("%d != %d", expected, actual) } } func TestMain(m *testing.M) { flag.Parse() if !testing.Verbose() { log.SetOutput(ioutil.Discard) } os.Exit(m.Run()) } panicparse-1.3.0/main.go000066400000000000000000000014251353500720100151110ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // panicparse: analyzes stack dump of Go processes and simplifies it. // // It is mostly useful on servers will large number of identical goroutines, // making the crash dump harder to read than strictly necessary. // // Colors: // - Magenta: first goroutine to be listed. // - Yellow: main package. // - Green: standard library. // - Red: other packages. // // Bright colors are used for exported symbols. package main import ( "fmt" "os" "github.com/maruel/panicparse/internal" ) func main() { if err := internal.Main(); err != nil { fmt.Fprintf(os.Stderr, "Failed: %s\n", err) os.Exit(1) } } panicparse-1.3.0/stack/000077500000000000000000000000001353500720100147415ustar00rootroot00000000000000panicparse-1.3.0/stack/bucket.go000066400000000000000000000052771353500720100165600ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack import ( "sort" ) // Similarity is the level at which two call lines arguments must match to be // considered similar enough to coalesce them. type Similarity int const ( // ExactFlags requires same bits (e.g. Locked). ExactFlags Similarity = iota // ExactLines requests the exact same arguments on the call line. ExactLines // AnyPointer considers different pointers a similar call line. AnyPointer // AnyValue accepts any value as similar call line. AnyValue ) // Aggregate merges similar goroutines into buckets. // // The buckets are ordered in library provided order of relevancy. You can // reorder at your chosing. func Aggregate(goroutines []*Goroutine, similar Similarity) []*Bucket { type count struct { ids []int first bool } b := map[*Signature]*count{} // O(n²). Fix eventually. for _, routine := range goroutines { found := false for key, c := range b { // When a match is found, this effectively drops the other goroutine ID. if key.similar(&routine.Signature, similar) { found = true c.ids = append(c.ids, routine.ID) c.first = c.first || routine.First if !key.equal(&routine.Signature) { // Almost but not quite equal. There's different pointers passed // around but the same values. Zap out the different values. newKey := key.merge(&routine.Signature) b[newKey] = c delete(b, key) } break } } if !found { // Create a copy of the Signature, since it will be mutated. key := &Signature{} *key = routine.Signature b[key] = &count{ids: []int{routine.ID}, first: routine.First} } } out := make(buckets, 0, len(b)) for signature, c := range b { sort.Ints(c.ids) out = append(out, &Bucket{Signature: *signature, IDs: c.ids, First: c.first}) } sort.Sort(out) return out } // Bucket is a stack trace signature and the list of goroutines that fits this // signature. type Bucket struct { Signature // IDs is the ID of each Goroutine with this Signature. IDs []int // First is true if this Bucket contains the first goroutine, e.g. the one // Signature that likely generated the panic() call, if any. First bool } // less does reverse sort. func (b *Bucket) less(r *Bucket) bool { if b.First || r.First { return b.First } return b.Signature.less(&r.Signature) } // // buckets is a list of Bucket sorted by repeation count. type buckets []*Bucket func (b buckets) Len() int { return len(b) } func (b buckets) Less(i, j int) bool { return b[i].less(b[j]) } func (b buckets) Swap(i, j int) { b[j], b[i] = b[i], b[j] } panicparse-1.3.0/stack/bucket_test.go000066400000000000000000000112111353500720100176000ustar00rootroot00000000000000// Copyright 2018 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack import ( "bytes" "io/ioutil" "reflect" "strings" "testing" ) func TestAggregateNotAggressive(t *testing.T) { // 2 goroutines with similar but not exact same signature. data := []string{ "panic: runtime error: index out of range", "", "goroutine 6 [chan receive]:", "main.func·001(0x11000000, 2)", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "", "goroutine 7 [chan receive]:", "main.func·001(0x21000000, 2)", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "", } c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), ioutil.Discard, false) if err != nil { t.Fatal(err) } actual := Aggregate(c.Goroutines, ExactLines) expected := []*Bucket{ { Signature: Signature{ State: "chan receive", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 72, Func: Func{Raw: "main.func·001"}, Args: Args{Values: []Arg{{Value: 0x11000000, Name: ""}, {Value: 2}}}, }, }, }, }, IDs: []int{6}, First: true, }, { Signature: Signature{ State: "chan receive", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 72, Func: Func{Raw: "main.func·001"}, Args: Args{Values: []Arg{{Value: 0x21000000, Name: "#1"}, {Value: 2}}}, }, }, }, }, IDs: []int{7}, }, } compareBuckets(t, expected, actual) } func TestAggregateExactMatching(t *testing.T) { // 2 goroutines with the exact same signature. data := []string{ "panic: runtime error: index out of range", "", "goroutine 6 [chan receive]:", "main.func·001()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "created by main.mainImpl", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:74 +0xeb", "", "goroutine 7 [chan receive]:", "main.func·001()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "created by main.mainImpl", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:74 +0xeb", "", } c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), &bytes.Buffer{}, false) if err != nil { t.Fatal(err) } actual := Aggregate(c.Goroutines, ExactLines) expected := []*Bucket{ { Signature: Signature{ State: "chan receive", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 72, Func: Func{Raw: "main.func·001"}, }, }, }, CreatedBy: Call{ SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 74, Func: Func{Raw: "main.mainImpl"}, }, }, IDs: []int{6, 7}, First: true, }, } compareBuckets(t, expected, actual) } func TestAggregateAggressive(t *testing.T) { // 3 goroutines with similar signatures. data := []string{ "panic: runtime error: index out of range", "", "goroutine 6 [chan receive, 10 minutes]:", "main.func·001(0x11000000, 2)", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "", "goroutine 7 [chan receive, 50 minutes]:", "main.func·001(0x21000000, 2)", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "", "goroutine 8 [chan receive, 100 minutes]:", "main.func·001(0x21000000, 2)", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:72 +0x49", "", } c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), ioutil.Discard, false) if err != nil { t.Fatal(err) } actual := Aggregate(c.Goroutines, AnyPointer) expected := []*Bucket{ { Signature: Signature{ State: "chan receive", SleepMin: 10, SleepMax: 100, Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 72, Func: Func{Raw: "main.func·001"}, Args: Args{Values: []Arg{{Value: 0x11000000, Name: "*"}, {Value: 2}}}, }, }, }, }, IDs: []int{6, 7, 8}, First: true, }, } compareBuckets(t, expected, actual) } func compareBuckets(t *testing.T, expected, actual []*Bucket) { if len(expected) != len(actual) { t.Fatalf("Different []Bucket length:\n- %v\n- %v", expected, actual) } for i := range expected { if !reflect.DeepEqual(expected[i], actual[i]) { t.Fatalf("Different Bucket:\n- %#v\n- %#v", expected[i], actual[i]) } } } panicparse-1.3.0/stack/context.go000066400000000000000000000534741353500720100167710ustar00rootroot00000000000000// Copyright 2018 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack import ( "bufio" "bytes" "errors" "fmt" "io" "os" "os/user" "path/filepath" "regexp" "runtime" "sort" "strconv" "strings" ) // Context is a parsing context. // // It contains the deduced GOROOT and GOPATH, if guesspaths is true. type Context struct { // Goroutines is the Goroutines found. // // They are in the order that they were printed. Goroutines []*Goroutine // GOROOT is the GOROOT as detected in the traceback, not the on the host. // // It can be empty if no root was determined, for example the traceback // contains only non-stdlib source references. // // Empty is guesspaths was false. GOROOT string // GOPATHs is the GOPATH as detected in the traceback, with the value being // the corresponding path mapped to the host. // // It can be empty if only stdlib code is in the traceback or if no local // sources were matched up. In the general case there is only one entry in // the map. // // Nil is guesspaths was false. GOPATHs map[string]string localgoroot string localgopaths []string } // ParseDump processes the output from runtime.Stack(). // // Returns nil *Context if no stack trace was detected. // // It pipes anything not detected as a panic stack trace from r into out. It // assumes there is junk before the actual stack trace. The junk is streamed to // out. // // If guesspaths is false, no guessing of GOROOT and GOPATH is done, and Call // entites do not have LocalSrcPath and IsStdlib filled in. func ParseDump(r io.Reader, out io.Writer, guesspaths bool) (*Context, error) { goroutines, err := parseDump(r, out) if len(goroutines) == 0 { return nil, err } c := &Context{ Goroutines: goroutines, localgoroot: runtime.GOROOT(), localgopaths: getGOPATHs(), } nameArguments(goroutines) // Corresponding local values on the host for Context. if guesspaths { c.findRoots() for _, r := range c.Goroutines { // Note that this is important to call it even if // c.GOROOT == c.localgoroot. r.updateLocations(c.GOROOT, c.localgoroot, c.GOPATHs) } } return c, err } // Private stuff. const ( lockedToThread = "locked to thread" elided = "...additional frames elided..." raceHeaderFooter = "==================" raceHeader = "WARNING: DATA RACE" ) // These are effectively constants. var ( // TODO(maruel): Handle corrupted stack cases: // - missed stack barrier // - found next stack barrier at 0x123; expected // - runtime: unexpected return pc for FUNC_NAME called from 0x123 reRoutineHeader = regexp.MustCompile("^([ \t]*)goroutine (\\d+) \\[([^\\]]+)\\]\\:$") reMinutes = regexp.MustCompile("^(\\d+) minutes$") reUnavail = regexp.MustCompile("^(?:\t| +)goroutine running on other thread; stack unavailable") // See gentraceback() in src/runtime/traceback.go for more information. // - Sometimes the source file comes up as "". It is the // compiler than generated these, not the runtime. // - The tab may be replaced with spaces when a user copy-paste it, handle // this transparently. // - "runtime.gopanic" is explicitly replaced with "panic" by gentraceback(). // - The +0x123 byte offset is printed when frame.pc > _func.entry. _func is // generated by the linker. // - The +0x123 byte offset is not included with generated code, e.g. unnamed // functions "func·006()" which is generally go func() { ... }() // statements. Since the _func is generated at runtime, it's probably why // _func.entry is not set. // - C calls may have fp=0x123 sp=0x123 appended. I think it normally happens // when a signal is not correctly handled. It is printed with m.throwing>0. // These are discarded. // - For cgo, the source file may be "??". reFile = regexp.MustCompile("^(?:\t| +)(\\?\\?|\\|.+\\.(?:c|go|s))\\:(\\d+)(?:| \\+0x[0-9a-f]+)(?:| fp=0x[0-9a-f]+ sp=0x[0-9a-f]+(?:| pc=0x[0-9a-f]+))$") // Sadly, it doesn't note the goroutine number so we could cascade them per // parenthood. reCreated = regexp.MustCompile("^created by (.+)$") reFunc = regexp.MustCompile("^(.+)\\((.*)\\)$") // See https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/tsan/rtl/tsan_report.cc // for the code generating these messages. Please note only the block in // #else // #if !SANITIZER_GO // is used. // TODO(maruel): " [failed to restore the stack]\n\n" // TODO(maruel): "Global var %s of size %zu at %p declared at %s:%zu\n" reRaceOperationHeader = regexp.MustCompile("^(Read|Write) at (0x[0-9a-f]+) by goroutine (\\d+):$") reRacePreviousOperationHeader = regexp.MustCompile("^Previous (read|write) at (0x[0-9a-f]+) by goroutine (\\d+):$") reRacePreviousOperationMainHeader = regexp.MustCompile("^Previous (read|write) at (0x[0-9a-f]+) by main goroutine:$") reRaceGoroutine = regexp.MustCompile("^Goroutine (\\d+) \\((running|finished)\\) created at:$") ) func parseDump(r io.Reader, out io.Writer) ([]*Goroutine, error) { scanner := bufio.NewScanner(r) scanner.Split(scanLines) // Do not enable race detection parsing yet, since it cannot be returned in // Context at the moment. s := scanningState{} for scanner.Scan() { line, err := s.scan(scanner.Text()) if line != "" { _, _ = io.WriteString(out, line) } if err != nil { return s.goroutines, err } } return s.goroutines, scanner.Err() } // scanLines is similar to bufio.ScanLines except that it: // - doesn't drop '\n' // - doesn't strip '\r' // - returns when the data is bufio.MaxScanTokenSize bytes func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { return 0, nil, nil } if i := bytes.IndexByte(data, '\n'); i >= 0 { return i + 1, data[0 : i+1], nil } if atEOF { return len(data), data, nil } if len(data) >= bufio.MaxScanTokenSize { // Returns the line even if it is not at EOF nor has a '\n', otherwise the // scanner will return bufio.ErrTooLong which is definitely not what we // want. return len(data), data, nil } return 0, nil, nil } // state is the state of the scan to detect and process a stack trace. type state int // Initial state is normal. Other states are when a stack trace is detected. const ( // Outside a stack trace. // to: gotRoutineHeader, raceHeader1 normal state = iota // Panic stack trace: // Empty line between goroutines. // from: gotFileCreated, gotFileFunc // to: gotRoutineHeader, normal betweenRoutine // Goroutine header was found, e.g. "goroutine 1 [running]:" // from: normal // to: gotUnavail, gotFunc gotRoutineHeader // Function call line was found, e.g. "main.main()" // from: gotRoutineHeader // to: gotFile gotFunc // Goroutine creation line was found, e.g. "created by main.glob..func4" // from: gotFileFunc // to: gotFileCreated gotCreated // File header was found, e.g. "\t/foo/bar/baz.go:116 +0x35" // from: gotFunc // to: gotFunc, gotCreated, betweenRoutine, normal gotFileFunc // File header was found, e.g. "\t/foo/bar/baz.go:116 +0x35" // from: gotCreated // to: betweenRoutine, normal gotFileCreated // State when the goroutine stack is instead is reUnavail. // from: gotRoutineHeader // to: betweenRoutine, gotCreated gotUnavail // Race detector: // Got "==================" // from: normal // to: normal, gotRaceHeader gotRaceHeader1 // Got "WARNING: DATA RACE" // from: gotRaceHeader1 // to: normal, gotRaceOperationHeader gotRaceHeader // A race operation was found, e.g. "Read at 0x00c0000e4030 by goroutine 7:" // from: gotRaceHeader // to: normal, gotRaceOperationFunc gotRaceOperationHeader // Function that caused the race, e.g. " main.panicRace.func1()" // from: gotRaceOperationHeader // to: normal, gotRaceOperationFile gotRaceOperationFunc // Function that caused the race, e.g. " main.panicRace.func1()" // from: gotRaceOperationFunc // to: normal, betweenRaces gotRaceOperationFile // Goroutine header, e.g. "Goroutine 7 (running) created at:" // from: betweenRaces // to: normal, gotRaceOperationHeader gotRaceGoroutineHeader // Function that caused the race, e.g. " main.panicRace.func1()" // from: gotRaceGoroutineHeader // to: normal, gotRaceGoroutineFile gotRaceGoroutineFunc // Function that caused the race, e.g. " main.panicRace.func1()" // from: gotRaceGoroutineFunc // to: normal, betweenRaces gotRaceGoroutineFile // Empty line between goroutines. // from: gotRaceOperationFile // to: normal, gotRaceOperationHeader betweenRaces ) type raceOp struct { write bool addr uint64 id int } // scanningState is the state of the scan to detect and process a stack trace // and stores the traces found. type scanningState struct { // Determines if race detection is enabled. Currently false since scan() // would swallow the race detector output, but the data is not part of // Context yet. raceDetectionEnabled bool // goroutines contains all the goroutines found. goroutines []*Goroutine state state prefix string races []raceOp } // scan scans one line, updates goroutines and move to the next state. func (s *scanningState) scan(line string) (string, error) { var cur *Goroutine if len(s.goroutines) != 0 { cur = s.goroutines[len(s.goroutines)-1] } trimmed := line if strings.HasSuffix(line, "\r\n") { trimmed = line[:len(line)-2] } else if strings.HasSuffix(line, "\n") { trimmed = line[:len(line)-1] } else { // There's two cases: // - It's the end of the stream and it's not terminating with EOL character. // - The line is longer than bufio.MaxScanTokenSize if s.state == normal { return line, nil } // Let it flow. It's possible the last line was trimmed and we still want to parse it. } if trimmed != "" && s.prefix != "" { // This can only be the case if s.state != normal or the line is empty. if !strings.HasPrefix(trimmed, s.prefix) { prefix := s.prefix s.state = normal s.prefix = "" return "", fmt.Errorf("inconsistent indentation: %q, expected %q", trimmed, prefix) } trimmed = trimmed[len(s.prefix):] } switch s.state { case normal: // We could look for '^panic:' but this is more risky, there can be a lot // of junk between this and the stack dump. fallthrough case betweenRoutine: // Look for a goroutine header. if match := reRoutineHeader.FindStringSubmatch(trimmed); match != nil { if id, err := strconv.Atoi(match[2]); err == nil { // See runtime/traceback.go. // ", \d+ minutes, locked to thread" items := strings.Split(match[3], ", ") sleep := 0 locked := false for i := 1; i < len(items); i++ { if items[i] == lockedToThread { locked = true continue } // Look for duration, if any. if match2 := reMinutes.FindStringSubmatch(items[i]); match2 != nil { sleep, _ = strconv.Atoi(match2[1]) } } g := &Goroutine{ Signature: Signature{ State: items[0], SleepMin: sleep, SleepMax: sleep, Locked: locked, }, ID: id, First: len(s.goroutines) == 0, } s.goroutines = append(s.goroutines, g) s.state = gotRoutineHeader s.prefix = match[1] return "", nil } } // Switch to race detection mode. if s.raceDetectionEnabled && trimmed == raceHeaderFooter { s.state = gotRaceHeader1 // Send the line to the user. return line, nil } // Fallthrough. s.state = normal s.prefix = "" return line, nil case gotRoutineHeader: if reUnavail.MatchString(trimmed) { // Generate a fake stack entry. cur.Stack.Calls = []Call{{SrcPath: ""}} // Next line is expected to be an empty line. s.state = gotUnavail return "", nil } call, err := parseFunc(trimmed) if call != nil { cur.Stack.Calls = append(cur.Stack.Calls, *call) s.state = gotFunc return "", err } return "", fmt.Errorf("expected a function after a goroutine header, got: %q", strings.TrimSpace(trimmed)) case gotFunc: // Look for a file. if match := reFile.FindStringSubmatch(trimmed); match != nil { num, err := strconv.Atoi(match[2]) if err != nil { return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed)) } // cur.Stack.Calls is guaranteed to have at least one item. i := len(cur.Stack.Calls) - 1 cur.Stack.Calls[i].SrcPath = match[1] cur.Stack.Calls[i].Line = num s.state = gotFileFunc return "", nil } return "", fmt.Errorf("expected a file after a function, got: %q", strings.TrimSpace(trimmed)) case gotCreated: // Look for a file. if match := reFile.FindStringSubmatch(trimmed); match != nil { num, err := strconv.Atoi(match[2]) if err != nil { return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed)) } cur.CreatedBy.SrcPath = match[1] cur.CreatedBy.Line = num s.state = gotFileCreated return "", nil } return "", fmt.Errorf("expected a file after a created line, got: %q", trimmed) case gotFileFunc: if match := reCreated.FindStringSubmatch(trimmed); match != nil { cur.CreatedBy.Func.Raw = match[1] s.state = gotCreated return "", nil } if elided == trimmed { cur.Stack.Elided = true // TODO(maruel): New state. return "", nil } call, err := parseFunc(trimmed) if call != nil { cur.Stack.Calls = append(cur.Stack.Calls, *call) s.state = gotFunc return "", err } if trimmed == "" { s.state = betweenRoutine return "", nil } // Back to normal state. s.state = normal s.prefix = "" return line, nil case gotFileCreated: if trimmed == "" { s.state = betweenRoutine return "", nil } s.state = normal s.prefix = "" return line, nil case gotUnavail: if trimmed == "" { s.state = betweenRoutine return "", nil } if match := reCreated.FindStringSubmatch(trimmed); match != nil { cur.CreatedBy.Func.Raw = match[1] s.state = gotCreated return "", nil } return "", fmt.Errorf("expected empty line after unavailable stack, got: %q", strings.TrimSpace(trimmed)) case gotRaceHeader1: if raceHeader == trimmed { s.state = gotRaceHeader // Send the line to the user. return line, nil } s.state = normal return line, nil case gotRaceHeader: if match := reRaceOperationHeader.FindStringSubmatch(trimmed); match != nil { w := match[1] == "Write" addr, err := strconv.ParseUint(match[2], 0, 64) if err != nil { return "", fmt.Errorf("failed to parse address on line: %q", strings.TrimSpace(trimmed)) } id, err := strconv.Atoi(match[3]) if err != nil { return "", fmt.Errorf("failed to parse goroutine id on line: %q", strings.TrimSpace(trimmed)) } s.races = append(s.races, raceOp{w, addr, id}) s.state = gotRaceOperationHeader return "", nil } s.state = normal return line, nil case gotRaceOperationHeader: call, err := parseFunc(trimmed) if call != nil { // TODO(maruel): Figure out. //cur.Stack.Calls = append(cur.Stack.Calls, *call) s.state = gotRaceOperationFunc return "", err } return "", fmt.Errorf("expected a function after a race operation, got: %q", trimmed) case gotRaceGoroutineHeader: call, err := parseFunc(strings.TrimLeft(trimmed, "\t ")) if call != nil { cur.Stack.Calls = append(cur.Stack.Calls, *call) s.state = gotRaceGoroutineFunc return "", err } return "", fmt.Errorf("expected a function after a race operation, got: %q", trimmed) case gotRaceOperationFunc: if match := reFile.FindStringSubmatch(trimmed); match != nil { _, err := strconv.Atoi(match[2]) if err != nil { return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed)) } /* TODO(maruel): Figure out. // cur.Stack.Calls is guaranteed to have at least one item. i := len(cur.Stack.Calls) - 1 cur.Stack.Calls[i].SrcPath = match[1] cur.Stack.Calls[i].Line = num */ s.state = gotRaceOperationFile return "", nil } return "", fmt.Errorf("expected a file after a race function, got: %q", trimmed) case gotRaceGoroutineFunc: if match := reFile.FindStringSubmatch(trimmed); match != nil { num, err := strconv.Atoi(match[2]) if err != nil { return "", fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(trimmed)) } // cur.Stack.Calls is guaranteed to have at least one item. i := len(cur.Stack.Calls) - 1 cur.Stack.Calls[i].SrcPath = match[1] cur.Stack.Calls[i].Line = num s.state = gotRaceGoroutineFile return "", nil } return "", fmt.Errorf("expected a file after a race function, got: %q", trimmed) case gotRaceOperationFile: if trimmed == "" { s.state = betweenRaces return "", nil } return "", fmt.Errorf("expected an empty line after a race file, got: %q", trimmed) case gotRaceGoroutineFile: if trimmed == "" { s.state = betweenRaces return "", nil } if trimmed == raceHeaderFooter { // Done. s.state = normal return "", nil } call, err := parseFunc(strings.TrimLeft(trimmed, "\t ")) if call != nil { // TODO(maruel): Process match. s.state = gotRaceGoroutineFunc return "", err } return "", fmt.Errorf("expected a function or the end after a race file, got: %q", trimmed) case betweenRaces: // Either Previous or Goroutine. if match := reRacePreviousOperationHeader.FindStringSubmatch(trimmed); match != nil { w := match[1] == "write" addr, err := strconv.ParseUint(match[2], 0, 64) if err != nil { return "", fmt.Errorf("failed to parse address on line: %q", strings.TrimSpace(trimmed)) } id, err := strconv.Atoi(match[3]) if err != nil { return "", fmt.Errorf("failed to parse goroutine id on line: %q", strings.TrimSpace(trimmed)) } s.races = append(s.races, raceOp{w, addr, id}) s.state = gotRaceOperationHeader return "", nil } if match := reRaceGoroutine.FindStringSubmatch(trimmed); match != nil { id, err := strconv.Atoi(match[1]) if err != nil { return "", fmt.Errorf("failed to parse goroutine id on line: %q", strings.TrimSpace(trimmed)) } g := &Goroutine{ Signature: Signature{State: match[2]}, ID: id, First: len(s.goroutines) == 0, } s.goroutines = append(s.goroutines, g) s.state = gotRaceGoroutineHeader return "", nil } return "", fmt.Errorf("expected an operator or goroutine, got: %q", trimmed) default: return "", errors.New("internal error") } } // parseFunc only return an error if also returning a Call. func parseFunc(line string) (*Call, error) { if match := reFunc.FindStringSubmatch(line); match != nil { call := &Call{Func: Func{Raw: match[1]}} for _, a := range strings.Split(match[2], ", ") { if a == "..." { call.Args.Elided = true continue } if a == "" { // Remaining values were dropped. break } v, err := strconv.ParseUint(a, 0, 64) if err != nil { return call, fmt.Errorf("failed to parse int on line: %q", strings.TrimSpace(line)) } call.Args.Values = append(call.Args.Values, Arg{Value: v}) } return call, nil } return nil, nil } // hasPathPrefix returns true if any of s is the prefix of p. func hasPathPrefix(p string, s map[string]string) bool { for prefix := range s { if strings.HasPrefix(p, prefix+"/") { return true } } return false } // getFiles returns all the source files deduped and ordered. func getFiles(goroutines []*Goroutine) []string { files := map[string]struct{}{} for _, g := range goroutines { for _, c := range g.Stack.Calls { files[c.SrcPath] = struct{}{} } } out := make([]string, 0, len(files)) for f := range files { out = append(out, f) } sort.Strings(out) return out } // splitPath splits a path into its components. // // The first item has its initial path separator kept. func splitPath(p string) []string { if p == "" { return nil } var out []string s := "" for _, c := range p { if c != '/' || (len(out) == 0 && strings.Count(s, "/") == len(s)) { s += string(c) } else if s != "" { out = append(out, s) s = "" } } if s != "" { out = append(out, s) } return out } // isFile returns true if the path is a valid file. func isFile(p string) bool { // TODO(maruel): Is it faster to open the file or to stat it? Worth a perf // test on Windows. i, err := os.Stat(p) return err == nil && !i.IsDir() } // rootedIn returns a root if the file split in parts is rooted in root. func rootedIn(root string, parts []string) string { //log.Printf("rootIn(%s, %v)", root, parts) for i := 1; i < len(parts); i++ { suffix := filepath.Join(parts[i:]...) if isFile(filepath.Join(root, suffix)) { return filepath.Join(parts[:i]...) } } return "" } // findRoots sets member GOROOT and GOPATHs. func (c *Context) findRoots() { c.GOPATHs = map[string]string{} for _, f := range getFiles(c.Goroutines) { // TODO(maruel): Could a stack dump have mixed cases? I think it's // possible, need to confirm and handle. //log.Printf(" Analyzing %s", f) if c.GOROOT != "" && strings.HasPrefix(f, c.GOROOT+"/") { continue } if hasPathPrefix(f, c.GOPATHs) { continue } parts := splitPath(f) if c.GOROOT == "" { if r := rootedIn(c.localgoroot, parts); r != "" { c.GOROOT = r //log.Printf("Found GOROOT=%s", c.GOROOT) continue } } found := false for _, l := range c.localgopaths { if r := rootedIn(l, parts); r != "" { //log.Printf("Found GOPATH=%s", r) c.GOPATHs[r] = l found = true break } } if !found { // If the source is not found, just too bad. //log.Printf("Failed to find locally: %s / %s", f, goroot) } } } func getGOPATHs() []string { var out []string for _, v := range filepath.SplitList(os.Getenv("GOPATH")) { // Disallow non-absolute paths? if v != "" { out = append(out, v) } } if len(out) == 0 { homeDir := "" u, err := user.Current() if err != nil { homeDir = os.Getenv("HOME") if homeDir == "" { panic(fmt.Sprintf("Could not get current user or $HOME: %s\n", err.Error())) } } else { homeDir = u.HomeDir } out = []string{homeDir + "go"} } return out } panicparse-1.3.0/stack/context_test.go000066400000000000000000000773151353500720100200300ustar00rootroot00000000000000// Copyright 2018 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack import ( "bufio" "bytes" "errors" "io" "io/ioutil" "os" "reflect" "strings" "testing" ) func TestParseDumpNothing(t *testing.T) { extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString("\n"), extra, true) if err != nil { t.Fatal(err) } if c != nil { t.Fatalf("unexpected %v", c) } } func TestParseDump1(t *testing.T) { // One call from main, one from stdlib, one from third party. // Create a long first line that will be ignored. It is to guard against // https://github.com/maruel/panicparse/issues/17. long := strings.Repeat("a", bufio.MaxScanTokenSize+1) data := []string{ long, "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/cockroachdb/cockroach/storage/engine._Cfunc_DBIterSeek()", " ??:0 +0x6d", "gopkg.in/yaml%2ev2.handleErr(0xc208033b20)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "reflect.Value.assignTo(0x570860, 0xc20803f3e0, 0x15)", " /goroot/src/reflect/value.go:2125 +0x368", "main.main()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:428 +0x27", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, true) if err != nil { t.Fatal(err) } compareString(t, long+"\npanic: reflect.Set: value of type\n\n", extra.String()) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "??", Func: Func{Raw: "github.com/cockroachdb/cockroach/storage/engine._Cfunc_DBIterSeek"}, }, { SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.handleErr"}, Args: Args{Values: []Arg{{Value: 0xc208033b20}}}, }, { SrcPath: "/goroot/src/reflect/value.go", Line: 2125, Func: Func{Raw: "reflect.Value.assignTo"}, Args: Args{Values: []Arg{{Value: 0x570860}, {Value: 0xc20803f3e0}, {Value: 0x15}}}, }, { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 428, Func: Func{Raw: "main.main"}, }, }, }, }, ID: 1, First: true, }, } for i := range expected { expected[i].updateLocations(c.GOROOT, c.localgoroot, c.GOPATHs) } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpLongWait(t *testing.T) { // One call from main, one from stdlib, one from third party. data := []string{ "panic: bleh", "", "goroutine 1 [chan send, 100 minutes]:", "gopkg.in/yaml%2ev2.handleErr(0xc208033b20)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "", "goroutine 2 [chan send, locked to thread]:", "gopkg.in/yaml%2ev2.handleErr(0xc208033b21)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "", "goroutine 3 [chan send, 101 minutes, locked to thread]:", "gopkg.in/yaml%2ev2.handleErr(0xc208033b22)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, true) if err != nil { t.Fatal(err) } compareString(t, "panic: bleh\n\n", extra.String()) expected := []*Goroutine{ { Signature: Signature{ State: "chan send", SleepMin: 100, SleepMax: 100, Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.handleErr"}, Args: Args{Values: []Arg{{Value: 0xc208033b20}}}, }, }, }, }, ID: 1, First: true, }, { Signature: Signature{ State: "chan send", Locked: true, Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.handleErr"}, Args: Args{Values: []Arg{{Value: 0xc208033b21, Name: "#1"}}}, }, }, }, }, ID: 2, }, { Signature: Signature{ State: "chan send", SleepMin: 101, SleepMax: 101, Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.handleErr"}, Args: Args{Values: []Arg{{Value: 0xc208033b22, Name: "#2"}}}, }, }, }, Locked: true, }, ID: 3, }, } for i := range expected { expected[i].updateLocations(c.GOROOT, c.localgoroot, c.GOPATHs) } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpAsm(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 16 [garbage collection]:", "runtime.switchtoM()", "\t/goroot/src/runtime/asm_amd64.s:198 fp=0xc20cfb80d8 sp=0xc20cfb80d0", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "garbage collection", Stack: Stack{ Calls: []Call{ { SrcPath: "/goroot/src/runtime/asm_amd64.s", Line: 198, Func: Func{Raw: "runtime.switchtoM"}, }, }, }, }, ID: 16, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpAsmGo1dot13(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 16 [garbage collection]:", "runtime.switchtoM()", "\t/goroot/src/runtime/asm_amd64.s:198 fp=0xc20cfb80d8 sp=0xc20cfb80d0 pc=0x5007be", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "garbage collection", Stack: Stack{ Calls: []Call{ { SrcPath: "/goroot/src/runtime/asm_amd64.s", Line: 198, Func: Func{Raw: "runtime.switchtoM"}, }, }, }, }, ID: 16, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpLineErr(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/maruel/panicparse/stack/stack.recurseType()", "\t/gopath/src/github.com/maruel/panicparse/stack/stack.go:12345678901234567890", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("failed to parse int on line: \"/gopath/src/github.com/maruel/panicparse/stack/stack.go:12345678901234567890\""), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{Calls: []Call{{Func: Func{Raw: "github.com/maruel/panicparse/stack/stack.recurseType"}}}}, }, ID: 1, First: true, }, } for i := range expected { expected[i].updateLocations(c.GOROOT, c.localgoroot, c.GOPATHs) } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpCreatedErr(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/maruel/panicparse/stack/stack.recurseType()", "\t/gopath/src/github.com/maruel/panicparse/stack/stack.go:1", "created by testing.RunTests", "\t/goroot/src/testing/testing.go:123456789012345678901 +0xa8b", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("failed to parse int on line: \"/goroot/src/testing/testing.go:123456789012345678901 +0xa8b\""), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 1, Func: Func{Raw: "github.com/maruel/panicparse/stack/stack.recurseType"}, }, }, }, CreatedBy: Call{ Func: Func{Raw: "testing.RunTests"}, }, }, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpValueErr(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/maruel/panicparse/stack/stack.recurseType(123456789012345678901)", "\t/gopath/src/github.com/maruel/panicparse/stack/stack.go:9", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("failed to parse int on line: \"github.com/maruel/panicparse/stack/stack.recurseType(123456789012345678901)\""), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{{Func: Func{Raw: "github.com/maruel/panicparse/stack/stack.recurseType"}}}, }, }, ID: 1, First: true, }, } for i := range expected { expected[i].updateLocations(c.GOROOT, c.localgoroot, c.GOPATHs) } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpInconsistentIndent(t *testing.T) { data := []string{ " goroutine 1 [running]:", " github.com/maruel/panicparse/stack/stack.recurseType()", " \t/gopath/src/github.com/maruel/panicparse/stack/stack.go:1", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New(`inconsistent indentation: " \t/gopath/src/github.com/maruel/panicparse/stack/stack.go:1", expected " "`), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ {Func: Func{Raw: "github.com/maruel/panicparse/stack/stack.recurseType"}}, }, }, }, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "", extra.String()) } func TestParseDumpOrderErr(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 16 [garbage collection]:", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "runtime.switchtoM()", "\t/goroot/src/runtime/asm_amd64.s:198 fp=0xc20cfb80d8 sp=0xc20cfb80d0", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("expected a function after a goroutine header, got: \"/gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6\""), err) expected := []*Goroutine{ { Signature: Signature{State: "garbage collection"}, ID: 16, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpElided(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 16 [garbage collection]:", "github.com/maruel/panicparse/stack/stack.recurseType(0x7f4fa9a3ec70, 0xc208062580, 0x7f4fa9a3e818, 0x50a820, 0xc20803a8a0)", "\t/gopath/src/github.com/maruel/panicparse/stack/stack.go:53 +0x845 fp=0xc20cfc66d8 sp=0xc20cfc6470", "...additional frames elided...", "created by testing.RunTests", "\t/goroot/src/testing/testing.go:555 +0xa8b", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "garbage collection", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 53, Func: Func{Raw: "github.com/maruel/panicparse/stack/stack.recurseType"}, Args: Args{ Values: []Arg{ {Value: 0x7f4fa9a3ec70}, {Value: 0xc208062580}, {Value: 0x7f4fa9a3e818}, {Value: 0x50a820}, {Value: 0xc20803a8a0}, }, }, }, }, Elided: true, }, CreatedBy: Call{ SrcPath: "/goroot/src/testing/testing.go", Line: 555, Func: Func{Raw: "testing.RunTests"}, }, }, ID: 16, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpSysCall(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 5 [syscall]:", "runtime.notetsleepg(0x918100, 0xffffffffffffffff, 0x1)", "\t/goroot/src/runtime/lock_futex.go:201 +0x52 fp=0xc208018f68 sp=0xc208018f40", "runtime.signal_recv(0x0)", "\t/goroot/src/runtime/sigqueue.go:109 +0x135 fp=0xc208018fa0 sp=0xc208018f68", "os/signal.loop()", "\t/goroot/src/os/signal/signal_unix.go:21 +0x1f fp=0xc208018fe0 sp=0xc208018fa0", "runtime.goexit()", "\t/goroot/src/runtime/asm_amd64.s:2232 +0x1 fp=0xc208018fe8 sp=0xc208018fe0", "created by os/signal.init·1", "\t/goroot/src/os/signal/signal_unix.go:27 +0x35", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "syscall", Stack: Stack{ Calls: []Call{ { SrcPath: "/goroot/src/runtime/lock_futex.go", Line: 201, Func: Func{Raw: "runtime.notetsleepg"}, Args: Args{ Values: []Arg{ {Value: 0x918100}, {Value: 0xffffffffffffffff}, {Value: 0x1}, }, }, }, { SrcPath: "/goroot/src/runtime/sigqueue.go", Line: 109, Func: Func{Raw: "runtime.signal_recv"}, Args: Args{ Values: []Arg{{}}, }, }, { SrcPath: "/goroot/src/os/signal/signal_unix.go", Line: 21, Func: Func{Raw: "os/signal.loop"}, }, { SrcPath: "/goroot/src/runtime/asm_amd64.s", Line: 2232, Func: Func{Raw: "runtime.goexit"}, }, }, }, CreatedBy: Call{ SrcPath: "/goroot/src/os/signal/signal_unix.go", Line: 27, Func: Func{Raw: "os/signal.init·1"}, }, }, ID: 5, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpUnavailCreated(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 24 [running]:", "\tgoroutine running on other thread; stack unavailable", "created by github.com/maruel/panicparse/stack.New", "\t/gopath/src/github.com/maruel/panicparse/stack/stack.go:131 +0x381", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{{SrcPath: ""}}, }, CreatedBy: Call{ SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 131, Func: Func{Raw: "github.com/maruel/panicparse/stack.New"}, }, }, ID: 24, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpUnavail(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 24 [running]:", "\tgoroutine running on other thread; stack unavailable", "", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{{SrcPath: ""}}, }, }, ID: 24, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpUnavailError(t *testing.T) { data := []string{ "panic: reflect.Set: value of type", "", "goroutine 24 [running]:", "\tgoroutine running on other thread; stack unavailable", "junk", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("expected empty line after unavailable stack, got: \"junk\""), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{{SrcPath: ""}}, }, }, ID: 24, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpNoOffset(t *testing.T) { data := []string{ "panic: runtime error: index out of range", "", "goroutine 37 [runnable]:", "github.com/maruel/panicparse/stack.func·002()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:110", "created by github.com/maruel/panicparse/stack.New", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:113 +0x43b", "", } c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), ioutil.Discard, false) if err != nil { t.Fatal(err) } expectedGR := []*Goroutine{ { Signature: Signature{ State: "runnable", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 110, Func: Func{Raw: "github.com/maruel/panicparse/stack.func·002"}, }, }, }, CreatedBy: Call{ SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 113, Func: Func{Raw: "github.com/maruel/panicparse/stack.New"}, }, }, ID: 37, First: true, }, } compareGoroutines(t, expectedGR, c.Goroutines) } func TestParseDumpHeaderError(t *testing.T) { // For coverage of scanLines. data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "junk", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("expected a function after a goroutine header, got: \"junk\""), err) expected := []*Goroutine{ { Signature: Signature{State: "running"}, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpFileError(t *testing.T) { // For coverage of scanLines. data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/maruel/panicparse/stack.func·002()", "junk", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("expected a file after a function, got: \"junk\""), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ {Func: Func{Raw: "github.com/maruel/panicparse/stack.func·002"}}, }, }, }, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpCreated(t *testing.T) { // For coverage of scanLines. data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/maruel/panicparse/stack.func·002()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:110", "created by github.com/maruel/panicparse/stack.New", "\t/gopath/src/github.com/maruel/panicparse/stack/stack.go:131 +0x381", "exit status 2", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 110, Func: Func{Raw: "github.com/maruel/panicparse/stack.func·002"}, }, }, }, CreatedBy: Call{ SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 131, Func: Func{Raw: "github.com/maruel/panicparse/stack.New"}, }, }, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\nexit status 2", extra.String()) } func TestParseDumpCreatedError(t *testing.T) { // For coverage of scanLines. data := []string{ "panic: reflect.Set: value of type", "", "goroutine 1 [running]:", "github.com/maruel/panicparse/stack.func·002()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:110", "created by github.com/maruel/panicparse/stack.New", "junk", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) compareErr(t, errors.New("expected a file after a created line, got: \"junk\""), err) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 110, Func: Func{Raw: "github.com/maruel/panicparse/stack.func·002"}, }, }, }, CreatedBy: Call{ Func: Func{Raw: "github.com/maruel/panicparse/stack.New"}, }, }, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) compareString(t, "panic: reflect.Set: value of type\n\n", extra.String()) } func TestParseDumpCCode(t *testing.T) { data := []string{ "SIGQUIT: quit", "PC=0x43f349", "", "goroutine 0 [idle]:", "runtime.epollwait(0x4, 0x7fff671c7118, 0xffffffff00000080, 0x0, 0xffffffff0028c1be, 0x0, 0x0, 0x0, 0x0, 0x0, ...)", " /goroot/src/runtime/sys_linux_amd64.s:400 +0x19", "runtime.netpoll(0x901b01, 0x0)", " /goroot/src/runtime/netpoll_epoll.go:68 +0xa3", "findrunnable(0xc208012000)", " /goroot/src/runtime/proc.c:1472 +0x485", "schedule()", " /goroot/src/runtime/proc.c:1575 +0x151", "runtime.park_m(0xc2080017a0)", " /goroot/src/runtime/proc.c:1654 +0x113", "runtime.mcall(0x432684)", " /goroot/src/runtime/asm_amd64.s:186 +0x5a", "", } c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), ioutil.Discard, false) if err != nil { t.Fatal(err) } expectedGR := []*Goroutine{ { Signature: Signature{ State: "idle", Stack: Stack{ Calls: []Call{ { SrcPath: "/goroot/src/runtime/sys_linux_amd64.s", Line: 400, Func: Func{Raw: "runtime.epollwait"}, Args: Args{ Values: []Arg{ {Value: 0x4}, {Value: 0x7fff671c7118}, {Value: 0xffffffff00000080}, {}, {Value: 0xffffffff0028c1be}, {}, {}, {}, {}, {}, }, Elided: true, }, }, { SrcPath: "/goroot/src/runtime/netpoll_epoll.go", Line: 68, Func: Func{Raw: "runtime.netpoll"}, Args: Args{Values: []Arg{{Value: 0x901b01}, {}}}, }, { SrcPath: "/goroot/src/runtime/proc.c", Line: 1472, Func: Func{Raw: "findrunnable"}, Args: Args{Values: []Arg{{Value: 0xc208012000}}}, }, { SrcPath: "/goroot/src/runtime/proc.c", Line: 1575, Func: Func{Raw: "schedule"}, }, { SrcPath: "/goroot/src/runtime/proc.c", Line: 1654, Func: Func{Raw: "runtime.park_m"}, Args: Args{Values: []Arg{{Value: 0xc2080017a0}}}, }, { SrcPath: "/goroot/src/runtime/asm_amd64.s", Line: 186, Func: Func{Raw: "runtime.mcall"}, Args: Args{Values: []Arg{{Value: 0x432684}}}, }, }, }, }, ID: 0, First: true, }, } compareGoroutines(t, expectedGR, c.Goroutines) } func TestParseDumpWithCarriageReturn(t *testing.T) { data := []string{ "goroutine 1 [running]:", "github.com/cockroachdb/cockroach/storage/engine._Cfunc_DBIterSeek()", " ??:0 +0x6d", "gopkg.in/yaml%2ev2.handleErr(0xc208033b20)", " /gopath/src/gopkg.in/yaml.v2/yaml.go:153 +0xc6", "reflect.Value.assignTo(0x570860, 0xc20803f3e0, 0x15)", " /goroot/src/reflect/value.go:2125 +0x368", "main.main()", " /gopath/src/github.com/maruel/panicparse/stack/stack.go:428 +0x27", "", } c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\r\n")), ioutil.Discard, false) if err != nil { t.Fatal(err) } expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "??", Func: Func{Raw: "github.com/cockroachdb/cockroach/storage/engine._Cfunc_DBIterSeek"}, }, { SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.handleErr"}, Args: Args{Values: []Arg{{Value: 0xc208033b20}}}, }, { SrcPath: "/goroot/src/reflect/value.go", Line: 2125, Func: Func{Raw: "reflect.Value.assignTo"}, Args: Args{Values: []Arg{{Value: 0x570860}, {Value: 0xc20803f3e0}, {Value: 0x15}}}, }, { SrcPath: "/gopath/src/github.com/maruel/panicparse/stack/stack.go", Line: 428, Func: Func{Raw: "main.main"}, }, }, }, }, ID: 1, First: true, }, } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpIndented(t *testing.T) { // goconvey is culprit of this. data := []string{ "Failures:", "", " * /home/maruel/go/src/foo/bar_test.go", " Line 209:", " Expected: '(*errors.errorString){s:\"context canceled\"}'", " Actual: 'nil'", " (Should resemble)!", " goroutine 8 [running]:", " foo/bar.TestArchiveFail.func1.2()", " /home/maruel/go/foo/bar_test.go:209 +0x469", " foo/bar.TestArchiveFail(0xc000338200)", " /home/maruel/go/src/foo/bar_test.go:155 +0xf1", " testing.tRunner(0xc000338200, 0x1615bf8)", " /home/maruel/golang/go/src/testing/testing.go:865 +0xc0", " created by testing.(*T).Run", " /home/maruel/golang/go/src/testing/testing.go:916 +0x35a", "", "", } extra := bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), &extra, false) if err != nil { t.Fatal(err) } compareString(t, strings.Join(data[:7], "\n")+"\n", extra.String()) expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "/home/maruel/go/foo/bar_test.go", Line: 209, Func: Func{Raw: "foo/bar.TestArchiveFail.func1.2"}, }, { SrcPath: "/home/maruel/go/src/foo/bar_test.go", Line: 155, Func: Func{Raw: "foo/bar.TestArchiveFail"}, Args: Args{Values: []Arg{{Value: 0xc000338200, Name: "#1"}}}, }, { SrcPath: "/home/maruel/golang/go/src/testing/testing.go", Line: 865, Func: Func{Raw: "testing.tRunner"}, Args: Args{Values: []Arg{{Value: 0xc000338200, Name: "#1"}, {Value: 0x1615bf8}}}, }, }, }, CreatedBy: Call{ SrcPath: "/home/maruel/golang/go/src/testing/testing.go", Line: 916, Func: Func{Raw: "testing.(*T).Run"}, }, }, ID: 8, First: true, }, } compareGoroutines(t, expected, c.Goroutines) } func TestParseDumpRace(t *testing.T) { // Generated with "panic race": data := []string{ "==================", "WARNING: DATA RACE", "Read at 0x00c0000e4030 by goroutine 7:", " main.panicRace.func1()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:37 +0x38", "", "Previous write at 0x00c0000e4030 by goroutine 6:", " main.panicRace.func1()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:37 +0x4e", "", "Goroutine 7 (running) created at:", " main.panicRace()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:35 +0x88", " main.main()", " /go/src/github.com/maruel/panicparse/cmd/panic/main.go:252 +0x2d9", "", "Goroutine 6 (running) created at:", " main.panicRace()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:35 +0x88", " main.main()", " /go/src/github.com/maruel/panicparse/cmd/panic/main.go:252 +0x2d9", "==================", "", } extra := &bytes.Buffer{} c, err := ParseDump(bytes.NewBufferString(strings.Join(data, "\n")), extra, false) if err != nil { t.Fatal(err) } // Confirm that it doesn't work yet. if c != nil { t.Fatal("expected c to be nil") } compareString(t, strings.Join(data, "\n"), extra.String()) } // This test should be deleted once Context state.raceDetectionEnabled is // removed and the race detector results is stored in Context. func TestRaceManual(t *testing.T) { // Generated with "panic race": data := []string{ "==================", "WARNING: DATA RACE", "Read at 0x00c0000e4030 by goroutine 7:", " main.panicRace.func1()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:37 +0x38", "", "Previous write at 0x00c0000e4030 by goroutine 6:", " main.panicRace.func1()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:37 +0x4e", "", "Goroutine 7 (running) created at:", " main.panicRace()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:35 +0x88", " main.main()", " /go/src/github.com/maruel/panicparse/cmd/panic/main.go:252 +0x2d9", "", "Goroutine 6 (running) created at:", " main.panicRace()", " /go/src/github.com/maruel/panicparse/cmd/panic/main_race.go:35 +0x88", " main.main()", " /go/src/github.com/maruel/panicparse/cmd/panic/main.go:252 +0x2d9", "==================", "", } extra := &bytes.Buffer{} expected := []*Goroutine{ { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "/go/src/github.com/maruel/panicparse/cmd/panic/main.go", Line: 252, Func: Func{Raw: "main.panicRace"}, }, }, }, }, ID: 7, First: true, }, { Signature: Signature{ State: "running", Stack: Stack{ Calls: []Call{ { SrcPath: "/go/src/github.com/maruel/panicparse/cmd/panic/main.go", Line: 252, Func: Func{Raw: "main.panicRace"}, }, }, }, }, ID: 6, }, } scanner := bufio.NewScanner(bytes.NewBufferString(strings.Join(data, "\n"))) scanner.Split(scanLines) s := scanningState{raceDetectionEnabled: true} for scanner.Scan() { line, err := s.scan(scanner.Text()) if line != "" { _, _ = io.WriteString(extra, line) } if err != nil { t.Fatal(err) } } compareGoroutines(t, expected, s.goroutines) expectedOps := []raceOp{{false, 0xc0000e4030, 7}, {true, 0xc0000e4030, 6}} if !reflect.DeepEqual(expectedOps, s.races) { t.Fatalf("%v", s.races) } } func TestSplitPath(t *testing.T) { if p := splitPath(""); p != nil { t.Fatalf("expected nil, got: %v", p) } } func TestGetGOPATHS(t *testing.T) { old := os.Getenv("GOPATH") defer func() { os.Setenv("GOPATH", old) }() os.Setenv("GOPATH", "") if p := getGOPATHs(); len(p) != 1 { t.Fatalf("expected only one path: %v", p) } } // func compareErr(t *testing.T, expected, actual error) { if actual == nil || expected.Error() != actual.Error() { t.Fatalf("%v != %v", expected, actual) } } func compareGoroutines(t *testing.T, expected, actual []*Goroutine) { if len(expected) != len(actual) { t.Fatalf("Different []*Goroutine length:\n- %v\n- %v", expected, actual) } for i := range expected { if !reflect.DeepEqual(expected[i], actual[i]) { t.Fatalf("Different Goroutine:\n- %#v\n- %#v", expected[i], actual[i]) } } } func compareString(t *testing.T, expected, actual string) { if expected != actual { t.Fatalf("%q != %q", expected, actual) } } panicparse-1.3.0/stack/example_test.go000066400000000000000000000040441353500720100177640ustar00rootroot00000000000000// Copyright 2018 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack_test import ( "bytes" "fmt" "io" "os" "github.com/maruel/panicparse/stack" ) const crash = `panic: oh no! goroutine 1 [running]: panic(0x0, 0x0) /home/user/src/golang/src/runtime/panic.go:464 +0x3e6 main.crash2(0x7fe50b49d028, 0xc82000a1e0) /home/user/go/src/github.com/maruel/panicparse/cmd/pp/main.go:45 +0x23 main.main() /home/user/go/src/github.com/maruel/panicparse/cmd/pp/main.go:50 +0xa6 ` func Example() { // Optional: Check for GOTRACEBACK being set, in particular if there is only // one goroutine returned. in := bytes.NewBufferString(crash) c, err := stack.ParseDump(in, os.Stdout, true) if err != nil { return } // Find out similar goroutine traces and group them into buckets. buckets := stack.Aggregate(c.Goroutines, stack.AnyValue) // Calculate alignment. srcLen := 0 pkgLen := 0 for _, bucket := range buckets { for _, line := range bucket.Signature.Stack.Calls { if l := len(line.SrcLine()); l > srcLen { srcLen = l } if l := len(line.Func.PkgName()); l > pkgLen { pkgLen = l } } } for _, bucket := range buckets { // Print the goroutine header. extra := "" if s := bucket.SleepString(); s != "" { extra += " [" + s + "]" } if bucket.Locked { extra += " [locked]" } if c := bucket.CreatedByString(false); c != "" { extra += " [Created by " + c + "]" } fmt.Printf("%d: %s%s\n", len(bucket.IDs), bucket.State, extra) // Print the stack lines. for _, line := range bucket.Stack.Calls { fmt.Printf( " %-*s %-*s %s(%s)\n", pkgLen, line.Func.PkgName(), srcLen, line.SrcLine(), line.Func.Name(), &line.Args) } if bucket.Stack.Elided { io.WriteString(os.Stdout, " (...)\n") } } // Output: // panic: oh no! // // 1: running // panic.go:464 panic(0, 0) // main main.go:45 crash2(0x7fe50b49d028, 0xc82000a1e0) // main main.go:50 main() } panicparse-1.3.0/stack/source.go000066400000000000000000000204761353500720100166010ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // This file contains the code to process sources, to be able to deduct the // original types. package stack import ( "bytes" "fmt" "go/ast" "go/parser" "go/token" "io/ioutil" "log" "math" "strings" ) // cache is a cache of sources on the file system. type cache struct { files map[string][]byte parsed map[string]*parsedFile } // Augment processes source files to improve calls to be more descriptive. // // It modifies goroutines in place. It requires calling ParseDump() with // guesspaths set to true to work properly. func Augment(goroutines []*Goroutine) { c := &cache{} for _, g := range goroutines { c.augmentGoroutine(g) } } // augmentGoroutine processes source files to improve call to be more // descriptive. // // It modifies the routine. func (c *cache) augmentGoroutine(goroutine *Goroutine) { if c.files == nil { c.files = map[string][]byte{} } if c.parsed == nil { c.parsed = map[string]*parsedFile{} } // For each call site, look at the next call and populate it. Then we can // walk back and reformat things. for i := range goroutine.Stack.Calls { c.load(goroutine.Stack.Calls[i].LocalSrcPath) } // Once all loaded, we can look at the next call when available. for i := 0; i < len(goroutine.Stack.Calls)-1; i++ { // Get the AST from the previous call and process the call line with it. if f := c.getFuncAST(&goroutine.Stack.Calls[i]); f != nil { processCall(&goroutine.Stack.Calls[i], f) } } } // Private stuff. // load loads a source file and parses the AST tree. Failures are ignored. func (c *cache) load(fileName string) { if _, ok := c.parsed[fileName]; ok { return } c.parsed[fileName] = nil if !strings.HasSuffix(fileName, ".go") { // Ignore C and assembly. c.files[fileName] = nil return } log.Printf("load(%s)", fileName) if _, ok := c.files[fileName]; !ok { var err error if c.files[fileName], err = ioutil.ReadFile(fileName); err != nil { log.Printf("Failed to read %s: %s", fileName, err) c.files[fileName] = nil return } } fset := token.NewFileSet() src := c.files[fileName] parsed, err := parser.ParseFile(fset, fileName, src, 0) if err != nil { log.Printf("Failed to parse %s: %s", fileName, err) return } // Convert the line number into raw file offset. offsets := []int{0, 0} start := 0 for l := 1; start < len(src); l++ { start += bytes.IndexByte(src[start:], '\n') + 1 offsets = append(offsets, start) } c.parsed[fileName] = &parsedFile{offsets, parsed} } func (c *cache) getFuncAST(call *Call) *ast.FuncDecl { if p := c.parsed[call.LocalSrcPath]; p != nil { return p.getFuncAST(call.Func.Name(), call.Line) } return nil } type parsedFile struct { lineToByteOffset []int parsed *ast.File } // getFuncAST gets the callee site function AST representation for the code // inside the function f at line l. func (p *parsedFile) getFuncAST(f string, l int) (d *ast.FuncDecl) { if len(p.lineToByteOffset) <= l { // The line number in the stack trace line does not exist in the file. That // can only mean that the sources on disk do not match the sources used to // build the binary. // TODO(maruel): This should be surfaced, so that source parsing is // completely ignored. return } // Walk the AST to find the lineToByteOffset that fits the line number. var lastFunc *ast.FuncDecl var found ast.Node // Inspect() goes depth first. This means for example that a function like: // func a() { // b := func() {} // c() // } // // Were we are looking at the c() call can return confused values. It is // important to look at the actual ast.Node hierarchy. ast.Inspect(p.parsed, func(n ast.Node) bool { if d != nil { return false } if n == nil { return true } if found != nil { // We are walking up. } if int(n.Pos()) >= p.lineToByteOffset[l] { // We are expecting a ast.CallExpr node. It can be harder to figure out // when there are multiple calls on a single line, as the stack trace // doesn't have file byte offset information, only line based. // gofmt will always format to one function call per line but there can // be edge cases, like: // a = A{Foo(), Bar()} d = lastFunc //p.processNode(call, n) return false } else if f, ok := n.(*ast.FuncDecl); ok { lastFunc = f } return true }) return } func name(n ast.Node) string { switch t := n.(type) { case *ast.InterfaceType: return "interface{}" case *ast.Ident: return t.Name case *ast.SelectorExpr: return t.Sel.Name case *ast.StarExpr: return "*" + name(t.X) default: return "" } } // fieldToType returns the type name and whether if it's an ellipsis. func fieldToType(f *ast.Field) (string, bool) { switch arg := f.Type.(type) { case *ast.ArrayType: return "[]" + name(arg.Elt), false case *ast.Ellipsis: return name(arg.Elt), true case *ast.FuncType: // Do not print the function signature to not overload the trace. return "func", false case *ast.Ident: return arg.Name, false case *ast.InterfaceType: return "interface{}", false case *ast.SelectorExpr: return arg.Sel.Name, false case *ast.StarExpr: return "*" + name(arg.X), false case *ast.MapType: return fmt.Sprintf("map[%s]%s", name(arg.Key), name(arg.Value)), false case *ast.ChanType: return fmt.Sprintf("chan %s", name(arg.Value)), false default: // TODO(maruel): Implement anything missing. return "", false } } // extractArgumentsType returns the name of the type of each input argument. func extractArgumentsType(f *ast.FuncDecl) ([]string, bool) { var fields []*ast.Field if f.Recv != nil { if len(f.Recv.List) != 1 { panic("Expect only one receiver; please fix panicparse's code") } // If it is an object receiver (vs a pointer receiver), its address is not // printed in the stack trace so it needs to be ignored. if _, ok := f.Recv.List[0].Type.(*ast.StarExpr); ok { fields = append(fields, f.Recv.List[0]) } } var types []string extra := false for _, arg := range append(fields, f.Type.Params.List...) { // Assert that extra is only set on the last item of fields? var t string t, extra = fieldToType(arg) mult := len(arg.Names) if mult == 0 { mult = 1 } for i := 0; i < mult; i++ { types = append(types, t) } } return types, extra } // processCall walks the function and populate call accordingly. func processCall(call *Call, f *ast.FuncDecl) { values := make([]uint64, len(call.Args.Values)) for i := range call.Args.Values { values[i] = call.Args.Values[i].Value } index := 0 pop := func() uint64 { if len(values) != 0 { x := values[0] values = values[1:] index++ return x } return 0 } popName := func() string { n := call.Args.Values[index].Name v := pop() if len(n) == 0 { return fmt.Sprintf("0x%x", v) } return n } types, extra := extractArgumentsType(f) for i := 0; len(values) != 0; i++ { var t string if i >= len(types) { if !extra { // These are unexpected value! Print them as hex. call.Args.Processed = append(call.Args.Processed, popName()) continue } t = types[len(types)-1] } else { t = types[i] } switch t { case "float32": call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float32frombits(uint32(pop())))) case "float64": call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%g", math.Float64frombits(pop()))) case "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64": call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%d", pop())) case "string": call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s, len=%d)", t, popName(), pop())) default: if strings.HasPrefix(t, "*") { call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName())) } else if strings.HasPrefix(t, "[]") { call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s len=%d cap=%d)", t, popName(), pop(), pop())) } else { // Assumes it's an interface. For now, discard the object value, which // is probably not a good idea. call.Args.Processed = append(call.Args.Processed, fmt.Sprintf("%s(%s)", t, popName())) pop() } } if len(values) == 0 && call.Args.Elided { return } } } panicparse-1.3.0/stack/source_go110_test.go000066400000000000000000000005041353500720100205350ustar00rootroot00000000000000// Copyright 2019 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // +build go1.11 package stack // See https://github.com/maruel/panicparse/issues/42 for explanation. func zapArguments() bool { return true } panicparse-1.3.0/stack/source_go111_test.go000066400000000000000000000005061353500720100205400ustar00rootroot00000000000000// Copyright 2019 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // +build !go1.11 package stack // See https://github.com/maruel/panicparse/issues/42 for explanation. func zapArguments() bool { return false } panicparse-1.3.0/stack/source_test.go000066400000000000000000000333751353500720100176420ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack import ( "bytes" "fmt" "io/ioutil" "os" "os/exec" "path/filepath" "reflect" "strings" "testing" ) func TestAugment(t *testing.T) { data := []struct { name string input string // Starting with go1.11, the stack trace do not contain much information // about the arguments and shows as elided. workaroundGo111Elided bool // Starting with go1.11, non-pointer call shows an elided argument, while // there was no argument listed before. workaroundGo111Extra bool expected Stack }{ { "Local function doesn't interfere", `func f(s string) { a := func(i int) int { return 1 + i } _ = a(3) panic("ooh") } func main() { f("yo") }`, false, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 7, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer, Name: ""}, {Value: 0x2}}, }, }, {SrcPath: "main.go", Line: 10, Func: Func{Raw: "main.main"}}, }, }, }, { "func", `func f(a func() string) { panic(a()) } func main() { f(func() string { return "ooh" }) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{Values: []Arg{{Value: pointer}}}, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "func ellipsis", `func f(a ...func() string) { panic(a[0]()) } func main() { f(func() string { return "ooh" }) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}, {Value: 0x1}, {Value: 0x1}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "interface{}", `func f(a []interface{}) { panic("ooh") } func main() { f(make([]interface{}, 5, 7)) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}, {Value: 0x5}, {Value: 0x7}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "[]int", `func f(a []int) { panic("ooh") } func main() { f(make([]int, 5, 7)) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}, {Value: 5}, {Value: 7}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "[]interface{}", `func f(a []interface{}) { panic(a[0].(string)) } func main() { f([]interface{}{"ooh"}) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}, {Value: 1}, {Value: 1}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "map[int]int", `func f(a map[int]int) { panic("ooh") } func main() { f(map[int]int{1: 2}) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "map[interface{}]interface{}", `func f(a map[interface{}]interface{}) { panic("ooh") } func main() { f(make(map[interface{}]interface{})) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "chan int", `func f(a chan int) { panic("ooh") } func main() { f(make(chan int)) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "chan interface{}", `func f(a chan interface{}) { panic("ooh") } func main() { f(make(chan interface{})) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "non-pointer method", `type S struct { } func (s S) f() { panic("ooh") } func main() { var s S s.f() }`, true, true, Stack{ Calls: []Call{ {SrcPath: "main.go", Line: 5, Func: Func{Raw: "main.S.f"}}, {SrcPath: "main.go", Line: 9, Func: Func{Raw: "main.main"}}, }, }, }, { "pointer method", `type S struct { } func (s *S) f() { panic("ooh") } func main() { var s S s.f() }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 5, Func: Func{Raw: "main.(*S).f"}, Args: Args{Values: []Arg{{Value: pointer}}}, }, {SrcPath: "main.go", Line: 9, Func: Func{Raw: "main.main"}}, }, }, }, { "string", `func f(s string) { panic(s) } func main() { f("ooh") }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{Values: []Arg{{Value: pointer}, {Value: 0x3}}}, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "string and int", `func f(s string, i int) { panic(s) } func main() { f("ooh", 42) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{Values: []Arg{{Value: pointer}, {Value: 0x3}, {Value: 42}}}, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "values are elided", `func f(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12 int, s13 interface{}) { panic("ooh") } func main() { f(0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 44, 45, nil) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{}, {}, {}, {}, {}, {}, {}, {}, {Value: 42}, {Value: 43}}, Elided: true, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "error", `import "errors" func f(err error) { panic(err.Error()) } func main() { f(errors.New("ooh")) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 4, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}, {Value: pointer}}, }, }, {SrcPath: "main.go", Line: 7, Func: Func{Raw: "main.main"}}, }, }, }, { "error unnamed", `import "errors" func f(error) { panic("ooh") } func main() { f(errors.New("ooh")) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 4, Func: Func{Raw: "main.f"}, Args: Args{ Values: []Arg{{Value: pointer}, {Value: pointer}}, }, }, {SrcPath: "main.go", Line: 7, Func: Func{Raw: "main.main"}}, }, }, }, { "float32", `func f(v float32) { panic("ooh") } func main() { f(0.5) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ // The value is NOT a pointer but floating point encoding is not // deterministic. Values: []Arg{{Value: pointer}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, { "float64", `func f(v float64) { panic("ooh") } func main() { f(0.5) }`, true, false, Stack{ Calls: []Call{ { SrcPath: "main.go", Line: 3, Func: Func{Raw: "main.f"}, Args: Args{ // The value is NOT a pointer but floating point encoding is not // deterministic. Values: []Arg{{Value: pointer}}, }, }, {SrcPath: "main.go", Line: 6, Func: Func{Raw: "main.main"}}, }, }, }, } for i, line := range data { // Marshal the code a bit to make it nicer. Inject 'package main'. lines := append([]string{"package main"}, strings.Split(line.input, "\n")...) for i := 2; i < len(lines); i++ { // Strip the 3 first tab characters. It's very adhoc but good enough here // and makes test failure much more readable. if lines[i][:3] != "\t\t\t" { t.Fatal("expected line to start with 3 tab characters") } lines[i] = lines[i][3:] } input := strings.Join(lines, "\n") // Run the command. _, content, clean := getCrash(t, input) // Analyze it. extra := bytes.Buffer{} c, err := ParseDump(bytes.NewBuffer(content), &extra, false) if err != nil { clean() t.Fatalf("failed to parse input for test %s: %v", line.name, err) } // On go1.4, there's one less space. actual := extra.String() if actual != "panic: ooh\n\nexit status 2\n" && actual != "panic: ooh\nexit status 2\n" { clean() t.Fatalf("Unexpected panic output:\n%#v", actual) } // On go1.11 with non-pointer method, it shows elided argument where there // used to be none before. It's only for test case "non-pointer method". if line.workaroundGo111Extra && zapArguments() { line.expected.Calls[0].Args.Elided = true } s := c.Goroutines[0].Signature.Stack t.Logf("Test #%d: %v", i, line.name) zapPointers(t, line.name, line.workaroundGo111Elided, &line.expected, &s) zapPaths(&s) clean() if !reflect.DeepEqual(line.expected, s) { t.Logf("Different (expected, then actual):\n- %#v\n- %#v", line.expected, s) t.Logf("Source code:\n%s", input) t.Logf("Output:\n%s", content) t.FailNow() } } } func TestAugmentDummy(t *testing.T) { goroutines := []*Goroutine{ { Signature: Signature{ Stack: Stack{ Calls: []Call{{SrcPath: "missing.go"}}, }, }, }, } Augment(goroutines) } func TestLoad(t *testing.T) { c := &cache{ files: map[string][]byte{"bad.go": []byte("bad content")}, parsed: map[string]*parsedFile{}, } c.load("foo.asm") c.load("bad.go") c.load("doesnt_exist.go") if l := len(c.parsed); l != 3 { t.Fatalf("expected 3, got %d", l) } if c.parsed["foo.asm"] != nil { t.Fatalf("foo.asm is not present; should not have been loaded") } if c.parsed["bad.go"] != nil { t.Fatalf("bad.go is not valid code; should not have been loaded") } if c.parsed["doesnt_exist.go"] != nil { t.Fatalf("doesnt_exist.go is not present; should not have been loaded") } if c.getFuncAST(&Call{SrcPath: "other"}) != nil { t.Fatalf("there's no 'other'") } } // const pointer = uint64(0xfffffffff) const pointerStr = "0xfffffffff" func overrideEnv(env []string, key, value string) []string { prefix := key + "=" for i, e := range env { if strings.HasPrefix(e, prefix) { env[i] = prefix + value return env } } return append(env, prefix+value) } func getCrash(t *testing.T, content string) (string, []byte, func()) { //p := getGOPATHs() //name, err := ioutil.TempDir(filepath.Join(p[0], "src"), "panicparse") name, err := ioutil.TempDir("", "panicparse") if err != nil { t.Fatalf("failed to create temporary directory: %v", err) } clean := func() { if err := os.RemoveAll(name); err != nil { t.Fatalf("failed to remove temporary directory %q: %v", name, err) } } main := filepath.Join(name, "main.go") if err := ioutil.WriteFile(main, []byte(content), 0500); err != nil { clean() t.Fatalf("failed to write %q: %v", main, err) } cmd := exec.Command("go", "run", main) // Use the Go 1.4 compatible format. cmd.Env = overrideEnv(os.Environ(), "GOTRACEBACK", "1") out, err := cmd.CombinedOutput() if err == nil { clean() t.Fatal("expected error since this is supposed to crash") } return main, out, clean } // zapPointers zaps out pointers. func zapPointers(t *testing.T, name string, workaroundGo111Elided bool, expected, s *Stack) { for i := range s.Calls { if i >= len(expected.Calls) { // When using GOTRACEBACK=2, it'll include runtime.main() and // runtime.goexit(). Ignore these since they could be changed in a future // version. s.Calls = s.Calls[:len(expected.Calls)] break } if workaroundGo111Elided && zapArguments() { // See https://github.com/maruel/panicparse/issues/42 for explanation. if len(expected.Calls[i].Args.Values) != 0 { expected.Calls[i].Args.Elided = true } expected.Calls[i].Args.Values = nil continue } for j := range s.Calls[i].Args.Values { if j >= len(expected.Calls[i].Args.Values) { break } if expected.Calls[i].Args.Values[j].Value == pointer { // Replace the pointer value. if s.Calls[i].Args.Values[j].Value == 0 { t.Fatalf("%s: Call %d, value %d, expected pointer, got 0", name, i, j) } old := fmt.Sprintf("0x%x", s.Calls[i].Args.Values[j].Value) s.Calls[i].Args.Values[j].Value = pointer for k := range s.Calls[i].Args.Processed { s.Calls[i].Args.Processed[k] = strings.Replace(s.Calls[i].Args.Processed[k], old, pointerStr, -1) } } } } } // zapPaths removes the directory part and only keep the base file name. func zapPaths(s *Stack) { for j := range s.Calls { s.Calls[j].SrcPath = filepath.Base(s.Calls[j].SrcPath) s.Calls[j].LocalSrcPath = "" } } panicparse-1.3.0/stack/stack.go000066400000000000000000000373331353500720100164060ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. // Package stack analyzes stack dump of Go processes and simplifies it. // // It is mostly useful on servers will large number of identical goroutines, // making the crash dump harder to read than strictly necessary. package stack import ( "fmt" "math" "net/url" "os" "path/filepath" "sort" "strings" "unicode" "unicode/utf8" ) // Func is a function call. // // Go stack traces print a mangled function call, this wrapper unmangle the // string before printing and adds other filtering methods. type Func struct { Raw string } // String is the fully qualified function name. // // Sadly Go is a bit confused when the package name doesn't match the directory // containing the source file and will use the directory name instead of the // real package name. func (f *Func) String() string { s, _ := url.QueryUnescape(f.Raw) return s } // Name is the naked function name. func (f *Func) Name() string { parts := strings.SplitN(filepath.Base(f.Raw), ".", 2) if len(parts) == 1 { return parts[0] } return parts[1] } // PkgName is the package name for this function reference. func (f *Func) PkgName() string { parts := strings.SplitN(filepath.Base(f.Raw), ".", 2) if len(parts) == 1 { return "" } s, _ := url.QueryUnescape(parts[0]) return s } // PkgDotName returns "." format. func (f *Func) PkgDotName() string { parts := strings.SplitN(filepath.Base(f.Raw), ".", 2) s, _ := url.QueryUnescape(parts[0]) if len(parts) == 1 { return parts[0] } if s != "" || parts[1] != "" { return s + "." + parts[1] } return "" } // IsExported returns true if the function is exported. func (f *Func) IsExported() bool { name := f.Name() parts := strings.Split(name, ".") r, _ := utf8.DecodeRuneInString(parts[len(parts)-1]) if unicode.ToUpper(r) == r { return true } return f.PkgName() == "main" && name == "main" } // Arg is an argument on a Call. type Arg struct { Value uint64 // Value is the raw value as found in the stack trace Name string // Name is a pseudo name given to the argument } // IsPtr returns true if we guess it's a pointer. It's only a guess, it can be // easily be confused by a bitmask. func (a *Arg) IsPtr() bool { // Assumes all pointers are above 16Mb and positive. return a.Value > 16*1024*1024 && a.Value < math.MaxInt64 } func (a *Arg) String() string { if a.Name != "" { return a.Name } if a.Value == 0 { return "0" } return fmt.Sprintf("0x%x", a.Value) } // Args is a series of function call arguments. type Args struct { Values []Arg // Values is the arguments as shown on the stack trace. They are mangled via simplification. Processed []string // Processed is the arguments generated from processing the source files. It can have a length lower than Values. Elided bool // If set, it means there was a trailing ", ..." } func (a *Args) String() string { var v []string if len(a.Processed) != 0 { v = make([]string, 0, len(a.Processed)) for _, item := range a.Processed { v = append(v, item) } } else { v = make([]string, 0, len(a.Values)) for _, item := range a.Values { v = append(v, item.String()) } } if a.Elided { v = append(v, "...") } return strings.Join(v, ", ") } // equal returns true only if both arguments are exactly equal. func (a *Args) equal(r *Args) bool { if a.Elided != r.Elided || len(a.Values) != len(r.Values) { return false } for i, l := range a.Values { if l != r.Values[i] { return false } } return true } // similar returns true if the two Args are equal or almost but not quite // equal. func (a *Args) similar(r *Args, similar Similarity) bool { if a.Elided != r.Elided || len(a.Values) != len(r.Values) { return false } if similar == AnyValue { return true } for i, l := range a.Values { switch similar { case ExactFlags, ExactLines: if l != r.Values[i] { return false } default: if l.IsPtr() != r.Values[i].IsPtr() || (!l.IsPtr() && l != r.Values[i]) { return false } } } return true } // merge merges two similar Args, zapping out differences. func (a *Args) merge(r *Args) Args { out := Args{ Values: make([]Arg, len(a.Values)), Elided: a.Elided, } for i, l := range a.Values { if l != r.Values[i] { out.Values[i].Name = "*" out.Values[i].Value = l.Value } else { out.Values[i] = l } } return out } // Call is an item in the stack trace. type Call struct { SrcPath string // Full path name of the source file as seen in the trace LocalSrcPath string // Full path name of the source file as seen in the host. Line int // Line number Func Func // Fully qualified function name (encoded). Args Args // Call arguments IsStdlib bool // true if it is a Go standard library function. This includes the 'go test' generated main executable. } // equal returns true only if both calls are exactly equal. func (c *Call) equal(r *Call) bool { return c.SrcPath == r.SrcPath && c.Line == r.Line && c.Func == r.Func && c.Args.equal(&r.Args) } // similar returns true if the two Call are equal or almost but not quite // equal. func (c *Call) similar(r *Call, similar Similarity) bool { return c.SrcPath == r.SrcPath && c.Line == r.Line && c.Func == r.Func && c.Args.similar(&r.Args, similar) } // merge merges two similar Call, zapping out differences. func (c *Call) merge(r *Call) Call { return Call{ SrcPath: c.SrcPath, Line: c.Line, Func: c.Func, Args: c.Args.merge(&r.Args), LocalSrcPath: c.LocalSrcPath, IsStdlib: c.IsStdlib, } } // SrcName returns the base file name of the source file. func (c *Call) SrcName() string { return filepath.Base(c.SrcPath) } // SrcLine returns "source.go:line", including only the base file name. func (c *Call) SrcLine() string { return fmt.Sprintf("%s:%d", c.SrcName(), c.Line) } // FullSrcLine returns "/path/to/source.go:line". // // This file path is mutated to look like the local path. func (c *Call) FullSrcLine() string { return fmt.Sprintf("%s:%d", c.SrcPath, c.Line) } // PkgSrc is one directory plus the file name of the source file. func (c *Call) PkgSrc() string { return filepath.Join(filepath.Base(filepath.Dir(c.SrcPath)), c.SrcName()) } // IsPkgMain returns true if it is in the main package. func (c *Call) IsPkgMain() bool { return c.Func.PkgName() == "main" } const testMainSrc = "_test" + string(os.PathSeparator) + "_testmain.go" // updateLocations initializes LocalSrcPath and IsStdlib. func (c *Call) updateLocations(goroot, localgoroot string, gopaths map[string]string) { if c.SrcPath != "" { // Always check GOROOT first, then GOPATH. if strings.HasPrefix(c.SrcPath, goroot) { // Replace remote GOROOT with local GOROOT. c.LocalSrcPath = filepath.Join(localgoroot, c.SrcPath[len(goroot):]) } else { // Replace remote GOPATH with local GOPATH. c.LocalSrcPath = c.SrcPath // TODO(maruel): Sort for deterministic behavior? for prefix, dest := range gopaths { if strings.HasPrefix(c.SrcPath, prefix) { c.LocalSrcPath = filepath.Join(dest, c.SrcPath[len(prefix):]) break } } } } // Consider _test/_testmain.go as stdlib since it's injected by "go test". c.IsStdlib = (goroot != "" && strings.HasPrefix(c.SrcPath, goroot)) || c.PkgSrc() == testMainSrc } // Stack is a call stack. type Stack struct { Calls []Call // Call stack. First is original function, last is leaf function. Elided bool // Happens when there's >100 items in Stack, currently hardcoded in package runtime. } // equal returns true on if both call stacks are exactly equal. func (s *Stack) equal(r *Stack) bool { if len(s.Calls) != len(r.Calls) || s.Elided != r.Elided { return false } for i := range s.Calls { if !s.Calls[i].equal(&r.Calls[i]) { return false } } return true } // similar returns true if the two Stack are equal or almost but not quite // equal. func (s *Stack) similar(r *Stack, similar Similarity) bool { if len(s.Calls) != len(r.Calls) || s.Elided != r.Elided { return false } for i := range s.Calls { if !s.Calls[i].similar(&r.Calls[i], similar) { return false } } return true } // merge merges two similar Stack, zapping out differences. func (s *Stack) merge(r *Stack) *Stack { // Assumes similar stacks have the same length. out := &Stack{ Calls: make([]Call, len(s.Calls)), Elided: s.Elided, } for i := range s.Calls { out.Calls[i] = s.Calls[i].merge(&r.Calls[i]) } return out } // less compares two Stack, where the ones that are less are more // important, so they come up front. // // A Stack with more private functions is 'less' so it is at the top. // Inversely, a Stack with only public functions is 'more' so it is at the // bottom. func (s *Stack) less(r *Stack) bool { lStdlib := 0 lPrivate := 0 for _, c := range s.Calls { if c.IsStdlib { lStdlib++ } else { lPrivate++ } } rStdlib := 0 rPrivate := 0 for _, s := range r.Calls { if s.IsStdlib { rStdlib++ } else { rPrivate++ } } if lPrivate > rPrivate { return true } if lPrivate < rPrivate { return false } if lStdlib > rStdlib { return false } if lStdlib < rStdlib { return true } // Stack lengths are the same. for x := range s.Calls { if s.Calls[x].Func.Raw < r.Calls[x].Func.Raw { return true } if s.Calls[x].Func.Raw > r.Calls[x].Func.Raw { return true } if s.Calls[x].PkgSrc() < r.Calls[x].PkgSrc() { return true } if s.Calls[x].PkgSrc() > r.Calls[x].PkgSrc() { return true } if s.Calls[x].Line < r.Calls[x].Line { return true } if s.Calls[x].Line > r.Calls[x].Line { return true } } return false } func (s *Stack) updateLocations(goroot, localgoroot string, gopaths map[string]string) { for i := range s.Calls { s.Calls[i].updateLocations(goroot, localgoroot, gopaths) } } // Signature represents the signature of one or multiple goroutines. // // It is effectively the stack trace plus the goroutine internal bits, like // it's state, if it is thread locked, which call site created this goroutine, // etc. type Signature struct { // Use git grep 'gopark(|unlock)\(' to find them all plus everything listed // in runtime/traceback.go. Valid values includes: // - chan send, chan receive, select // - finalizer wait, mark wait (idle), // - Concurrent GC wait, GC sweep wait, force gc (idle) // - IO wait, panicwait // - semacquire, semarelease // - sleep, timer goroutine (idle) // - trace reader (blocked) // Stuck cases: // - chan send (nil chan), chan receive (nil chan), select (no cases) // Runnable states: // - idle, runnable, running, syscall, waiting, dead, enqueue, copystack, // Scan states: // - scan, scanrunnable, scanrunning, scansyscall, scanwaiting, scandead, // scanenqueue State string CreatedBy Call // Which other goroutine which created this one. SleepMin int // Wait time in minutes, if applicable. SleepMax int // Wait time in minutes, if applicable. Stack Stack Locked bool // Locked to an OS thread. } // equal returns true only if both signatures are exactly equal. func (s *Signature) equal(r *Signature) bool { if s.State != r.State || !s.CreatedBy.equal(&r.CreatedBy) || s.Locked != r.Locked || s.SleepMin != r.SleepMin || s.SleepMax != r.SleepMax { return false } return s.Stack.equal(&r.Stack) } // similar returns true if the two Signature are equal or almost but not quite // equal. func (s *Signature) similar(r *Signature, similar Similarity) bool { if s.State != r.State || !s.CreatedBy.similar(&r.CreatedBy, similar) { return false } if similar == ExactFlags && s.Locked != r.Locked { return false } return s.Stack.similar(&r.Stack, similar) } // merge merges two similar Signature, zapping out differences. func (s *Signature) merge(r *Signature) *Signature { min := s.SleepMin if r.SleepMin < min { min = r.SleepMin } max := s.SleepMax if r.SleepMax > max { max = r.SleepMax } return &Signature{ State: s.State, // Drop right side. CreatedBy: s.CreatedBy, // Drop right side. SleepMin: min, SleepMax: max, Stack: *s.Stack.merge(&r.Stack), Locked: s.Locked || r.Locked, // TODO(maruel): This is weirdo. } } // less compares two Signature, where the ones that are less are more // important, so they come up front. A Signature with more private functions is // 'less' so it is at the top. Inversely, a Signature with only public // functions is 'more' so it is at the bottom. func (s *Signature) less(r *Signature) bool { if s.Stack.less(&r.Stack) { return true } if r.Stack.less(&s.Stack) { return false } if s.Locked && !r.Locked { return true } if r.Locked && !s.Locked { return false } if s.State < r.State { return true } if s.State > r.State { return false } return false } // SleepString returns a string "N-M minutes" if the goroutine(s) slept for a // long time. // // Returns an empty string otherwise. func (s *Signature) SleepString() string { if s.SleepMax == 0 { return "" } if s.SleepMin != s.SleepMax { return fmt.Sprintf("%d~%d minutes", s.SleepMin, s.SleepMax) } return fmt.Sprintf("%d minutes", s.SleepMax) } // CreatedByString return a short context about the origin of this goroutine // signature. func (s *Signature) CreatedByString(fullPath bool) string { created := s.CreatedBy.Func.PkgDotName() if created == "" { return "" } created += " @ " if fullPath { created += s.CreatedBy.FullSrcLine() } else { created += s.CreatedBy.SrcLine() } return created } func (s *Signature) updateLocations(goroot, localgoroot string, gopaths map[string]string) { s.CreatedBy.updateLocations(goroot, localgoroot, gopaths) s.Stack.updateLocations(goroot, localgoroot, gopaths) } // Goroutine represents the state of one goroutine, including the stack trace. type Goroutine struct { Signature // It's stack trace, internal bits, state, which call site created it, etc. ID int // Goroutine ID. First bool // First is the goroutine first printed, normally the one that crashed. } // Private stuff. // nameArguments is a post-processing step where Args are 'named' with numbers. func nameArguments(goroutines []*Goroutine) { // Set a name for any pointer occurring more than once. type object struct { args []*Arg inPrimary bool id int } objects := map[uint64]object{} // Enumerate all the arguments. for i := range goroutines { for j := range goroutines[i].Stack.Calls { for k := range goroutines[i].Stack.Calls[j].Args.Values { arg := goroutines[i].Stack.Calls[j].Args.Values[k] if arg.IsPtr() { objects[arg.Value] = object{ args: append(objects[arg.Value].args, &goroutines[i].Stack.Calls[j].Args.Values[k]), inPrimary: objects[arg.Value].inPrimary || i == 0, } } } } // CreatedBy.Args is never set. } order := make(uint64Slice, 0, len(objects)/2) for k, obj := range objects { if len(obj.args) > 1 && obj.inPrimary { order = append(order, k) } } sort.Sort(order) nextID := 1 for _, k := range order { for _, arg := range objects[k].args { arg.Name = fmt.Sprintf("#%d", nextID) } nextID++ } // Now do the rest. This is done so the output is deterministic. order = make(uint64Slice, 0, len(objects)) for k := range objects { order = append(order, k) } sort.Sort(order) for _, k := range order { // Process the remaining pointers, they were not referenced by primary // thread so will have higher IDs. if objects[k].inPrimary { continue } for _, arg := range objects[k].args { arg.Name = fmt.Sprintf("#%d", nextID) } nextID++ } } type uint64Slice []uint64 func (a uint64Slice) Len() int { return len(a) } func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] } panicparse-1.3.0/stack/stack_test.go000066400000000000000000000113711353500720100174370ustar00rootroot00000000000000// Copyright 2015 Marc-Antoine Ruel. All rights reserved. // Use of this source code is governed under the Apache License, Version 2.0 // that can be found in the LICENSE file. package stack import ( "flag" "io/ioutil" "log" "os" "path/filepath" "testing" ) func TestCallPkg1(t *testing.T) { c := Call{ SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.handleErr"}, Args: Args{Values: []Arg{{Value: 0xc208033b20}}}, } compareString(t, "yaml.go", c.SrcName()) compareString(t, filepath.Join("yaml.v2", "yaml.go"), c.PkgSrc()) compareString(t, "gopkg.in/yaml.v2.handleErr", c.Func.String()) compareString(t, "handleErr", c.Func.Name()) // This is due to directory name not matching the package name. compareString(t, "yaml.v2", c.Func.PkgName()) compareBool(t, false, c.Func.IsExported()) compareBool(t, false, c.IsStdlib) compareBool(t, false, c.IsPkgMain()) } func TestCallPkg2(t *testing.T) { c := Call{ SrcPath: "/gopath/src/gopkg.in/yaml.v2/yaml.go", Line: 153, Func: Func{Raw: "gopkg.in/yaml%2ev2.(*decoder).unmarshal"}, Args: Args{Values: []Arg{{Value: 0xc208033b20}}}, } compareString(t, "yaml.go", c.SrcName()) compareString(t, filepath.Join("yaml.v2", "yaml.go"), c.PkgSrc()) // TODO(maruel): Using '/' for this function is inconsistent on Windows // w.r.t. other functions. compareString(t, "gopkg.in/yaml.v2.(*decoder).unmarshal", c.Func.String()) compareString(t, "(*decoder).unmarshal", c.Func.Name()) // This is due to directory name not matching the package name. compareString(t, "yaml.v2", c.Func.PkgName()) compareBool(t, false, c.Func.IsExported()) compareBool(t, false, c.IsStdlib) compareBool(t, false, c.IsPkgMain()) } func TestCallStdlib(t *testing.T) { c := Call{ SrcPath: "/goroot/src/reflect/value.go", Line: 2125, Func: Func{Raw: "reflect.Value.assignTo"}, Args: Args{Values: []Arg{{Value: 0x570860}, {Value: 0xc20803f3e0}, {Value: 0x15}}}, } c.updateLocations("/goroot", "/goroot", nil) compareString(t, "value.go", c.SrcName()) compareString(t, "value.go:2125", c.SrcLine()) compareString(t, filepath.Join("reflect", "value.go"), c.PkgSrc()) compareString(t, "reflect.Value.assignTo", c.Func.String()) compareString(t, "Value.assignTo", c.Func.Name()) compareString(t, "reflect", c.Func.PkgName()) compareBool(t, false, c.Func.IsExported()) compareBool(t, true, c.IsStdlib) compareBool(t, false, c.IsPkgMain()) } func TestCallMain(t *testing.T) { c := Call{ SrcPath: "/gopath/src/github.com/maruel/panicparse/cmd/pp/main.go", Line: 428, Func: Func{Raw: "main.main"}, } compareString(t, "main.go", c.SrcName()) compareString(t, "main.go:428", c.SrcLine()) compareString(t, filepath.Join("pp", "main.go"), c.PkgSrc()) compareString(t, "main.main", c.Func.String()) compareString(t, "main", c.Func.Name()) compareString(t, "main", c.Func.PkgName()) compareBool(t, true, c.Func.IsExported()) compareBool(t, false, c.IsStdlib) compareBool(t, true, c.IsPkgMain()) } func TestCallC(t *testing.T) { c := Call{ SrcPath: "/goroot/src/runtime/proc.c", Line: 1472, Func: Func{Raw: "findrunnable"}, Args: Args{Values: []Arg{{Value: 0xc208012000}}}, } c.updateLocations("/goroot", "/goroot", nil) compareString(t, "proc.c", c.SrcName()) compareString(t, "proc.c:1472", c.SrcLine()) compareString(t, filepath.Join("runtime", "proc.c"), c.PkgSrc()) compareString(t, "findrunnable", c.Func.String()) compareString(t, "findrunnable", c.Func.Name()) compareString(t, "", c.Func.PkgName()) compareBool(t, false, c.Func.IsExported()) compareBool(t, true, c.IsStdlib) compareBool(t, false, c.IsPkgMain()) } func TestArgs(t *testing.T) { a := Args{ Values: []Arg{ {Value: 0x4}, {Value: 0x7fff671c7118}, {Value: 0xffffffff00000080}, {}, {Value: 0xffffffff0028c1be}, {}, {}, {}, {}, {}, }, Elided: true, } compareString(t, "0x4, 0x7fff671c7118, 0xffffffff00000080, 0, 0xffffffff0028c1be, 0, 0, 0, 0, 0, ...", a.String()) } func TestFuncAnonymous(t *testing.T) { f := Func{Raw: "main.func·001"} compareString(t, "main.func·001", f.String()) compareString(t, "main.func·001", f.PkgDotName()) compareString(t, "func·001", f.Name()) compareString(t, "main", f.PkgName()) compareBool(t, false, f.IsExported()) } func TestFuncGC(t *testing.T) { f := Func{Raw: "gc"} compareString(t, "gc", f.String()) compareString(t, "gc", f.PkgDotName()) compareString(t, "gc", f.Name()) compareString(t, "", f.PkgName()) compareBool(t, false, f.IsExported()) } // func compareBool(t *testing.T, expected, actual bool) { if expected != actual { t.Fatalf("%t != %t", expected, actual) } } func TestMain(m *testing.M) { flag.Parse() if !testing.Verbose() { log.SetOutput(ioutil.Discard) } os.Exit(m.Run()) }