util-0.5.2/0000755000175000017500000000000014503242340012016 5ustar nileshnileshutil-0.5.2/exec/0000755000175000017500000000000014503242340012742 5ustar nileshnileshutil-0.5.2/exec/LICENSE0000644000175000017500000000210314503242340013743 0ustar nileshnileshCopyright (c) 2013 Wei Shen (shenwei356@gmail.com) The MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.util-0.5.2/exec/README.md0000644000175000017500000000057114503242340014224 0ustar nileshnileshexec ======== Package exec runs external commands. It wraps os.exec to allow using full command string as arguments, and provides functions of providing (stdin, stdout, stderr) channel for (stdin, stdout, stderr) pipe. ***Attention, this package is experimental***. This package is imported by [crun of go edition](https://github.com/shenwei356/crun/blob/master/go/crun.go) util-0.5.2/exec/exec.go0000644000175000017500000000710514503242340014220 0ustar nileshnilesh// Package exec runs external commands. It wraps os.exec to // allow using full command string as arguments, // and provides functions // of providing (stdin, stdout, stderr) channel for // (stdin, stdout, stderr) pipe. // // Attention, this package is experimental. package exec import ( "bufio" "errors" "io" "os/exec" "strings" ) type Cmd struct { *exec.Cmd } // Command returns the Cmd struct to execute the command. // No need to split the path and arguments. // The error may be caused by unclosed quote. func Command(name string) (*Cmd, error) { path, argv, err := parseCommandName(name) if err != nil { return nil, err } return &Cmd{exec.Command(path, argv...)}, nil } // StdoutChannel returns a channel that will be connected to // the command's standard error when the command starts. // It closes when StdoutPipe closed. func (c *Cmd) StdoutChannel() (chan string, error) { ch := make(chan string, 100) pipe, err := c.StdoutPipe() if err != nil { return nil, err } reader := bufio.NewReader(pipe) go func() { for { str, err := reader.ReadString('\n') if err != nil { if err == io.EOF { if str != "" { ch <- str } } close(ch) return } if str != "" { ch <- str } } }() return ch, nil } // StderrChannel returns a channel that will be connected to // the command's standard error when the command starts. // It closes when StderrPipe closed. func (c *Cmd) StderrChannel() (chan string, error) { ch := make(chan string, 100) pipe, err := c.StderrPipe() if err != nil { return nil, err } reader := bufio.NewReader(pipe) go func() { for { str, err := reader.ReadString('\n') if err != nil { if err == io.EOF { if str != "" { ch <- str } } close(ch) return } if str != "" { ch <- str } } }() return ch, nil } // StdinChannel returns a channel that will be connected to // the command's standard error when the command starts. func (c *Cmd) StdinChannel() (chan string, error) { ch := make(chan string, 100) pipe, err := c.StdinPipe() if err != nil { return nil, err } writer := bufio.NewWriter(pipe) go func() { for { select { case str := <-ch: writer.WriteString(str) } } }() return ch, nil } // parseCommandName split the full command into path and arguments. func parseCommandName(name string) (string, []string, error) { if len(strings.Trim(name, " ")) == 0 { return "", nil, errors.New("no command given") } var ( quoted bool = false quotation rune tmp []rune = make([]rune, 0) argv []string = make([]string, 0) ) for _, b := range name { switch b { case ' ': if quoted { tmp = append(tmp, b) } else { if len(strings.Trim(string(tmp), " ")) > 0 { argv = append(argv, string(tmp)) } tmp = make([]rune, 0) } case '"': if quoted { if quotation == '"' { quoted, quotation = false, '_' argv = append(argv, string(tmp)) tmp = make([]rune, 0) } else { tmp = append(tmp, b) } } else { quoted, quotation = true, '"' } case '\'': if quoted { if quotation == '\'' { quoted, quotation = false, '_' argv = append(argv, string(tmp)) tmp = make([]rune, 0) } else { tmp = append(tmp, b) } } else { quoted, quotation = true, '\'' } default: tmp = append(tmp, b) } } if len(strings.Trim(string(tmp), " ")) > 0 { argv = append(argv, string(tmp)) } path := argv[0] var arg []string if len(argv) > 1 { arg = argv[1:] } if quoted { return path, arg, errors.New("unclosed quote") } return path, arg, nil } util-0.5.2/randutil/0000755000175000017500000000000014503242340013640 5ustar nileshnileshutil-0.5.2/randutil/shuffle.go0000644000175000017500000000025414503242340015624 0ustar nileshnileshpackage randutil import "math/rand" // Shuffle shuffles a slice of int func Shuffle(s []int) { for i := range s { j := rand.Intn(i + 1) s[i], s[j] = s[j], s[i] } } util-0.5.2/.gitignore0000755000175000017500000000041114503242340014005 0ustar nileshnilesh# Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.directory util-0.5.2/go.sum0000644000175000017500000000732614503242340013161 0ustar nileshnileshgithub.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shenwei356/breader v0.3.2 h1:GLy2clIMck6FdTwj8WLnmhv0PW/7Pp+Wcx7TVEHG0ks= github.com/shenwei356/breader v0.3.2/go.mod h1:BimwolkMTIr/O4iX7xXtjEB1z5y39G+8I5Tsm9guC3E= github.com/shenwei356/natsort v0.0.0-20190418160752-600d539c017d h1:eeXLHcXyGEr72V1SOSEI7vSzUOTJvHutwF7Ykm+hscQ= github.com/shenwei356/natsort v0.0.0-20190418160752-600d539c017d/go.mod h1:SiiGiRFyRtV7S9RamOrmQR5gpGIRhWJM1w0EtmuQ1io= github.com/shenwei356/xopen v0.2.2 h1:g1v3YjiIky9k6oN4qmnU1bDciAHnSrmOn2sMTE5pChY= github.com/shenwei356/xopen v0.2.2/go.mod h1:6EQUa6I7Zsl2GQKqcL9qGLrTzVE+oZyly+uhzovQYSk= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 h1:DRC1ubdb3ZmyyIeCSTxjZIQAnpLPfKVgYrLETQuOPjo= github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553/go.mod h1:Rj7Csq/tZ/egz+Ltc2IVpsA5309AmSMEswjkTZmq2Xc= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= util-0.5.2/stats/0000755000175000017500000000000014503242340013154 5ustar nileshnileshutil-0.5.2/stats/quantile_test.go0000644000175000017500000000543614503242340016374 0ustar nileshnileshpackage stats import ( "math" "math/rand" "testing" stats2 "github.com/montanaflynn/stats" ) type testCase struct { data []float64 q1, median, q3 float64 min, max float64 } var cases = []testCase{ { data: []float64{}, median: 0, q1: 0, q3: 0, min: 0, max: 0, }, { data: []float64{2}, median: 2, q1: 2, q3: 2, min: 2, max: 2, }, { data: []float64{1, 2}, median: 1.5, q1: 1, q3: 2, min: 1, max: 2, }, { data: []float64{1, 2, 3}, median: 2, q1: 1.5, q3: 2.5, min: 1, max: 3, }, { data: []float64{1, 2, 3, 4}, median: 2.5, q1: 1.5, q3: 3.5, min: 1, max: 4, }, { data: []float64{2, 3, 4, 5, 6, 7, 8, 9}, median: 5.5, q1: 3.5, q3: 7.5, min: 2, max: 9, }, { data: []float64{0.5, 0.6, 0.7, 0.8, 0.8, 0.85, 0.9}, median: 0.8, q1: 0.65, q3: 0.825, min: 0.5, max: 0.9, }, { data: []float64{1, 0.8, 0.8, 0.85, 0.9}, median: 0.85, q1: 0.8, q3: 0.9, min: 0.8, max: 1, }, } func Test(t *testing.T) { for i, _case := range cases { rand.Shuffle(len(_case.data), func(i, j int) { _case.data[i], _case.data[j] = _case.data[j], _case.data[i] }) stats := NewQuantiler() for _, l := range _case.data { stats.Add(l) } if stats.Count() != uint64(len(_case.data)) { t.Errorf("case %d: count mismatch", i) } min := stats.Min() if min != _case.min { t.Errorf("case %d: min mismatch: %f != %f", i, min, _case.min) } max := stats.Max() if max != _case.max { t.Errorf("case %d: max mismatch: %f != %f", i, max, _case.max) } median := stats.Median() if math.Abs(median-_case.median) > 0.001 { t.Errorf("case %d: median mismatch: %f != %f", i, median, _case.median) } q1 := stats.Q1() if math.Abs(q1-_case.q1) > 0.001 { t.Errorf("case %d: q1 mismatch: %f != %f", i, q1, _case.q1) } q3 := stats.Q3() if math.Abs(q3-_case.q3) > 0.001 { t.Errorf("case %d: q3 mismatch: %f != %f", i, q3, _case.q3) } } } var cases2 = []testCase{ { data: []float64{}, }, { data: []float64{0.8}, }, { data: []float64{1, 2}, }, { data: []float64{1, 2, 3}, }, { data: []float64{1, 2, 3, 3, 4, 5, 6, 7, 5}, }, { data: []float64{0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 7, 4, 2, 1, 4, 5, 6, 6, 4, 2, 2, 4, 10}, }, } func Test2(t *testing.T) { for i, _case := range cases2 { rand.Shuffle(len(_case.data), func(i, j int) { _case.data[i], _case.data[j] = _case.data[j], _case.data[i] }) stats := NewQuantiler() for _, l := range _case.data { stats.Add(l) } p90 := stats.Percentile(90) pp90, _ := stats2.Percentile(_case.data, 90) if math.Abs(p90-pp90) > 0.001 { t.Errorf("case %d: p90 mismatch: %f != %f", i, p90, pp90) } } } util-0.5.2/stats/quantile.go0000644000175000017500000001172714503242340015335 0ustar nileshnileshpackage stats import ( "math" "github.com/twotwotwo/sorts" ) type valueCount struct { Value, Count float64 } type valueCounts []valueCount func (c valueCounts) Len() int { return len(c) } func (c valueCounts) Less(i, j int) bool { return c[i].Value < c[j].Value } func (c valueCounts) Swap(i, j int) { c[i], c[j] = c[j], c[i] } type Quantiler struct { count map[float64]float64 // value -> count n uint64 // n min, max, sum float64 // sum // for sorting counts []valueCount // value, count accCounts []valueCount // value, accumulative count sorted bool } // NewQuantiler initializes a Quantiler func NewQuantiler() *Quantiler { return &Quantiler{count: make(map[float64]float64, 1024), min: math.MaxFloat64} } // Add adds a new element func (stats *Quantiler) Add(value float64) { stats.n++ stats.sum += value stats.count[value]++ if value > stats.max { stats.max = value } if value < stats.min { stats.min = value } stats.sorted = false } func (stats *Quantiler) sort() { stats.counts = make([]valueCount, 0, len(stats.count)) for value, count := range stats.count { stats.counts = append(stats.counts, valueCount{value, count}) } sorts.Quicksort(valueCounts(stats.counts)) stats.accCounts = make([]valueCount, len(stats.count)) for i, data := range stats.counts { if i == 0 { stats.accCounts[i] = valueCount{data.Value, data.Count} } else { stats.accCounts[i] = valueCount{data.Value, data.Count + stats.accCounts[i-1].Count} } } stats.sorted = true } // Count returns number of elements func (stats *Quantiler) Count() uint64 { return stats.n } // Min returns the minimum value func (stats *Quantiler) Min() float64 { if stats.n == 0 { return 0 } return stats.min } // Max returns the maxinimum length func (stats *Quantiler) Max() float64 { return stats.max } // Sum returns the sum func (stats *Quantiler) Sum() float64 { return stats.sum } // Mean returns mean func (stats *Quantiler) Mean() float64 { return float64(stats.sum) / float64(stats.n) } // Q2 returns Q2 func (stats *Quantiler) Q2() float64 { return stats.Median() } // Median returns median func (stats *Quantiler) Median() float64 { if !stats.sorted { stats.sort() } if len(stats.counts) == 0 { return 0 } if len(stats.counts) == 1 { return float64(stats.counts[0].Value) } even := stats.n&1 == 0 // %2 == 0 var iMedianL, iMedianR uint64 // 0-based if even { iMedianL = uint64(stats.n/2) - 1 // 3 iMedianR = uint64(stats.n / 2) // 4 } else { iMedianL = uint64(stats.n / 2) } return stats.getValue(even, iMedianL, iMedianR) } // Q1 returns Q1 func (stats *Quantiler) Q1() float64 { if !stats.sorted { stats.sort() } if len(stats.counts) == 0 { return 0 } if len(stats.counts) == 1 { return stats.counts[0].Value } even := stats.n&1 == 0 // %2 == 0 var iMedianL, iMedianR uint64 // 0-based var n uint64 if even { n = stats.n / 2 } else { n = (stats.n + 1) / 2 } even = n%2 == 0 if even { iMedianL = uint64(n/2) - 1 iMedianR = uint64(n / 2) } else { iMedianL = uint64(n / 2) } return stats.getValue(even, iMedianL, iMedianR) } // Q3 returns Q3 func (stats *Quantiler) Q3() float64 { if !stats.sorted { stats.sort() } if len(stats.counts) == 0 { return 0 } if len(stats.counts) == 1 { return stats.counts[0].Value } even := stats.n&1 == 0 // %2 == 0 var iMedianL, iMedianR uint64 // 0-based var mean, n uint64 if even { n = stats.n / 2 mean = n } else { n = (stats.n + 1) / 2 mean = stats.n / 2 } even = n%2 == 0 if even { iMedianL = uint64(n/2) - 1 + mean iMedianR = uint64(n/2) + mean } else { iMedianL = uint64(n/2) + mean } return stats.getValue(even, iMedianL, iMedianR) } func (stats *Quantiler) getValue(even bool, iMedianL uint64, iMedianR uint64) float64 { var accCount float64 var flag bool var prev float64 for _, data := range stats.accCounts { accCount = data.Count if flag { // the middle two having different value. // example: 1, 2, 3, 4 or 1, 2 return (data.Value + prev) / 2 } if accCount >= float64(iMedianL+1) { if even { if accCount >= float64(iMedianR+1) { // having >=2 of same value in the middle. // example: 2, 2, 2, 3, 3, 4, 8, 8 return data.Value } flag = true prev = data.Value } else { // right here return data.Value } } } // never happen // panic("should never happen") return 0 } func (stats *Quantiler) Percentile(percent float64) float64 { if percent <= 0 || percent > 100 { panic("invalid percentile") } if !stats.sorted { stats.sort() } if len(stats.counts) == 0 { return 0 } if len(stats.counts) == 1 { return float64(stats.counts[0].Value) } i0 := float64(stats.n) * percent / 100 i := math.Floor(i0) even := math.Abs(i0-i) > 0.001 var iMedianL, iMedianR uint64 // 0-based if even { iMedianL = uint64(i) - 1 iMedianR = uint64(i) } else { iMedianL = uint64(i - 1) } return stats.getValue(even, iMedianL, iMedianR) } util-0.5.2/stringutil/0000755000175000017500000000000014503242340014222 5ustar nileshnileshutil-0.5.2/stringutil/String2ByteSlice.go0000644000175000017500000000333114503242340017705 0ustar nileshnileshpackage stringutil import ( "bytes" "github.com/shenwei356/natsort" ) // String2ByteSlice is for sortint of string-[]byte pairs type String2ByteSlice struct { Key string Value []byte } // String2ByteSliceList is list of string2ByteSlice type String2ByteSliceList []String2ByteSlice // NaturalOrder is the global variable for sorting String2ByteSlice var NaturalOrder = false // IgnoreCase for ignoring case when sorting in natural order var IgnoreCase = false func (list String2ByteSliceList) Len() int { return len(list) } func (list String2ByteSliceList) Less(i, j int) bool { if NaturalOrder { return natsort.Compare(list[i].Key, list[j].Key, IgnoreCase) } return list[i].Key < list[j].Key } func (list String2ByteSliceList) Swap(i, j int) { list[i], list[j] = list[j], list[i] } // ReversedString2ByteSliceList is reversed String2ByteSliceList type ReversedString2ByteSliceList struct { String2ByteSliceList } // Less ... func (list ReversedString2ByteSliceList) Less(i, j int) bool { if NaturalOrder { return !natsort.Compare(list.String2ByteSliceList[i].Key, list.String2ByteSliceList[j].Key, IgnoreCase) } return list.String2ByteSliceList[i].Key > list.String2ByteSliceList[j].Key } // ByValue ... type ByValue struct { String2ByteSliceList } // Less ... func (list ByValue) Less(i, j int) bool { c := bytes.Compare(list.String2ByteSliceList[i].Value, list.String2ByteSliceList[j].Value) if c == -1 { return true } return false } // ReversedByValue ... type ReversedByValue struct { String2ByteSliceList } // Less ... func (list ReversedByValue) Less(i, j int) bool { c := bytes.Compare(list.String2ByteSliceList[j].Value, list.String2ByteSliceList[i].Value) if c == -1 { return true } return false } util-0.5.2/stringutil/StringCount.go0000644000175000017500000000334414503242340017034 0ustar nileshnileshpackage stringutil import "sort" // StringCount is a struct store count of Key type StringCount struct { Key string Count int } // StringCountList is slice of Keycount type StringCountList []StringCount func (b StringCountList) Len() int { return len(b) } func (b StringCountList) Less(i, j int) bool { // return b[i].Count < b[j].Count // This will return unwanted result: return b[i].Count < b[j].Count || b[i].Key < b[j].Key if b[i].Count < b[j].Count { return true } if b[i].Count == b[j].Count { if b[i].Key < b[j].Key { return true } return false } return false } func (b StringCountList) Swap(i, j int) { b[i], b[j] = b[j], b[i] } // ReversedStringCountList is Reversed StringCountList type ReversedStringCountList struct { StringCountList } // Less is different from the Less of StringCountList func (b ReversedStringCountList) Less(i, j int) bool { // return b.StringCountList[i].Count > b.StringCountList[j].Count if b.StringCountList[i].Count > b.StringCountList[j].Count { return true } if b.StringCountList[i].Count == b.StringCountList[j].Count { if b.StringCountList[i].Key < b.StringCountList[j].Key { return true } return false } return false } // CountOfString returns the count of Key for a Key slice func CountOfString(s []string) map[string]int { count := make(map[string]int) for _, b := range s { count[b]++ } return count } // SortCountOfString sorts count of Key func SortCountOfString(count map[string]int, reverse bool) StringCountList { countList := make(StringCountList, len(count)) i := 0 for b, c := range count { countList[i] = StringCount{b, c} i++ } if reverse { sort.Sort(ReversedStringCountList{countList}) } else { sort.Sort(countList) } return countList } util-0.5.2/stringutil/util.go0000644000175000017500000000354214503242340015532 0ustar nileshnileshpackage stringutil import ( "bytes" "regexp" "unsafe" "github.com/shenwei356/util/byteutil" ) // Split splits a byte slice by given letters func Split(slice string, letters string) []string { result := byteutil.Split([]byte(slice), []byte(letters)) result2 := []string{} for _, s := range result { result2 = append(result2, string(s)) } return result2 } // Str2Bytes convert string to byte slice. Warning: it's unsafe!!! func Str2Bytes(s string) []byte { x := (*[2]uintptr)(unsafe.Pointer(&s)) h := [3]uintptr{x[0], x[1], x[1]} return *(*[]byte)(unsafe.Pointer(&h)) } // ReverseStringSlice reverses StringSlice func ReverseStringSlice(s []string) []string { // make a copy of s l := len(s) t := make([]string, l) for i := 0; i < l; i++ { t[i] = s[i] } // reverse for i, j := 0, l-1; i < j; i, j = i+1, j-1 { t[i], t[j] = t[j], t[i] } return t } // ReverseStringSliceInplace reverses StringSlice func ReverseStringSliceInplace(s []string) { for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { s[i], s[j] = s[j], s[i] } } // EscapeSymbols escape custom symbols func EscapeSymbols(s, symbols string) string { m := make(map[rune]struct{}) for _, c := range symbols { m[c] = struct{}{} } var buf bytes.Buffer var ok bool for _, c := range s { if _, ok = m[c]; ok { buf.WriteByte('\\') } buf.WriteRune(c) } return buf.String() } // UnEscaper returns a function for unescaping string func UnEscaper() func(s string) string { var re = regexp.MustCompile(`\\([abfnrtv'"?])`) var m = map[string]string{ `\a`: "\a", `\b`: "\b", `\f`: "\f", `\n`: "\n", `\r`: "\r", `\t`: "\t", `\v`: "\v", `\\`: "\\", `\'`: "'", `\"`: "\"", `\?`: "?", } var mapping = func(key string) string { if v, ok := m[key]; ok { return v } return key } return func(s string) string { return re.ReplaceAllStringFunc(s, mapping) } } util-0.5.2/stringutil/multiKeySort.go0000644000175000017500000000543314503242340017231 0ustar nileshnileshpackage stringutil import ( "math" "strconv" "strings" "github.com/shenwei356/natsort" ) // SortType defines the sort type type SortType struct { Index int IgnoreCase bool Natural bool // natural order Number bool UserDefined bool Reverse bool Levels map[string]int } // MultiKeyStringSlice sort [][]string by multiple keys type MultiKeyStringSlice struct { SortTypes *[]SortType Value []string } // MultiKeyStringSliceList is slice of MultiKeyStringSlice type MultiKeyStringSliceList []MultiKeyStringSlice func (list MultiKeyStringSliceList) Len() int { return len(list) } func (list MultiKeyStringSliceList) Swap(i, j int) { list[i], list[j] = list[j], list[i] } func (list MultiKeyStringSliceList) Less(i, j int) bool { var err error var v int for _, t := range *list[i].SortTypes { if t.Natural { if t.IgnoreCase { v = strings.Compare(strings.ToLower(list[i].Value[t.Index]), strings.ToLower(list[j].Value[t.Index])) } else { v = strings.Compare(list[i].Value[t.Index], list[j].Value[t.Index]) } if v == 0 { continue } if natsort.Compare(list[i].Value[t.Index], list[j].Value[t.Index], t.IgnoreCase) { v = -1 } else { v = 1 } } else if t.Number { var a, b float64 a, err = strconv.ParseFloat(removeComma(list[i].Value[t.Index]), 64) if err != nil || math.IsNaN(a) { a = math.MaxFloat64 } b, err = strconv.ParseFloat(removeComma(list[j].Value[t.Index]), 64) if err != nil || math.IsNaN(b) { b = math.MaxFloat64 } if a < b { v = -1 } else if a == b { v = 0 } else { v = 1 } } else if t.UserDefined { var a, b int var okA, okB bool if t.IgnoreCase { a, okA = t.Levels[strings.ToLower(list[i].Value[t.Index])] b, okB = t.Levels[strings.ToLower(list[j].Value[t.Index])] } else { a, okA = t.Levels[list[i].Value[t.Index]] b, okB = t.Levels[list[j].Value[t.Index]] } if okA { if okB { if a < b { v = -1 } else if a == b { v = 0 } else { v = 1 } } else { v = -1 } } else if okB { v = 1 } else { v = strings.Compare(list[i].Value[t.Index], list[j].Value[t.Index]) } } else { if t.IgnoreCase { v = strings.Compare(strings.ToLower(list[i].Value[t.Index]), strings.ToLower(list[j].Value[t.Index])) } else { v = strings.Compare(list[i].Value[t.Index], list[j].Value[t.Index]) } } if v == 0 { } else if v < 0 { if t.Reverse { return false } return true } else { if t.Reverse { return true } return false } } return true } func removeComma(s string) string { newSlice := []byte{} for i := 0; i < len(s); i++ { switch s[i] { case ',': default: newSlice = append(newSlice, s[i]) } } return string(newSlice) } util-0.5.2/bytesize/0000755000175000017500000000000014503242340013654 5ustar nileshnileshutil-0.5.2/bytesize/bytesize_test.go0000644000175000017500000000271214503242340017102 0ustar nileshnileshpackage bytesize import ( // "fmt" "testing" ) // test Right cases func Test1(t *testing.T) { bytes := []ByteSize{3, 1000, 1024, 1203, 132434, 41234134, 132413241324, 13e15} data := make(map[ByteSize]string) for _, b := range bytes { // 132434 : "129.33 KB" data[b] = b.String() } for _, s := range data { // size == 132433.92 size, err := Parse([]byte(s)) if err != nil && err.Error() != ErrText { t.Error("UNKNOWN ERROR TYPE") } // size == "129.33 KB" if s != size.String() { t.Error("FAILED") } } } // test more func Test2(t *testing.T) { data := make(map[string]ByteSize) data["1234.2 kb"] = 1263820.80 // lower case data["-1234.2 kb"] = -1263820.80 // lower case data[" 1234.2 kb "] = 1263820.80 // space data["1234.2 k"] = 1263820.80 // simple unit data["1234.2 "] = 1234.2 // no unit data[" kb "] = -1 // illegal value data["- kb"] = -1 // illegal value data["1234.2 aB"] = -1 // illegal unit data["1234.2 Packages"] = -1 // illegal unit data["1234.2 P."] = -1 // illegal unit for s, info := range data { size, err := Parse([]byte(s)) if err != nil { // fmt.Printf("%s\t%s\n", s, err) if err.Error() != ErrText || info != -1 { t.Error("unknown error type or test sample error") } } else { // check value if size != info { t.Error("value error") } } // fmt.Printf("%s\t%.2f\t%.2f\n", s, info, size) } } util-0.5.2/bytesize/bytesize.go0000644000175000017500000001103014503242340016034 0ustar nileshnilesh// Copyright 2014 Wei Shen (shenwei356@gmail.com). All rights reserved. // Use of this source code is governed by a MIT-license // that can be found in the LICENSE file. // Package bytesize provides a way to show readable values of byte size // by reediting the code from http://golang.org/doc/effective_go.html. // It could also parsing byte size text to ByteSize object. package bytesize import ( "errors" "fmt" "regexp" "strconv" "strings" ) // ByteSize stands for byte size. Division operation is needed, so it uses float64 instead of uint64 type ByteSize float64 // const for bytesize. B is also specified. const ( B ByteSize = 1 << (10 * iota) KB MB GB TB PB EB ZB YB ) // FullUnit decides output "10 GB" (true) or "10 G" (false). var FullUnit = true // Print readable values of byte size func (b ByteSize) String() string { if FullUnit { switch { case b >= YB: return fmt.Sprintf("%.2f YB", b/YB) case b >= ZB: return fmt.Sprintf("%.2f ZB", b/ZB) case b >= EB: return fmt.Sprintf("%.2f EB", b/EB) case b >= PB: return fmt.Sprintf("%.2f PB", b/PB) case b >= TB: return fmt.Sprintf("%.2f TB", b/TB) case b >= GB: return fmt.Sprintf("%.2f GB", b/GB) case b >= MB: return fmt.Sprintf("%.2f MB", b/MB) case b >= KB: return fmt.Sprintf("%.2f KB", b/KB) } return fmt.Sprintf("%.2f B", b) } switch { case b >= YB: return fmt.Sprintf("%.2f Y", b/YB) case b >= ZB: return fmt.Sprintf("%.2f Z", b/ZB) case b >= EB: return fmt.Sprintf("%.2f E", b/EB) case b >= PB: return fmt.Sprintf("%.2f P", b/PB) case b >= TB: return fmt.Sprintf("%.2f T", b/TB) case b >= GB: return fmt.Sprintf("%.2f G", b/GB) case b >= MB: return fmt.Sprintf("%.2f M", b/MB) case b >= KB: return fmt.Sprintf("%.2f K", b/KB) } return fmt.Sprintf("%.2f", b) } // BytesizeRegexp is the regexp object for ByteSize Text. The REGEXP is: // // (?i)^\s*([\-?[\d\.]+)\s*([KMGTPEZY]?B|[BKMGTPEZY]|)\s?$ // // Example: // // data["1234.2 kb"] = 1263820.80 // lower case // data["-1234.2 kb"] = -1263820.80 // lower case // data[" 1234.2 kb "] = 1263820.80 // space // data["1234.2 k"] = 1263820.80 // simple unit // data["1234.2 "] = 1234.2 // no unit // data[" kb "] = -1 // illegal value // data["- kb"] = -1 // illegal value // data["1234.2 aB"] = -1 // illegal unit // data["1234.2 Packages"] = -1 // illegal unit // var BytesizeRegexp = regexp.MustCompile(`(?i)^\s*(\-?[\d\.]+)\s*([KMGTPEZY]?B|[BKMGTPEZY]|)\s*$`) // ErrText is error information for Illegal byte size text var ErrText = "illegal bytesize text" // Parse parses ByteSize Text to ByteSize object // // Example // // size, err := bytesize.Parse([]byte("1.5 KB")) // if err != nil { // fmt.Println(err) // } // fmt.Printf("%.0f bytes\n", size) // func Parse(sizeText []byte) (ByteSize, error) { if !BytesizeRegexp.Match(sizeText) { return 0, errors.New(ErrText) } // parse value and unit subs := BytesizeRegexp.FindSubmatch(sizeText) // no need to check ParseFloat error. BytesizeRegexp could ensure this size, _ := strconv.ParseFloat(string(subs[1]), 64) unit := strings.ToUpper(string(subs[2])) switch unit { case "B", "": size = size * float64(B) case "KB", "K": size = size * float64(KB) case "MB", "M": size = size * float64(MB) case "GB", "G": size = size * float64(GB) case "TB", "T": size = size * float64(TB) case "PB", "P": size = size * float64(PB) case "EB", "E": size = size * float64(EB) case "ZB", "Z": size = size * float64(ZB) case "YB", "Y": size = size * float64(YB) } // fmt.Printf("%s\t=%.2f=\t=%s=\n", sizeText, size, unit) return ByteSize(size), nil } // ParseByteSize parses byte size from string. func ParseByteSize(val string) (int, error) { val = strings.Trim(val, " \t\r\n") if val == "" { return 0, nil } var u int64 var noUnit bool switch val[len(val)-1] { case 'B', 'b': u = 1 case 'K', 'k': u = 1 << 10 case 'M', 'm': u = 1 << 20 case 'G', 'g': u = 1 << 30 default: noUnit = true u = 1 } var size float64 var err error if noUnit { size, err = strconv.ParseFloat(val, 10) if err != nil { return 0, fmt.Errorf("invalid byte size: %s", val) } if size < 0 { size = 0 } return int(size), nil } if len(val) == 1 { // no value return 0, nil } size, err = strconv.ParseFloat(strings.Trim(val[0:len(val)-1], " \t\r\n"), 10) if err != nil { return 0, fmt.Errorf("invalid byte size: %s", val) } if size < 0 { size = 0 } return int(size * float64(u)), nil } util-0.5.2/bytesize/LICENSE0000644000175000017500000000210314503242340014655 0ustar nileshnileshCopyright (c) 2013 Wei Shen (shenwei356@gmail.com) The MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.util-0.5.2/bytesize/README.md0000644000175000017500000000220114503242340015126 0ustar nileshnileshbytesize ======== Package for providing a way to show readable values of byte sizes by reediting the code from http://golang.org/doc/effective_go.html. It could also parsing byte size text to ByteSize object. Usage ------- fmt.Printf("1024 bytes\t%v\n", bytesize.ByteSize(1024)) fmt.Printf("13146111 bytes\t%v\n", bytesize.ByteSize(13146111)) // parsing size, err := bytesize.Parse([]byte("1.5 KB")) if err != nil { fmt.Println(err) } fmt.Printf("\n%.0f bytes\n", size) Result: 1024 bytes 1.00 KB 13146111 bytes 12.54 MB 1536 bytes REGEXP for ByteSize Text ---------------------------- (?i)^\s*([\-\d\.]+)\s*([KMGTPEZY]?B|[BKMGTPEZY]|)\s*$ Example: data["1234.2 kb"] = 1263820.80 lower case data["-1234.2 kb"] = -1263820.80 lower case data[" 1234.2 kb "] = 1263820.80 space data["1234.2 k"] = 1263820.80 simple unit data["1234.2 "] = 1234.2 no unit data[" kb "] = -1 illegal value data["- kb"] = -1 illegal value data["1234.2 aB"] = -1 illegal unit data["1234.2 Packages"] = -1 illegal unit util-0.5.2/bytesize/.directory0000755000175000017500000000007414503242340015665 0ustar nileshnilesh[Dolphin] Timestamp=2016,11,15,20,4,27 Version=3 ViewMode=1 util-0.5.2/struct/0000755000175000017500000000000014503242340013342 5ustar nileshnileshutil-0.5.2/struct/sa/0000755000175000017500000000000014503242340013745 5ustar nileshnileshutil-0.5.2/struct/sa/sa.go0000644000175000017500000000073014503242340014677 0ustar nileshnileshpackage sa import "sort" // SuffixArray returns the suffix array of s func SuffixArray(s []byte) []int { n := len(s) suffixMap := make(map[string]int, n) for i := 0; i < n; i++ { suffixMap[string(s[i:])] = i } suffixes := make([]string, n) i := 0 for suffix := range suffixMap { suffixes[i] = suffix i++ } indice := make([]int, n) i = 0 sort.Strings(suffixes) for _, suffix := range suffixes { indice[i] = suffixMap[suffix] i++ } return indice } util-0.5.2/struct/sa/sa_test.go0000644000175000017500000000035114503242340015735 0ustar nileshnileshpackage sa import ( "reflect" "testing" ) func TestSuffixArray(t *testing.T) { s := []byte("banana$") sa := SuffixArray(s) if !reflect.DeepEqual(sa, []int{6, 5, 3, 1, 0, 4, 2}) { t.Error("Test failed: TestSuffixArray") } } util-0.5.2/struct/stack/0000755000175000017500000000000014503242340014447 5ustar nileshnileshutil-0.5.2/struct/stack/stack.go0000644000175000017500000000127114503242340016104 0ustar nileshnileshpackage stack // Stack struct type Stack []interface{} // Empty tell if it is empty func (s Stack) Empty() bool { return len(s) == 0 } // Peek return the last element func (s Stack) Peek() interface{} { return s[len(s)-1] } // Put puts element to stack func (s *Stack) Put(i interface{}) { (*s) = append((*s), i) } // Pop pops element from the stack func (s *Stack) Pop() interface{} { d := (*s)[len(*s)-1] (*s) = (*s)[:len(*s)-1] return d } /* func main() { var s stack for i := 0; i < 3; i++ { s.Put(i) fmt.Printf("len=%d\n", len(s)) fmt.Printf("peek=%d\n", s.Peek()) } for !s.Empty() { i := s.Pop() fmt.Printf("len=%d\n", len(s)) fmt.Printf("pop=%d\n", i) } } */ util-0.5.2/LICENSE0000644000175000017500000000210314503242340013017 0ustar nileshnileshCopyright (c) 2013 Wei Shen (shenwei356@gmail.com) The MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.util-0.5.2/numbers/0000755000175000017500000000000014503242340013471 5ustar nileshnileshutil-0.5.2/numbers/base.go0000644000175000017500000000110614503242340014730 0ustar nileshnileshpackage numbers type Uint64Slice []uint64 func (s Uint64Slice) Len() int { return len(s) } func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] } func (s Uint64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s *Uint64Slice) Push(x interface{}) { *s = append(*s, x.(uint64)) } func (s *Uint64Slice) Pop() interface{} { old := *s n := len(old) x := old[n-1] *s = old[0 : n-1] return x } func Equal(s, t []uint64) bool { if len(s) != len(t) { return false } for i, v := range s { if v != t[i] { return false } } return true } util-0.5.2/numbers/uniq.go0000644000175000017500000000171714503242340015002 0ustar nileshnileshpackage numbers import "sort" // Uniq removes duplicated elements in the list, and returns a new one. func Uniq(list *[]uint64) *[]uint64 { if len(*list) == 0 { return &[]uint64{} } else if len(*list) == 1 { return &[]uint64{(*list)[0]} } sort.Sort(Uint64Slice(*list)) s := make([]uint64, 0, len(*list)) p := (*list)[0] s = append(s, p) for _, v := range (*list)[1:] { if v != p { s = append(s, v) } p = v } return &s } // UniqInplace is faster than Uniq for short slice (<1000). func UniqInplace(list *[]uint64) { if len(*list) == 0 || len(*list) == 1 { return } sort.Sort(Uint64Slice(*list)) var i, j int var p, v uint64 var flag bool p = (*list)[0] for i = 1; i < len(*list); i++ { v = (*list)[i] if v == p { if !flag { j = i // mark insertion position flag = true } continue } if flag { // need to insert to previous position (*list)[j] = v j++ } p = v } if j > 0 { *list = (*list)[:j] } } util-0.5.2/numbers/uniq_test.go0000644000175000017500000000250014503242340016030 0ustar nileshnileshpackage numbers import ( "math/rand" "testing" ) var data []*[]uint64 var data2 []*[]uint64 func init() { N := 100000 // N lists n := 300 // n elements for a list data = make([]*[]uint64, N) data2 = make([]*[]uint64, N) for i := 0; i < N; i++ { _n := rand.Intn(n) if _n < 0 { _n = -_n } _data := make([]uint64, _n) for j := 0; j < _n; j++ { _data[j] = uint64(float64(rand.Intn(_n)) / float64(1)) } _data2 := make([]uint64, _n) copy(_data2, _data) data[i] = &_data data2[i] = &_data2 } } func TestUniq(t *testing.T) { for _, _data := range data { u1 := Uniq(_data) c := make([]uint64, len(*_data)) copy(c, *_data) UniqInplace(&c) // fmt.Printf("original: %v\n", _data) // fmt.Printf(" Inplace: %v\n", c) // fmt.Printf(" uniq: %v\n", *u1) if !Equal(*u1, c) { // fmt.Printf("original: %v\n", _data) // fmt.Printf(" Inplace: %v\n", c) // fmt.Printf(" uniq: %v\n", *u1) t.Error("error") } } } var result *[]uint64 func BenchmarkUniq(b *testing.B) { var _result *[]uint64 // for i := 0; i < b.N; i++ { for _, _data := range data { _result = Uniq(_data) } // } result = _result } func BenchmarkUniqInplace(b *testing.B) { var _result *[]uint64 // for i := 0; i < b.N; i++ { for _, _data := range data2 { UniqInplace(_data) } // } result = _result } util-0.5.2/pathutil/0000755000175000017500000000000014503242340013650 5ustar nileshnileshutil-0.5.2/pathutil/path.go0000644000175000017500000000325114503242340015134 0ustar nileshnileshpackage pathutil import ( "fmt" "os" "regexp" ) // Exists checks if a file or directory exists. func Exists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } // LinkExists checks if link exists. func LinkExists(path string) (bool, error) { _, err := os.Lstat(path) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } // DirExists checks if a path exists and is a directory. func DirExists(path string) (bool, error) { fi, err := os.Stat(path) if err == nil && fi.IsDir() { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } // IsEmpty checks if a given path is empty. func IsEmpty(path string) (bool, error) { if b, _ := Exists(path); !b { return false, fmt.Errorf("%q path does not exist", path) } fi, err := os.Stat(path) if err != nil { return false, err } if fi.IsDir() { f, err := os.Open(path) defer f.Close() if err != nil { return false, err } list, err := f.Readdir(-1) // f.Close() - see bug fix above return len(list) == 0, nil } return fi.Size() == 0, nil } // IsDir checks if a given path is a directory. func IsDir(path string) (bool, error) { fi, err := os.Stat(path) if err != nil { return false, err } return fi.IsDir(), nil } // ReInvalidPathChars is used to remove invalid path characters var ReInvalidPathChars = regexp.MustCompile(`[<>:"/\\\|?\*]+`) // RemoveInvalidPathChars removes invalid characters for path func RemoveInvalidPathChars(s string, repl string) string { return ReInvalidPathChars.ReplaceAllString(s, repl) } util-0.5.2/go.mod0000644000175000017500000000123414503242340013124 0ustar nileshnileshmodule github.com/shenwei356/util go 1.17 require ( github.com/montanaflynn/stats v0.6.6 github.com/pkg/errors v0.9.1 github.com/shenwei356/breader v0.3.2 github.com/shenwei356/natsort v0.0.0-20190418160752-600d539c017d github.com/spf13/cobra v1.4.0 github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 ) require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/klauspost/compress v1.15.12 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/shenwei356/xopen v0.2.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ulikunitz/xz v0.5.10 // indirect ) util-0.5.2/math/0000755000175000017500000000000014503242340012747 5ustar nileshnileshutil-0.5.2/math/float.go0000644000175000017500000000102214503242340014376 0ustar nileshnileshpackage math import "math" // Round returns round of float64 func Round(f float64, n int) float64 { pow10N := math.Pow10(n) return math.Trunc((f+0.5/pow10N)*pow10N) / pow10N } const MaxUint = ^uint(0) const MinUint = 0 const MaxInt = int(MaxUint >> 1) const MinInt = -MaxInt - 1 func MinInts(a int, vals ...int) int { min := a for _, v := range vals { if v < min { min = v } } return min } func MaxInts(a int, vals ...int) int { max := a for _, v := range vals { if v > max { max = v } } return max } util-0.5.2/.gitattributes0000755000175000017500000000074314503242340014720 0ustar nileshnilesh# Auto detect text files and perform LF normalization * text=auto # Custom for Visual Studio *.cs diff=csharp *.sln merge=union *.csproj merge=union *.vbproj merge=union *.fsproj merge=union *.dbproj merge=union # Standard to msysgit *.doc diff=astextplain *.DOC diff=astextplain *.docx diff=astextplain *.DOCX diff=astextplain *.dot diff=astextplain *.DOT diff=astextplain *.pdf diff=astextplain *.PDF diff=astextplain *.rtf diff=astextplain *.RTF diff=astextplain util-0.5.2/byteutil/0000755000175000017500000000000014503242340013657 5ustar nileshnileshutil-0.5.2/byteutil/util_test.go0000644000175000017500000000073614503242340016230 0ustar nileshnileshpackage byteutil import "testing" func TestSubSlice(t *testing.T) { s := []byte("0123456789") if true && string(SubSlice(s, 0, 0)) == "0123456789" && string(SubSlice(s, 0, 1)) == "0" && string(SubSlice(s, 1, 2)) == "1" && string(SubSlice(s, -2, -1)) == "8" && string(SubSlice(s, len(s)-1, len(s))) == "9" && string(SubSlice(s, -1, 0)) == "9" && // different from python string(SubSlice(s, 7, -1)) == "78" && true { } else { t.Error("SubSlice error") } } util-0.5.2/byteutil/struct.go0000644000175000017500000000111714503242340015532 0ustar nileshnileshpackage byteutil import "bytes" // SliceOfByteSlice is [][]byte type SliceOfByteSlice [][]byte func (s SliceOfByteSlice) Len() int { return len(s) } func (s SliceOfByteSlice) Less(i, j int) bool { c := bytes.Compare(s[i], s[j]) if c == -1 { return true } return false } func (s SliceOfByteSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // ByteSlice is []byte type ByteSlice []byte func (bs ByteSlice) Len() int { return len(bs) } func (bs ByteSlice) Less(i, j int) bool { return bs[i] < bs[j] } func (bs ByteSlice) Swap(i, j int) { bs[i], bs[j] = bs[j], bs[i] } util-0.5.2/byteutil/byte_coder_test.go0000644000175000017500000000076614503242340017375 0ustar nileshnileshpackage byteutil import ( "testing" ) func TestByteCoder(t *testing.T) { coder, err := NewByteCoder([]byte("acgtryswkmbdhvACGTRYSWKMBDHV")) if err != nil { t.Error(err) } dna2int, err := coder.Encode([]byte("Jj")) if err != ErrInvalideLetter { t.Error(err) } dna2int, err = coder.Encode([]byte("acTg")) if err != nil { t.Error(err) } int2dna, err := coder.Decode(dna2int) if err != nil { t.Error(err) } if string(int2dna) != "acTg" { t.Errorf("ByteCoder test error") } } util-0.5.2/byteutil/alphabet.go0000644000175000017500000000106214503242340015765 0ustar nileshnileshpackage byteutil import "sort" // Alphabet returns the alphabet of a byte slice func Alphabet(s []byte) []byte { count := CountOfByte(s) letters := make([]byte, len(count)) i := 0 for b := range count { letters[i] = b i++ } sort.Sort(ByteSlice(letters)) return letters } // AlphabetFromCountOfByte returns the alphabet of a byte slice from count func AlphabetFromCountOfByte(count map[byte]int) []byte { letters := make([]byte, len(count)) i := 0 for b := range count { letters[i] = b i++ } sort.Sort(ByteSlice(letters)) return letters } util-0.5.2/byteutil/byte_coder.go0000644000175000017500000000411214503242340016323 0ustar nileshnileshpackage byteutil import ( "fmt" "sort" ) var ( // ErrInvalideLetter means invalid letter ErrInvalideLetter = fmt.Errorf("ByteCoder: invalid letter") // ErrInvalideCode means invalid code ErrInvalideCode = fmt.Errorf("ByteCoder: invalid code") ) // ByteCoder is used to convert betweeen byte and int type ByteCoder struct { Alphabet []byte alphabetQuerySlice []byte bytes2int []int int2bytes []byte } // NewByteCoder Create a ByteCoder type func NewByteCoder(alphabet []byte) (*ByteCoder, error) { if len(alphabet) == 0 { return nil, fmt.Errorf("ByteCoder: alphabet should not be empty") } m := make(map[byte]struct{}, len(alphabet)) for _, a := range alphabet { m[a] = struct{}{} } max := -1 var b int for a := range m { b = int(a) if max < b { max = b } } alphabet2 := make([]byte, len(m)) slice := make([]byte, max+1) i := 0 for a := range m { slice[a-'\x00'] = a alphabet2[i] = a i++ } sort.Sort(ByteSlice(alphabet2)) bytes2int := make([]int, max+1) int2bytes := make([]byte, len(m)) for i, a := range alphabet2 { bytes2int[a-'\x00'] = i int2bytes[i] = a } return &ByteCoder{Alphabet: alphabet2, alphabetQuerySlice: slice, bytes2int: bytes2int, int2bytes: int2bytes}, nil } func (coder *ByteCoder) String() string { return fmt.Sprintf(`ByteCoder: alphabet:"%s" num:%d`, coder.Alphabet, len(coder.Alphabet)) } // Encode converts []byte to []int func (coder *ByteCoder) Encode(s []byte) ([]int, error) { code := make([]int, len(s)) for i, b := range s { if int(b) > len(coder.alphabetQuerySlice) { return nil, ErrInvalideLetter } v := coder.alphabetQuerySlice[b-'\x00'] if v == 0 { return nil, ErrInvalideLetter } code[i] = coder.bytes2int[v] } return code, nil } // Decode convert []int to []byte func (coder *ByteCoder) Decode(code []int) ([]byte, error) { bytes := make([]byte, len(code)) for i, b := range code { if b >= len(coder.int2bytes) { return nil, ErrInvalideCode } v := coder.int2bytes[b] if v == 0 { return nil, ErrInvalideCode } bytes[i] = v } return bytes, nil } util-0.5.2/byteutil/count_test.go0000644000175000017500000000152714503242340016402 0ustar nileshnileshpackage byteutil import "testing" func TestCountOfByteAndAlphabet(t *testing.T) { s := []byte("abcdefadfwefasdga") count := CountOfByte(s) alphabet := Alphabet(s) sum := 0 for _, letter := range alphabet { sum += count[letter] } if sum != len(s) { t.Error("Test failed: TestCountOfByteAndAlphabet") } } func TestSortCountOfByte(t *testing.T) { s := []byte("cccaaadd") countList := SortCountOfByte(CountOfByte(s), true) // fmt.Println(countList) // if !(countList[0].Count == 3 && (countList[0].Key == 'a' || countList[0].Key == 'c')) { if !(countList[0].Count == 3 && countList[0].Key == 'a') { t.Error("Test failed: TestSortCountOfByte") } countList = SortCountOfByte(CountOfByte(s), false) // fmt.Println(countList) if !(countList[0].Key == 'd' && countList[0].Count == 2) { t.Error("Test failed: TestSortCountOfByte") } } util-0.5.2/byteutil/util.go0000644000175000017500000001042314503242340015163 0ustar nileshnileshpackage byteutil import ( "bytes" // "fmt" "unsafe" ) // ReverseByteSlice reverses a byte slice func ReverseByteSlice(s []byte) []byte { // make a copy of s l := len(s) t := make([]byte, l) for i := 0; i < l; i++ { t[i] = s[i] } // reverse for i, j := 0, l-1; i < j; i, j = i+1, j-1 { t[i], t[j] = t[j], t[i] } return t } // ReverseByteSliceInplace reverses a byte slice func ReverseByteSliceInplace(s []byte) { // reverse for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { s[i], s[j] = s[j], s[i] } } var _newline = []byte{'\n'} // WrapByteSlice wraps byte slice func WrapByteSlice(s []byte, width int) []byte { if width < 1 { return s } l := len(s) if l == 0 { return s } var lines int if l%width == 0 { lines = l/width - 1 } else { lines = int(l / width) } // var buffer bytes.Buffer buffer := bytes.NewBuffer(make([]byte, 0, l+lines)) var start, end int for i := 0; i <= lines; i++ { start = i * width end = (i + 1) * width if end > l { end = l } buffer.Write(s[start:end]) if i < lines { // buffer.WriteString("\n") buffer.Write(_newline) } } return buffer.Bytes() } // WrapByteSlice2 wraps byte slice, it reuses the bytes.Buffer func WrapByteSlice2(s []byte, width int, buffer *bytes.Buffer) ([]byte, *bytes.Buffer) { if width < 1 { return s, buffer } l := len(s) if l == 0 { return s, buffer } var lines int if l%width == 0 { lines = l/width - 1 } else { lines = int(l / width) } if buffer == nil { buffer = bytes.NewBuffer(make([]byte, 0, l+lines)) } else { buffer.Reset() } var start, end int for i := 0; i <= lines; i++ { start = i * width end = (i + 1) * width if end > l { end = l } buffer.Write(s[start:end]) if i < lines { buffer.Write(_newline) } } return buffer.Bytes(), buffer } // SubSlice provides similar slice indexing as python with one exception // that end could be equal to 0. // So we could get the last element by SubSlice(s, -1, 0) // or get the whole element by SubSlice(s, 0, 0) func SubSlice(slice []byte, start int, end int) []byte { if start == 0 && end == 0 { return slice } if start == end || (start < 0 && end > 0) { return []byte{} } l := len(slice) s, e := start, end if s < 0 { s = l + s if s < 1 { s = 0 } } if e < 0 { e = l + e if e < 0 { e = 0 } } if e == 0 || e > l { e = l } return slice[s:e] } // ByteToLower lowers a byte func ByteToLower(b byte) byte { if b <= '\u007F' { if 'A' <= b && b <= 'Z' { b += 'a' - 'A' } return b } return b } // ByteToUpper upper a byte func ByteToUpper(b byte) byte { if b <= '\u007F' { if 'a' <= b && b <= 'z' { b -= 'a' - 'A' } return b } return b } // MakeQuerySlice is used to replace map. // see: http://blog.shenwei.me/map-is-not-the-fastest-in-go/ func MakeQuerySlice(letters []byte) []byte { max := -1 for i := 0; i < len(letters); i++ { j := int(letters[i]) if max < j { max = j } } querySlice := make([]byte, max+1) for i := 0; i < len(letters); i++ { querySlice[int(letters[i])] = letters[i] } return querySlice } // Split splits a byte slice by giveen letters. // It's much faster than regexp.Split func Split(slice []byte, letters []byte) [][]byte { querySlice := MakeQuerySlice(letters) results := [][]byte{} tmp := []byte{} var j int var value byte var sliceSize = len(querySlice) for _, b := range slice { j = int(b) if j >= sliceSize { // not delimiter byte tmp = append(tmp, b) continue } value = querySlice[j] if value == 0 { // not delimiter byte tmp = append(tmp, b) continue } else { if len(tmp) > 0 { results = append(results, tmp) tmp = []byte{} } } } if len(tmp) > 0 { results = append(results, tmp) } return results } // Bytes2Str convert byte slice to string without GC. Warning: it's unsafe!!! func Bytes2Str(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } // CountBytes counts given ASCII characters in a byte slice func CountBytes(seq, letters []byte) int { if len(letters) == 0 || len(seq) == 0 { return 0 } // do not use map querySlice := make([]byte, 256) for i := 0; i < len(letters); i++ { querySlice[int(letters[i])] = letters[i] } var g byte var n int for i := 0; i < len(seq); i++ { g = querySlice[int(seq[i])] if g > 0 { // not gap n++ } } return n } util-0.5.2/byteutil/count.go0000644000175000017500000000324114503242340015336 0ustar nileshnileshpackage byteutil import "sort" // ByteCount is a struct store count of byte type ByteCount struct { Key byte Count int } // ByteCountList is slice of ByteCount type ByteCountList []ByteCount func (b ByteCountList) Len() int { return len(b) } func (b ByteCountList) Less(i, j int) bool { // return b[i].Count < b[j].Count // This will return unwanted result: return b[i].Count < b[j].Count || b[i].Key < b[j].Key if b[i].Count < b[j].Count { return true } if b[i].Count == b[j].Count { if b[i].Key < b[j].Key { return true } return false } return false } func (b ByteCountList) Swap(i, j int) { b[i], b[j] = b[j], b[i] } // ReversedByteCountList is Reversed ByteCountList type ReversedByteCountList struct { ByteCountList } // Less is different from the Less of ByteCountList func (b ReversedByteCountList) Less(i, j int) bool { // return b.ByteCountList[i].Count > b.ByteCountList[j].Count if b.ByteCountList[i].Count > b.ByteCountList[j].Count { return true } if b.ByteCountList[i].Count == b.ByteCountList[j].Count { if b.ByteCountList[i].Key < b.ByteCountList[j].Key { return true } return false } return false } // CountOfByte returns the count of byte for a byte slice func CountOfByte(s []byte) map[byte]int { count := make(map[byte]int) for _, b := range s { count[b]++ } return count } // SortCountOfByte sorts count of byte func SortCountOfByte(count map[byte]int, reverse bool) ByteCountList { countList := make(ByteCountList, len(count)) i := 0 for b, c := range count { countList[i] = ByteCount{b, c} i++ } if reverse { sort.Sort(ReversedByteCountList{countList}) } else { sort.Sort(countList) } return countList } util-0.5.2/README.md0000644000175000017500000000121514503242340013274 0ustar nileshnilesh# util [![GoDoc](https://godoc.org/github.com/shenwei356/util?status.svg)](https://godoc.org/github.com/shenwei356/util) [![Go Report Card](https://goreportcard.com/badge/github.com/shenwei356/util)](https://goreportcard.com/report/github.com/shenwei356/util) Utility packges ## Install This package is "go-gettable", just: go get -u github.com/shenwei356/util Copyright (c) 2013-2020, Wei Shen (shenwei356@gmail.com) ## More See the README of sub package. ## Documentation [See documentation on godoc for more detail](https://godoc.org/github.com/shenwei356/util). [MIT License](https://github.com/shenwei356/util/blob/master/LICENSE) util-0.5.2/cliutil/0000755000175000017500000000000014503242340013463 5ustar nileshnileshutil-0.5.2/cliutil/others.go0000644000175000017500000000500714503242340015320 0ustar nileshnileshpackage cliutil import ( "bufio" "fmt" "os" "strings" "github.com/pkg/errors" "github.com/shenwei356/breader" ) // ReadKVs parse two-column (key\tvalue) tab-delimited file(s). func ReadKVs(file string, ignoreCase bool) (map[string]string, error) { type KV [2]string fn := func(line string) (interface{}, bool, error) { if len(line) > 0 && line[len(line)-1] == '\n' { line = line[:len(line)-1] } if len(line) > 0 && line[len(line)-1] == '\r' { line = line[:len(line)-1] } if len(line) == 0 { return nil, false, nil } items := strings.Split(line, "\t") if len(items) < 2 { return nil, false, nil } if ignoreCase { return KV([2]string{strings.ToLower(items[0]), items[1]}), true, nil } return KV([2]string{items[0], items[1]}), true, nil } kvs := make(map[string]string) reader, err := breader.NewBufferedReader(file, 4, 100, fn) if err != nil { return kvs, err } var items KV var data interface{} for chunk := range reader.Ch { if chunk.Err != nil { return kvs, err } for _, data = range chunk.Data { items = data.(KV) kvs[items[0]] = items[1] } } return kvs, nil } // DropCR removes last "\r" if it is. func DropCR(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\r' { return data[0 : len(data)-1] } return data } // DropLF removes "\n" func DropLF(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\n' { return data[0 : len(data)-1] } return data } func GetFileList(args []string, checkFile bool) []string { files := make([]string, 0, 1000) if len(args) == 0 { files = append(files, "-") } else { for _, file := range args { if isStdin(file) { continue } if !checkFile { continue } if _, err := os.Stat(file); os.IsNotExist(err) { CheckError(errors.Wrap(err, file)) } } files = args } return files } func GetFileListFromFile(file string, checkFile bool) ([]string, error) { fh, err := os.Open(file) if err != nil { return nil, fmt.Errorf("read file list from '%s': %s", file, err) } var _file string lists := make([]string, 0, 1000) scanner := bufio.NewScanner(fh) for scanner.Scan() { _file = scanner.Text() if strings.TrimSpace(_file) == "" { continue } if checkFile && !isStdin(_file) { if _, err = os.Stat(_file); os.IsNotExist(err) { return lists, fmt.Errorf("check file '%s': %s", _file, err) } } lists = append(lists, _file) } if err = scanner.Err(); err != nil { return nil, fmt.Errorf("read file list from '%s': %s", file, err) } return lists, nil } util-0.5.2/cliutil/cobra.go0000644000175000017500000000730514503242340015105 0ustar nileshnileshpackage cliutil import ( "fmt" "os" "strconv" "github.com/shenwei356/util/stringutil" "github.com/spf13/cobra" ) func isStdin(file string) bool { return file == "-" } func CheckError(err error) { if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(-1) } } func GetFlagInt(cmd *cobra.Command, flag string) int { value, err := cmd.Flags().GetInt(flag) CheckError(err) return value } func GetFlagUint8(cmd *cobra.Command, flag string) uint8 { value, err := cmd.Flags().GetUint8(flag) CheckError(err) return value } func GetFlagUint32(cmd *cobra.Command, flag string) uint32 { value, err := cmd.Flags().GetUint32(flag) CheckError(err) return value } func GetFlagUint64(cmd *cobra.Command, flag string) uint64 { value, err := cmd.Flags().GetUint64(flag) CheckError(err) return value } func GetFlagPositiveInt(cmd *cobra.Command, flag string) int { value, err := cmd.Flags().GetInt(flag) CheckError(err) if value <= 0 { CheckError(fmt.Errorf("value of flag --%s should be greater than 0", flag)) } return value } func GetFlagPositiveFloat64(cmd *cobra.Command, flag string) float64 { value, err := cmd.Flags().GetFloat64(flag) CheckError(err) if value <= 0 { CheckError(fmt.Errorf("value of flag --%s should be greater than 0", flag)) } return value } func GetFlagNonNegativeInt(cmd *cobra.Command, flag string) int { value, err := cmd.Flags().GetInt(flag) CheckError(err) if value < 0 { CheckError(fmt.Errorf("value of flag --%s should be greater than or equal to 0", flag)) } return value } func GetFlagNonNegativeFloat64(cmd *cobra.Command, flag string) float64 { value, err := cmd.Flags().GetFloat64(flag) CheckError(err) if value < 0 { CheckError(fmt.Errorf("value of flag --%s should be greater than or equal to ", flag)) } return value } func GetFlagBool(cmd *cobra.Command, flag string) bool { value, err := cmd.Flags().GetBool(flag) CheckError(err) return value } func GetFlagString(cmd *cobra.Command, flag string) string { value, err := cmd.Flags().GetString(flag) CheckError(err) return value } func GetFlagNonEmptyString(cmd *cobra.Command, flag string) string { value, err := cmd.Flags().GetString(flag) CheckError(err) if value == "" { CheckError(fmt.Errorf("value of flag --%s should not be empty", flag)) } return value } func GetFlagCommaSeparatedStrings(cmd *cobra.Command, flag string) []string { value, err := cmd.Flags().GetString(flag) CheckError(err) return stringutil.Split(value, ",") } func GetFlagSemicolonSeparatedStrings(cmd *cobra.Command, flag string) []string { value, err := cmd.Flags().GetString(flag) CheckError(err) return stringutil.Split(value, ";") } func GetFlagCommaSeparatedInts(cmd *cobra.Command, flag string) []int { filedsStrList := GetFlagCommaSeparatedStrings(cmd, flag) fields := make([]int, len(filedsStrList)) for i, value := range filedsStrList { v, err := strconv.Atoi(value) if err != nil { CheckError(fmt.Errorf("value of flag --%s should be comma separated integers", flag)) } fields[i] = v } return fields } func GetFlagRune(cmd *cobra.Command, flag string) rune { value, err := cmd.Flags().GetString(flag) CheckError(err) if len(value) > 1 { CheckError(fmt.Errorf("value of flag --%s should has length of 1", flag)) } var v rune for _, r := range value { v = r break } return v } func GetFlagFloat64(cmd *cobra.Command, flag string) float64 { value, err := cmd.Flags().GetFloat64(flag) CheckError(err) return value } func GetFlagInt64(cmd *cobra.Command, flag string) int64 { value, err := cmd.Flags().GetInt64(flag) CheckError(err) return value } func GetFlagStringSlice(cmd *cobra.Command, flag string) []string { value, err := cmd.Flags().GetStringSlice(flag) CheckError(err) return value }