pax_global_header00006660000000000000000000000064132746755650014536gustar00rootroot0000000000000052 comment=7db4be5742aecc82f99aa44d27ce25a3600a7a80 golang-github-araddon-gou-0.0~git20180509.7db4be5/000077500000000000000000000000001327467556500212105ustar00rootroot00000000000000golang-github-araddon-gou-0.0~git20180509.7db4be5/.gitignore000066400000000000000000000003741327467556500232040ustar00rootroot00000000000000# Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe golang-github-araddon-gou-0.0~git20180509.7db4be5/LICENSE.md000066400000000000000000000021141327467556500226120ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2012-2014 Aaron Raddon and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.golang-github-araddon-gou-0.0~git20180509.7db4be5/README.md000066400000000000000000000065201327467556500224720ustar00rootroot00000000000000gou - Go Utilities =========================== Go Utilities (logging, json) JsonHelper =============== A Go Json Helper, focused on Type coercion, and json path query. ```go package main import . "github.com/araddon/gou" import . "github.com/araddon/gou/goutest" import "testing" func TestJsonHelper() { var jsonData := []byte(`{ "name":"aaron", "nullstring":null, "ints":[1,2,3,4], "int":1, "intstr":"1", "int64":1234567890, "MaxSize" : 1048576, "strings":["string1"], "stringscsv":"string1,string2", "nested":{ "nest":"string2", "strings":["string1"], "int":2, "list":["value"], "nest2":{ "test":"good" } }, "nested2":[ {"sub":2} ], "period.name":"value" }` jh := NewJsonHelper(jsonData) // String method Assert(jh.String("name") == "aaron", t, "should get 'aaron' %s", jh.String("name")) // Int Method Assert(jh.Int("int") == 1, t, "get int ") // Selecting items from an array Assert(jh.Int("ints[0]") == 1, t, "get int from array %d", jh.Int("ints[0]")) Assert(jh.Int("ints[2]") == 3, t, "get int from array %d", jh.Int("ints[0]")) // Getting arrays Assert(len(jh.Ints("ints")) == 4, t, "get int array %v", jh.Ints("ints")) // Type coercion to Int64 Assert(jh.Int64("int64") == 1234567890, t, "get int") Assert(jh.Int("nested.int") == 2, t, "get int") // Path based selection Assert(jh.String("nested.nest") == "string2", t, "should get string %s", jh.String("nested.nest")) Assert(jh.String("nested.nest2.test") == "good", t, "should get string %s", jh.String("nested.nest2.test")) Assert(jh.String("nested.list[0]") == "value", t, "get string from array") Assert(jh.Int("nested2[0].sub") == 2, t, "get int from obj in array %d", jh.Int("nested2[0].sub")) // casing? Assert(jh.Int("MaxSize") == 1048576, t, "get int, test capitalization? ") sl := jh.Strings("strings") Assert(len(sl) == 1 && sl[0] == "string1", t, "get strings ") sl = jh.Strings("stringscsv") Assert(len(sl) == 2 && sl[0] == "string1", t, "get strings ") // Safe gets i64, ok := jh.Int64Safe("int64") Assert(ok, t, "int64safe ok") Assert(i64 == 1234567890, t, "int64safe value") i, ok := jh.IntSafe("int") Assert(ok, t, "intsafe ok") Assert(i == 1, t, "intsafe value") l := jh.List("nested2") Assert(len(l) == 1, t, "get list") jhm := jh.Helpers("nested2") Assert(len(jhm) == 1, t, "get list of helpers") Assert(jhm[0].Int("sub") == 2, t, "Should get list of helpers") // Now lets test xpath type syntax Assert(jh.Int("/MaxSize") == 1048576, t, "get int, test capitalization? ") Assert(jh.String("/nested/nest") == "string2", t, "should get string %s", jh.String("/nested/nest")) Assert(jh.String("/nested/list[0]") == "value", t, "get string from array") // note this one has period in name Assert(jh.String("/period.name") == "value", t, "test period in name ") } ``` Logging =============== Yet Another Go Logger, configureable logging. ```go package main import "github.com/araddon/gou" import "flag" var logLevel *string = flag.String("logging", "debug", "Which log level: [debug,info,warn,error,fatal]") func main() { flag.Parse() gou.SetupLogging(*logLevel) // logging methods gou.Debug("hello", thing, " more ", stuff) gou.Error("hello") gou.Errorf("hello %v", thing) } ``` License =============== MIT License golang-github-araddon-gou-0.0~git20180509.7db4be5/coerce.go000066400000000000000000000173411327467556500230050ustar00rootroot00000000000000package gou import ( "encoding/json" "fmt" "math" "strconv" "strings" ) // Coerce types (string,int,int64, float, []byte) into String type func CoerceString(v interface{}) (string, error) { switch val := v.(type) { case string: if val == "null" || val == "NULL" { return "", nil } return val, nil case int: return strconv.Itoa(val), nil case int32: return strconv.FormatInt(int64(val), 10), nil case int64: return strconv.FormatInt(val, 10), nil case uint32: return strconv.FormatUint(uint64(val), 10), nil case uint64: return strconv.FormatUint(val, 10), nil case float32: return strconv.FormatFloat(float64(val), 'f', -1, 32), nil case float64: return strconv.FormatFloat(val, 'f', -1, 64), nil case bool: if val { return "true", nil } return "false", nil case []byte: if string(val) == "null" || string(val) == "NULL" { return "", nil } return string(val), nil case json.RawMessage: if string(val) == "null" || string(val) == "NULL" { return "", nil } return string(val), nil } return "", fmt.Errorf("Could not coerce to string: %v", v) } // Coerce type to string, returning zero length string if error or nil func CoerceStringShort(v interface{}) string { val, _ := CoerceString(v) return val } // CoerceStrings Coerce type to strings, will split on comma by default. func CoerceStrings(v interface{}) []string { switch val := v.(type) { case string: if val == "" { return nil } return strings.Split(val, ",") case []string: return val case []interface{}: sva := make([]string, 0) for _, av := range val { switch aval := av.(type) { case string: sva = append(sva, aval) default: sv, err := CoerceString(av) if err == nil && sv != "" { sva = append(sva, sv) } } } return sva } return []string{CoerceStringShort(v)} } func CoerceFloats(v interface{}) []float64 { switch val := v.(type) { case float64: return []float64{val} case []string: fa := make([]float64, 0) for _, av := range val { f, err := CoerceFloat(av) if err == nil && !math.IsNaN(f) { fa = append(fa, f) } } return fa case []interface{}: fa := make([]float64, 0) for _, av := range val { switch aval := av.(type) { case float64: fa = append(fa, aval) default: f, err := CoerceFloat(av) if err == nil && !math.IsNaN(f) { fa = append(fa, f) } } } return fa } fv, err := CoerceFloat(v) if err == nil { return []float64{fv} } return nil } func CoerceFloat(v interface{}) (float64, error) { switch val := v.(type) { case int: return float64(val), nil case int32: return float64(val), nil case int64: return float64(val), nil case uint32: return float64(val), nil case uint64: return float64(val), nil case float64: return val, nil case string: if len(val) > 0 { if iv, err := strconv.ParseFloat(val, 64); err == nil { return iv, nil } } case []byte: if len(val) > 0 { if iv, err := strconv.ParseFloat(string(val), 64); err == nil { return iv, nil } } case json.RawMessage: if len(val) > 0 { if iv, err := strconv.ParseFloat(string(val), 64); err == nil { return iv, nil } } case nil: return math.NaN(), nil } return 0, fmt.Errorf("Could not Coerce Value: %v", v) } func CoerceFloatShort(v interface{}) float64 { val, _ := CoerceFloat(v) return val } func CoerceInt64(v interface{}) (int64, error) { val, ok := valToInt64(v) if ok { return val, nil } return 0, fmt.Errorf("Could not coerce to int64: %v", v) } func CoerceInt64Short(v interface{}) int64 { val, ok := valToInt64(v) if ok { return val } return 0 } func CoerceInt(v interface{}) (int, error) { val, ok := valToInt(v) if ok { return val, nil } return 0, fmt.Errorf("Could not coerce to int64: %v", v) } func CoerceIntShort(v interface{}) int { val, ok := valToInt(v) if ok { return val } return 0 } func CoerceInts(v interface{}) []int { switch val := v.(type) { case []string: iva := make([]int, 0) for _, av := range val { avAsInt, ok := valToInt(av) if ok { iva = append(iva, avAsInt) } } return iva case []interface{}: iva := make([]int, 0) for _, av := range val { avAsInt, ok := valToInt(av) if ok { iva = append(iva, avAsInt) } } return iva } return []int{CoerceIntShort(v)} } // Coerce a val(interface{}) into a Uint64 func CoerceUint(v interface{}) (uint64, error) { u64, ok := valToUint64(v) if !ok { return 0, fmt.Errorf("Could not Coerce %v", v) } return u64, nil } // Coerce a Val(interface{}) into Uint64 func CoerceUintShort(v interface{}) uint64 { val, _ := CoerceUint(v) return val } // Given any numeric type (float*, int*, uint*, string) return an int. Returns false if it would // overflow or if the the argument is not numeric. func valToInt(i interface{}) (int, bool) { i64, ok := valToInt64(i) if !ok { return -1, false } if i64 > MaxInt || i64 < MinInt { return -1, false } return int(i64), true } // Given any simple type (float*, int*, uint*, string, []byte, json.RawMessage) return an int64. // Returns false if it would overflow or if the the argument is not numeric. func valToInt64(i interface{}) (int64, bool) { switch x := i.(type) { case float32: return int64(x), true case float64: return int64(x), true case uint8: return int64(x), true case uint16: return int64(x), true case uint32: return int64(x), true case uint64: if x > math.MaxInt64 { return 0, false } return int64(x), true case int8: return int64(x), true case int16: return int64(x), true case int32: return int64(x), true case int64: return int64(x), true case int: return int64(x), true case uint: if uint64(x) > math.MaxInt64 { return 0, false } return int64(x), true case string: if len(x) > 0 { if iv, err := strconv.ParseInt(x, 10, 64); err == nil { return iv, true } if iv, err := strconv.ParseFloat(x, 64); err == nil { return valToInt64(iv) } } case []byte: if len(x) > 0 { if iv, err := strconv.ParseInt(string(x), 10, 64); err == nil { return iv, true } if iv, err := strconv.ParseFloat(string(x), 64); err == nil { return valToInt64(iv) } } case json.RawMessage: if len(x) > 0 { if iv, err := strconv.ParseInt(string(x), 10, 64); err == nil { return iv, true } if iv, err := strconv.ParseFloat(string(x), 64); err == nil { return valToInt64(iv) } } } return 0, false } // Given any simple type (float*, int*, uint*, string, []byte, json.RawMessage) return an int64. // Returns false if it would overflow or if the the argument is not numeric. func valToUint64(i interface{}) (uint64, bool) { switch x := i.(type) { case float32: return uint64(x), true case float64: return uint64(x), true case uint8: return uint64(x), true case uint16: return uint64(x), true case uint32: return uint64(x), true case uint64: return x, true case int8: return uint64(x), true case int16: return uint64(x), true case int32: return uint64(x), true case int64: return uint64(x), true case int: return uint64(x), true case uint: return uint64(x), true case string: if len(x) > 0 { if uiv, err := strconv.ParseUint(x, 10, 64); err == nil { return uiv, true } if fv, err := strconv.ParseFloat(x, 64); err == nil { return uint64(fv), true } } case []byte: if len(x) > 0 { if uiv, err := strconv.ParseUint(string(x), 10, 64); err == nil { return uiv, true } if fv, err := strconv.ParseFloat(string(x), 64); err == nil { return uint64(fv), true } } case json.RawMessage: if len(x) > 0 { if uiv, err := strconv.ParseUint(string(x), 10, 64); err == nil { return uiv, true } if fv, err := strconv.ParseFloat(string(x), 64); err == nil { return uint64(fv), true } } } return 0, false } golang-github-araddon-gou-0.0~git20180509.7db4be5/coerce_test.go000066400000000000000000000031661327467556500240440ustar00rootroot00000000000000package gou import ( "testing" "github.com/stretchr/testify/assert" ) func TestCoerce(t *testing.T) { data := map[string]interface{}{ "int": 4, "float": 45.3, "string": "22", "stringf": "22.2", "boolt": true, "emptystring": "", } assert.True(t, CoerceStringShort(data["int"]) == "4", "get int as string") assert.True(t, CoerceStringShort(data["float"]) == "45.3", "get float as string: %v", data["float"]) assert.True(t, CoerceStringShort(data["string"]) == "22", "get string as string: %v", data["string"]) assert.True(t, CoerceStringShort(data["stringf"]) == "22.2", "get stringf as string: %v", data["stringf"]) assert.Equal(t, "true", CoerceStringShort(data["boolt"])) assert.True(t, CoerceIntShort(data["int"]) == 4, "get int as int: %v", data["int"]) assert.True(t, CoerceIntShort(data["float"]) == 45, "get float as int: %v", data["float"]) assert.True(t, CoerceIntShort(data["string"]) == 22, "get string as int: %v", data["string"]) assert.True(t, CoerceIntShort(data["stringf"]) == 22, "get stringf as int: %v", data["stringf"]) assert.Equal(t, 0, len(CoerceStrings(data["emptystring"])), "get emptystring as []string: %v", data["emptystring"]) assert.Equal(t, []string{"22"}, CoerceStrings(data["string"]), "get string as []string: %v", data["string"]) assert.Equal(t, []string{"4"}, CoerceStrings(data["int"]), "get int as []string: %v", data["int"]) assert.Equal(t, []float64{float64(4)}, CoerceFloats(data["int"]), "get int as []float64: %v", data["int"]) assert.Equal(t, []float64{float64(45.3)}, CoerceFloats(data["float"]), "get float as []float64: %v", data["float"]) } golang-github-araddon-gou-0.0~git20180509.7db4be5/http.go000066400000000000000000000110321327467556500225130ustar00rootroot00000000000000package gou import ( "bytes" "encoding/json" "io" "io/ioutil" "net/http" "net/url" ) // Simple Fetch Wrapper, given a url it returns bytes func Fetch(url string) (ret []byte, err error) { resp, err := http.Get(url) defer func() { if resp != nil && resp.Body != nil { resp.Body.Close() } }() if err != nil { Log(WARN, err.Error()) return } ret, err = ioutil.ReadAll(resp.Body) if err != nil { return } return } // Simple Fetch Wrapper, given a url it returns bytes and response func FetchResp(url string) (ret []byte, err error, resp *http.Response) { resp, err = http.Get(url) defer func() { if resp != nil && resp.Body != nil { resp.Body.Close() } }() if err != nil { Log(WARN, err.Error()) } if resp == nil || resp.Body == nil { return } ret, err = ioutil.ReadAll(resp.Body) return } // Simple Fetch Wrapper, given a url it returns Helper, error // Sends as type application/json, interprets whatever datatype is sent in appropriately func JsonHelperHttp(method, urlStr string, data interface{}) (JsonHelper, error) { var body io.Reader if data != nil { switch val := data.(type) { case string: body = bytes.NewReader([]byte(val)) case io.Reader: body = val case url.Values: body = bytes.NewReader([]byte(val.Encode())) default: by, err := json.Marshal(data) if err != nil { return nil, err } body = bytes.NewReader(by) } } req, err := http.NewRequest(method, urlStr, body) if err != nil { return nil, err } req.Header.Add("Accept", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() jh, err := NewJsonHelperReader(resp.Body) return jh, err } // posts an application/json to url with body // ie: type = application/json func PostJson(postUrl string, data interface{}) (ret string, err error, resp *http.Response) { var buf io.Reader if data != nil { switch val := data.(type) { case string: buf = bytes.NewBufferString(val) case []byte: buf = bytes.NewReader(val) case json.RawMessage: buf = bytes.NewReader([]byte(val)) case io.Reader: buf = val case url.Values: buf = bytes.NewBufferString(val.Encode()) default: by, err := json.Marshal(data) if err != nil { return "", err, nil } buf = bytes.NewReader(by) } } resp, err = http.Post(postUrl, "application/json", buf) defer func() { if resp != nil && resp.Body != nil { resp.Body.Close() } }() if err != nil { Log(WARN, err.Error()) return "", err, resp } bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err, resp } return string(bodyBytes), nil, resp } // issues http delete an application/json to url with body func DeleteJson(url, body string) (ret string, err error, resp *http.Response) { //Post(url string, bodyType string, body io.Reader) buf := bytes.NewBufferString(body) Debug(buf.Len()) req, err := http.NewRequest("DELETE", url, buf) if err != nil { Debug(err) return } req.Header.Add("Content-Type", "application/json") resp, err = http.DefaultClient.Do(req) //(url, "application/json", buf) defer func() { if resp != nil && resp.Body != nil { resp.Body.Close() } }() if err != nil { Log(WARN, err.Error()) return "", err, resp } data, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err, resp } return string(data), nil, resp } // posts a www-form encoded form to url with body func PostForm(url, body string) (ret string, err error, resp *http.Response) { //Post(url string, bodyType string, body io.Reader) buf := bytes.NewBufferString(body) resp, err = http.Post(url, "application/x-www-form-urlencoded", buf) defer func() { if resp != nil && resp.Body != nil { resp.Body.Close() } }() if err != nil { Log(WARN, url, " ", body, " ", err.Error()) return "", err, resp } data, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err, resp } return string(data), nil, resp } // issues http put an application/json to url with optional body func PutJson(url, body string) (ret string, err error, resp *http.Response) { buf := bytes.NewBufferString(body) req, err := http.NewRequest("PUT", url, buf) if err != nil { Debug(err) return } req.Header.Add("Content-Type", "application/json") resp, err = http.DefaultClient.Do(req) defer func() { if resp != nil && resp.Body != nil { resp.Body.Close() } }() if err != nil { Log(WARN, err.Error()) return "", err, resp } data, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err, resp } return string(data), nil, resp } golang-github-araddon-gou-0.0~git20180509.7db4be5/jsonhelper.go000066400000000000000000000304311327467556500237110ustar00rootroot00000000000000package gou import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "math" "net/http" "net/url" "strconv" "strings" "unicode/utf8" ) // Convert a slice of bytes into an array by ensuring it is wrapped // with [] func MakeJsonList(b []byte) []byte { if !bytes.HasPrefix(b, []byte{'['}) { b = append([]byte{'['}, b...) b = append(b, ']') } return b } func JsonString(v interface{}) string { b, err := json.Marshal(v) if err != nil { return `""` } return string(b) } func firstNonWsRune(by []byte) (r rune, ok bool) { for { if len(by) == 0 { return 0, false } r, numBytes := utf8.DecodeRune(by) switch r { case '\t', '\n', '\r', ' ': by = by[numBytes:] // advance past the current whitespace rune and continue continue case utf8.RuneError: // This is returned when invalid UTF8 is found return 0, false } return r, true } } // Determines if the bytes is a json array, only looks at prefix // not parsing the entire thing func IsJson(by []byte) bool { firstRune, ok := firstNonWsRune(by) if !ok { return false } if firstRune == '[' || firstRune == '{' { return true } return false } // Determines if the bytes is a json array, only looks at prefix // not parsing the entire thing func IsJsonArray(by []byte) bool { firstRune, ok := firstNonWsRune(by) if !ok { return false } if firstRune == '[' { return true } return false } func IsJsonObject(by []byte) bool { firstRune, ok := firstNonWsRune(by) if !ok { return false } if firstRune == '{' { return true } return false } type JsonRawWriter struct { bytes.Buffer } func (m *JsonRawWriter) MarshalJSON() ([]byte, error) { return m.Bytes(), nil } func (m *JsonRawWriter) Raw() json.RawMessage { return json.RawMessage(m.Bytes()) } // A wrapper around a map[string]interface{} to facilitate coercion // of json data to what you want // // allows usage such as this // // jh := NewJsonHelper([]byte(`{ // "name":"string", // "ints":[1,5,9,11], // "int":1, // "int64":1234567890, // "MaxSize" : 1048576, // "strings":["string1"], // "nested":{ // "nest":"string2", // "strings":["string1"], // "int":2, // "list":["value"], // "nest2":{ // "test":"good" // } // }, // "nested2":[ // {"sub":5} // ] // }`) // // i := jh.Int("nested.int") // 2 // i2 := jh.Int("ints[1]") // 5 array position 1 from [1,5,9,11] // s := jh.String("nested.nest") // "string2" // type JsonHelper map[string]interface{} func NewJsonHelper(b []byte) JsonHelper { jh := make(JsonHelper) json.Unmarshal(b, &jh) return jh } func NewJsonHelperReader(r io.Reader) (jh JsonHelper, err error) { jh = make(JsonHelper) err = json.NewDecoder(r).Decode(&jh) return } func NewJsonHelpers(b []byte) []JsonHelper { var jhl []JsonHelper json.Unmarshal(MakeJsonList(b), &jhl) return jhl } func NewJsonHelperMapString(m map[string]string) JsonHelper { jh := make(JsonHelper) for k, v := range m { jh[k] = v } return jh } // Make a JsonHelper from http response. This will automatically // close the response body func NewJsonHelperFromResp(resp *http.Response) (JsonHelper, error) { jh := make(JsonHelper) if resp == nil || resp.Body == nil { return jh, fmt.Errorf("No response or response body to read") } defer resp.Body.Close() respBytes, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } if len(respBytes) == 0 { return jh, fmt.Errorf("No data in response") } if err := json.Unmarshal(respBytes, &jh); err != nil { return jh, err } return jh, nil } func jsonList(v interface{}) []interface{} { switch v.(type) { case []interface{}: return v.([]interface{}) } return nil } func jsonEntry(name string, v interface{}) (interface{}, bool) { switch val := v.(type) { case map[string]interface{}: if root, ok := val[name]; ok { return root, true } else { return nil, false } case JsonHelper: return val.Get(name), true case []interface{}: return v, true default: Debug("no type? ", name, " ", v) return nil, false } } // Get the key (or keypath) value as interface, mostly used // internally through String, etc methods // // jh.Get("name.subname") // jh.Get("name/subname") // jh.Get("name.arrayname[1]") // jh.Get("name.arrayname[]") func (j JsonHelper) Get(n string) interface{} { if len(j) == 0 { return nil } var parts []string if strings.Contains(n, "/") { parts = strings.Split(n, "/") if strings.HasPrefix(n, "/") && len(parts) > 0 { parts = parts[1:] } } else { parts = strings.Split(n, ".") } var root interface{} var err error var ok, isList, listEntry bool var ln, st, idx int for ict, name := range parts { isList = strings.HasSuffix(name, "[]") listEntry = strings.HasSuffix(name, "]") && !isList ln, idx = len(name), -1 if isList || listEntry { st = strings.Index(name, "[") idx, err = strconv.Atoi(name[st+1 : ln-1]) name = name[:st] } if ict == 0 { root, ok = j[name] } else { root, ok = jsonEntry(name, root) } if !ok { if len(parts) > 0 { // lets ensure the actual json-value doesn't have period in key root, ok = j[n] if !ok { return nil } else { return root } } else { return nil } } if isList { return jsonList(root) } else if listEntry && err == nil { if lst := jsonList(root); lst != nil && len(lst) > idx { root = lst[idx] } else { return nil } } } return root } // Get a Helper from a string path func (j JsonHelper) Helper(n string) JsonHelper { v := j.Get(n) if v == nil { return nil } switch vt := v.(type) { case map[string]interface{}: cn := JsonHelper{} for n, val := range vt { cn[n] = val } return cn case map[string]string: cn := JsonHelper{} for n, val := range vt { cn[n] = val } return cn case JsonHelper: return vt default: //Infof("wrong type: %T", v) } return nil } // Get list of Helpers at given name. Trys to coerce into // proper Helper type func (j JsonHelper) Helpers(n string) []JsonHelper { v := j.Get(n) if v == nil { return nil } switch val := v.(type) { case []map[string]interface{}: hl := make([]JsonHelper, 0) for _, mapVal := range val { hl = append(hl, mapVal) } return hl case []interface{}: jhl := make([]JsonHelper, 0) for _, item := range val { if jh, ok := item.(map[string]interface{}); ok { jhl = append(jhl, jh) } } return jhl } return nil } // Gets slice of interface{} func (j JsonHelper) List(n string) []interface{} { v := j.Get(n) switch val := v.(type) { case []string: il := make([]interface{}, len(val)) for i, val := range val { il[i] = val } return il case []interface{}: return val } return nil } func (j JsonHelper) String(n string) string { if v := j.Get(n); v != nil { val, _ := CoerceString(v) return val } return "" } func (j JsonHelper) Strings(n string) []string { if v := j.Get(n); v != nil { return CoerceStrings(v) } return nil } func (j JsonHelper) Ints(n string) []int { v := j.Get(n) if v == nil { return nil } return CoerceInts(v) } func (j JsonHelper) StringSafe(n string) (string, bool) { v := j.Get(n) if v != nil { if s, ok := v.(string); ok { return s, ok } } return "", false } func (j JsonHelper) Int(n string) int { i, ok := j.IntSafe(n) if !ok { return -1 } return i } func (j JsonHelper) IntSafe(n string) (int, bool) { v := j.Get(n) return valToInt(v) } func (j JsonHelper) Int64(n string) int64 { i64, ok := j.Int64Safe(n) if !ok { return -1 } return i64 } func (j JsonHelper) Int64Safe(n string) (int64, bool) { v := j.Get(n) return valToInt64(v) } func (j JsonHelper) Float64(n string) float64 { v := j.Get(n) f64, err := CoerceFloat(v) if err != nil { return math.NaN() } return f64 } func (j JsonHelper) Float64Safe(n string) (float64, bool) { v := j.Get(n) if v == nil { return math.NaN(), true } fv, err := CoerceFloat(v) if err != nil { return math.NaN(), false } return fv, true } func (j JsonHelper) Uint64(n string) uint64 { v := j.Get(n) if v != nil { return CoerceUintShort(v) } return 0 } func (j JsonHelper) Uint64Safe(n string) (uint64, bool) { v := j.Get(n) if v != nil { if uv, err := CoerceUint(v); err == nil { return uv, true } } return 0, false } func (j JsonHelper) BoolSafe(n string) (val bool, ok bool) { v := j.Get(n) if v != nil { switch v.(type) { case bool: return v.(bool), true case string: if s := v.(string); len(s) > 0 { if b, err := strconv.ParseBool(s); err == nil { return b, true } } } } return false, false } func (j JsonHelper) Bool(n string) bool { val, ok := j.BoolSafe(n) if !ok { return false } return val } func (j JsonHelper) Map(n string) map[string]interface{} { v := j.Get(n) if v == nil { return nil } m, ok := v.(map[string]interface{}) if !ok { return nil } return m } func (j JsonHelper) MapSafe(n string) (map[string]interface{}, bool) { v := j.Get(n) if v == nil { return nil, false } m, ok := v.(map[string]interface{}) if !ok { return nil, false } return m, true } func (j JsonHelper) PrettyJson() []byte { jsonPretty, _ := json.MarshalIndent(j, " ", " ") return jsonPretty } func (j JsonHelper) Keys() []string { keys := make([]string, 0) for key := range j { keys = append(keys, key) } return keys } func (j JsonHelper) HasKey(name string) bool { if val := j.Get(name); val != nil { return true } return false } // GobDecode overwrites the receiver, which must be a pointer, // with the value represented by the byte slice, which was written // by GobEncode, usually for the same concrete type. // GobDecode([]byte) error func (j *JsonHelper) GobDecode(data []byte) error { var mv map[string]interface{} if err := json.Unmarshal(data, &mv); err != nil { return err } *j = JsonHelper(mv) return nil } func (j *JsonHelper) GobEncode() ([]byte, error) { by, err := json.Marshal(j) return by, err } // The following consts are from http://code.google.com/p/go-bit/ (Apache licensed). It // lets us figure out how wide go ints are, and determine their max and min values. // Note the use of << to create an untyped constant. const bitsPerWord = 32 << uint(^uint(0)>>63) // Implementation-specific size of int and uint in bits. const BitsPerWord = bitsPerWord // either 32 or 64 // Implementation-specific integer limit values. const ( MaxInt = 1<<(BitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 MinInt = -MaxInt - 1 // either -1 << 31 or -1 << 63 MaxUint = 1< 0 { // uv[k] = sva // } case map[string]bool: // what to do? Info("not implemented: [string]bool") case map[string]interface{}: if len(x) > 0 { if err := flattenJsonMap(uv, x, k+"."); err != nil { return err } } case string: uv.Set(k, x) case bool: if x == true { uv.Set(k, "t") } else { uv.Set(k, "f") } case int: uv.Set(k, strconv.FormatInt(int64(x), 10)) case int8: uv.Set(k, strconv.FormatInt(int64(x), 10)) case int16: uv.Set(k, strconv.FormatInt(int64(x), 10)) case int32: uv.Set(k, strconv.FormatInt(int64(x), 10)) case int64: uv.Set(k, strconv.FormatInt(x, 10)) case uint: uv.Set(k, strconv.FormatUint(uint64(x), 10)) case uint8: uv.Set(k, strconv.FormatUint(uint64(x), 10)) case uint16: uv.Set(k, strconv.FormatUint(uint64(x), 10)) case uint32: uv.Set(k, strconv.FormatUint(uint64(x), 10)) case uint64: uv.Set(k, strconv.FormatUint(x, 10)) case float32: uv.Set(k, strconv.FormatFloat(float64(x), 'f', -1, 64)) case float64: uv.Set(k, strconv.FormatFloat(x, 'f', -1, 64)) default: // what types don't we support? // []interface{} } return nil } golang-github-araddon-gou-0.0~git20180509.7db4be5/jsonhelper_test.go000066400000000000000000000150231327467556500247500ustar00rootroot00000000000000package gou import ( "bytes" "encoding/gob" "encoding/json" "math" "strings" "testing" "github.com/stretchr/testify/assert" ) // go test -bench=".*" // go test -run="(Util)" var ( jh JsonHelper ) func init() { SetupLogging("debug") SetColorOutput() //SetLogger(log.New(os.Stderr, "", log.Ltime|log.Lshortfile), "debug") // create test data json.Unmarshal([]byte(`{ "name":"aaron", "nullstring":null, "ints":[1,2,3,4], "int":1, "intstr":"1", "int64":1234567890, "float64":123.456, "float64str":"123.456", "float64null": null, "MaxSize" : 1048576, "strings":["string1"], "stringscsv":"string1,string2", "nested":{ "nest":"string2", "strings":["string1"], "int":2, "list":["value"], "nest2":{ "test":"good" } }, "nested2":[ {"sub":2} ], "period.name":"value" }`), &jh) } func TestJsonRawWriter(t *testing.T) { var buf bytes.Buffer buf.WriteString(`"hello"`) raw := json.RawMessage(buf.Bytes()) bya, _ := json.Marshal(&buf) Debug(string(bya)) bya, _ = json.Marshal(&raw) Debug(string(bya)) /* bya, err := json.Marshal(buf) assert.True(t,string(bya) == `"hello"`, t, "Should be hello but was %s", string(bya)) Debug(string(buf.Bytes()), err) var jrw JsonRawWriter jrw.WriteString(`"hello"`) Debug(jrw.Raw()) bya, err = json.Marshal(jrw.Raw()) assert.True(t,string(bya) == `"hello"`, t, "Should be hello but was %s", string(bya)) Debug(string(jrw.Bytes()), err) */ } func TestJsonHelper(t *testing.T) { assert.True(t, jh.String("name") == "aaron", "should get 'aaron' %s", jh.String("name")) assert.True(t, jh.String("nullstring") == "", "should get '' %s", jh.String("nullstring")) assert.True(t, jh.Int("int") == 1, "get int ") assert.True(t, jh.Int("ints[0]") == 1, "get int from array %d", jh.Int("ints[0]")) assert.True(t, jh.Int("ints[2]") == 3, "get int from array %d", jh.Int("ints[0]")) assert.True(t, len(jh.Ints("ints")) == 4, "get int array %v", jh.Ints("ints")) assert.True(t, jh.Int64("int64") == 1234567890, "get int") assert.True(t, jh.Int("nested.int") == 2, "get int") assert.True(t, jh.String("nested.nest") == "string2", "should get string %s", jh.String("nested.nest")) assert.True(t, jh.String("nested.nest2.test") == "good", "should get string %s", jh.String("nested.nest2.test")) assert.True(t, jh.String("nested.list[0]") == "value", "get string from array") assert.True(t, jh.Int("nested2[0].sub") == 2, "get int from obj in array %d", jh.Int("nested2[0].sub")) assert.True(t, jh.Int("MaxSize") == 1048576, "get int, test capitalization? ") sl := jh.Strings("strings") assert.True(t, len(sl) == 1 && sl[0] == "string1", "get strings ") sl = jh.Strings("stringscsv") assert.True(t, len(sl) == 2 && sl[0] == "string1", "get strings ") i64, ok := jh.Int64Safe("int64") assert.True(t, ok, t, "int64safe ok") assert.True(t, i64 == 1234567890, "int64safe value") u64, ok := jh.Uint64Safe("int64") assert.True(t, ok, "uint64safe ok") assert.True(t, u64 == 1234567890, "int64safe value") _, ok = jh.Uint64Safe("notexistent") assert.True(t, !ok, "should not be ok") _, ok = jh.Uint64Safe("name") assert.True(t, !ok, "should not be ok") i, ok := jh.IntSafe("int") assert.True(t, ok, "intsafe ok") assert.True(t, i == 1, "intsafe value") l := jh.List("nested2") assert.True(t, len(l) == 1, "get list") fv, ok := jh.Float64Safe("name") assert.True(t, !ok, "floatsafe not ok") fv, ok = jh.Float64Safe("float64") assert.True(t, ok, "floatsafe ok") assert.True(t, CloseEnuf(fv, 123.456), "floatsafe value %v", fv) fv = jh.Float64("float64") assert.True(t, CloseEnuf(fv, 123.456), "floatsafe value %v", fv) fv, ok = jh.Float64Safe("float64str") assert.True(t, ok, "floatsafe ok") assert.True(t, CloseEnuf(fv, 123.456), "floatsafe value %v", fv) fv = jh.Float64("float64str") assert.True(t, CloseEnuf(fv, 123.456), "floatsafe value %v", fv) fv, ok = jh.Float64Safe("float64null") assert.True(t, ok, "float64null ok") assert.True(t, math.IsNaN(fv), "float64null expected Nan but got %v", fv) fv = jh.Float64("float64null") assert.True(t, math.IsNaN(fv), "float64null expected Nan but got %v", fv) jhm := jh.Helpers("nested2") assert.True(t, len(jhm) == 1, "get list of helpers") assert.True(t, jhm[0].Int("sub") == 2, "Should get list of helpers") } func TestJsonCoercion(t *testing.T) { assert.True(t, jh.Int("intstr") == 1, "get string as int %s", jh.String("intstr")) assert.True(t, jh.String("int") == "1", "get int as string %s", jh.String("int")) assert.True(t, jh.Int("notint") == -1, "get non existent int = 0??? ") } func TestJsonPathNotation(t *testing.T) { // Now lets test xpath type syntax assert.True(t, jh.Int("/MaxSize") == 1048576, "get int, test capitalization? ") assert.True(t, jh.String("/nested/nest") == "string2", "should get string %s", jh.String("/nested/nest")) assert.True(t, jh.String("/nested/list[0]") == "value", "get string from array") // note this one has period in name assert.True(t, jh.String("/period.name") == "value", "test period in name ") } func TestFromReader(t *testing.T) { raw := `{"testing": 123}` reader := strings.NewReader(raw) jh, err := NewJsonHelperReader(reader) assert.True(t, err == nil, "Unexpected error decoding json: %s", err) assert.True(t, jh.Int("testing") == 123, "Unexpected value in json: %d", jh.Int("testing")) } func TestJsonHelperGobEncoding(t *testing.T) { raw := `{"testing": 123,"name":"bob & more"}` reader := strings.NewReader(raw) jh, err := NewJsonHelperReader(reader) assert.True(t, err == nil, "Unexpected error decoding gob: %s", err) assert.True(t, jh.Int("testing") == 123, "Unexpected value in gob: %d", jh.Int("testing")) var buf bytes.Buffer err = gob.NewEncoder(&buf).Encode(&jh) assert.True(t, err == nil, err) var jhNew JsonHelper err = gob.NewDecoder(&buf).Decode(&jhNew) assert.True(t, err == nil, err) assert.True(t, jhNew.Int("testing") == 123, "Unexpected value in gob: %d", jhNew.Int("testing")) assert.True(t, jhNew.String("name") == "bob & more", "Unexpected value in gob: %d", jhNew.String("name")) buf2 := bytes.Buffer{} gt := GobTest{"Hello", jh} err = gob.NewEncoder(&buf2).Encode(>) assert.True(t, err == nil, err) var gt2 GobTest err = gob.NewDecoder(&buf2).Decode(>2) assert.True(t, err == nil, err) assert.True(t, gt2.Name == "Hello", "Unexpected value in gob: %d", gt2.Name) assert.True(t, gt2.Data.Int("testing") == 123, "Unexpected value in gob: %d", gt2.Data.Int("testing")) assert.True(t, gt2.Data.String("name") == "bob & more", "Unexpected value in gob: %d", gt2.Data.String("name")) } type GobTest struct { Name string Data JsonHelper } golang-github-araddon-gou-0.0~git20180509.7db4be5/log.go000066400000000000000000000351541327467556500223300ustar00rootroot00000000000000package gou import ( "context" "fmt" "log" "os" "runtime" "strings" "sync" "time" ) const ( NOLOGGING = -1 FATAL = 0 ERROR = 1 WARN = 2 INFO = 3 DEBUG = 4 ) /* https://github.com/mewkiz/pkg/tree/master/term RED = '\033[0;1;31m' GREEN = '\033[0;1;32m' YELLOW = '\033[0;1;33m' BLUE = '\033[0;1;34m' MAGENTA = '\033[0;1;35m' CYAN = '\033[0;1;36m' WHITE = '\033[0;1;37m' DARK_MAGENTA = '\033[0;35m' ANSI_RESET = '\033[0m' LogColor = map[int]string{FATAL: "\033[0m\033[37m", ERROR: "\033[0m\033[31m", WARN: "\033[0m\033[33m", INFO: "\033[0m\033[32m", DEBUG: "\033[0m\033[34m"} \e]PFdedede */ var ( LogLevel int = ERROR EMPTY struct{} ErrLogLevel int = ERROR logger *log.Logger customLogger LoggerCustom loggerErr *log.Logger LogColor = map[int]string{FATAL: "\033[0m\033[37m", ERROR: "\033[0m\033[31m", WARN: "\033[0m\033[33m", INFO: "\033[0m\033[35m", DEBUG: "\033[0m\033[34m"} LogPrefix = map[int]string{ FATAL: "[FATAL] ", ERROR: "[ERROR] ", WARN: "[WARN] ", INFO: "[INFO] ", DEBUG: "[DEBUG] ", } logContextKey = "log_prefix" escapeNewlines bool = false postFix = "" //\033[0m LogLevelWords map[string]int = map[string]int{"fatal": 0, "error": 1, "warn": 2, "info": 3, "debug": 4, "none": -1} logThrottles = make(map[string]*Throttler) throttleMu sync.Mutex ) // LoggerCustom defines custom interface for logger implementation type LoggerCustom interface { Log(depth, logLevel int, msg string, fields map[string]interface{}) } // Setup default logging to Stderr, equivalent to: // // gou.SetLogger(log.New(os.Stderr, "", log.Ltime|log.Lshortfile), "debug") func SetupLogging(lvl string) { SetLogger(log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile|log.Lmicroseconds), strings.ToLower(lvl)) } // Setup default logging to Stderr, equivalent to: // // gou.SetLogger(log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile|log.Lmicroseconds), level) func SetupLoggingLong(lvl string) { SetLogger(log.New(os.Stderr, "", log.LstdFlags|log.Llongfile|log.Lmicroseconds), strings.ToLower(lvl)) } // SetupLoggingFile writes logs to the file object parameter. func SetupLoggingFile(f *os.File, lvl string) { SetLogger(log.New(f, "", log.LstdFlags|log.Lshortfile|log.Lmicroseconds), strings.ToLower(lvl)) } // SetCustomLogger sets the logger to a custom logger func SetCustomLogger(cl LoggerCustom) { customLogger = cl } // GetCustomLogger returns the custom logger if initialized func GetCustomLogger() LoggerCustom { return customLogger } // Setup colorized output if this is a terminal func SetColorIfTerminal() { if IsTerminal() { SetColorOutput() } } // Setup colorized output func SetColorOutput() { for lvl, color := range LogColor { LogPrefix[lvl] = color } postFix = "\033[0m" } //Set whether to escape newline characters in log messages func SetEscapeNewlines(en bool) { escapeNewlines = en } // Setup default log output to go to a dev/null // // log.SetOutput(new(DevNull)) func DiscardStandardLogger() { log.SetOutput(new(DevNull)) } // you can set a logger, and log level,most common usage is: // // gou.SetLogger(log.New(os.Stdout, "", log.LstdFlags), "debug") // // loglevls: debug, info, warn, error, fatal // Note, that you can also set a separate Error Log Level func SetLogger(l *log.Logger, logLevel string) { logger = l LogLevelSet(logLevel) } func GetLogger() *log.Logger { return logger } // you can set a logger, and log level. this is for errors, and assumes // you are logging to Stderr (seperate from stdout above), allowing you to seperate // debug&info logging from errors // // gou.SetLogger(log.New(os.Stderr, "", log.LstdFlags), "debug") // // loglevls: debug, info, warn, error, fatal func SetErrLogger(l *log.Logger, logLevel string) { loggerErr = l if lvl, ok := LogLevelWords[logLevel]; ok { ErrLogLevel = lvl } } func GetErrLogger() *log.Logger { return logger } // sets the log level from a string func LogLevelSet(levelWord string) { if lvl, ok := LogLevelWords[levelWord]; ok { LogLevel = lvl } } // NewContext returns a new Context carrying contextual log message // that gets prefixed to log statements. func NewContext(ctx context.Context, msg string) context.Context { return context.WithValue(ctx, logContextKey, msg) } // NewContextWrap returns a new Context carrying contextual log message // that gets prefixed to log statements. func NewContextWrap(ctx context.Context, msg string) context.Context { logContext, ok := ctx.Value(logContextKey).(string) if ok { return context.WithValue(ctx, logContextKey, fmt.Sprintf("%s %s", logContext, msg)) } return context.WithValue(ctx, logContextKey, msg) } // FromContext extracts the Log Context prefix from context func FromContext(ctx context.Context) string { logContext, _ := ctx.Value(logContextKey).(string) return logContext } // Log at debug level func Debug(v ...interface{}) { if LogLevel >= 4 { DoLog(3, DEBUG, fmt.Sprint(v...)) } } // Debug log formatted func Debugf(format string, v ...interface{}) { if LogLevel >= 4 { DoLog(3, DEBUG, fmt.Sprintf(format, v...)) } } // Debug log formatted context writer func DebugCtx(ctx context.Context, format string, v ...interface{}) { if LogLevel >= 4 { lc := FromContext(ctx) if len(lc) > 0 { format = fmt.Sprintf("%s %s", lc, format) } DoLog(3, DEBUG, fmt.Sprintf(format, v...)) } } func DebugT(lineCt int) { if LogLevel >= 4 { DoLog(3, DEBUG, fmt.Sprint("\n", PrettyStack(lineCt))) } } // Log at info level func Info(v ...interface{}) { if LogLevel >= 3 { DoLog(3, INFO, fmt.Sprint(v...)) } } // info log formatted func Infof(format string, v ...interface{}) { if LogLevel >= 3 { DoLog(3, INFO, fmt.Sprintf(format, v...)) } } // Info log formatted context writer func InfoCtx(ctx context.Context, format string, v ...interface{}) { if LogLevel >= 3 { lc := FromContext(ctx) if len(lc) > 0 { format = fmt.Sprintf("%s %s", lc, format) } DoLog(3, INFO, fmt.Sprintf(format, v...)) } } // Info Trace func InfoT(lineCt int) { if LogLevel >= 3 { DoLog(3, INFO, fmt.Sprint("\n", PrettyStack(lineCt))) } } // Log at warn level func Warn(v ...interface{}) { if LogLevel >= 2 { DoLog(3, WARN, fmt.Sprint(v...)) } } // Warn log formatted func Warnf(format string, v ...interface{}) { if LogLevel >= 2 { DoLog(3, WARN, fmt.Sprintf(format, v...)) } } // Warn log formatted context writer func WarnCtx(ctx context.Context, format string, v ...interface{}) { if LogLevel >= 2 { lc := FromContext(ctx) if len(lc) > 0 { format = fmt.Sprintf("%s %s", lc, format) } DoLog(3, WARN, fmt.Sprintf(format, v...)) } } // Warn Trace func WarnT(lineCt int) { if LogLevel >= 2 { DoLog(3, WARN, fmt.Sprint("\n", PrettyStack(lineCt))) } } // Log at error level func Error(v ...interface{}) { if LogLevel >= 1 { DoLog(3, ERROR, fmt.Sprint(v...)) } } // Error log formatted func Errorf(format string, v ...interface{}) { if LogLevel >= 1 { DoLog(3, ERROR, fmt.Sprintf(format, v...)) } } // Error log formatted context writer func ErrorCtx(ctx context.Context, format string, v ...interface{}) { if LogLevel >= 1 { lc := FromContext(ctx) if len(lc) > 0 { format = fmt.Sprintf("%s %s", lc, format) } DoLog(3, ERROR, fmt.Sprintf(format, v...)) } } // Log this error, and return error object func LogErrorf(format string, v ...interface{}) error { err := fmt.Errorf(format, v...) if LogLevel >= 1 { DoLog(3, ERROR, err.Error()) } return err } // Log to logger if setup // // Log(ERROR, "message") func Log(logLvl int, v ...interface{}) { if LogLevel >= logLvl { DoLog(3, logLvl, fmt.Sprint(v...)) } } // Log to logger if setup, grab a stack trace and add that as well // // u.LogTracef(u.ERROR, "message %s", varx) // func LogTracef(logLvl int, format string, v ...interface{}) { if LogLevel >= logLvl { // grab a stack trace stackBuf := make([]byte, 6000) stackBufLen := runtime.Stack(stackBuf, false) stackTraceStr := string(stackBuf[0:stackBufLen]) parts := strings.Split(stackTraceStr, "\n") if len(parts) > 1 { v = append(v, strings.Join(parts[3:], "\n")) } DoLog(3, logLvl, fmt.Sprintf(format+"\n%v", v...)) } } // Log to logger if setup, grab a stack trace and add that as well // // u.LogTracef(u.ERROR, "message %s", varx) // func LogTraceDf(logLvl, lineCt int, format string, v ...interface{}) { if LogLevel >= logLvl { // grab a stack trace stackBuf := make([]byte, 6000) stackBufLen := runtime.Stack(stackBuf, false) stackTraceStr := string(stackBuf[0:stackBufLen]) parts := strings.Split(stackTraceStr, "\n") if len(parts) > 1 { if (len(parts) - 3) > lineCt { parts = parts[3 : 3+lineCt] parts2 := make([]string, 0, len(parts)/2) for i := 1; i < len(parts); i = i + 2 { parts2 = append(parts2, parts[i]) } v = append(v, strings.Join(parts2, "\n")) //v = append(v, strings.Join(parts[3:3+lineCt], "\n")) } else { v = append(v, strings.Join(parts[3:], "\n")) } } DoLog(3, logLvl, fmt.Sprintf(format+"\n%v", v...)) } } func PrettyStack(lineCt int) string { stackBuf := make([]byte, 10000) stackBufLen := runtime.Stack(stackBuf, false) stackTraceStr := string(stackBuf[0:stackBufLen]) parts := strings.Split(stackTraceStr, "\n") if len(parts) > 3 { parts = parts[2:] parts2 := make([]string, 0, len(parts)/2) for i := 3; i < len(parts)-1; i++ { if !strings.HasSuffix(parts[i], ")") && !strings.HasPrefix(parts[i], "/usr/local") { parts2 = append(parts2, parts[i]) } } if len(parts2) > lineCt { return strings.Join(parts2[0:lineCt], "\n") } return strings.Join(parts2, "\n") } return stackTraceStr } // Throttle logging based on key, such that key would never occur more than // @limit times per hour // // LogThrottleKey(u.ERROR, 1,"error_that_happens_a_lot" "message %s", varx) // func LogThrottleKey(logLvl, limit int, key, format string, v ...interface{}) { if LogLevel >= logLvl { throttleMu.Lock() th, ok := logThrottles[key] if !ok { th = NewThrottler(limit, 3600*time.Second) logThrottles[key] = th } skip, throttleCount := th.Throttle() if skip { throttleMu.Unlock() return } throttleMu.Unlock() prefix := "" if throttleCount > 0 { prefix = fmt.Sprintf("LogsThrottled[%d] ", throttleCount) } DoLog(3, logLvl, prefix+fmt.Sprintf(format, v...)) } } // Throttle logging based on @format as a key, such that key would never occur more than // @limit times per hour // // LogThrottle(u.ERROR, 1, "message %s", varx) // func LogThrottle(logLvl, limit int, format string, v ...interface{}) { if LogLevel >= logLvl { throttleMu.Lock() th, ok := logThrottles[format] if !ok { th = NewThrottler(limit, 3600*time.Second) logThrottles[format] = th } var throttleCount int32 skip, throttleCount := th.Throttle() if skip { throttleMu.Unlock() return } throttleMu.Unlock() prefix := "" if throttleCount > 0 { prefix = fmt.Sprintf("LogsThrottled[%d] ", throttleCount) } DoLog(3, logLvl, prefix+fmt.Sprintf(format, v...)) } } // Throttle logging based on @format as a key, such that key would never occur more than // @limit times per hour // // LogThrottleD(5, u.ERROR, 1, "message %s", varx) // func LogThrottleD(depth, logLvl, limit int, format string, v ...interface{}) { if LogLevel >= logLvl { throttleMu.Lock() th, ok := logThrottles[format] if !ok { th = NewThrottler(limit, 3600*time.Second) logThrottles[format] = th } skip, throttleCount := th.Throttle() if skip { throttleMu.Unlock() return } throttleMu.Unlock() prefix := fmt.Sprintf("Log Throttled[%d] ", throttleCount) DoLog(depth, logLvl, prefix+fmt.Sprintf(format, v...)) } } // Log to logger if setup // Logf(ERROR, "message %d", 20) func Logf(logLvl int, format string, v ...interface{}) { if LogLevel >= logLvl { DoLog(3, logLvl, fmt.Sprintf(format, v...)) } } func LogFieldsf(logLvl int, fields map[string]interface{}, format string, v ...interface{}) { if LogLevel >= logLvl { DoLogFields(3, logLvl, fmt.Sprintf(format, v...), fields) } } // Log to logger if setup // LogP(ERROR, "prefix", "message", anyItems, youWant) func LogP(logLvl int, prefix string, v ...interface{}) { if ErrLogLevel >= logLvl && loggerErr != nil { loggerErr.Output(3, prefix+LogPrefix[logLvl]+fmt.Sprint(v...)+postFix) } else if LogLevel >= logLvl && logger != nil { logger.Output(3, prefix+LogPrefix[logLvl]+fmt.Sprint(v...)+postFix) } } // Log to logger if setup with a prefix // LogPf(ERROR, "prefix", "formatString %s %v", anyItems, youWant) func LogPf(logLvl int, prefix string, format string, v ...interface{}) { if ErrLogLevel >= logLvl && loggerErr != nil { loggerErr.Output(3, prefix+LogPrefix[logLvl]+fmt.Sprintf(format, v...)+postFix) } else if LogLevel >= logLvl && logger != nil { logger.Output(3, prefix+LogPrefix[logLvl]+fmt.Sprintf(format, v...)+postFix) } } // When you want to use the log short filename flag, and want to use // the lower level logging functions (say from an *Assert* type function) // you need to modify the stack depth: // // func init() {} // SetLogger(log.New(os.Stderr, "", log.Ltime|log.Lshortfile|log.Lmicroseconds), lvl) // } // // func assert(t *testing.T, myData) { // // we want log line to show line that called this assert, not this line // LogD(5, DEBUG, v...) // } func LogD(depth int, logLvl int, v ...interface{}) { if LogLevel >= logLvl { DoLog(depth, logLvl, fmt.Sprint(v...)) } } // Low level log with depth , level, message and logger func DoLog(depth, logLvl int, msg string) { DoLogFields(depth+1, logLvl, msg, nil) } // DoLogFields allows the inclusion of additional context for logrus logs // file and line number are included in the fields by default func DoLogFields(depth, logLvl int, msg string, fields map[string]interface{}) { if escapeNewlines { msg = EscapeNewlines(msg) } if customLogger == nil { // Use standard logger if ErrLogLevel >= logLvl && loggerErr != nil { loggerErr.Output(depth, LogPrefix[logLvl]+msg+postFix) } else if LogLevel >= logLvl && logger != nil { logger.Output(depth, LogPrefix[logLvl]+msg+postFix) } } else { customLogger.Log(depth, logLvl, msg, fields) } } type winsize struct { Row uint16 Col uint16 Xpixel uint16 Ypixel uint16 } const ( _TIOCGWINSZ = 0x5413 // OSX 1074295912 ) // http://play.golang.org/p/5LIA41Iqfp // Dummy discard, satisfies io.Writer without importing io or os. type DevNull struct{} func (DevNull) Write(p []byte) (int, error) { return len(p), nil } // Replace standard newline characters with escaped newlines so long msgs will // remain one line. func EscapeNewlines(str string) string { return strings.Replace(str, "\n", "\\n", -1) } golang-github-araddon-gou-0.0~git20180509.7db4be5/log_test.go000066400000000000000000000015051327467556500233600ustar00rootroot00000000000000package gou import ( "io/ioutil" "os" "strings" "testing" "time" ) func TestSetupLogToFile(t *testing.T) { tmpf, err := ioutil.TempFile("", "goutest") if err != nil { t.Fatalf("error creating log file: %v\n", err) } defer os.Remove(tmpf.Name()) SetupLoggingFile(tmpf, "debug") logStr := "hihi" Infof(logStr) // Flush file buffer to disk err = tmpf.Sync() if err != nil { t.Errorf("error syncing tmpf: %v", err) } time.Sleep(1 * time.Second) // Read tmp file and confirm log message was written bytes, err := ioutil.ReadFile(tmpf.Name()) if err != nil { t.Errorf("error reading temp file[%s]: %v\n", tmpf.Name(), err) } logFileBytes := string(bytes) if !strings.Contains(logFileBytes, logStr) { t.Logf("logfile:\n%s", logFileBytes) t.Errorf("%s not found in logfile %s\n", logStr, tmpf.Name()) } } golang-github-araddon-gou-0.0~git20180509.7db4be5/log_unix.go000066400000000000000000000007461327467556500233720ustar00rootroot00000000000000// +build !windows package gou import ( "syscall" "unsafe" ) // Determine is this process is running in a Terminal or not? func IsTerminal() bool { ws := &winsize{} isTerm := true defer func() { if r := recover(); r != nil { isTerm = false } }() // This blows up on windows retCode, _, _ := syscall.Syscall(syscall.SYS_IOCTL, uintptr(syscall.Stdin), uintptr(_TIOCGWINSZ), uintptr(unsafe.Pointer(ws))) if int(retCode) == -1 { return false } return isTerm } golang-github-araddon-gou-0.0~git20180509.7db4be5/log_windows.go000066400000000000000000000002671327467556500240770ustar00rootroot00000000000000// +build windows package gou // Determine is this process is running in a Terminal or not? func IsTerminal() bool { return false // TODO Needs correct implementation on Windows } golang-github-araddon-gou-0.0~git20180509.7db4be5/testutil.go000066400000000000000000000026231327467556500234170ustar00rootroot00000000000000package gou import ( "os" "time" ) var ( //finished chan bool lastTest time.Time = time.Now() stopper func() = func() {} ) // Wait for condition (defined by func) to be true // this is mostly for testing, but a utility to // create a ticker checking back every 100 ms to see // if something (the supplied check func) is done // // WaitFor(func() bool { // return ctr.Ct == 0 // },10) // timeout (in seconds) is the last arg func WaitFor(check func() bool, timeoutSecs int) { timer := time.NewTicker(100 * time.Millisecond) for { select { case <-timer.C: if check() { timer.Stop() return } case <-time.After(time.Duration(timeoutSecs) * time.Second): return } } } // Use this in combo with StopCheck() for test functions that must start // processes such as func SetStopper(f func()) { stopper = f } // take two floats, compare, need to be within 2% func CloseEnuf(a, b float64) bool { c := a / b if c > .98 && c < 1.02 { return true } return false } // take two ints, compare, need to be within 5% func CloseInt(a, b int) bool { c := float64(a) / float64(b) if c >= .95 && c <= 1.05 { return true } return false } func StartTest() { lastTest = time.Now() } func StopCheck() { t := time.Now() if lastTest.Add(time.Millisecond*1000).UnixNano() < t.UnixNano() { Log(INFO, "Stopping Test ", lastTest.Unix()) //finished <- true stopper() os.Exit(0) } } golang-github-araddon-gou-0.0~git20180509.7db4be5/testutil_test.go000066400000000000000000000004701327467556500244540ustar00rootroot00000000000000package gou import ( "testing" "time" ) func TestWaitFor(t *testing.T) { isDone := false foundDone := false go func() { time.Sleep(time.Second * 1) isDone = true }() WaitFor(func() bool { if isDone == true { foundDone = true } return isDone == true }, 2) if !foundDone { t.Fail() } } golang-github-araddon-gou-0.0~git20180509.7db4be5/throttle.go000066400000000000000000000035101327467556500234030ustar00rootroot00000000000000package gou import ( "time" ) type Throttler struct { // Limit to this events/per maxPer float64 per float64 count int32 // Last Event last time.Time // How many events are allowed left to happen? // Starts at limit, decrements down allowance float64 } // new Throttler that will tell you to limit or not based // on given @max events @per duration func NewThrottler(max int, per time.Duration) *Throttler { return &Throttler{ maxPer: float64(max), allowance: float64(max), count: int32(0), last: time.Now(), per: per.Seconds(), } } // Should we limit this because we are above rate? // Returns a bool of whether to throttle the message, and a count // of previous log messages throttled since last log message. func (r *Throttler) ThrottleAdd(ct int32) (bool, int32) { if r.maxPer == 0 { return false, 0 } // http://stackoverflow.com/questions/667508/whats-a-good-rate-limiting-algorithm now := time.Now() elapsed := float64(now.Sub(r.last).Nanoseconds()) / 1e9 // seconds r.last = now r.allowance += elapsed * (r.maxPer / r.per) //Infof("maxRate: %v cur: %v elapsed:%-6.6f incr: %v", r.maxPer, int(r.allowance), elapsed, elapsed*float64(r.maxPer)) if r.allowance > r.maxPer { r.allowance = r.maxPer } if r.allowance < 1.0 { r.count += ct // increment throttled log count return true, r.count // do throttle/limit } tmpCount := r.count r.count = 0 // reset count r.allowance -= 1.0 return false, tmpCount // dont throttle, return previous throttle count } // Should we limit this because we are above rate? // Returns a bool of whether to throttle the message, and a count // of previous log messages throttled since last log message. func (r *Throttler) Throttle() (bool, int32) { return r.ThrottleAdd(1) } func (r *Throttler) ThrottleCount() int32 { return r.count } golang-github-araddon-gou-0.0~git20180509.7db4be5/throttle_test.go000066400000000000000000000033601327467556500244450ustar00rootroot00000000000000package gou import ( "testing" "time" "github.com/stretchr/testify/assert" ) func TestThrottleer(t *testing.T) { th := NewThrottler(10, 10*time.Second) for i := 0; i < 10; i++ { thb, tc := th.Throttle() assert.True(t, thb == false, "Should not throttle %v", i) assert.True(t, tc < 10, "Throttle count should remain below 10 %v", tc) time.Sleep(time.Millisecond * 10) } throttled := 0 th = NewThrottler(10, 1*time.Second) // We are going to loop 20 times, first 10 should make it, next 10 throttled for i := 0; i < 20; i++ { LogThrottleKey(WARN, 10, "throttle", "hello %v", i) thb, tc := th.Throttle() if thb { throttled += 1 assert.True(t, int(tc) == i-9, "Throttle count should rise %v, i: %d", tc, i) } } assert.True(t, throttled == 10, "Should throttle 10 of 20 requests: %v", throttled) // Now sleep for 1 second so that we should // no longer be throttled time.Sleep(time.Second * 2) thb, _ := th.Throttle() assert.True(t, thb == false, "We should not have been throttled") } func TestThrottler2(t *testing.T) { th := NewThrottler(10, 1*time.Second) tkey := "throttle2" throttleMu.Lock() logThrottles[tkey] = th throttleMu.Unlock() th, ok := logThrottles[tkey] if !ok { t.Errorf("Throttle key %s not created!", tkey) } // We are going to loop 20 times, first 10 should make it, next 10 throttled for i := 0; i < 20; i++ { LogThrottleKey(WARN, 10, tkey, "hello %v", i) } throttleMu.Lock() th = logThrottles[tkey] tcount := th.ThrottleCount() assert.True(t, tcount == 10, "Should throttle 10 of 20 requests: %v", tcount) throttleMu.Unlock() // Now sleep for 1 second so that we should // no longer be throttled time.Sleep(time.Second * 1) LogThrottleKey(WARN, 10, tkey, "hello again %v", 20) } golang-github-araddon-gou-0.0~git20180509.7db4be5/uid.go000066400000000000000000000037451327467556500223310ustar00rootroot00000000000000package gou import ( "crypto/md5" "crypto/rand" "encoding/binary" "fmt" "io" "os" "strconv" "sync/atomic" "time" ) const ( //2013-2-3 ourEpoch = uint32(1359931242) ) func init() { initHostPidId() } /* Special thanks to ideas from Mgo, and Noeqd, this is somewhat inbetween them https://github.com/bmizerany/noeqd It is a roughly sortable UID, but uses machine specific info (host, processid) as part of the uid so each machine *will* have unique id's The host+processid is 3 bytes */ // uidCounter is an atomically incremented each time we created // a new uid within given ms time window var uidCounter uint32 = 0 // hostPidId stores the generated hostPid var hostPidId []byte // initHostPidId generates a machine-process specific id by using hostname // and processid func initHostPidId() { var sum [4]byte hostB := sum[:] host, err := os.Hostname() if err != nil { // if we cannot get hostname, just use a random set of bytes _, err2 := io.ReadFull(rand.Reader, hostB) if err2 != nil { panic(fmt.Errorf("cannot get hostname: %v; %v", err, err2)) } } else { hw := md5.New() hw.Write([]byte(host)) copy(hostB, hw.Sum(nil)) } pid := os.Getpid() hostI := binary.BigEndian.Uint32(hostB) uid := uint32(pid) + uint32(hostI) binary.BigEndian.PutUint32(hostB, uid) b := make([]byte, 4) binary.BigEndian.PutUint32(b, uid) hostPidId = b[:] } // uid is a 64 bit int uid type Uid uint64 // Create a new uint64 unique id func NewUid() uint64 { b := make([]byte, 8) ts := uint32(time.Now().Unix()) - ourEpoch // Timestamp, 4 bytes, big endian binary.BigEndian.PutUint32(b, ts) //Debugf("ts=%v b=%v", ts, b) // first 3 bytes of host/pid b[4] = hostPidId[2] b[5] = hostPidId[3] b[6] = hostPidId[3] // Increment, 2 bytes, big endian i := atomic.AddUint32(&uidCounter, 1) //b[6] = byte(i >> 8) b[7] = byte(i) ui := binary.BigEndian.Uint64(b) //Debugf("ui=%d b=%v ", ui, b) return ui } func (u *Uid) String() string { return strconv.FormatUint(uint64(*u), 10) } golang-github-araddon-gou-0.0~git20180509.7db4be5/uid_test.go000066400000000000000000000001551327467556500233600ustar00rootroot00000000000000package gou import ( "testing" ) func TestUid(t *testing.T) { u := NewUid() Debug(u) Debug(NewUid()) }