pax_global_header00006660000000000000000000000064131626535600014521gustar00rootroot0000000000000052 comment=dbc1e22735aa6ed7bd9579a407c17bc7c4a4e046 logrus-logstash-hook-0.4.1/000077500000000000000000000000001316265356000156165ustar00rootroot00000000000000logrus-logstash-hook-0.4.1/.gitignore000066400000000000000000000004261316265356000176100ustar00rootroot00000000000000# Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test .idea # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.test *.prof *.iml logrus-logstash-hook-0.4.1/.travis.yml000066400000000000000000000003721316265356000177310ustar00rootroot00000000000000language: go sudo: false matrix: include: - go: 1.3 - go: 1.4 - go: 1.5 - go: 1.6 - go: tip install: - # Skip script: - go get -t -v ./... - diff -u <(echo -n) <(gofmt -d .) - go tool vet . - go test -v -race ./... logrus-logstash-hook-0.4.1/CHANGELOG.md000066400000000000000000000006701316265356000174320ustar00rootroot00000000000000# Changelog ## 0.4 * Update the name of the package from `logrus_logstash` to `logrustash` * Add TimeFormat to Hook * Replace the old logrus package path: `github.com/Sirupsen/logrus` with `github.com/sirupsen/logrus` ## 0.3 * Fix the Logstash format to set `@version` to `"1"` * Add unit-tests to logstash.go * Remove the assert package * Add prefix filtering ## Before that (major changes) * Update LICENSE to MIT from GPL logrus-logstash-hook-0.4.1/LICENSE000066400000000000000000000020661316265356000166270ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2016 Boaz Shuster Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.logrus-logstash-hook-0.4.1/README.md000066400000000000000000000056461316265356000171100ustar00rootroot00000000000000# Logstash hook for logrus :walrus: [![Build Status](https://travis-ci.org/bshuster-repo/logrus-logstash-hook.svg?branch=master)](https://travis-ci.org/bshuster-repo/logrus-logstash-hook) Use this hook to send the logs to [Logstash](https://www.elastic.co/products/logstash) over both UDP and TCP. ## Usage ```go package main import ( "github.com/sirupsen/logrus" "github.com/bshuster-repo/logrus-logstash-hook" ) func main() { log := logrus.New() hook, err := logrustash.NewHook("tcp", "172.17.0.2:9999", "myappName") if err != nil { log.Fatal(err) } log.Hooks.Add(hook) ctx := log.WithFields(logrus.Fields{ "method": "main", }) ... ctx.Info("Hello World!") } ``` This is how it will look like: ```ruby { "@timestamp" => "2016-02-29T16:57:23.000Z", "@version" => "1", "level" => "info", "message" => "Hello World!", "method" => "main", "host" => "172.17.0.1", "port" => 45199, "type" => "myappName" } ``` ## Hook Fields Fields can be added to the hook, which will always be in the log context. This can be done when creating the hook: ```go hook, err := logrustash.NewHookWithFields("tcp", "172.17.0.2:9999", "myappName", logrus.Fields{ "hostname": os.Hostname(), "serviceName": "myServiceName", }) ``` Or afterwards: ```go hook.WithFields(logrus.Fields{ "hostname": os.Hostname(), "serviceName": "myServiceName", }) ``` This allows you to set up the hook so logging is available immediately, and add important fields as they become available. Single fields can be added/updated using 'WithField': ```go hook.WithField("status", "running") ``` ## Field prefix The hook allows you to send logging to logstash and also retain the default std output in text format. However to keep this console output readable some fields might need to be omitted from the default non-hooked log output. Each hook can be configured with a prefix used to identify fields which are only to be logged to the logstash connection. For example if you don't want to see the hostname and serviceName on each log line in the console output you can add a prefix: ```go hook, err := logrustash.NewHookWithFields("tcp", "172.17.0.2:9999", "myappName", logrus.Fields{ "_hostname": os.Hostname(), "_serviceName": "myServiceName", }) ... hook.WithPrefix("_") ``` There are also constructors available which allow you to specify the prefix from the start. The std-out will not have the '\_hostname' and '\_servicename' fields, and the logstash output will, but the prefix will be dropped from the name. # Authors Name | Github | Twitter | ------------ | --------- | ---------- | Boaz Shuster | ripcurld0 | @ripcurld0 | # License MIT. logrus-logstash-hook-0.4.1/logstash.go000066400000000000000000000103111316265356000177650ustar00rootroot00000000000000package logrustash import ( "net" "strings" "github.com/sirupsen/logrus" ) // Hook represents a connection to a Logstash instance type Hook struct { conn net.Conn appName string alwaysSentFields logrus.Fields hookOnlyPrefix string TimeFormat string } // NewHook creates a new hook to a Logstash instance, which listens on // `protocol`://`address`. func NewHook(protocol, address, appName string) (*Hook, error) { return NewHookWithFields(protocol, address, appName, make(logrus.Fields)) } // NewHookWithConn creates a new hook to a Logstash instance, using the supplied connection func NewHookWithConn(conn net.Conn, appName string) (*Hook, error) { return NewHookWithFieldsAndConn(conn, appName, make(logrus.Fields)) } // NewHookWithFields creates a new hook to a Logstash instance, which listens on // `protocol`://`address`. alwaysSentFields will be sent with every log entry. func NewHookWithFields(protocol, address, appName string, alwaysSentFields logrus.Fields) (*Hook, error) { return NewHookWithFieldsAndPrefix(protocol, address, appName, alwaysSentFields, "") } // NewHookWithFieldsAndPrefix creates a new hook to a Logstash instance, which listens on // `protocol`://`address`. alwaysSentFields will be sent with every log entry. prefix is used to select fields to filter func NewHookWithFieldsAndPrefix(protocol, address, appName string, alwaysSentFields logrus.Fields, prefix string) (*Hook, error) { conn, err := net.Dial(protocol, address) if err != nil { return nil, err } return NewHookWithFieldsAndConnAndPrefix(conn, appName, alwaysSentFields, prefix) } // NewHookWithFieldsAndConn creates a new hook to a Logstash instance using the supplied connection func NewHookWithFieldsAndConn(conn net.Conn, appName string, alwaysSentFields logrus.Fields) (*Hook, error) { return NewHookWithFieldsAndConnAndPrefix(conn, appName, alwaysSentFields, "") } //NewHookWithFieldsAndConnAndPrefix creates a new hook to a Logstash instance using the suppolied connection and prefix func NewHookWithFieldsAndConnAndPrefix(conn net.Conn, appName string, alwaysSentFields logrus.Fields, prefix string) (*Hook, error) { return &Hook{conn: conn, appName: appName, alwaysSentFields: alwaysSentFields, hookOnlyPrefix: prefix}, nil } //NewFilterHook makes a new hook which does not forward to logstash, but simply enforces the prefix rules func NewFilterHook() *Hook { return NewFilterHookWithPrefix("") } //NewFilterHookWithPrefix make a new hook which does not forward to logstash, but simply enforces the specified prefix func NewFilterHookWithPrefix(prefix string) *Hook { return &Hook{conn: nil, appName: "", alwaysSentFields: make(logrus.Fields), hookOnlyPrefix: prefix} } func (h *Hook) filterHookOnly(entry *logrus.Entry) { if h.hookOnlyPrefix != "" { for key := range entry.Data { if strings.HasPrefix(key, h.hookOnlyPrefix) { delete(entry.Data, key) } } } } //WithPrefix sets a prefix filter to use in all subsequent logging func (h *Hook) WithPrefix(prefix string) { h.hookOnlyPrefix = prefix } func (h *Hook) WithField(key string, value interface{}) { h.alwaysSentFields[key] = value } func (h *Hook) WithFields(fields logrus.Fields) { //Add all the new fields to the 'alwaysSentFields', possibly overwriting exising fields for key, value := range fields { h.alwaysSentFields[key] = value } } func (h *Hook) Fire(entry *logrus.Entry) error { //make sure we always clear the hookonly fields from the entry defer h.filterHookOnly(entry) // Add in the alwaysSentFields. We don't override fields that are already set. for k, v := range h.alwaysSentFields { if _, inMap := entry.Data[k]; !inMap { entry.Data[k] = v } } //For a filteringHook, stop here if h.conn == nil { return nil } formatter := LogstashFormatter{Type: h.appName} if h.TimeFormat != "" { formatter.TimestampFormat = h.TimeFormat } dataBytes, err := formatter.FormatWithPrefix(entry, h.hookOnlyPrefix) if err != nil { return err } if _, err = h.conn.Write(dataBytes); err != nil { return err } return nil } func (h *Hook) Levels() []logrus.Level { return []logrus.Level{ logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel, logrus.DebugLevel, } } logrus-logstash-hook-0.4.1/logstash_formatter.go000066400000000000000000000033521316265356000220570ustar00rootroot00000000000000package logrustash import ( "encoding/json" "fmt" "strings" "time" "github.com/sirupsen/logrus" ) // Formatter generates json in logstash format. // Logstash site: http://logstash.net/ type LogstashFormatter struct { Type string // if not empty use for logstash type field. // TimestampFormat sets the format used for timestamps. TimestampFormat string } func (f *LogstashFormatter) Format(entry *logrus.Entry) ([]byte, error) { return f.FormatWithPrefix(entry, "") } func (f *LogstashFormatter) FormatWithPrefix(entry *logrus.Entry, prefix string) ([]byte, error) { fields := make(logrus.Fields) for k, v := range entry.Data { //remvove the prefix when sending the fields to logstash if prefix != "" && strings.HasPrefix(k, prefix) { k = strings.TrimPrefix(k, prefix) } switch v := v.(type) { case error: // Otherwise errors are ignored by `encoding/json` // https://github.com/Sirupsen/logrus/issues/377 fields[k] = v.Error() default: fields[k] = v } } fields["@version"] = "1" timeStampFormat := f.TimestampFormat if timeStampFormat == "" { timeStampFormat = time.RFC3339 } fields["@timestamp"] = entry.Time.Format(timeStampFormat) // set message field v, ok := entry.Data["message"] if ok { fields["fields.message"] = v } fields["message"] = entry.Message // set level field v, ok = entry.Data["level"] if ok { fields["fields.level"] = v } fields["level"] = entry.Level.String() // set type field if f.Type != "" { v, ok = entry.Data["type"] if ok { fields["fields.type"] = v } fields["type"] = f.Type } serialized, err := json.Marshal(fields) if err != nil { return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) } return append(serialized, '\n'), nil } logrus-logstash-hook-0.4.1/logstash_formatter_test.go000066400000000000000000000031131316265356000231110ustar00rootroot00000000000000package logrustash import ( "bytes" "encoding/json" "fmt" "net/url" "testing" "github.com/sirupsen/logrus" ) func TestLogstashFormatter(t *testing.T) { lf := LogstashFormatter{Type: "abc"} fields := logrus.Fields{ "message": "def", "level": "ijk", "type": "lmn", "one": 1, "pi": 3.14, "bool": true, "error": &url.Error{Op: "Get", URL: "http://example.com", Err: fmt.Errorf("The error")}, } entry := logrus.WithFields(fields) entry.Message = "msg" entry.Level = logrus.InfoLevel b, _ := lf.Format(entry) var data map[string]interface{} dec := json.NewDecoder(bytes.NewReader(b)) dec.UseNumber() dec.Decode(&data) // base fields if data["@timestamp"] == "" { t.Error("expected @timestamp to be not empty") } tt := []struct { expected string key string }{ // base fields {"1", "@version"}, {"abc", "type"}, {"msg", "message"}, {"info", "level"}, {"Get http://example.com: The error", "error"}, // substituted fields {"def", "fields.message"}, {"ijk", "fields.level"}, {"lmn", "fields.type"}, } for _, te := range tt { if te.expected != data[te.key] { t.Errorf("expected data[%s] to be '%s' but got '%s'", te.key, te.expected, data[te.key]) } } // formats if json.Number("1") != data["one"] { t.Errorf("expected one to be '%v' but got '%v'", json.Number("1"), data["one"]) } if json.Number("3.14") != data["pi"] { t.Errorf("expected pi to be '%v' but got '%v'", json.Number("3.14"), data["pi"]) } if true != data["bool"] { t.Errorf("expected bool to be '%v' but got '%v'", true, data["bool"]) } } logrus-logstash-hook-0.4.1/logstash_test.go000066400000000000000000000200341316265356000210270ustar00rootroot00000000000000package logrustash import ( "bytes" "encoding/json" "fmt" "net" "reflect" "testing" "time" "github.com/sirupsen/logrus" ) func TestLegostashHook(t *testing.T) { type Expct struct { appName string hookOnlyPrefix string alwaysSentFields logrus.Fields } tt := []struct { expected Expct initFunc func() (*Hook, error) }{ {Expct{"bla", "", nil}, func() (*Hook, error) { return NewHook("udp", "localhost:9999", "bla") }}, {Expct{"bzz", "", nil}, func() (*Hook, error) { udpConn, err := net.Dial("udp", "localhost:9999") if err != nil { return nil, err } return NewHookWithConn(udpConn, "bzz") }}, {Expct{"blk", "", logrus.Fields{"id": 1}}, func() (*Hook, error) { return NewHookWithFields("udp", "localhost:9999", "blk", logrus.Fields{"id": 1}) }}, {Expct{"prefix", "-->", logrus.Fields{"id": 1}}, func() (*Hook, error) { return NewHookWithFieldsAndPrefix("udp", "localhost:9999", "prefix", logrus.Fields{"id": 1}, "-->") }}, {Expct{"fieldsconn", "", logrus.Fields{"id": 5}}, func() (*Hook, error) { udpConn, err := net.Dial("udp", "localhost:9999") if err != nil { return nil, err } return NewHookWithFieldsAndConn(udpConn, "fieldsconn", logrus.Fields{"id": 5}) }}, {Expct{"zz", "~~>", logrus.Fields{"id": "bal"}}, func() (*Hook, error) { udpConn, err := net.Dial("udp", "localhost:9999") if err != nil { return nil, err } return NewHookWithFieldsAndConnAndPrefix(udpConn, "zz", logrus.Fields{"id": "bal"}, "~~>") }}, } for _, te := range tt { h, err := te.initFunc() if err != nil { t.Error(err) } if h == nil { t.Error("expected hook to be not nil") } if h.conn == nil { t.Error("expected conn to be not nil") } if h.appName != te.expected.appName { t.Errorf("expected appName to be '%s' but got '%s'", te.expected.appName, h.appName) } if h.alwaysSentFields == nil { t.Error("expected alwaysSentFields to be not nil") } if te.expected.alwaysSentFields != nil && !reflect.DeepEqual(te.expected.alwaysSentFields, h.alwaysSentFields) { t.Errorf("expected alwaysSentFields to be '%v' but got '%v'", te.expected.alwaysSentFields, h.alwaysSentFields) } if h.hookOnlyPrefix != te.expected.hookOnlyPrefix { t.Error("expected hookOnlyPrefix to be an empty string") } } } func TestNewFiltering(t *testing.T) { type Expct struct { prefix string appName string } tt := []struct { expected Expct initFunc func() *Hook }{ {Expct{"", ""}, func() *Hook { return NewFilterHook() }}, {Expct{"~~~>", ""}, func() *Hook { return NewFilterHookWithPrefix("~~~>") }}, } for _, te := range tt { h := te.initFunc() if h.conn != nil { t.Error("expected conn to be nil") } if h.alwaysSentFields == nil { t.Error("expected alwaysSentFields to be not nil") } if h.hookOnlyPrefix != te.expected.prefix { t.Errorf("expected prefix to be '%s' but got '%s'", te.expected.prefix, h.hookOnlyPrefix) } } } func TestSettingAttributes(t *testing.T) { tt := []struct { setFunc func(*Hook) expctFunc func(*Hook) error }{ {func(h *Hook) { h.WithPrefix("mprefix1") }, func(h *Hook) error { if h.hookOnlyPrefix != "mprefix1" { return fmt.Errorf("expected hookOnlyPrefix to be '%s' but got '%s'", "mprefix1", h.hookOnlyPrefix) } return nil }}, {func(h *Hook) { h.WithField("name", "muha") }, func(h *Hook) error { nField := logrus.Fields{"name": "muha"} if !reflect.DeepEqual(h.alwaysSentFields, nField) { return fmt.Errorf("expected hookOnlyPrefix to be '%s' but got '%s'", nField, h.hookOnlyPrefix) } return nil }}, {func(h *Hook) { h.WithFields(logrus.Fields{"filename": "app.log", "owner": "mick"}) }, func(h *Hook) error { nField := logrus.Fields{"name": "test-me!", "filename": "app.log", "owner": "mick"} if !reflect.DeepEqual(h.alwaysSentFields, nField) { return fmt.Errorf("expected hookOnlyPrefix to be '%s' but got '%s'", nField, h.hookOnlyPrefix) } return nil }}, } for _, te := range tt { hook := NewFilterHook() hook.alwaysSentFields = logrus.Fields{"name": "test-me!"} te.setFunc(hook) if err := te.expctFunc(hook); err != nil { t.Error(err) } } } func TestFilterHookOnly(t *testing.T) { tt := []struct { entry *logrus.Entry prefix string expected logrus.Fields }{ {&logrus.Entry{Data: logrus.Fields{"name": "slimshady"}}, "", logrus.Fields{"name": "slimshady"}}, {&logrus.Entry{Data: logrus.Fields{"_name": "slimshady", "nick": "blabla"}}, "_", logrus.Fields{"nick": "blabla"}}, } for _, te := range tt { hook := NewFilterHookWithPrefix(te.prefix) hook.filterHookOnly(te.entry) if !reflect.DeepEqual(te.entry.Data, te.expected) { t.Errorf("expected entry data to be '%v' but got '%v'", te.expected, te.entry.Data) } } } type AddrMock struct { } func (a AddrMock) Network() string { return "" } func (a AddrMock) String() string { return "" } type ConnMock struct { buff *bytes.Buffer } func (c ConnMock) Read(b []byte) (int, error) { return c.buff.Read(b) } func (c ConnMock) Write(b []byte) (int, error) { return c.buff.Write(b) } func (c ConnMock) Close() error { return nil } func (c ConnMock) LocalAddr() net.Addr { return AddrMock{} } func (c ConnMock) RemoteAddr() net.Addr { return AddrMock{} } func (c ConnMock) SetDeadline(t time.Time) error { return nil } func (c ConnMock) SetReadDeadline(t time.Time) error { return nil } func (c ConnMock) SetWriteDeadline(t time.Time) error { return nil } func TestFire(t *testing.T) { conn := ConnMock{buff: bytes.NewBufferString("")} hook := &Hook{ conn: conn, appName: "fire_test", alwaysSentFields: logrus.Fields{"test-name": "fire-test", "->ignore": "haaa", "override": "no"}, hookOnlyPrefix: "->", } entry := &logrus.Entry{ Message: "hello world!", Data: logrus.Fields{"override": "yes"}, Level: logrus.DebugLevel, } if err := hook.Fire(entry); err != nil { t.Error(err) } var res map[string]string if err := json.NewDecoder(conn.buff).Decode(&res); err != nil { t.Error(err) } expected := map[string]string{ "@timestamp": "0001-01-01T00:00:00Z", "@version": "1", "ignore": "haaa", "level": "debug", "message": "hello world!", "override": "yes", "test-name": "fire-test", "type": "fire_test", } if !reflect.DeepEqual(expected, res) { t.Errorf("expected message to be '%v' but got '%v'", expected, res) } } func TestFireFilterHook(t *testing.T) { hook := &Hook{ appName: "fire_hook_test", alwaysSentFields: logrus.Fields{"test-name": "fire-test-hook", "_ignore": "haaa", "override": "no"}, hookOnlyPrefix: "_", } entry := &logrus.Entry{ Message: "hello world!", Data: logrus.Fields{"override": "yes"}, Level: logrus.DebugLevel, } if err := hook.Fire(entry); err != nil { t.Error(err) } expected := &logrus.Entry{ Message: "hello world!", Data: logrus.Fields{"test-name": "fire-test-hook", "override": "yes"}, Level: logrus.DebugLevel, } if !reflect.DeepEqual(expected, entry) { t.Errorf("expected message to be '%v' but got '%v'", expected, entry) } } func TestLevels(t *testing.T) { hook := &Hook{} expected := []logrus.Level{ logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel, logrus.DebugLevel, } res := hook.Levels() if !reflect.DeepEqual(expected, res) { t.Errorf("expected levels to be '%v' but got '%v'", expected, res) } } func TestLogstashTimeStampFormat(t *testing.T) { conn := ConnMock{buff: bytes.NewBufferString("")} hook := &Hook{ conn: conn, TimeFormat: time.Kitchen, } fTime := time.Date(2009, time.November, 10, 3, 4, 0, 0, time.UTC) if err := hook.Fire(&logrus.Entry{Time: fTime}); err != nil { t.Errorf("expected fire to not return error: %s", err) } var res map[string]string if err := json.NewDecoder(conn.buff).Decode(&res); err != nil { t.Error(err) } if value, ok := res["@timestamp"]; !ok { t.Error("expected result to have '@timestamp'") } else if value != "3:04AM" { t.Errorf("expected time to be '%s' but got '%s'", "3:04AM", value) } }