diff --git a/.gitignore b/.gitignore index b5c379a..0d5cb3f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ # Output of the go coverage tool. *.out coverage.html +cover.html # Random pieces of test data I accumuate while writing this program. testlog diff --git a/integration-tests/loggers_test.go b/integration-tests/loggers_test.go index 24bc47d..af5fae7 100644 --- a/integration-tests/loggers_test.go +++ b/integration-tests/loggers_test.go @@ -135,6 +135,33 @@ func TestLoggers(t *testing.T) { l.Error("line 3", exampleError) }, }, + { + name: "bunyan", + ins: &parse.InputSchema{ + LevelKey: "level", + MessageKey: "msg", + TimeKey: "time", + LevelFormat: parse.BunyanV0LevelParser, + TimeFormat: parse.DefaultTimeParser, + Strict: true, + DeleteKeys: []string{"v"}, + }, + f: func(buf *bytes.Buffer) { + // This is a node library. We could bundle webpack and a JS + // interpreter, but I just ran this program and copied in the + // output. + + // var bunyan = require("bunyan"); + // var log = bunyan.createLogger({ name: "test" }); + // log.info("line 1"); + // log.info({ string: "value", int: 42, object: { foo: "bar" } }, "line 2"); + // log.info({ error: "whoa" }, "line 3"); + buf.Write([]byte(`{"level":30,"msg":"line 1","time":"2021-03-09T17:44:26.203Z","v":0} +{"level":30,"string":"value","int":42,"object":{"foo":"bar"},"msg":"line 2","time":"2021-03-09T17:44:26.204Z","v":0} +{"level":30,"error":"whoa","msg":"line 3","time":"2021-03-09T17:44:26.204Z","v":0} +`)) + }, + }, } f := &ignoreTimeFormatter{ diff --git a/pkg/parse/default_parsers.go b/pkg/parse/default_parsers.go index eec9955..0dd3ec8 100644 --- a/pkg/parse/default_parsers.go +++ b/pkg/parse/default_parsers.go @@ -91,6 +91,28 @@ func LagerLevelParser(in interface{}) (Level, error) { } } +// BunyanV0LLevelParser maps bunyan's float64 levels to log levels. +func BunyanV0LevelParser(in interface{}) (Level, error) { + x, ok := in.(float64) + if !ok { + return LevelUnknown, fmt.Errorf("invalid bunyan log level %T(%v), want float64", in, in) + } + if x <= 10 { + return LevelTrace, nil + } else if x <= 20 { + return LevelDebug, nil + } else if x <= 30 { + return LevelInfo, nil + } else if x <= 40 { + return LevelWarn, nil + } else if x <= 50 { + return LevelError, nil + } else if x <= 60 { + return LevelFatal, nil + } + return LevelUnknown, fmt.Errorf("invalid bunyan log level %v", x) +} + // DefaultLevelParser uses common strings to determine the log level. Case does not matter; info is // the same log level as INFO. func DefaultLevelParser(in interface{}) (Level, error) { diff --git a/pkg/parse/default_parsers_test.go b/pkg/parse/default_parsers_test.go index 0be56dc..9274359 100644 --- a/pkg/parse/default_parsers_test.go +++ b/pkg/parse/default_parsers_test.go @@ -80,6 +80,14 @@ func TestLevelParsers(t *testing.T) { {float64(1), LagerLevelParser, LevelInfo, false}, {float64(2), LagerLevelParser, LevelError, false}, {float64(3), LagerLevelParser, LevelFatal, false}, + {float64(10), BunyanV0LevelParser, LevelTrace, false}, + {float64(20), BunyanV0LevelParser, LevelDebug, false}, + {float64(30), BunyanV0LevelParser, LevelInfo, false}, + {float64(40), BunyanV0LevelParser, LevelWarn, false}, + {float64(50), BunyanV0LevelParser, LevelError, false}, + {float64(60), BunyanV0LevelParser, LevelFatal, false}, + {"foo", BunyanV0LevelParser, LevelUnknown, true}, + {float64(61), BunyanV0LevelParser, LevelUnknown, true}, } for i, test := range testData { got, err := test.parser(test.in) diff --git a/pkg/parse/parse.go b/pkg/parse/parse.go index fb73bbc..3fb12e9 100644 --- a/pkg/parse/parse.go +++ b/pkg/parse/parse.go @@ -48,6 +48,10 @@ type InputSchema struct { // If true, print an error when non-JSON lines appear in the input. If false, treat them // as normal messages with as much information extracted as possible. Strict bool + + // DeleteKeys contains a list of keys to delete; used when the log lines contain version + // information that is used for guessing the schema. + DeleteKeys []string } // OutputFormatter describes an object that actually does the output formatting. Methods take a @@ -345,6 +349,18 @@ func (s *InputSchema) guessSchema(l *line) { s.MessageKey = "message" return } + if has("time") && has("level") && has("v") && has("msg") { + // bunyan + if v, ok := l.fields["v"].(float64); ok && v == 0 { + s.TimeKey = "time" + s.TimeFormat = DefaultTimeParser // RFC3339 + s.LevelKey = "level" + s.LevelFormat = BunyanV0LevelParser + s.MessageKey = "msg" + s.DeleteKeys = append(s.DeleteKeys, "v") + return + } + } if has("time") && has("level") && has("msg") { // logrus default json encoder s.TimeKey = "time" @@ -430,6 +446,9 @@ func (s *InputSchema) ReadLine(l *line) error { } else { pushError(fmt.Errorf("no level key %q in incoming log", s.LevelKey)) } + for _, k := range s.DeleteKeys { + delete(l.fields, k) + } return retErr }