diff --git a/.gitignore b/.gitignore index 32f86af..53d4585 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea vendor -coverage.txt \ No newline at end of file +coverage.txt +gommon-build \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 3cf8faf..96074f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ sudo: false go: - "1.10" + - "1.11" - tip install: diff --git a/Makefile b/Makefile index ddcc406..95706f0 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,13 @@ FLAGS = -X main.version=$(VERSION) -X main.commit=$(BUILD_COMMIT) -X main.buildT install: go install -ldflags "$(FLAGS)" ./cmd/gommon +.PHONY: build +build: + go build -ldflags "$(FLAGS)" -o gommon-build ./cmd/gommon + .PHONY: fmt fmt: - gofmt -d -l -w $(PKGST) + goimports -d -l -w $(PKGST) .PHONY: generate generate: diff --git a/cmd/gommon/main.go b/cmd/gommon/main.go index c9fc9c7..a97be2c 100644 --- a/cmd/gommon/main.go +++ b/cmd/gommon/main.go @@ -16,7 +16,8 @@ import ( "github.com/dyweb/gommon/util/logutil" ) -var log = dlog.NewApplicationLogger() +var log, logReg = dlog.NewApplicationLoggerAndRegistry("gommon") + var verbose = false var ( version string @@ -34,8 +35,8 @@ func main() { Long: "Generate go files for gommon", PersistentPreRun: func(cmd *cobra.Command, args []string) { if verbose { - dlog.SetLevelRecursive(log, dlog.DebugLevel) - dlog.EnableSourceRecursive(log) + dlog.SetLevel(logReg, dlog.DebugLevel) + dlog.EnableSource(logReg) } }, Run: func(cmd *cobra.Command, args []string) { @@ -135,6 +136,6 @@ func genCmd() *cobra.Command { } func init() { - log.AddChild(logutil.Registry) - dlog.SetHandlerRecursive(log, cli.New(os.Stderr, true)) + logReg.AddRegistry(logutil.Registry()) + dlog.SetHandler(logReg, cli.New(os.Stderr, true)) } diff --git a/generator/generate.go b/generator/generate.go index 2993d35..96b4e8a 100644 --- a/generator/generate.go +++ b/generator/generate.go @@ -8,7 +8,7 @@ import ( "path/filepath" "strings" - "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v2" "github.com/dyweb/gommon/errors" "github.com/dyweb/gommon/noodle" diff --git a/generator/pkg.go b/generator/pkg.go index 173cda7..ee0f07e 100644 --- a/generator/pkg.go +++ b/generator/pkg.go @@ -15,7 +15,7 @@ const ( DefaultGeneratedFile = "gommon_generated.go" ) -var log = logutil.NewPackageLogger() +var log, logReg = logutil.NewPackageLoggerAndRegistry() type ConfigFile struct { // Loggers is helper methods on struct for gommon/log to build a tree for logger, this is subject to change diff --git a/generator/shell.go b/generator/shell.go index 48d11e4..133d503 100644 --- a/generator/shell.go +++ b/generator/shell.go @@ -4,7 +4,7 @@ import ( "os" "os/exec" - "github.com/kballard/go-shellquote" + shellquote "github.com/kballard/go-shellquote" "github.com/dyweb/gommon/errors" "github.com/dyweb/gommon/util/fsutil" diff --git a/log/CHANGELOG.md b/log/CHANGELOG.md new file mode 100644 index 0000000..12e83eb --- /dev/null +++ b/log/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## 2018-12-08 + +- remove logger relation ship from logger struct, use logger registry instead + +## 2018-11-25 + +- reduce `Handler` interface method from 6 to 1, just a `HandleLog(level Level, now time.Time, msg string, source string, context Fields, fields Fields)` +- add `Print`, `Printf` for drop in replacement of standard library like non leveled logging library \ No newline at end of file diff --git a/log/TODO.md b/log/TODO.md index a97bdcd..25f84d6 100644 --- a/log/TODO.md +++ b/log/TODO.md @@ -1,5 +1,9 @@ # TODO +## 2018-11-24 + +- [ ] [#90](https://github.com/dyweb/gommon/pull/90) the long pending benchmark and refactor for adding context (fields to logger instance) + ## 2018-05-02 - [x] [#67](https://github.com/dyweb/gommon/issues/67) add example in doc diff --git a/log/_benchmarks/.gitignore b/log/_benchmarks/.gitignore new file mode 100644 index 0000000..da9ff88 --- /dev/null +++ b/log/_benchmarks/.gitignore @@ -0,0 +1,3 @@ +*.out +# binary generated for profile +*.test \ No newline at end of file diff --git a/log/_benchmarks/Makefile b/log/_benchmarks/Makefile new file mode 100644 index 0000000..95aab99 --- /dev/null +++ b/log/_benchmarks/Makefile @@ -0,0 +1,12 @@ +# TODO: the dependency is from global go path ... +bench: + go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out +bench-gommon: + go test -run none -bench=".*/gommon" -benchtime 3s -benchmem -memprofile p.out +bench-gommon-no-fields: + go test -run none -bench="BenchmarkWithoutFieldsText/gommon.F" -benchtime 3s -benchmem -memprofile p.out +bench-gommon-no-context-with-fields: + go test -run none -bench="BenchmarkNoContextWithFieldsJSON/gommon.F" -benchtime 3s -benchmem -memprofile p.out +pprof-ui: +# TODO: need to give it binary path otherwise it will throw error + go tool pprof -http=:8080 p.out \ No newline at end of file diff --git a/log/_benchmarks/README.md b/log/_benchmarks/README.md new file mode 100644 index 0000000..3fbecb5 --- /dev/null +++ b/log/_benchmarks/README.md @@ -0,0 +1,16 @@ +# Benchmark + +## Log libraries + +- [ ] TODO: k8s fork for glog https://github.com/kubernetes/klog , +they are also considering parent children logger https://github.com/kubernetes/klog/issues/22 + +## Ref + +- zap https://github.com/uber-go/zap/tree/master/benchmarks + - first clone and put the repo to $GOPATH/src/go.uber.org/zap they are not using github repo as import path +- zerolog https://github.com/rs/logbench +- https://hackernoon.com/does-logging-cause-cpu-load-a-test-of-all-the-golang-logging-libraries-34052240f90d + - it starts a http server to log and use a client to hit the server + - [x] it measure system call using `sudo strace -c -t -p $(pid)` and see context switches +- https://medium.com/justforfunc/analyzing-the-performance-of-go-functions-with-benchmarks-60b8162e61c6 \ No newline at end of file diff --git a/log/_benchmarks/benchmark_test.go b/log/_benchmarks/benchmark_test.go new file mode 100644 index 0000000..ce6b411 --- /dev/null +++ b/log/_benchmarks/benchmark_test.go @@ -0,0 +1,618 @@ +package benchmarks + +import ( + "errors" + "io/ioutil" + "testing" + + stdlog "log" + + // zap + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + // zerolog + "github.com/rs/zerolog" + + // apex + apexlog "github.com/apex/log" + apexlogconsole "github.com/apex/log/handlers/cli" // TODO: this relies on so many color packages .... + apexlogjson "github.com/apex/log/handlers/json" + + // logrus + "github.com/sirupsen/logrus" + + // klog, fork of glog by k8s + "k8s.io/klog" + + dlog "github.com/dyweb/gommon/log" + "github.com/dyweb/gommon/log/handlers/json" +) + +type ZapDiscard struct { +} + +func (z *ZapDiscard) Write(b []byte) (int, error) { + return ioutil.Discard.Write(b) +} + +func (z *ZapDiscard) Sync() error { + return nil +} + +func newZapJsonLogger(lvl zapcore.Level) *zap.Logger { + ec := zap.NewProductionEncoderConfig() + ec.EncodeDuration = zapcore.NanosDurationEncoder + ec.EncodeTime = zapcore.EpochNanosTimeEncoder + enc := zapcore.NewJSONEncoder(ec) + return zap.New(zapcore.NewCore( + enc, + &ZapDiscard{}, + lvl, + )) +} + +func newZapConsoleLogger(lvl zapcore.Level) *zap.Logger { + ec := zap.NewProductionEncoderConfig() + ec.EncodeDuration = zapcore.NanosDurationEncoder + ec.EncodeTime = zapcore.EpochNanosTimeEncoder + enc := zapcore.NewConsoleEncoder(ec) + return zap.New(zapcore.NewCore( + enc, + &ZapDiscard{}, + lvl, + )) +} + +func newZerologJsonLogger() zerolog.Logger { + // TODO: this may not be the ideal way to init zero logger, see the author's benchmark + // https://github.com/rs/logbench/blob/master/zerolog_test.go + return zerolog.New(ioutil.Discard).With().Timestamp().Logger() +} + +// https://github.com/rs/zerolog/tree/master#pretty-logging +func newZerologConsoleLogger() zerolog.Logger { + return zerolog.New(zerolog.ConsoleWriter{Out: ioutil.Discard}).With().Timestamp().Logger() +} + +func newApexJsonLogger(lvl apexlog.Level) *apexlog.Logger { + return &apexlog.Logger{ + Handler: apexlogjson.New(ioutil.Discard), + Level: lvl, + } +} + +func newApexConsoleLogger(lvl apexlog.Level) *apexlog.Logger { + return &apexlog.Logger{ + Handler: apexlogconsole.New(ioutil.Discard), + Level: lvl, + } +} + +// TODO: use logrus entry might be more reasonable? +func newLogrusJsonLogger(lvl logrus.Level) *logrus.Logger { + return &logrus.Logger{ + Out: ioutil.Discard, + Formatter: &logrus.JSONFormatter{}, + Level: lvl, + } +} + +func newLogrusConsoleLogger(lvl logrus.Level) *logrus.Logger { + return &logrus.Logger{ + Out: ioutil.Discard, + Formatter: &logrus.TextFormatter{DisableColors: true}, + Level: lvl, + } +} + +// disabled should have not allocation +func BenchmarkDisabledLevelNoFormat(b *testing.B) { + b.Log("logging at a disabled level") + msg := "If you support level you should not see me and should not cause allocation, I know I talk too much" + b.Run("gommon", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.ErrorLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // TODO: it has 16B allocation due to parameter is interface, size of interface is int64(type), int64(ptr) + // https://research.swtch.com/interfaces + logger.Info(msg) + } + }) + }) + b.Run("gommon.F", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.ErrorLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.InfoF(msg) + } + }) + }) + b.Run("gommon.check", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.ErrorLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if logger.IsInfoEnabled() { + logger.Info(msg) + } + } + }) + }) + b.Run("zap", func(b *testing.B) { + logger := newZapConsoleLogger(zap.ErrorLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zap.check", func(b *testing.B) { + logger := newZapConsoleLogger(zap.ErrorLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if m := logger.Check(zap.InfoLevel, msg); m != nil { + m.Write() + } + } + }) + }) + b.Run("zap.sugar", func(b *testing.B) { + logger := newZapConsoleLogger(zap.ErrorLevel).Sugar() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zerolog", func(b *testing.B) { + logger := newZerologConsoleLogger().Level(zerolog.ErrorLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info().Msg(msg) + } + }) + }) + b.Run("apex", func(b *testing.B) { + logger := newApexConsoleLogger(apexlog.ErrorLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("logrus", func(b *testing.B) { + logger := newLogrusConsoleLogger(logrus.ErrorLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + //b.Run("klog", func(b *testing.B) { + // // TODO: it seems glog can't create individual logger instance? + // klog.SetOutput(ioutil.Discard) + // //klog.InitFlags() + // b.RunParallel(func(pb *testing.PB) { + // for pb.Next() { + // klog.Info(msg) + // } + // }) + //}) +} + +// no fields and don't call *f method for Printf style text formatting +func BenchmarkWithoutFieldsText(b *testing.B) { + b.ReportAllocs() + b.Log("logging a single line text like stdlog without format and fields") + msg := "TODO: is fixed length msg really a good idea, we should give dynamic length with is more real world" + + b.Run("gommon", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("gommon.F", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.InfoF(msg) + } + }) + }) + b.Run("std", func(b *testing.B) { + logger := stdlog.New(ioutil.Discard, "", stdlog.Ldate|stdlog.Ltime) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Print(msg) + } + }) + }) + b.Run("zap", func(b *testing.B) { + logger := newZapConsoleLogger(zap.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zap.sugar", func(b *testing.B) { + logger := newZapConsoleLogger(zap.InfoLevel).Sugar() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zerolog", func(b *testing.B) { + logger := newZerologConsoleLogger() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info().Msg(msg) + } + }) + }) + b.Run("apex", func(b *testing.B) { + logger := newApexConsoleLogger(apexlog.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("logrus", func(b *testing.B) { + logger := newLogrusConsoleLogger(logrus.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("klog", func(b *testing.B) { + // TODO: it seems glog can't create individual logger instance? + klog.SetOutput(ioutil.Discard) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + klog.Info(msg) // I think klog's buffer pool is the reason + } + }) + }) +} + +func BenchmarkWithoutFieldsTextFormat(b *testing.B) { + b.ReportAllocs() + b.Log("logging a single line text like stdlog with format but without fields") + format := "TODO: is fixed length msg really a good idea? we should give dynamic length with is more real world %d %s %s" + i1 := 10086 + s1 := "sub str aaaaa" + err := errors.New("some error") + + b.Run("gommon", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Infof(format, i1, s1, err) + } + }) + }) + b.Run("zap.sugar", func(b *testing.B) { + logger := newZapConsoleLogger(zap.InfoLevel).Sugar() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Infof(format, i1, s1, err) + } + }) + }) + // TODO: seems zerolog console logger also don't have *f variant + b.Run("apex", func(b *testing.B) { + logger := newApexConsoleLogger(apexlog.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Infof(format, i1, s1, err) + } + }) + }) + b.Run("logrus", func(b *testing.B) { + logger := newLogrusConsoleLogger(logrus.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Infof(format, i1, s1, err) + } + }) + }) + b.Run("klog", func(b *testing.B) { + // TODO: it seems glog can't create individual logger instance? + klog.SetOutput(ioutil.Discard) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + klog.Infof(format, i1, s1, err) + } + }) + }) +} + +func BenchmarkWithoutFieldsJSON(b *testing.B) { + b.ReportAllocs() + b.Log("logging without fields and without printf, use json output") + msg := "TODO: is fixed length msg really a good idea, we should give dynamic length with is more real world" + b.Run("gommon", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("gommon.F", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.InfoF(msg) + } + }) + }) + b.Run("zap", func(b *testing.B) { + logger := newZapJsonLogger(zap.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zap.sugar", func(b *testing.B) { + logger := newZapJsonLogger(zap.InfoLevel).Sugar() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zerolog", func(b *testing.B) { + logger := newZerologJsonLogger() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info().Msg(msg) + } + }) + }) + b.Run("apex", func(b *testing.B) { + logger := newApexJsonLogger(apexlog.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("logrus", func(b *testing.B) { + logger := newLogrusJsonLogger(logrus.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) +} + +func BenchmarkCallerJSON(b *testing.B) { + b.ReportAllocs() + b.Log("logging without fields and without printf, use json output and enable log file line") + msg := "TODO: is fixed length msg really a good idea, we should give dynamic length with is more real world" + b.Run("gommon", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + logger.EnableSource() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("gommon.F", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + logger.EnableSource() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.InfoF(msg) + } + }) + }) +} + +func BenchmarkWithContextNoFieldsJSON(b *testing.B) { + b.ReportAllocs() + b.Log("logging with context attached to logger (entry/event) no text format, no fields, use json output") + msg := "TODO: is fixed length msg really a good idea, we should give dynamic length with is more real world" + b.Run("gommon", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + // TODO: generate unified fields for all logging libraries + logger.AddFields(dlog.Int("i1", 1), dlog.Str("s1", "v1")) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("gommon.F", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + // TODO: generate unified fields for all logging libraries + logger.AddFields(dlog.Int("i1", 1), dlog.Str("s1", "v1")) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.InfoF(msg) + } + }) + }) + b.Run("zap", func(b *testing.B) { + logger := newZapJsonLogger(zap.InfoLevel). + With(zap.Int("i1", 1), zap.String("s1", "v1")) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zap.sugar", func(b *testing.B) { + logger := newZapJsonLogger(zap.InfoLevel). + With(zap.Int("i1", 1), zap.String("s1", "v1")). + Sugar() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("zerolog", func(b *testing.B) { + logger := newZerologJsonLogger(). + With().Int("i1", 1).Str("s1", "v1").Logger() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info().Msg(msg) + } + }) + }) + b.Run("apex", func(b *testing.B) { + logger := newApexJsonLogger(apexlog.InfoLevel). + WithFields(apexlog.Fields{ + "i1": 1, + "s1": "v1", + }) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) + b.Run("logrus", func(b *testing.B) { + logger := newLogrusJsonLogger(logrus.InfoLevel). + WithFields(logrus.Fields{ + "i1": 1, + "s1": "v1", + }) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg) + } + }) + }) +} + +func BenchmarkNoContextWithFieldsJSON(b *testing.B) { + b.ReportAllocs() + b.Log("logging with fields at log site, no context attached to logger (entry/event) no text format, use json output") + msg := "TODO: is fixed length msg really a good idea, we should give dynamic length with is more real world" + b.Run("gommon.F", func(b *testing.B) { + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(json.New(ioutil.Discard)) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // TODO: generate unified fields for all logging libraries + logger.InfoF(msg, + dlog.Int("i1", 1), + dlog.Str("s1", "v1"), + ) + } + }) + }) + b.Run("zap", func(b *testing.B) { + logger := newZapJsonLogger(zap.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info(msg, zap.Int("i1", 1), zap.String("s1", "v1")) + } + }) + }) + b.Run("zap.sugar", func(b *testing.B) { + logger := newZapJsonLogger(zap.InfoLevel).Sugar() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Infow(msg, "i1", 1, "s1", "v1") + } + }) + }) + b.Run("zerolog", func(b *testing.B) { + logger := newZerologJsonLogger() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.Info().Int("i1", 1).Str("s1", "v1").Msg(msg) + } + }) + }) + b.Run("apex", func(b *testing.B) { + logger := newApexJsonLogger(apexlog.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.WithFields(apexlog.Fields{ + "i1": 1, + "s1": "v1", + }).Info(msg) + } + }) + }) + b.Run("logrus", func(b *testing.B) { + logger := newLogrusJsonLogger(logrus.InfoLevel) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + logger.WithFields(logrus.Fields{ + "i1": 1, + "s1": "v1", + }).Info(msg) + } + }) + }) +} diff --git a/log/_benchmarks/doc.go b/log/_benchmarks/doc.go new file mode 100644 index 0000000..0878121 --- /dev/null +++ b/log/_benchmarks/doc.go @@ -0,0 +1,2 @@ +// Package benchmarks provides benchmark using go test and simulated real environment using pprof +package benchmarks diff --git a/log/_benchmarks/example_test.go b/log/_benchmarks/example_test.go new file mode 100644 index 0000000..bfc3001 --- /dev/null +++ b/log/_benchmarks/example_test.go @@ -0,0 +1,217 @@ +package benchmarks + +import ( + stdlog "log" + "os" + "testing" + + "k8s.io/klog" + + // zap + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/rs/zerolog" + + // apex + apexlog "github.com/apex/log" + apexlogconsole "github.com/apex/log/handlers/cli" // TODO: this relies on so many color packages .... + apexlogjson "github.com/apex/log/handlers/json" + + "github.com/sirupsen/logrus" +) + +// this file shows how to use different logging library and +// what it looks like when logging using different formats + +// the order is +// - gommon +// - std +// - zap +// - zerolog +// - apex +// - logrus + +// log in standard library, no level and field support +func TestStd(t *testing.T) { + logger := stdlog.New(os.Stdout, "", stdlog.LstdFlags) + logger.Print("a", 1) + logger.Printf("%s %d", "a", 1) + logger.Println("a", 1) + //2018/11/23 21:13:59 a1 + //2018/11/23 21:13:59 a 1 + //2018/11/23 21:13:59 a 1 +} + +// zap https://github.com/uber-go/zap +func TestZap(t *testing.T) { + // https://github.com/sandipb/zap-examples + + t.Run("json", func(t *testing.T) { + t.Run("production logger", func(t *testing.T) { + logger, err := zap.NewProduction() + if err != nil { + t.Fatal(err) + return + } + logger.Info("hi") + // {"level":"info","ts":1542779858.448864,"caller":"_benchmarks/example_test.go:19","msg":"hi"} + }) + + t.Run("production config", func(t *testing.T) { + // TODO: this config is from zap's own benchmark, it does not have caller enabled, strange, seems need key in config + // https://github.com/sandipb/zap-examples/tree/master/src/customlogger#customizing-the-encoder + // it seems you need to add they keys in config ... + ec := zap.NewProductionEncoderConfig() + ec.EncodeDuration = zapcore.NanosDurationEncoder + ec.EncodeTime = zapcore.EpochNanosTimeEncoder + enc := zapcore.NewJSONEncoder(ec) + logger := zap.New(zapcore.NewCore( + enc, + os.Stderr, + zapcore.InfoLevel, + )) + logger.Info("this is a message") + // {"level":"info","ts":1542778510834696318,"msg":"this is a message"} + logger.Named("jack").Info("this is a named message") + // {"level":"info","ts":1542778510834708500,"logger":"jack","msg":"this is a named message"} + logger.Named("jack").Named("marry").Info("what's my name") + // {"level":"info","ts":1542781224287444070,"logger":"jack.marry","msg":"what's my name"} + + // TODO: might move context to top level since it's an important feature + t.Run("context", func(t *testing.T) { + logger.With(zap.Int("count", 1), zap.String("str", `"need escape"`)).Info("ha") + // {"level":"info","ts":1542779234131875417,"msg":"ha","count":1,"str":"\"need escape\""} + ctxLogger := logger.With(zap.Int("count", 1)) + ctxLogger.Info("this is the log") + // {"level":"info","ts":1542779282702624079,"msg":"this is the log","count":1} + ctxLogger.With(zap.Bool("b", true)).Info("yep") + // {"level":"info","ts":1542779322646305013,"msg":"yep","count":1,"b":true} + + // NOTE: it will have duplicated key + ctxLogger.With(zap.Bool("a", true), zap.Bool("a", false)).Info("dup?") + // {"level":"info","ts":1542783708445651389,"msg":"dup?","count":1,"a":true,"a":false} + + // TODO: https://github.com/sandipb/zap-examples/tree/master/src/customlogger#changing-logger-behavior-on-the-fly + // things like AddCaller, AddStacktrace seems no longer exists + }) + }) + }) + + t.Run("console", func(t *testing.T) { + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + return + } + logger.Info("text based") + // 2018-11-20T22:56:08.273-0800 INFO _benchmarks/example_test.go:64 text based + // NOTE: fields are still encoded as json ... + logger.With(zap.Int("a", 1), zap.String("b", "aaa")).Info("ho ho ho") + // 2018-11-20T22:56:08.273-0800 INFO _benchmarks/example_test.go:65 ho ho ho {"a": 1, "b": "aaa"} + }) +} + +// zerolog https://github.com/rs/zerolog +func TestZerolog(t *testing.T) { + t.Run("json", func(t *testing.T) { + logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + logger.Info().Msg("show some info") + logger.Info().Str("f1", "v1").Msg("have key-value") + //{"level":"info","time":"2018-11-23T21:25:39-08:00","message":"show some info"} + //{"level":"info","f1":"v1","time":"2018-11-23T21:25:39-08:00","message":"have key-value"} + + t.Run("context", func(t *testing.T) { + ctxLogger := logger.With().Str("base", "value").Logger() + ctxLogger.Info().Msg("inherit context") + ctxLogger.Info().Str("extra", "value").Msg("extra field") + //{"level":"info","base":"value","time":"2018-11-23T21:25:39-08:00","message":"inherit context"} + //{"level":"info","base":"value","extra":"value","time":"2018-11-23T21:25:39-08:00","message":"extra field"} + }) + }) + t.Run("console", func(t *testing.T) { + logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stdout}).With().Timestamp().Logger() + logger.Info().Msg("level have color? yes") // info is yellow + logger.Warn().Msg("warn is yellow?") // warn is red + logger.Info().Str("f1", "v1").Msg("field has color? no") + //9:29PM INF level have color? yes + //9:29PM WRN warn is yellow? + //9:29PM INF field has color? no f1=v1 + }) +} + +// apex/log https://github.com/apex/log +func TestApex(t *testing.T) { + // TODO: its fields are not flat w/ level and message? + t.Run("json", func(t *testing.T) { + logger := apexlog.Logger{ + Handler: apexlogjson.New(os.Stdout), + Level: apexlog.InfoLevel, + } + logger.Info("hi") + logger.WithField("f1", "v1").Info("have field") + //{"fields":{},"level":"info","timestamp":"2018-11-23T21:43:00.637421923-08:00","message":"hi"} + //{"fields":{"f1":"v1"},"level":"info","timestamp":"2018-11-23T21:43:00.637500665-08:00","message":"have filed"} + + t.Run("context", func(t *testing.T) { + ctxEntry := logger.WithField("base", "value") + ctxEntry.Info("inherit context") + ctxEntry.WithField("extra", "value").Info("extra field") + //{"fields":{"base":"value"},"level":"info","timestamp":"2018-11-23T21:43:00.637524216-08:00","message":"inherit context"} + //{"fields":{"base":"value","extra":"value"},"level":"info","timestamp":"2018-11-23T21:43:00.637534403-08:00","message":"extra field"} + }) + + }) + t.Run("console", func(t *testing.T) { + logger := apexlog.Logger{ + Handler: apexlogconsole.New(os.Stdout), + Level: apexlog.InfoLevel, + } + logger.Info("level have color?") // NOTE: it seems color is disabled when not tty and become a dot .... + logger.WithField("f1", "v1").Info("field have color?") + // • level have color? + // • field have color? f1=v1 + }) +} + +func TestLogrus(t *testing.T) { + t.Run("json", func(t *testing.T) { + logger := logrus.New() + logger.SetOutput(os.Stdout) + logger.SetLevel(logrus.InfoLevel) + logger.SetFormatter(&logrus.JSONFormatter{}) + //logger.SetReportCaller() + + logger.Info("hi") + logger.WithField("f1", "v1").Info("have field") + //{"level":"info","msg":"hi","time":"2018-11-23T22:51:07-08:00"} + //{"f1":"v1","level":"info","msg":"have field","time":"2018-11-23T22:51:07-08:00"} + + t.Run("context", func(t *testing.T) { + ctxEntry := logger.WithField("base", "value") + ctxEntry.Info("inherit context") + ctxEntry.WithField("extra", "value").Info("extra field") + //{"base":"value","level":"info","msg":"inherit context","time":"2018-11-23T23:19:56-08:00"} + //{"base":"value","extra":"value","level":"info","msg":"extra field","time":"2018-11-23T23:19:56-08:00"} + }) + }) + + t.Run("console", func(t *testing.T) { + logger := logrus.New() + logger.SetOutput(os.Stdout) + logger.SetLevel(logrus.InfoLevel) + logger.SetFormatter(&logrus.TextFormatter{ForceColors: true}) + + logger.Info("level have color? yes, when tty or forced") + logger.WithField("f1", "v1").Info("field has color? yes") + //INFO[0000] level have color? yes, when tty or forced + //INFO[0000] field has color? f1=v1 + }) +} + +func TestKlog(t *testing.T) { + t.Run("console", func(t *testing.T) { + klog.SetOutput(os.Stderr) + klog.Info("just log something") + }) +} diff --git a/log/_benchmarks/playground/field_interface/.gitignore b/log/_benchmarks/playground/field_interface/.gitignore new file mode 100644 index 0000000..a7636f2 --- /dev/null +++ b/log/_benchmarks/playground/field_interface/.gitignore @@ -0,0 +1 @@ +field_interface \ No newline at end of file diff --git a/log/_benchmarks/playground/field_interface/escape.txt b/log/_benchmarks/playground/field_interface/escape.txt new file mode 100644 index 0000000..6c9ce85 --- /dev/null +++ b/log/_benchmarks/playground/field_interface/escape.txt @@ -0,0 +1,23 @@ +go build -gcflags "-m -m" . +# github.com/dyweb/gommon/log/_benchmarks/playground/field_interface +./field.go:32:6: cannot inline main: function too complex: cost 315 exceeds budget 80 +./field.go:38:30: inlining call to log.NewTestLogger func(log.Level) *log.Logger { var log.l·3 *log.Logger; log.l·3 = ; log.l·3 = &log.Logger literal; return log.l·3 } +./field.go:39:37: inlining call to log.NewIOHandler func(io.Writer) log.Handler { return log.Handler(&log.IOHandler literal) } +./field.go:42:28: inlining call to log.Int func(string, int) log.Field { return log.Field literal } +./field.go:43:28: inlining call to log.Int func(string, int) log.Field { return log.Field literal } +./field.go:43:14: inlining call to log.(*Logger).NoopF method(*log.Logger) func(string, ...log.Field) { } +./field.go:38:30: &log.Logger literal escapes to heap +./field.go:38:30: from log.l·3 (assigned) at ./field.go:38:30 +./field.go:38:30: from ~R0 (assign-pair) at ./field.go:38:30 +./field.go:38:30: from logger (assigned) at ./field.go:38:9 +./field.go:38:30: from logger (passed to call[argument escapes]) at ./field.go:39:19 +./field.go:39:37: log.Handler(&log.IOHandler literal) escapes to heap +./field.go:39:37: from ~R0 (assign-pair) at ./field.go:39:37 +./field.go:39:37: from log.Handler(~R0) (passed to call[argument escapes]) at ./field.go:39:19 +./field.go:39:37: &log.IOHandler literal escapes to heap +./field.go:39:37: from log.Handler(&log.IOHandler literal) (interface-converted) at ./field.go:39:37 +./field.go:39:37: from ~R0 (assign-pair) at ./field.go:39:37 +./field.go:39:37: from log.Handler(~R0) (passed to call[argument escapes]) at ./field.go:39:19 +./field.go:42:14: ... argument escapes to heap +./field.go:42:14: from ... argument (passed to call[argument escapes]) at ./field.go:42:14 +./field.go:43:14: main []log.Field literal does not escape diff --git a/log/_benchmarks/playground/field_interface/field.go b/log/_benchmarks/playground/field_interface/field.go new file mode 100644 index 0000000..4c3ab57 --- /dev/null +++ b/log/_benchmarks/playground/field_interface/field.go @@ -0,0 +1,45 @@ +package main + +import ( + "io/ioutil" + + dlog "github.com/dyweb/gommon/log" +) + +// +//type myLogger struct { +// h handler +//} +// +////go:noinline +//func (l *myLogger) info(s string) { +// l.h.log(s) +//} +// +//type handler interface { +// log(s string) +//} +// +//type printer struct { +//} +// +////go:noinline +//func (p *printer) log(s string) { +// // do nothing +// os.Stdout.Write([]byte(s)) +//} + +// go build -gcflags "-m -m" . +func main() { + //mLogger := myLogger{ + // h: &printer{}, + //} + //mLogger.info("a") + + logger := dlog.NewTestLogger(dlog.InfoLevel) + logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + + logger.InfoF("a") // no slice of fields, no heap alloc + logger.InfoF("a", dlog.Int("a", 1)) // escaped + logger.NoopF("a", dlog.Int("a", 1)) // NoopF don't call any interface using the fields given, so no heap +} diff --git a/log/_benchmarks/playground/slice_on_stack/.gitignore b/log/_benchmarks/playground/slice_on_stack/.gitignore new file mode 100644 index 0000000..683d83d --- /dev/null +++ b/log/_benchmarks/playground/slice_on_stack/.gitignore @@ -0,0 +1,2 @@ +# binary +slice_on_stack diff --git a/log/_benchmarks/playground/slice_on_stack/README.md b/log/_benchmarks/playground/slice_on_stack/README.md new file mode 100644 index 0000000..5a099c8 --- /dev/null +++ b/log/_benchmarks/playground/slice_on_stack/README.md @@ -0,0 +1,6 @@ +# Slice on stack + +- slice must have static length, i.e. `make(0, i)` where `i` is variable will cause it to escape +- no `parameter to indirect call` + - I am not sure I understand it correctly, but is seems when use methods from an interface, it's parameter to indirect + call because it's opaque to the compiler so it can't analysis what happens to the bytes slice \ No newline at end of file diff --git a/log/_benchmarks/playground/slice_on_stack/allocate_location.go b/log/_benchmarks/playground/slice_on_stack/allocate_location.go new file mode 100644 index 0000000..cc07000 --- /dev/null +++ b/log/_benchmarks/playground/slice_on_stack/allocate_location.go @@ -0,0 +1,98 @@ +package main + +import ( + "io" +) + +type handle struct { + w io.Writer + + // value receiver + f f + fp *f + + // pointer receiver + f2 f2 + f2p *f2 + + // interface + fi fi +} + +type f struct { +} + +//go:noinline +func (f f) Write(b []byte) { + b[0] = 'c' +} + +type f2 struct { +} + +type fi interface { + Write(b []byte) +} + +//go:noinline +func (f *f2) Write(b []byte) { + b[0] = 'c' +} + +func (h *handle) Log(s string) { + l1 := make([]byte, 0, 100) // constant type, but still escape to heap, + l1 = append(l1, s...) + h.w.Write(l1) // parameter to indirect call +} + +func (h *handle) Log2(s string) { + l2 := make([]byte, 0, len(s)) // escape (non-constant size) + l2 = append(l2, s...) + h.w.Write(l2) +} + +func (h *handle) Log3(s string) { + l3 := make([]byte, 0, 100) // don't escape to heap + l3 = append(l3, s...) + h.f.Write(l3) +} + +func (h *handle) Log4(s string) { + l4 := make([]byte, 0, 100) // don't escape to heap TODO: because receiver is not pointer? + l4 = append(l4, s...) + h.fp.Write(l4) +} + +func (h *handle) Log5(s string) { + l5 := make([]byte, 0, 100) // still don't escape to heap ... + l5 = append(l5, s...) + h.f2.Write(l5) +} + +func (h *handle) Log6(s string) { + l6 := make([]byte, 0, 100) // still don't escape + l6 = append(l6, s...) + h.f2p.Write(l6) +} + +func (h *handle) Log7(s string) { + l7 := make([]byte, 0, 100) // escape ... + l7 = append(l7, s...) + h.fi.Write(l7) +} + +//go:noinline +func (h *handle) Compute(i int) { + c1 := make([]byte, 0, 100) // don't escape to heap + c1[i] = 'c' +} + +//go:noinline +func (h *handle) Compute2(i int) { + c1 := make([]byte, i) // escape to heap, unknown size + c1[i-1] = 'c' +} + +// go build -gcflags "-m -m" . +func main() { +} diff --git a/log/_benchmarks/playground/slice_on_stack/escape.txt b/log/_benchmarks/playground/slice_on_stack/escape.txt new file mode 100644 index 0000000..09f6c1f --- /dev/null +++ b/log/_benchmarks/playground/slice_on_stack/escape.txt @@ -0,0 +1,61 @@ +go build -gcflags "-m -m" . +# github.com/dyweb/gommon/log/_benchmarks/playground/slice_on_stack +./allocate_location.go:26:6: cannot inline f.Write: marked go:noinline +./allocate_location.go:38:6: cannot inline (*f2).Write: marked go:noinline +./allocate_location.go:42:6: cannot inline (*handle).Log: function too complex: cost 97 exceeds budget 80 +./allocate_location.go:48:6: cannot inline (*handle).Log2: function too complex: cost 98 exceeds budget 80 +./allocate_location.go:54:6: cannot inline (*handle).Log3: function too complex: cost 97 exceeds budget 80 +./allocate_location.go:60:6: cannot inline (*handle).Log4: function too complex: cost 98 exceeds budget 80 +./allocate_location.go:66:6: cannot inline (*handle).Log5: function too complex: cost 98 exceeds budget 80 +./allocate_location.go:72:6: cannot inline (*handle).Log6: function too complex: cost 97 exceeds budget 80 +./allocate_location.go:78:6: cannot inline (*handle).Log7: function too complex: cost 97 exceeds budget 80 +./allocate_location.go:85:6: cannot inline (*handle).Compute: marked go:noinline +./allocate_location.go:91:6: cannot inline (*handle).Compute2: marked go:noinline +./allocate_location.go:97:6: can inline main as: func() { } +./allocate_location.go:26:18: f.Write b does not escape +./allocate_location.go:38:7: (*f2).Write f does not escape +./allocate_location.go:38:20: (*f2).Write b does not escape +./allocate_location.go:43:12: make([]byte, 0, 100) escapes to heap +./allocate_location.go:43:12: from l1 (assigned) at ./allocate_location.go:43:5 +./allocate_location.go:43:12: from h.w.Write(l1) (parameter to indirect call) at ./allocate_location.go:45:11 +./allocate_location.go:42:7: leaking param content: h +./allocate_location.go:42:7: from h.w (dot of pointer) at ./allocate_location.go:45:3 +./allocate_location.go:42:7: from h.w.Write(l1) (receiver in indirect call) at ./allocate_location.go:45:11 +./allocate_location.go:42:22: (*handle).Log s does not escape +./allocate_location.go:49:12: make([]byte, 0, len(s)) escapes to heap +./allocate_location.go:49:12: from make([]byte, 0, len(s)) (non-constant size) at ./allocate_location.go:49:12 +./allocate_location.go:48:7: leaking param content: h +./allocate_location.go:48:7: from h.w (dot of pointer) at ./allocate_location.go:51:3 +./allocate_location.go:48:7: from h.w.Write(l2) (receiver in indirect call) at ./allocate_location.go:51:11 +./allocate_location.go:48:23: (*handle).Log2 s does not escape +./allocate_location.go:54:7: (*handle).Log3 h does not escape +./allocate_location.go:54:23: (*handle).Log3 s does not escape +./allocate_location.go:55:12: (*handle).Log3 make([]byte, 0, 100) does not escape +./allocate_location.go:60:7: (*handle).Log4 h does not escape +./allocate_location.go:60:23: (*handle).Log4 s does not escape +./allocate_location.go:61:12: (*handle).Log4 make([]byte, 0, 100) does not escape +./allocate_location.go:66:7: (*handle).Log5 h does not escape +./allocate_location.go:66:23: (*handle).Log5 s does not escape +./allocate_location.go:67:12: (*handle).Log5 make([]byte, 0, 100) does not escape +./allocate_location.go:69:6: (*handle).Log5 h.f2 does not escape +./allocate_location.go:72:7: (*handle).Log6 h does not escape +./allocate_location.go:72:23: (*handle).Log6 s does not escape +./allocate_location.go:73:12: (*handle).Log6 make([]byte, 0, 100) does not escape +./allocate_location.go:79:12: make([]byte, 0, 100) escapes to heap +./allocate_location.go:79:12: from l7 (assigned) at ./allocate_location.go:79:5 +./allocate_location.go:79:12: from h.fi.Write(l7) (parameter to indirect call) at ./allocate_location.go:81:12 +./allocate_location.go:78:7: leaking param content: h +./allocate_location.go:78:7: from h.fi (dot of pointer) at ./allocate_location.go:81:3 +./allocate_location.go:78:7: from h.fi.Write(l7) (receiver in indirect call) at ./allocate_location.go:81:12 +./allocate_location.go:78:23: (*handle).Log7 s does not escape +./allocate_location.go:85:7: (*handle).Compute h does not escape +./allocate_location.go:86:12: (*handle).Compute make([]byte, 0, 100) does not escape +./allocate_location.go:92:12: make([]byte, i) escapes to heap +./allocate_location.go:92:12: from make([]byte, i) (non-constant size) at ./allocate_location.go:92:12 +./allocate_location.go:91:7: (*handle).Compute2 h does not escape +:1: (*f).Write .this does not escape +./allocate_location.go:26:18: (*f).Write b does not escape +./allocate_location.go:34:8: leaking param: b +./allocate_location.go:34:8: from .this.Write(b) (parameter to indirect call) at :1 +:1: leaking param: .this +:1: from .this.Write(b) (receiver in indirect call) at :1 diff --git a/log/_benchmarks/playground/slice_or_variadic/.gitignore b/log/_benchmarks/playground/slice_or_variadic/.gitignore new file mode 100644 index 0000000..d4f4671 --- /dev/null +++ b/log/_benchmarks/playground/slice_or_variadic/.gitignore @@ -0,0 +1 @@ +slice_or_variadic \ No newline at end of file diff --git a/log/_benchmarks/playground/slice_or_variadic/bench.txt b/log/_benchmarks/playground/slice_or_variadic/bench.txt new file mode 100644 index 0000000..345ca82 --- /dev/null +++ b/log/_benchmarks/playground/slice_or_variadic/bench.txt @@ -0,0 +1,10 @@ + go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks/playground/slice_or_variadic +BenchmarkVariadic-8 1000000000 6.50 ns/op 0 B/op 0 allocs/op +BenchmarkSlice-8 1000000000 6.80 ns/op 0 B/op 0 allocs/op +BenchmarkVariadicStruct-8 500000000 10.4 ns/op 0 B/op 0 allocs/op +BenchmarkSliceStruct-8 500000000 9.44 ns/op 0 B/op 0 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks/playground/slice_or_variadic 26.720s diff --git a/log/_benchmarks/playground/slice_or_variadic/escape.txt b/log/_benchmarks/playground/slice_or_variadic/escape.txt new file mode 100644 index 0000000..4003fed --- /dev/null +++ b/log/_benchmarks/playground/slice_or_variadic/escape.txt @@ -0,0 +1,13 @@ +go build -gcflags "-m -m" . +# github.com/dyweb/gommon/log/_benchmarks/playground/slice_or_variadic +./param_allocate.go:9:6: cannot inline fVariadic: unhandled op FOR +./param_allocate.go:17:6: cannot inline fVariadicSt: unhandled op FOR +./param_allocate.go:25:6: cannot inline fSlice: unhandled op FOR +./param_allocate.go:33:6: cannot inline fSliceSt: unhandled op FOR +./param_allocate.go:42:6: cannot inline main: function too complex: cost 172 exceeds budget 80 +./param_allocate.go:9:16: fVariadic nums does not escape +./param_allocate.go:17:18: fVariadicSt nums does not escape +./param_allocate.go:25:13: fSlice nums does not escape +./param_allocate.go:33:15: fSliceSt nums does not escape +./param_allocate.go:43:11: main ... argument does not escape +./param_allocate.go:44:14: main []int literal does not escape diff --git a/log/_benchmarks/playground/slice_or_variadic/param_allocate.go b/log/_benchmarks/playground/slice_or_variadic/param_allocate.go new file mode 100644 index 0000000..f4881a6 --- /dev/null +++ b/log/_benchmarks/playground/slice_or_variadic/param_allocate.go @@ -0,0 +1,45 @@ +package main + +type st struct { + x int + s string + i interface{} +} + +func fVariadic(nums ...int) int { + s := 0 + for i := 0; i < len(nums); i++ { + s += nums[i] + } + return s +} + +func fVariadicSt(nums ...st) int { + s := 0 + for i := 0; i < len(nums); i++ { + s += nums[i].x + } + return s +} + +func fSlice(nums []int) int { + s := 0 + for i := 0; i < len(nums); i++ { + s += nums[i] + } + return s +} + +func fSliceSt(nums []st) int { + s := 0 + for i := 0; i < len(nums); i++ { + s += nums[i].x + } + return s +} + +// go build -gcflags "-m -m" . +func main() { + fVariadic(1, 2, 3) + fSlice([]int{1, 2, 3}) +} diff --git a/log/_benchmarks/playground/slice_or_variadic/param_test.go b/log/_benchmarks/playground/slice_or_variadic/param_test.go new file mode 100644 index 0000000..96b1989 --- /dev/null +++ b/log/_benchmarks/playground/slice_or_variadic/param_test.go @@ -0,0 +1,48 @@ +package main + +import "testing" + +// go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out + +func BenchmarkVariadic(b *testing.B) { + s := 0 + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + s += fVariadic(1, 2, 3) + } + }) +} + +func BenchmarkSlice(b *testing.B) { + s := 0 + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + s += fSlice([]int{1, 2, 3}) + } + }) +} + +func BenchmarkVariadicStruct(b *testing.B) { + s := 0 + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + s += fVariadicSt(st{x: 1}, st{x: 2}) + } + }) +} + +func BenchmarkSliceStruct(b *testing.B) { + s := 0 + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + s += fSliceSt([]st{ + {x: 1}, + {x: 2}, + }) + } + }) +} diff --git a/log/_benchmarks/results/2018-09-22.md b/log/_benchmarks/results/2018-09-22.md new file mode 100644 index 0000000..d12dffe --- /dev/null +++ b/log/_benchmarks/results/2018-09-22.md @@ -0,0 +1,16 @@ +# 2018-09-22 + +zap and zerolog are from global GOPATH, using go1.11 + +````text +go test -bench=. +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkWithoutFieldsJSON/gommon-8 20000000 96.9 ns/op +BenchmarkWithoutFieldsJSON/Zap-8 10000000 176 ns/op +BenchmarkWithoutFieldsJSON/Zap.Sugar-8 5000000 257 ns/op +BenchmarkWithoutFieldsJSON/zerorlog-8 20000000 118 ns/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 8.031s +```` \ No newline at end of file diff --git a/log/_benchmarks/results/2018-11-17.md b/log/_benchmarks/results/2018-11-17.md new file mode 100644 index 0000000..a557108 --- /dev/null +++ b/log/_benchmarks/results/2018-11-17.md @@ -0,0 +1,100 @@ +# 2018-11-17 + +zap and zerolog are from global GOPATH, using go1.11 + +based on https://godoc.org/golang.org/x/tools/benchmark/parse + +- N number of iteration +- ns/op nano second per iteration +- B/op bytes allocated per iteration +- allocs/op allocs per iteration +- MB/s need to use `b.SetBytes` in test to have this working... + +````text +go test -bench=. -benchmem +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkWithoutFieldsJSON/gommon-8 10000000 126 ns/op 496 B/op 4 allocs/op +BenchmarkWithoutFieldsJSON/Zap-8 10000000 205 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/Zap.Sugar-8 3000000 353 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/zerorlog-8 10000000 131 ns/op 0 B/op 0 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 6.617s +```` + +Need to see where the allocation comes from, need to focus on changing the interface of gommon first + +- most allocation comes from the `formatHead` method of json handler + - but how other library reduce allocation? using pool maybe? + +````text + Total: 17.55GB 17.55GB (flat, cum) 66.94% + 61 . . s.Sync() + 62 . . } + 63 . . } + 64 . . + 65 . . func formatHead(level log.Level, time time.Time, msg string) []byte { + 66 6.20GB 6.20GB b := make([]byte, 0, 5+4+10+len(msg)) + 67 . . b = append(b, `{"l":"`...) + 68 . . b = append(b, level.String()...) + 69 . . b = append(b, `","t":`...) + 70 . . b = strconv.AppendInt(b, time.Unix(), 10) + 71 . . b = append(b, `,"m":"`...) + 72 11.35GB 11.35GB b = append(b, msg...) + 73 . . b = append(b, '"') + 74 . . return b + 75 . . } + 76 . . + 77 . . func formatFields(b []byte, fields log.Fields) []byte { +```` + +After increase size of init slice and accept slice as input, the allocation is reduced by 1 + +````go +// handlers/json/handler.go +func formatHead(dst []byte, level log.Level, time int64, msg string) []byte { + dst = append(dst, `{"l":"`...) + dst = append(dst, level.String()...) + dst = append(dst, `","t":`...) + dst = strconv.AppendInt(dst, time, 10) + dst = append(dst, `,"m":"`...) + dst = append(dst, msg...) + dst = append(dst, '"') + return dst +} +```` + +After include logrus and apex/log + +````text +BenchmarkWithoutFieldsText/gommon-8 30000000 129 ns/op 272 B/op 3 allocs/op +BenchmarkWithoutFieldsText/std-8 10000000 401 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsText/zap-8 20000000 273 ns/op 72 B/op 3 allocs/op +BenchmarkWithoutFieldsText/zap.sugar-8 20000000 309 ns/op 200 B/op 5 allocs/op +BenchmarkWithoutFieldsText/zerolog-8 3000000 1395 ns/op 2011 B/op 36 allocs/op +BenchmarkWithoutFieldsText/apex-8 3000000 1551 ns/op 320 B/op 15 allocs/op +BenchmarkWithoutFieldsText/logrus-8 1000000 3209 ns/op 769 B/op 15 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 79.6 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/zap-8 20000000 179 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/zap.sugar-8 20000000 259 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/zerolog-8 50000000 113 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/apex-8 2000000 1920 ns/op 512 B/op 10 allocs/op +BenchmarkWithoutFieldsJSON/logrus-8 2000000 2990 ns/op 1218 B/op 22 allocs/op +```` + +## Conclusion + +- [ ] `formatHead` need to accept by and return byte in all handlers, the hard coded length for level and time need adjustment + +## Commands + +Ref + +- https://www.ardanlabs.com/blog/2017/05/language-mechanics-on-escape-analysis.html + - `go build -gcflags "-m -m" .` +- https://github.com/ardanlabs/gotraining/blob/master/topics/go/profiling/memcpu/README.md + - `go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out` +- https://segment.com/blog/allocation-efficiency-in-high-performance-go-services/ +- https://rakyll.org/pprof-ui/ + - `go tool pprof -http=:8080 p.out` \ No newline at end of file diff --git a/log/_benchmarks/results/2018-11-23.md b/log/_benchmarks/results/2018-11-23.md new file mode 100644 index 0000000..61c6098 --- /dev/null +++ b/log/_benchmarks/results/2018-11-23.md @@ -0,0 +1,37 @@ +# 2018-11-23 + +- klog (glog) only have 1 alloc per op, btw: stdlog only has 2 +- even with disabled level, due to `args... interface{}` convert to interface cause 1 allocation on heap + - [ ] TODO: why convert to `interface{}` cause allocation on heap ... escape somewhere? + - http://commaok.xyz/post/interface-allocs/ does not seems to help much ... + +````text +go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 7.37 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.23 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap-8 500000000 11.2 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.check-8 500000000 11.2 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.sugar-8 500000000 9.21 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/zerolog-8 5000000000 1.69 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/apex-8 2000000000 2.56 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/logrus-8 1000000000 7.04 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 140 ns/op 272 B/op 3 allocs/op +BenchmarkWithoutFieldsText/std-8 10000000 405 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsText/zap-8 20000000 283 ns/op 72 B/op 3 allocs/op +BenchmarkWithoutFieldsText/zap.sugar-8 10000000 333 ns/op 200 B/op 5 allocs/op +BenchmarkWithoutFieldsText/zerolog-8 3000000 1449 ns/op 2011 B/op 36 allocs/op +BenchmarkWithoutFieldsText/apex-8 3000000 1569 ns/op 320 B/op 15 allocs/op +BenchmarkWithoutFieldsText/logrus-8 1000000 3190 ns/op 769 B/op 15 allocs/op +BenchmarkWithoutFieldsText/klog-8 10000000 643 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 87.4 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/zap-8 30000000 184 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/zap.sugar-8 20000000 278 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/zerolog-8 50000000 114 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/apex-8 2000000 2098 ns/op 512 B/op 10 allocs/op +BenchmarkWithoutFieldsJSON/logrus-8 1000000 3060 ns/op 1218 B/op 22 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 122.702s +```` \ No newline at end of file diff --git a/log/_benchmarks/results/2018-11-24.md b/log/_benchmarks/results/2018-11-24.md new file mode 100644 index 0000000..95881f1 --- /dev/null +++ b/log/_benchmarks/results/2018-11-24.md @@ -0,0 +1,38 @@ +# 2018-11-24 + +- add `InfoF` into benchmark, it does not have `interface{}` thus no allocation when log level is disabled + - the 7ns/op is for the 1 allocation for `interface{}` I guess, it is same for gommon and logrus + +````text +go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 7.33 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.F-8 5000000000 0.57 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.23 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap-8 500000000 11.6 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.check-8 500000000 12.0 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.sugar-8 500000000 9.12 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/zerolog-8 5000000000 1.64 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/apex-8 2000000000 2.56 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/logrus-8 1000000000 7.08 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 137 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsText/gommon.F-8 50000000 88.1 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsText/std-8 10000000 403 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsText/zap-8 20000000 274 ns/op 72 B/op 3 allocs/op +BenchmarkWithoutFieldsText/zap.sugar-8 20000000 307 ns/op 200 B/op 5 allocs/op +BenchmarkWithoutFieldsText/zerolog-8 3000000 1392 ns/op 2012 B/op 36 allocs/op +BenchmarkWithoutFieldsText/apex-8 2000000 1558 ns/op 320 B/op 15 allocs/op +BenchmarkWithoutFieldsText/logrus-8 1000000 3161 ns/op 769 B/op 15 allocs/op +BenchmarkWithoutFieldsText/klog-8 10000000 611 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 81.4 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 38.6 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsJSON/zap-8 30000000 175 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/zap.sugar-8 20000000 257 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/zerolog-8 50000000 117 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/apex-8 1000000 3168 ns/op 512 B/op 10 allocs/op +BenchmarkWithoutFieldsJSON/logrus-8 1000000 3480 ns/op 1218 B/op 22 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 131.258s +```` \ No newline at end of file diff --git a/log/_benchmarks/results/2018-11-25.md b/log/_benchmarks/results/2018-11-25.md new file mode 100644 index 0000000..d88a8ab --- /dev/null +++ b/log/_benchmarks/results/2018-11-25.md @@ -0,0 +1,213 @@ +# 2018-11-25 + +````text +go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 6.92 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.F-8 5000000000 0.58 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.22 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap-8 500000000 10.9 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.check-8 500000000 11.2 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.sugar-8 500000000 9.32 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/zerolog-8 5000000000 1.66 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/apex-8 2000000000 2.56 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/logrus-8 1000000000 6.91 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 132 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsText/gommon.F-8 50000000 88.2 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsText/std-8 10000000 401 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsText/zap-8 20000000 275 ns/op 72 B/op 3 allocs/op +BenchmarkWithoutFieldsText/zap.sugar-8 20000000 311 ns/op 200 B/op 5 allocs/op +BenchmarkWithoutFieldsText/zerolog-8 3000000 1422 ns/op 2011 B/op 36 allocs/op +BenchmarkWithoutFieldsText/apex-8 2000000 1538 ns/op 320 B/op 15 allocs/op +BenchmarkWithoutFieldsText/logrus-8 1000000 3154 ns/op 769 B/op 15 allocs/op +BenchmarkWithoutFieldsText/klog-8 10000000 615 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsTextFormat/gommon-8 20000000 210 ns/op 360 B/op 4 allocs/op +BenchmarkWithoutFieldsTextFormat/zap.sugar-8 10000000 369 ns/op 241 B/op 6 allocs/op +BenchmarkWithoutFieldsTextFormat/apex-8 2000000 1944 ns/op 488 B/op 18 allocs/op +BenchmarkWithoutFieldsTextFormat/logrus-8 1000000 4012 ns/op 1001 B/op 18 allocs/op +BenchmarkWithoutFieldsTextFormat/klog-8 10000000 634 ns/op 24 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 107 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 63.5 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsJSON/zap-8 30000000 182 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/zap.sugar-8 20000000 260 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/zerolog-8 50000000 116 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/apex-8 2000000 2440 ns/op 512 B/op 10 allocs/op +BenchmarkWithoutFieldsJSON/logrus-8 1000000 3867 ns/op 1218 B/op 22 allocs/op +BenchmarkWithContextFieldsJSON/gommon-8 30000000 158 ns/op 608 B/op 4 allocs/op +BenchmarkWithContextFieldsJSON/gommon.F-8 30000000 110 ns/op 480 B/op 2 allocs/op +BenchmarkWithContextFieldsJSON/zap-8 30000000 187 ns/op 0 B/op 0 allocs/op +BenchmarkWithContextFieldsJSON/zap.sugar-8 20000000 258 ns/op 128 B/op 2 allocs/op +BenchmarkWithContextFieldsJSON/zerolog-8 50000000 113 ns/op 0 B/op 0 allocs/op +BenchmarkWithContextFieldsJSON/apex-8 1000000 4718 ns/op 1089 B/op 19 allocs/op +BenchmarkWithContextFieldsJSON/logrus-8 1000000 5014 ns/op 1362 B/op 25 allocs/op +BenchmarkNoContextWithFieldsJSON/gommon.F-8 30000000 128 ns/op 608 B/op 3 allocs/op +BenchmarkNoContextWithFieldsJSON/zap-8 20000000 282 ns/op 128 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/zap.sugar-8 20000000 296 ns/op 256 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/zerolog-8 30000000 123 ns/op 0 B/op 0 allocs/op +BenchmarkNoContextWithFieldsJSON/apex-8 1000000 4123 ns/op 1545 B/op 23 allocs/op +BenchmarkNoContextWithFieldsJSON/logrus-8 1000000 4560 ns/op 1844 B/op 29 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 228.248s +```` + +gommon only + +````text + make bench-gommon +go test -run none -bench=".*/gommon" -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 6.81 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.F-8 5000000000 0.57 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.22 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 129 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsText/gommon.F-8 50000000 89.6 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsTextFormat/gommon-8 20000000 202 ns/op 360 B/op 4 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 103 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 61.9 ns/op 160 B/op 1 allocs/op +BenchmarkWithContextFieldsJSON/gommon-8 30000000 146 ns/op 608 B/op 4 allocs/op +BenchmarkWithContextFieldsJSON/gommon.F-8 50000000 103 ns/op 480 B/op 2 allocs/op +BenchmarkNoContextWithFieldsJSON/gommon.F-8 30000000 119 ns/op 608 B/op 3 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 49.553s +```` + +gommon, after allocating more space for fields, `.F` reduced to one alloc + +````text +make bench-gommon +go test -run none -bench=".*/gommon" -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 6.91 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.F-8 5000000000 0.57 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.23 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 130 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsText/gommon.F-8 50000000 87.4 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsTextFormat/gommon-8 20000000 205 ns/op 360 B/op 4 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 101 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 62.2 ns/op 160 B/op 1 allocs/op +BenchmarkWithContextFieldsJSON/gommon-8 30000000 117 ns/op 352 B/op 3 allocs/op +BenchmarkWithContextFieldsJSON/gommon.F-8 50000000 74.0 ns/op 224 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/gommon.F-8 50000000 92.2 ns/op 352 B/op 2 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 48.130s +```` + +The allocation comes from ~~creating new slice~~ arguments escaped to heap because down the road they are passed as parameter +to interface method calls + +````text +BenchmarkWithContextFieldsJSON/gommon.F-8 50000000 74.0 ns/op 224 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/gommon.F-8 50000000 92.2 ns/op 352 B/op 2 allocs/op +```` + +````go + 533 6.05GB 16.61GB logger.InfoF(msg, dlog.Fields{ + 534 . . dlog.Int("i1", 1), dlog.Str("s1", "v1"), +```` + +Found the reason + +- first there is a typo ... `BenchmarkWithContextFieldsJSON` should be `BenchmarkWithContextNoFieldsJSON` +- second is handler is interface, since fields are passed to handler, it will have the `parameter to indirect call` problem, + - [ ] the 128 bytes is because we have two fields? and size of each field is 64 byte (or is same as zap's) +- zerolog does not have the problem because when its field method like `Int` it didn't call any method any interface, +it just encode to bytes directly, which is why zerolog's text is extremely slow, because it need to decode all the encoded ... + +````go +var ( + enc = json.Encoder{} +) + +func (e *Event) Int(key string, i int) *Event { + if e == nil { + return e + } + e.buf = enc.AppendInt(enc.AppendKey(e.buf, key), i) + return e +} + +func (Encoder) AppendInt(dst []byte, val int) []byte { + return strconv.AppendInt(dst, int64(val), 10) +} + +// Write transforms the JSON input with formatters and appends to w.Out. +func (w ConsoleWriter) Write(p []byte) (n int, err error) { + var evt map[string]interface{} + p = decodeIfBinaryToBytes(p) + d := json.NewDecoder(bytes.NewReader(p)) + d.UseNumber() + err = d.Decode(&evt) + if err != nil { + return n, fmt.Errorf("cannot decode event: %s", err) + } + + for _, p := range w.PartsOrder { + w.writePart(buf, evt, p) + } + + w.writeFields(evt, buf) + + buf.WriteByte('\n') + buf.WriteTo(w.Out) + return len(p), nil +} +```` + +````text +make bench +go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 8.35 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.F-8 5000000000 0.58 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.23 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap-8 500000000 13.2 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.check-8 300000000 14.6 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/zap.sugar-8 300000000 11.3 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/zerolog-8 5000000000 1.74 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/apex-8 2000000000 2.60 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/logrus-8 1000000000 7.40 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 145 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsText/gommon.F-8 50000000 99.1 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsText/std-8 10000000 446 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsText/zap-8 20000000 293 ns/op 72 B/op 3 allocs/op +BenchmarkWithoutFieldsText/zap.sugar-8 20000000 327 ns/op 200 B/op 5 allocs/op +BenchmarkWithoutFieldsText/zerolog-8 3000000 1414 ns/op 2012 B/op 36 allocs/op +BenchmarkWithoutFieldsText/apex-8 3000000 1549 ns/op 320 B/op 15 allocs/op +BenchmarkWithoutFieldsText/logrus-8 1000000 3469 ns/op 769 B/op 15 allocs/op +BenchmarkWithoutFieldsText/klog-8 10000000 674 ns/op 16 B/op 1 allocs/op +BenchmarkWithoutFieldsTextFormat/gommon-8 20000000 283 ns/op 360 B/op 4 allocs/op +BenchmarkWithoutFieldsTextFormat/zap.sugar-8 10000000 379 ns/op 241 B/op 6 allocs/op +BenchmarkWithoutFieldsTextFormat/apex-8 2000000 1976 ns/op 488 B/op 18 allocs/op +BenchmarkWithoutFieldsTextFormat/logrus-8 1000000 4055 ns/op 1001 B/op 18 allocs/op +BenchmarkWithoutFieldsTextFormat/klog-8 10000000 690 ns/op 24 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 30000000 135 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 70.7 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsJSON/zap-8 30000000 186 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/zap.sugar-8 20000000 263 ns/op 128 B/op 2 allocs/op +BenchmarkWithoutFieldsJSON/zerolog-8 50000000 115 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsJSON/apex-8 2000000 2010 ns/op 512 B/op 10 allocs/op +BenchmarkWithoutFieldsJSON/logrus-8 1000000 3138 ns/op 1218 B/op 22 allocs/op +BenchmarkWithContextNoFieldsJSON/gommon-8 30000000 127 ns/op 352 B/op 3 allocs/op +BenchmarkWithContextNoFieldsJSON/gommon.F-8 50000000 75.9 ns/op 224 B/op 1 allocs/op +BenchmarkWithContextNoFieldsJSON/zap-8 30000000 186 ns/op 0 B/op 0 allocs/op +BenchmarkWithContextNoFieldsJSON/zap.sugar-8 20000000 262 ns/op 128 B/op 2 allocs/op +BenchmarkWithContextNoFieldsJSON/zerolog-8 50000000 113 ns/op 0 B/op 0 allocs/op +BenchmarkWithContextNoFieldsJSON/apex-8 1000000 3609 ns/op 1089 B/op 19 allocs/op +BenchmarkWithContextNoFieldsJSON/logrus-8 1000000 4560 ns/op 1362 B/op 25 allocs/op +BenchmarkNoContextWithFieldsJSON/gommon.F-8 50000000 96.8 ns/op 352 B/op 2 allocs/op +BenchmarkNoContextWithFieldsJSON/zap-8 20000000 263 ns/op 128 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/zap.sugar-8 20000000 298 ns/op 256 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/zerolog-8 30000000 124 ns/op 0 B/op 0 allocs/op +BenchmarkNoContextWithFieldsJSON/apex-8 1000000 4498 ns/op 1545 B/op 23 allocs/op +BenchmarkNoContextWithFieldsJSON/logrus-8 1000000 4678 ns/op 1844 B/op 29 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 236.741s +```` \ No newline at end of file diff --git a/log/_benchmarks/results/2018-12-08.md b/log/_benchmarks/results/2018-12-08.md new file mode 100644 index 0000000..c39beb4 --- /dev/null +++ b/log/_benchmarks/results/2018-12-08.md @@ -0,0 +1,31 @@ +# 2018-12-08 + +- add caller to benchmark, time increased a lot when log file line is enabled, but it's still much smaller than apex/log & logrus (they don't even have log file line enabled) + - also the caller struct is not causing allocation + +````text +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 60.7 ns/op 160 B/op 1 allocs/op +BenchmarkCallerJSON/gommon.F-8 20000000 236 ns/op 240 B/op 1 allocs/op +```` + +````text +go test -run none -bench=".*/gommon" -benchtime 3s -benchmem -memprofile p.out +goos: linux +goarch: amd64 +pkg: github.com/dyweb/gommon/log/_benchmarks +BenchmarkDisabledLevelNoFormat/gommon-8 1000000000 6.77 ns/op 16 B/op 1 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.F-8 5000000000 0.58 ns/op 0 B/op 0 allocs/op +BenchmarkDisabledLevelNoFormat/gommon.check-8 5000000000 0.23 ns/op 0 B/op 0 allocs/op +BenchmarkWithoutFieldsText/gommon-8 30000000 127 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsText/gommon.F-8 50000000 86.9 ns/op 160 B/op 1 allocs/op +BenchmarkWithoutFieldsTextFormat/gommon-8 20000000 201 ns/op 360 B/op 4 allocs/op +BenchmarkWithoutFieldsJSON/gommon-8 50000000 101 ns/op 288 B/op 3 allocs/op +BenchmarkWithoutFieldsJSON/gommon.F-8 100000000 60.7 ns/op 160 B/op 1 allocs/op +BenchmarkCallerJSON/gommon-8 20000000 279 ns/op 368 B/op 3 allocs/op +BenchmarkCallerJSON/gommon.F-8 20000000 236 ns/op 240 B/op 1 allocs/op +BenchmarkWithContextNoFieldsJSON/gommon-8 30000000 114 ns/op 352 B/op 3 allocs/op +BenchmarkWithContextNoFieldsJSON/gommon.F-8 50000000 74.3 ns/op 224 B/op 1 allocs/op +BenchmarkNoContextWithFieldsJSON/gommon.F-8 50000000 92.1 ns/op 352 B/op 2 allocs/op +PASS +ok github.com/dyweb/gommon/log/_benchmarks 58.463s +```` \ No newline at end of file diff --git a/log/_examples/simple/.gitignore b/log/_examples/simple/.gitignore new file mode 100644 index 0000000..bd8888a --- /dev/null +++ b/log/_examples/simple/.gitignore @@ -0,0 +1,2 @@ +# the binary generate when using go build w/ gcflags +simple \ No newline at end of file diff --git a/log/_examples/simple/main.go b/log/_examples/simple/main.go index 6a718f3..bd0296b 100644 --- a/log/_examples/simple/main.go +++ b/log/_examples/simple/main.go @@ -9,42 +9,38 @@ import ( "github.com/dyweb/gommon/log/handlers/json" ) -var log = dlog.NewApplicationLogger() +var log, logReg = dlog.NewApplicationLoggerAndRegistry("simple") // simply log to stderr func main() { if len(os.Args) > 1 { if os.Args[1] == "json" { - dlog.SetHandlerRecursive(log, json.New(os.Stderr)) + dlog.SetHandler(logReg, json.New(os.Stderr)) } if os.Args[1] == "cli" { - dlog.SetHandlerRecursive(log, cli.New(os.Stderr, false)) + dlog.SetHandler(logReg, cli.New(os.Stderr, false)) } if os.Args[1] == "cli-d" { - dlog.SetHandlerRecursive(log, cli.New(os.Stderr, true)) + dlog.SetHandler(logReg, cli.New(os.Stderr, true)) } } - dlog.SetLevelRecursive(log, dlog.DebugLevel) + dlog.SetLevel(logReg, dlog.DebugLevel) log.Debug("show me the meaning of being lonely") log.Info("this is love!") + log.Print("print is info level") log.Warnf("this is love %d", 2) - log.InfoF("this love", dlog.Fields{ - dlog.Int("num", 2), - dlog.Str("foo", "bar"), - }) - log.EnableSource() + log.InfoF("this love", dlog.Int("num", 2), dlog.Str("foo", "bar")) + dlog.EnableSource(logReg) + // TODO: show skip caller using a util func log.Info("show me the line") log.Infof("show the line %d", 2) - log.InfoF("show the line", dlog.Fields{ - dlog.Int("num", 2), - dlog.Str("foo", "bar"), - }) + log.InfoF("show the line", dlog.Int("num", 2), dlog.Str("foo", "bar")) log.DisableSource() - log.WarnF("I will sleep", dlog.Fields{ - dlog.Int("duration", 1), - }) - time.Sleep(time.Second) + log.WarnF("I will sleep", dlog.Int("duration", 1)) + time.Sleep(10 * time.Millisecond) log.Info("no more line number") + log.AddField(dlog.Str("f1", "v1")) + log.Info("should have some extra context") // TODO: panic and fatal } diff --git a/log/caller.go b/log/caller.go new file mode 100644 index 0000000..d3a6ad6 --- /dev/null +++ b/log/caller.go @@ -0,0 +1,32 @@ +package log + +import "runtime" + +// caller.go contains logic for getting log file line + +// Caller is the result from runtime.Caller +type Caller struct { + // File is the full file path without any trimming, it would be UnknownFile if caller is not found + File string + Line int +} + +var emptyCaller = Caller{File: "", Line: 0} + +// EmptyCaller is mainly used for testing handler, it contains a empty file and line 0 +func EmptyCaller() Caller { + return emptyCaller +} + +const UnknownFile = "" + +// caller gets source location at runtime, in the future we may generate it at compile time to reduce the +// overhead, though I am not sure what the overhead is without actual benchmark and profiling +// Also I think the complexity added does not worth the performance benefits it gives +func caller(skip int) Caller { + _, file, line, ok := runtime.Caller(2 + skip) + if !ok { + return Caller{UnknownFile, 1} + } + return Caller{File: file, Line: line} +} diff --git a/log/caller_test.go b/log/caller_test.go new file mode 100644 index 0000000..1cbc63a --- /dev/null +++ b/log/caller_test.go @@ -0,0 +1,31 @@ +package log + +import ( + "testing" + + "github.com/dyweb/gommon/util/testutil" + "github.com/stretchr/testify/assert" +) + +// test if skip caller works + +var lgCallerSkip = NewPackageLogger() + +func utilReportError(m string) { + l := lgCallerSkip.Copy().SetCallerSkip(1) + l.InfoF(m) +} + +func TestLogger_SetCallerSkip(t *testing.T) { + th := NewTestHandler() + lgCallerSkip.SetHandler(th) + lgCallerSkip.EnableSource() + + utilReportError("mie") + + th.HasLog(InfoLevel, "mie") + l, ok := th.getLogByMessage("mie") + assert.True(t, ok) + assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/caller_test.go", l.source.File) + assert.Equal(t, 24, l.source.Line) +} diff --git a/log/doc.go b/log/doc.go index 8cea4f0..de99984 100644 --- a/log/doc.go +++ b/log/doc.go @@ -1,17 +1,5 @@ -/* -Package log provides structured logging with fine grained control -over libraries using a tree hierarchy of loggers. - -Conventions - -1. no direct use of the log package, MUST create new logger. - -2. library/application MUST have a library/application logger as their registry. - -3. every package MUST have a package level logger as child of the registry, normally defined in pkg.go - -4. logger is a registry and can contain children. - -5. instance of struct should have their own logger as children of package logger -*/ +// Package log provides structured logging with fine grained control +// over libraries using a tree of logger registry +// +// TODO: add convention and usage package log diff --git a/log/doc/README.md b/log/doc/README.md index 9949be4..e543d28 100644 --- a/log/doc/README.md +++ b/log/doc/README.md @@ -1,25 +1,6 @@ # gommon/log -## Survey - -https://github.com/avelino/awesome-go#logging - -- [solr](solr.md) the last straw that drive us to log v2 -- [std/log](std-log.md) -- [sirupsen/logrus](logrus.md) -- [apex/log](apex-log.md) -- [uber-go/zap](zap.md) -- [log15](log15.md) -- [gokit/log](gokit-log.md) -- [nanolog](nanolog.md) -- [glog](glog.md) -- [seelog](seelog.md) -- [log4j](log4j.md) -- [ ] TODO: might check open tracing as well, instrument like code should be put into other package - -Logging library used by popular go projects - -- k8s, [CockroachDB](https://github.com/cockroachdb/cockroach/tree/master/pkg/util/log) glog +- [Survey](survey) ## Goals diff --git a/log/doc/design/2018-09-05-clean-up.md b/log/doc/design/2018-09-05-clean-up.md new file mode 100644 index 0000000..13c1e99 --- /dev/null +++ b/log/doc/design/2018-09-05-clean-up.md @@ -0,0 +1,98 @@ +# 2018-09-05 + +Haven't really write clear doc about the design decisions made along the way, so start from now + +## Issues + +Handler + +- `MultiHandler` for fan out [#87](https://github.com/dyweb/gommon/issues/87) +- add parser for generated log [#89](https://github.com/dyweb/gommon/issues/89) + +Context (tree hierarchy) + +- Use fields attached to logger [#79](https://github.com/dyweb/gommon/issues/79) +- Simplify relationship of loggers [#78](https://github.com/dyweb/gommon/issues/78) +- A tree of logger has GC problem [#33](https://github.com/dyweb/gommon/issues/33) + +Runtime + +- allow specify caller skip [#86](https://github.com/dyweb/gommon/issues/86) +- debug log in init never show [#60](https://github.com/dyweb/gommon/issues/60) + +Performance + +- add benchmark code [#88](https://github.com/dyweb/gommon/issues/88) + +## Background + +**History** + +In the early design and implementation, gommon/log is basically following other libraries, +v1 followed logrus (using structured logging, have entry, formatter and writer). +v2 followed zap (using typed fields) and apex/log (using handler to replace formatter and writer). + +**Goals** + +Performance was a goal, though I always say it's not a major goal, but that can't explain those ugly magic number +and manually inline in handlers. However, without out measurement, those things could lead to poor performance. +Also performance is sometimes opposite with (strict) correctness, like if you don't do escape on fields, it will +be faster but if will break json format if you log a field with key `"contains quote"`, and if you don't de-dup fields +by key, careless user may log same key twice. + +Observability or fine grained control is another goal, this is mainly from using Solr during my internship at PayPal. +Solr has an admin page to control logging level of individual package, which is really amazing. +Considering when people report bug of an application and attach the log, +it will be filled with useless information when debug level is turned on for every library, +however if you can control level of single package (including those from dependency), +you can just ask user to run it with specific log flags like `--log-debug-packages=gommon,go.ice` + +**Current** + +Current implementation (before this refactor) is already usable (for a long time). +However, it's far from production ready, I need a 'new' gommon for a side project at work, +an advanced e2e test framework that writes log to both file and stdout and eventually ship to ELK. + +There are something I really like about current implementation. + +First is simplicity, unlike zap which pass struct around with slice of pointers (interfaces) in struct field, +all a handler needed is its function parameters, no entry struct. + +Second is using generator, a logging library, especially a leveled logging library, +a bunch of code is duplicated, `Info` has no difference with `Debug`. +Go does not have marco, but you can use `text/template` and then call format to have a poor man's generator. +A counter example is logrus, one thing surprised me about logrus is when you call `logrus.Infof` it is calling `logrus.Info(fmt.Sprintf)`, +this add extra call stack and makes finding source line for logrus hard, you need to traverse callers until you find one that is not logrus. + +There are something I really don't like about current implementation as well. + +First is over design, the tree of logger is inspired by solr but it goes too extreme, when each struct logger is +child of a package logger, the children map will contain a bunch of trash and may even cause those struct can't be +garbage collected. + +Second is blind optimization on performance with sacrifice on correctness, using magic number for initial slice size +when formatting is not wise, and don't escape for cli handler is fine, for json handler it's damaging. + +Third is no design doc before implementation, normally after the implementation I forgot the detail in 1 week if not 1 day, +and for some other libraries like noodle, I even [forgot how to use it](https://github.com/dyweb/gommon/pull/83). + +## Implementation + +This section is written two weeks after the start of the doc due to I can't free myself from working stuff (I can't blame +anyone for that though, was having a good time before the ddl approached) + +Before starting implementation, I first need to sort out of the dependency between the issues I am going to solve +in this milestone, it's not just I can pick the hardest/easiest, interface/functionality change may cause previous +effort in vain, and the order matters. + +I decided to go with benchmark first, there are two reasons. First writing benchmark force me to use other people's library +and I may have overlooked something when I am looking at godoc and source code. Second, benchmark can be used throughout +rest of the implementation in this milestone, especially when it comes to fields, tree of logger, handler interface and +implementation, there were many blind optimization that makes the code hard to maintain, but I have no clue if they +are following my assumption or the opposite. + +After benchmark is partially settled (we don't support inherit parent logger's fields so can't replica zap's benchmark fully), +We can start rework on fields, first is log should combine fields in logger and ad-hoc fields, this requires change in +handler interface and implementations (which was mainly copy and paste ...) + +At last is the tree of logger problem and improve logging source line (i.e. allow skip caller, using fields maybe) diff --git a/log/doc/design/2018-09-21-benchmark.md b/log/doc/design/2018-09-21-benchmark.md new file mode 100644 index 0000000..df9a3ea --- /dev/null +++ b/log/doc/design/2018-09-21-benchmark.md @@ -0,0 +1,23 @@ +# 2018-09-21 Benchmark + +There are three things, write the benchmark, run the benchmark, draw the graph. It's possible to collect profile as well, +i.e. use logger on a http server and load test the server, collect profile using pprof. + +The main problem for putting benchmark in this repo is it will result in very large dependency, but if I don't put it +in this repo, then the develop would be very painful, need to copy unreleased file to the benchmark repo. + +## Ref + +Benchmarks + +- zap https://github.com/uber-go/zap/tree/master/benchmarks + - first clone and put the repo to $GOPATH/src/go.uber.org/zap they are not using github repo as import path +- zerolog https://github.com/rs/logbench +- https://hackernoon.com/does-logging-cause-cpu-load-a-test-of-all-the-golang-logging-libraries-34052240f90d + - [x] it measure system call using `sudo strace -c -t -p $(pid)` +- https://medium.com/justforfunc/analyzing-the-performance-of-go-functions-with-benchmarks-60b8162e61c6 +- https://godoc.org/golang.org/x/perf + +Tools + +- https://godoc.org/golang.org/x/tools/benchmark/parse parse benchmark result \ No newline at end of file diff --git a/log/doc/design/2018-09-21-fields-and-handler-interface.md b/log/doc/design/2018-09-21-fields-and-handler-interface.md new file mode 100644 index 0000000..f45fc8a --- /dev/null +++ b/log/doc/design/2018-09-21-fields-and-handler-interface.md @@ -0,0 +1,3 @@ +# 2018-09-21 + +- 2018-11-18 already forgot what I was trying to write at that time, how to handle fields maybe \ No newline at end of file diff --git a/log/doc/design/2018-11-18-design-continued.md b/log/doc/design/2018-11-18-design-continued.md new file mode 100644 index 0000000..a72af6e --- /dev/null +++ b/log/doc/design/2018-11-18-design-continued.md @@ -0,0 +1,62 @@ +# 2018-11-18 + +Haven't worked on gommon for a while, thanks to the [go training](https://github.com/ardanlabs/gotraining), +idea about go performance increased a bit. Especially what is allocation in go, previously I never really thought +about what is on heap and what is on stack + +Continue on [2018-09-05](2018-09-05-clean-up.md) the basic steps are following + +- finish benchmark regardless of results + - try many log libraries in all the ways they allowed, structured, printf, try their interface and see their output + - zap + - zerolog + - glog? + - stdlog + - logrus + - apex/log + - if there are trivial optimization, optimize (or just keep track) along the way (though the may be in vain once we decided to change the public interface) +- list what are the use cases for gommon/log +- generate the modification plan +- execute plan +- another round of benchmark + +## Using other logging libraries + +- use fields to add context +- pass context down (i.e. store context in logger) +- source, file and line number + +For adding context, there are two ways in existing library + +- 'fast' libraries encode context to byte slice right away (result in they can't remove duplicated fields and sort fields added later) +- create an entry that holds both fields and reference to logger, attach all the logging methods `Debug` to the entry + - logger's `Debug` is just `newEntry().Debug` + +### Zap + +Keep context (attach fields) + +`logger.With(zap.Int("count", 1))` + +- it will clone and return a new logger when adding new fields + - also clone a core and encode fields right away into the core + - core (similar to handler) **encode context data** (fields) to avoid encode it several times (exchange space for speed) + - thus it will have duplicated fields if give same key with different data + +### Zerolog + +`logger.With().Str("k", "v").Logger()` + +- when `Str` is called, it will encode field and append to `[]byte` + +### Apex + +`logger.WithField("f1", "v1").Info("have field")` + +- same as logrus + +### Logrus + +`logger.WithField("f1", "v1").Info("have field")` + +- create a new entry that hold pointer to logger, add fields to the new entry \ No newline at end of file diff --git a/log/doc/design/2018-11-24-fields-and-handler-interface.md b/log/doc/design/2018-11-24-fields-and-handler-interface.md new file mode 100644 index 0000000..909b36f --- /dev/null +++ b/log/doc/design/2018-11-24-fields-and-handler-interface.md @@ -0,0 +1,163 @@ +# 2018-11-24 Fields and Handler interface + +Follow up on the [survey](2018-11-18-design-continued.md) after trying out how other library support adding fields to 'logger'. + +There are two ways, create a new entry that hold reference to the original logger or create a new logger. +I don't like the entry way, in this way the entry becomes the real logger, because all the user facing log methods like +`Debug`, `Info` all lands on entry instead of the logger. +For the logger way, you need to clone the logger so the fields are added instead of updated, however for zap and zerolog, +in order to increase speed, they encode the new fields into bytes right away. zerolog is actually more entry way, +it is using `Event`, event copy logger's context (which is bytes) and writer (basically io.Writer) + +Actually the `Event` in zerolog is still different from `Entry` in logrus, in zerolog you are keep updating the context +of current event when you add fields, while in logrus you are creating new entry when adding fields. + +````go +// apex/log/entry.go +// Entry represents a single log entry. +type Entry struct { + Logger *Logger `json:"-"` + Fields Fields `json:"fields"` + Level Level `json:"level"` + Timestamp time.Time `json:"timestamp"` + Message string `json:"message"` + start time.Time + fields []Fields +} + +// apex/log/entry.go +func (e *Entry) Debug(msg string) { + e.Logger.log(DebugLevel, e, msg) +} +```` + +````go +// logrus/entry.go +func (entry *Entry) Debug(args ...interface{}) { + if entry.Logger.IsLevelEnabled(DebugLevel) { + entry.log(DebugLevel, fmt.Sprint(args...)) + } +} +```` + +````go +// zap/logger.go + +// With creates a child logger and adds structured context to it. Fields added +// to the child don't affect the parent, and vice versa. +func (log *Logger) With(fields ...Field) *Logger { + if len(fields) == 0 { + return log + } + l := log.clone() + l.core = l.core.With(fields) + return l +} +```` + +````go +// zerolog/event.go + +// Event represents a log event. It is instanced by one of the level method of +// Logger and finalized by the Msg or Msgf method. +type Event struct { + buf []byte + w LevelWriter + level Level + done func(msg string) + ch []Hook // hooks from context +} + + +// Str adds the field key with val as a string to the *Event context. +func (e *Event) Str(key, val string) *Event { + if e == nil { + return e + } + e.buf = enc.AppendString(enc.AppendKey(e.buf, key), val) + return e +} +```` + +## Proposed change set + +- enable fields inside logger struct to keep context + - `AddField(f Field)` to add field in place, normally used for struct and method logger + - ~~`Entry()` to return a log entry that similar to entry in logrus & event in zerolog~~ +- ~~`Entry` copies handler, config from the logger when it is created and never update, it should be short lived~~ + - `AddField` + - only have `Info` `Info` but no `InfoF` like logger + - I don't think we really need entry when we have InfoF ... + +TODO + +- [x] ~~change identity from pointer to struct~~ still keep as pointer so we can have nil value + - for function that returns identity, return value instead of pointer + - [ ] might change it to a string? +- [ ] enable fields +- [ ] allow add fields (don't allow remove) +- [ ] change handler interface or logger to merge fields with adhoc fields + - I don't want to create new slice just for merging the fields, and logger need to check if its context is empty + + +- now it becomes permutation + - source + - fields + - context +- we have total 2 * 2 * 2 ... 8 methods to implement, well ... such copy and paste and if else ... + +````go +type Handler interface { + HandleLogWithFields(level Level, time time.Time, msg string, fields Fields) + // context are fields from the logger + HandleLogWithContextFields(lvl Level, time time.Time, msg string, context Fields, fields Fields) +} +```` + +Also I decided to trim do the interface, it may cause some extra performance overhead due to more if else inside handler,. +but this avoid copy paste code in handler implementations, also make handler func possible + +Before simplify interface + +````go +// Handler formats log message and writes to underlying storage, stdout, file, remote server etc. +// It MUST be thread safe because logger calls handler concurrently without any locking. +// There is NO log entry struct in gommon/log, which is used in many logging packages, the reason is +// if extra field is added to the interface, compiler would throw error on stale handler implementations. +type Handler interface { + // HandleLog accepts level, log time, formatted log message + HandleLog(level Level, now time.Time, msg string) + // HandleLogWithSource accepts formatted source line of log i.e., http.go:13 + // TODO: pass frame instead of string so handler can use trace for error handling? + HandleLogWithSource(level Level, now time.Time, msg string, source string) + // HandleLogWithFields accepts fields with type hint, + // implementation should inspect the type field instead of using reflection + HandleLogWithFields(level Level, now time.Time, msg string, fields Fields) + // HandleLogWithSourceFields accepts both source and fields + HandleLogWithSourceFields(level Level, now time.Time, msg string, source string, fields Fields) + // HandleLogWithContextFields get context from logger, which is also fields + HandleLogWithContextFields(level Level, now time.Time, msg string, context Fields, fields Fields) + // HandleLogWithSourceContextFields contains everything + HandleLogWithSourceContextFields(level Level, now time.Time, msg string, source string, context Fields, fields Fields) + // Flush writes the buffer to underlying storage + Flush() +} +```` + +After simplify interface + +````go +// Handler formats log message and writes to underlying storage, stdout, file, remote server etc. +// It MUST be thread safe because logger calls handler concurrently without any locking. +// There is NO log entry struct in gommon/log, which is used in many logging packages, the reason is +// if extra field is added to the interface, compiler would throw error on stale handler implementations. +type Handler interface { + // HandleLog requires level, now, msg, all the others are optional + // source is file:line, i.e. main.go:18 TODO: pass frame instead of string so handler can use trace for error handling? + // context are fields attached to the logger instance + // fields are ad-hoc fields from logger method like DebugF(msg, fields) + HandleLog(level Level, now time.Time, msg string, source string, context Fields, fields Fields) + // Flush writes the buffer to underlying storage + Flush() +} +```` \ No newline at end of file diff --git a/log/doc/design/2018-11-25-benchmark-with-fields.md b/log/doc/design/2018-11-25-benchmark-with-fields.md new file mode 100644 index 0000000..d566c33 --- /dev/null +++ b/log/doc/design/2018-11-25-benchmark-with-fields.md @@ -0,0 +1,29 @@ +# 2018-11-25 Benchmark with fields + +Finally, gommon/log support fields in logger, which is often called context + +## zap + +Disabled + +- disabled w/o fields +- disabled accumulated context (fields added to logger) +- disabled adding fields (adding fields when log) + +Not disabled + +- w/o fields +- accumulated context +- adding fields + +## zerolog + +https://github.com/rs/logbench + +It's more cleaner than zap's no copy and paste + +enable/disabled are top level + +- no context +- with context +- different type of fields \ No newline at end of file diff --git a/log/doc/design/2018-11-25-json.md b/log/doc/design/2018-11-25-json.md new file mode 100644 index 0000000..5d38289 --- /dev/null +++ b/log/doc/design/2018-11-25-json.md @@ -0,0 +1,320 @@ +# 2018-11-25 json + +## Survey + +How does other libraries handling json escape, gommon wasn't handling it at all so it will break easily, might want +- https://github.com/json-iterator/go/blob/master/stream_str.go#L311 + +Common + +- need to call `utf8.DecodeRuneInString` for handling utf8 when there are characters that need escape + +````go +// encoding/json/encode.go + +// NOTE: keep in sync with stringBytes below. +func (e *encodeState) string(s string, escapeHTML bool) { + e.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if htmlSafeSet[b] || (!escapeHTML && safeSet[b]) { + i++ + continue + } + if start < i { + e.WriteString(s[start:i]) + } + switch b { + case '\\', '"': + e.WriteByte('\\') + e.WriteByte(b) + case '\n': + e.WriteByte('\\') + e.WriteByte('n') + case '\r': + e.WriteByte('\\') + e.WriteByte('r') + case '\t': + e.WriteByte('\\') + e.WriteByte('t') + default: + // This encodes bytes < 0x20 except for \t, \n and \r. + // If escapeHTML is set, it also escapes <, >, and & + // because they can lead to security holes when + // user-controlled strings are rendered into JSON + // and served to some browsers. + e.WriteString(`\u00`) + e.WriteByte(hex[b>>4]) + e.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + e.WriteString(s[start:i]) + } + e.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + e.WriteString(s[start:i]) + } + e.WriteString(`\u202`) + e.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + e.WriteString(s[start:]) + } + e.WriteByte('"') +} +```` + +- zap + +````go +// zap/zapcore/json_encoder.go + +// safeAddString JSON-escapes a string and appends it to the internal buffer. +// Unlike the standard library's encoder, it doesn't attempt to protect the +// user from browser vulnerabilities or JSONP-related problems. +func (enc *jsonEncoder) safeAddString(s string) { + for i := 0; i < len(s); { + if enc.tryAddRuneSelf(s[i]) { + i++ + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if enc.tryAddRuneError(r, size) { + i++ + continue + } + enc.buf.AppendString(s[i : i+size]) + i += size + } +} + +// tryAddRuneSelf appends b if it is valid UTF-8 character represented in a single byte. +func (enc *jsonEncoder) tryAddRuneSelf(b byte) bool { + if b >= utf8.RuneSelf { + return false + } + if 0x20 <= b && b != '\\' && b != '"' { + enc.buf.AppendByte(b) + return true + } + switch b { + case '\\', '"': + enc.buf.AppendByte('\\') + enc.buf.AppendByte(b) + case '\n': + enc.buf.AppendByte('\\') + enc.buf.AppendByte('n') + case '\r': + enc.buf.AppendByte('\\') + enc.buf.AppendByte('r') + case '\t': + enc.buf.AppendByte('\\') + enc.buf.AppendByte('t') + default: + // Encode bytes < 0x20, except for the escape sequences above. + enc.buf.AppendString(`\u00`) + enc.buf.AppendByte(_hex[b>>4]) + enc.buf.AppendByte(_hex[b&0xF]) + } + return true +} + +func (enc *jsonEncoder) tryAddRuneError(r rune, size int) bool { + if r == utf8.RuneError && size == 1 { + enc.buf.AppendString(`\ufffd`) + return true + } + return false +} +```` + +- zerolog + +````go +// rs/zerolog/internal/json/string.go + +// AppendString encodes the input string to json and appends +// the encoded string to the input byte slice. +// +// The operation loops though each byte in the string looking +// for characters that need json or utf8 encoding. If the string +// does not need encoding, then the string is appended in it's +// entirety to the byte slice. +// If we encounter a byte that does need encoding, switch up +// the operation and perform a byte-by-byte read-encode-append. +func (Encoder) AppendString(dst []byte, s string) []byte { + // Start with a double quote. + dst = append(dst, '"') + // Loop through each character in the string. + for i := 0; i < len(s); i++ { + // Check if the character needs encoding. Control characters, slashes, + // and the double quote need json encoding. Bytes above the ascii + // boundary needs utf8 encoding. + if !noEscapeTable[s[i]] { + // We encountered a character that needs to be encoded. Switch + // to complex version of the algorithm. + dst = appendStringComplex(dst, s, i) + return append(dst, '"') + } + } + // The string has no need for encoding an therefore is directly + // appended to the byte slice. + dst = append(dst, s...) + // End with a double quote + return append(dst, '"') +} + +// appendStringComplex is used by appendString to take over an in +// progress JSON string encoding that encountered a character that needs +// to be encoded. +func appendStringComplex(dst []byte, s string, i int) []byte { + start := 0 + for i < len(s) { + b := s[i] + if b >= utf8.RuneSelf { + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + // In case of error, first append previous simple characters to + // the byte slice if any and append a remplacement character code + // in place of the invalid sequence. + if start < i { + dst = append(dst, s[start:i]...) + } + dst = append(dst, `\ufffd`...) + i += size + start = i + continue + } + i += size + continue + } + if noEscapeTable[b] { + i++ + continue + } + // We encountered a character that needs to be encoded. + // Let's append the previous simple characters to the byte slice + // and switch our operation to read and encode the remainder + // characters byte-by-byte. + if start < i { + dst = append(dst, s[start:i]...) + } + switch b { + case '"', '\\': + dst = append(dst, '\\', b) + case '\b': + dst = append(dst, '\\', 'b') + case '\f': + dst = append(dst, '\\', 'f') + case '\n': + dst = append(dst, '\\', 'n') + case '\r': + dst = append(dst, '\\', 'r') + case '\t': + dst = append(dst, '\\', 't') + default: + dst = append(dst, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF]) + } + i++ + start = i + } + if start < len(s) { + dst = append(dst, s[start:]...) + } + return dst +} +```` + +- https://github.com/json-iterator/go/blob/master/stream_str.go#L310-L372 + +````go +// https://github.com/json-iterator/go/blob/master/stream_str.go#L310-L372 + +// WriteString write string to stream without html escape +func (stream *Stream) WriteString(s string) { + valLen := len(s) + stream.buf = append(stream.buf, '"') + // write string, the fast path, without utf8 and escape support + i := 0 + for ; i < valLen; i++ { + c := s[i] + if c > 31 && c != '"' && c != '\\' { + stream.buf = append(stream.buf, c) + } else { + break + } + } + if i == valLen { + stream.buf = append(stream.buf, '"') + return + } + writeStringSlowPath(stream, i, s, valLen) +} + +func writeStringSlowPath(stream *Stream, i int, s string, valLen int) { + start := i + // for the remaining parts, we process them char by char + for i < valLen { + if b := s[i]; b < utf8.RuneSelf { + if safeSet[b] { + i++ + continue + } + if start < i { + stream.WriteRaw(s[start:i]) + } + switch b { + case '\\', '"': + stream.writeTwoBytes('\\', b) + case '\n': + stream.writeTwoBytes('\\', 'n') + case '\r': + stream.writeTwoBytes('\\', 'r') + case '\t': + stream.writeTwoBytes('\\', 't') + default: + // This encodes bytes < 0x20 except for \t, \n and \r. + // If escapeHTML is set, it also escapes <, >, and & + // because they can lead to security holes when + // user-controlled strings are rendered into JSON + // and served to some browsers. + stream.WriteRaw(`\u00`) + stream.writeTwoBytes(hex[b>>4], hex[b&0xF]) + } + i++ + start = i + continue + } + i++ + continue + } + if start < len(s) { + stream.WriteRaw(s[start:]) + } + stream.writeByte('"') +} +```` \ No newline at end of file diff --git a/log/doc/design/2018-12-02-tree-of-logger.md b/log/doc/design/2018-12-02-tree-of-logger.md new file mode 100644 index 0000000..cb0bdf1 --- /dev/null +++ b/log/doc/design/2018-12-02-tree-of-logger.md @@ -0,0 +1,361 @@ +# 2018-12-02 Tree of logger + +## Issues + +- [#33](https://github.com/dyweb/gommon/issues/33) Maintain a Tree of Loggers is hard when struct has their own loggers +- [#78](https://github.com/dyweb/gommon/issues/78) Simplify the relationship of loggers + +## Background + +This is the original motivation of having gommon, have hierarchy between loggers and give application +fine grained control over its dependency libraries' log levels. +Most existing log libraries either requires library authors to expose logging configuration (i.e. pass a logger when +creating a client) or use a global logging level. +This result in hard time for developer & user to filter out useful message during development and when report errors. +A ideal scenario in pseudo code would be `server --log-debug=Server.WritePosts,Server.DeletePosts` where only some +package has debug level enabled + +````text +// user report on already released code +// server --log-debug=Server.WritePosts,Server.DeletePosts +func (s *Server) WritePosts() { + log := s.logger.Method() // a method logger + log.Debug("very huge text") +} + +// developer wrote a buggy code in a new method, hard code log level in code for the time being +func (s *Server) WritePosts() { + // log := s.logger // normal logger + log := s.logger.Copy().SetLevel(Debug) + log.Debug("very huge text") +} +```` + +A good example would be the Java world, where you can use a xml file to config log of multiple packages. +[cihub/seelog](https://github.com/cihub/seelog) is a very close implementation in go. +Also the main inspiration was the log level config UI in solr + +![solr-log-ui](../survey/solr-log-admin.png) + +In summary the problem I wanted to solve by introducing a tree of logger is to +allow config log level of individual package/struct so that the log is filtered +from producer and no external filtering tool is required. + +## Previous Solutions + +The original solution in log v1 is a filter inspired by logrus's hook, +it keeps a set (actually it's a map) of package names, and entry struct +has a field called Pkg. Actually I think I never used it during the lifespan +of log v1, in v2 I started model after apex/log and uber-go/zap so hooks +are omitted + +````go +// Filter determines if the entry should be logged +type Filter interface { + Accept(entry *Entry) bool + FilterName() string + FilterDescription() string +} + +var _ Filter = (*PkgFilter)(nil) + +// PkgFilter only allows entry without `pkg` field or `pkg` value in the allow set to pass +// TODO: we should support level +// TODO: a more efficient way might be trie tree and use `/` to divide package into segments instead of using character +type PkgFilter struct { + allow st.Set +} + +// Accept checks if the entry.Pkg (NOT entry.Fields["pkg"]) is in the white list +func (filter *PkgFilter) Accept(entry *Entry) bool { + return filter.allow.Contains(entry.Pkg) +} + +// FilterName implements Filter interface +func (filter *PkgFilter) FilterName() string { + return "PkgFilter" +} + +func (filter *PkgFilter) FilterDescription() string { + return "Filter log based on their pkg tag value, it is logged if it does not have pkg field or in whitelist" +} +```` + +Then follow the idea in the solr UI I decided to make **a tree of logger** +and allow each logger has their own logging level, because instead of +filtering based on package, you can simply turn one package to a more +verbose level while keep other package at their original level. +Thus I allow the loggers to contains children and identity, +parent children relationship is kept by forcing all the loggers created +using factory function. +Identity is obtained when create the logger using factory function using +runtime, I used some dirty trick to split method and function. +Then to set log level one have to traverse the logger tree to set all the +loggers, each package expose its top level logger, which is also a registry. + +````go +type Logger struct { + // mu is a Mutex instead of RWMutex because it's only for avoid concurrent write, + // for performance reason and the natural of logging, reading stale config is not a big problem, + // so we don't check mutex on read operation (i.e. log message) and allow race condition + mu sync.Mutex + h Handler + level Level + source bool + children map[string][]*Logger + + id *Identity // use nil so we can have logger without identity +} + +// +// TODO: allow release a child logger, this will be a trouble if we created 1,000 Client struct with its own logger... +func (l *Logger) AddChild(child *Logger) { + l.mu.Lock() + if l.children == nil { + l.children = make(map[string][]*Logger, 1) + } + // children are group by their identity, i.e a package logger may have many struct logger of same struct because + // that struct is used in multiple goroutines, those loggers have different pointer for identity, but they should + // have same source line, so we use SourceLocation as key + k := child.id.SourceLocation() + children := l.children[k] + // avoid putting same pointer twice, though it should never happen if AddChild is called correctly + exists := false + for _, c := range children { + if c == child { + exists = true + break + } + } + if !exists { + l.children[k] = append(children, child) + } + l.mu.Unlock() +} + +func SetLevelRecursive(root *Logger, level Level) { + visited := make(map[*Logger]bool) + PreOrderDFS(root, visited, func(l *Logger) { + // TODO: remove it after we have tested it .... + //fmt.Println(l.Identity().String()) + l.SetLevel(level) + }) +} + +func NewPackageLogger() *Logger { + return NewPackageLoggerWithSkip(1) +} + +func NewFunctionLogger(packageLogger *Logger) *Logger { + id := NewIdentityFromCaller(1) + l := &Logger{ + id: &id, + } + return newLogger(packageLogger, l) +} + +func NewStructLogger(packageLogger *Logger, loggable LoggableStruct) *Logger { + id := loggable.LoggerIdentity(func() Identity { + return NewIdentityFromCaller(1) + }) + l := &Logger{ + id: &id, + } + l = newLogger(packageLogger, l) + loggable.SetLogger(l) + return l +} + +func NewMethodLogger(structLogger *Logger) *Logger { + id := NewIdentityFromCaller(1) + l := &Logger{ + id: &id, + } + return newLogger(structLogger, l) +} +```` + +Keeping this parent child relationship introduce problem to gc because +after lifespan of a struct/method has expired, their logger is still referenced +by their parent logger in the map. + +````go +var log = NewPackageLogger() + +func handleEcho(w http.ResponseWriter, r *http.Request) { + logger := NewFunctionLogger(log) + logger.Info("handling echo") +} + +func main() { + m := http.NewServerMux() + m.HandlerFunc("/echo", handleEcho) + http.ListenAndServe(m) +} +```` + +In the example above, every time the echo func is called, a new entry is added +to the package level logger, the logger is useless after the func is finished +because next time it is going to create a new one, but because of +the children map in its parent, the logger will be referenced and can't be +garbage collected, not to mention have a huge map with pointer value will +also cause problem to garbage collector. + +Besides the performance (it's actually memory leak) +We are also mixing the logger with logger registry but saving the map in registry. +One observation is package/library level logger are singleton, because +I just define them as `var log` in `pkg.go`, so they will be there in the entire +application lifecycle and there will be a fixed number of them. +Function and method logger will have many, it's a common practice to create +a new logger with fields attached based on function parameters, and they won't +run for a long time. +Struct is tricky, some struct like Server will likely have same span as entire +application. While others like Worker will only live for a short time. + +````go +var log = NewPackageLogger() + +func findUser(u string) { + logger := NewFunctionLogger(log).AddField(dlog.Str("user", u)) + logger.Info("start finding user") + logger.Warn("user not found") +} + +// long running struct like server +type Server struct { + logger *dlog.Logger +} + +func (s *Server) Echo(w http.ResponseWriter, r *http.Request) { + myName := r.Query().Get("name") + logger := NewMethodLogger(s.logger).AddField(dlog.Str("name", myName)) + logger.Info("let's say hi") +} + +// short running +type Worker struct { + logger *dlog.Logger +} + +func NewWorker() *Worker { + w := Worker{} + dlog.NewStructLogger(w) // need to generate getter setter using gommon +} + +func (s *Worker) Fetch() { + // wget google.com .... +} + +func Work() { + var wg sync.WaitGroup + wg.Add(100) + go func() { + w := NewWoker() + w.Fetch() + wg.Done() + } + wg.Wait() +} +```` + +## Proposed solutions + +Based on previous solutions we can find the following patterns + +- keep the parent children relationship for all the loggers is unrealistic (gc) and unnecessary (a lot of them are short lived, just copy the parent's level and handler is fine) +- set level only gives some basic control, we can have more complex control based on logger location (the identity we have now) and caller location, this logic can be implemented in handler +- we can move logger relationship out into registry and only register important loggers like library, package and long running struct + +Some performance concerns + +- we will put identity caller into handler interface, identity and caller should be put as struct, +pass pointer to struct may cause heap allocation, pass struct need to deal with empty identity ... + +The new design for tree of logger and filter log when generating log has the following part + +- registry that keeps a tree of registry and loggers +- handler that accept identity, user can implement any logic inside that handler + +````text +type Logger struct { + identity *Identity // could be nil, though most time it should not be except in benchmark and test +} + +type Registry { + childRegistry map[string]*Registry // use pointer instead of struct because Registry also use slice, if it is only using map, we can use struct because map is actually a pointer to underlying hashmap struct + childLoggers []*Logger +} + +func SetLevel(rootRegistry Registry, lvl Level) { + DfsRegistry(rootRegistry, func(reg *Registry) { + for _, l := range reg.Logger { + l.SetLevel(lvl) + } + }) + // or + DfsLogger(rootRegistry, func(l *Logger) { + l.SetLevel(lvl) + }) +} + +type Handler interface { + HandleLog(loggerIdentity *Identity, caller Caller, level Level, msg string, context []Field, fields []Field) +} + +type FileLineFilter struct { + blockFile string + blockLine int + h Handler +} + +func (fl *FileLineFilter) HandleLog(loggerIdentity *Identity, caller Caller, level Level, msg string, context []Field, fields []Field) { + if caller.File == fl.blockFile { + return + } + if caller.Line == fl.blockLine { + return + } + // go through + h(loggerIdentity, caller, msg, context, fields) +} + +// TODO: need an example that check identity and level, if debug and is an allowed struct, go through +```` + + +````text +// gommon/config/config.go +var log, logRegistry = NewPackageLoggerWithRegistry() + +func NewConfigLoader() ConfigLoader { + l := ConfigLoader + l.logger = dlog.NewStructLogger(log, l) // NOTE: package logger is now only used for copy level and handler, no longer used for registry +} + +// ayi/web/server.go +var log, logRegistry = NewPackageLoggerWithRegistry() + +func NewServer() Server { + s := Server{} + s.logger = dlog.NewStructLogger(log, s) + logRegistry.AddLogger(s.logger) // server is long running and we know when it shuts down +} + +func (s *Server) Echo(w http.ResponseWriter, r *http.Request) { + logger := s.logger.MethodLogger() // add method field automatically + +} +```` + +## Implementation + +- [x] remove the parent children logic from logger +- [x] add traverse registry and logger + - [x] add traverse log in PreOrderDfs +- [ ] fix logic for func and method logger + - [ ] what was I trying to fix? creating sub logger or? +- [ ] check if the skip caller is correct when create logger registry, unit test is in same package so it's always correct +- [ ] check if identity of log registry is correct +- [ ] check if identity of logger is correct +- [x] change caller to struct + - [ ] it may have some performance impact \ No newline at end of file diff --git a/log/doc/survey/README.md b/log/doc/survey/README.md new file mode 100644 index 0000000..a7f7794 --- /dev/null +++ b/log/doc/survey/README.md @@ -0,0 +1,32 @@ +# Survey + +https://github.com/avelino/awesome-go#logging + +High performance + +- [uber-go/zap](zap.md) structured logging with high performance +- [rs/zerolog](zerolog.md) high performance, only focus on json logging +- [nanolog](nanolog.md) use binary format, only store the changed part (like prepare statement in database) + +Simple + +- [std/log](std-log.md) standard library log package +- [glog](glog.md) leveled only, but can sample based on number of hits on a certain file:line +- [gokit/log](gokit-log.md) extreme simple interface + +Structured + +- [sirupsen/logrus](logrus.md) structured logging, poor performance +- [apex/log](apex-log.md) use handler instead of formatter + writer +- [log15](log15.md) lazy evaluation + +Java(ish) + +- [solr](solr.md) the last straw that drive us to log v2, gives you [a tree graph to control log level of ALL the packages](solr-log-admin.png), including dependencies +- [seelog](seelog.md) javaish, fine grained control log filtering (by func, file etc.) +- [log4j](log4j.md) java logger +- [ ] TODO: might check open tracing as well, instrument like code should be put into other package + +Logging library used by popular go projects + +- k8s, [CockroachDB](https://github.com/cockroachdb/cockroach/tree/master/pkg/util/log) glog \ No newline at end of file diff --git a/log/doc/apex-log.md b/log/doc/survey/apex-log.md similarity index 100% rename from log/doc/apex-log.md rename to log/doc/survey/apex-log.md diff --git a/log/doc/glog.md b/log/doc/survey/glog.md similarity index 100% rename from log/doc/glog.md rename to log/doc/survey/glog.md diff --git a/log/doc/gokit-log.md b/log/doc/survey/gokit-log.md similarity index 100% rename from log/doc/gokit-log.md rename to log/doc/survey/gokit-log.md diff --git a/log/doc/log15.md b/log/doc/survey/log15.md similarity index 100% rename from log/doc/log15.md rename to log/doc/survey/log15.md diff --git a/log/doc/log4j.md b/log/doc/survey/log4j.md similarity index 100% rename from log/doc/log4j.md rename to log/doc/survey/log4j.md diff --git a/log/doc/logrus.md b/log/doc/survey/logrus.md similarity index 100% rename from log/doc/logrus.md rename to log/doc/survey/logrus.md diff --git a/log/doc/nanolog.md b/log/doc/survey/nanolog.md similarity index 100% rename from log/doc/nanolog.md rename to log/doc/survey/nanolog.md diff --git a/log/doc/seelog.md b/log/doc/survey/seelog.md similarity index 100% rename from log/doc/seelog.md rename to log/doc/survey/seelog.md diff --git a/log/doc/solr-log-admin.png b/log/doc/survey/solr-log-admin.png similarity index 100% rename from log/doc/solr-log-admin.png rename to log/doc/survey/solr-log-admin.png diff --git a/log/doc/solr.md b/log/doc/survey/solr.md similarity index 100% rename from log/doc/solr.md rename to log/doc/survey/solr.md diff --git a/log/doc/std-log.md b/log/doc/survey/std-log.md similarity index 100% rename from log/doc/std-log.md rename to log/doc/survey/std-log.md diff --git a/log/doc/survey/zap-benchmark.md b/log/doc/survey/zap-benchmark.md new file mode 100644 index 0000000..aa62b84 --- /dev/null +++ b/log/doc/survey/zap-benchmark.md @@ -0,0 +1,78 @@ +# Zap benchmark + +https://github.com/at15/zap/tree/master/benchmarks + +- disabled level, no fields + - zap has a check to avoid logging when it is enabled +- disabled level, fields attached to logger instance +- disabled level, fields attached when call +- log without field, pure message and use `f` for formatting +- log with fields attached to logger instance +- log with fields attached when call + - [ ] TODO: what about combine fields, logger + call, it seems in zap, when you attach fields to logger, they are encoded right away + +````text +⇒ go test -bench=. +goos: linux +goarch: amd64 +pkg: go.uber.org/zap/benchmarks +BenchmarkDisabledWithoutFields/Zap-8 100000000 11.5 ns/op +BenchmarkDisabledWithoutFields/Zap.Check-8 100000000 11.4 ns/op +BenchmarkDisabledWithoutFields/Zap.Sugar-8 200000000 9.51 ns/op +BenchmarkDisabledWithoutFields/Zap.SugarFormatting-8 20000000 63.9 ns/op +BenchmarkDisabledWithoutFields/apex/log-8 1000000000 2.59 ns/op +BenchmarkDisabledWithoutFields/sirupsen/logrus-8 200000000 7.62 ns/op +BenchmarkDisabledWithoutFields/rs/zerolog-8 1000000000 2.40 ns/op +BenchmarkDisabledAccumulatedContext/Zap-8 100000000 11.7 ns/op +BenchmarkDisabledAccumulatedContext/Zap.Check-8 100000000 11.4 ns/op +BenchmarkDisabledAccumulatedContext/Zap.Sugar-8 200000000 9.56 ns/op +BenchmarkDisabledAccumulatedContext/Zap.SugarFormatting-8 20000000 65.1 ns/op +BenchmarkDisabledAccumulatedContext/apex/log-8 2000000000 1.06 ns/op +BenchmarkDisabledAccumulatedContext/sirupsen/logrus-8 200000000 7.24 ns/op +BenchmarkDisabledAccumulatedContext/rs/zerolog-8 1000000000 2.36 ns/op +BenchmarkDisabledAddingFields/Zap-8 10000000 139 ns/op +BenchmarkDisabledAddingFields/Zap.Check-8 100000000 11.6 ns/op +BenchmarkDisabledAddingFields/Zap.Sugar-8 20000000 67.9 ns/op +BenchmarkDisabledAddingFields/apex/log-8 5000000 242 ns/op +BenchmarkDisabledAddingFields/sirupsen/logrus-8 3000000 421 ns/op +BenchmarkDisabledAddingFields/rs/zerolog-8 30000000 47.9 ns/op +BenchmarkWithoutFields/Zap-8 10000000 147 ns/op +BenchmarkWithoutFields/Zap.Check-8 10000000 146 ns/op +BenchmarkWithoutFields/Zap.CheckSampled-8 30000000 44.0 ns/op +BenchmarkWithoutFields/Zap.Sugar-8 10000000 222 ns/op +BenchmarkWithoutFields/Zap.SugarFormatting-8 300000 3766 ns/op +BenchmarkWithoutFields/apex/log-8 1000000 1992 ns/op +BenchmarkWithoutFields/go-kit/kit/log-8 5000000 296 ns/op +BenchmarkWithoutFields/inconshreveable/log15-8 500000 3279 ns/op +BenchmarkWithoutFields/sirupsen/logrus-8 2000000 707 ns/op +BenchmarkWithoutFields/go.pedge.io/lion-8 3000000 452 ns/op +BenchmarkWithoutFields/stdlib.Println-8 3000000 427 ns/op +BenchmarkWithoutFields/stdlib.Printf-8 500000 3162 ns/op +BenchmarkWithoutFields/rs/zerolog-8 20000000 110 ns/op +BenchmarkWithoutFields/rs/zerolog.Formatting-8 300000 4158 ns/op +BenchmarkWithoutFields/rs/zerolog.Check-8 10000000 122 ns/op +BenchmarkAccumulatedContext/Zap-8 10000000 158 ns/op +BenchmarkAccumulatedContext/Zap.Check-8 10000000 157 ns/op +BenchmarkAccumulatedContext/Zap.CheckSampled-8 30000000 45.7 ns/op +BenchmarkAccumulatedContext/Zap.Sugar-8 10000000 233 ns/op +BenchmarkAccumulatedContext/Zap.SugarFormatting-8 500000 3762 ns/op +BenchmarkAccumulatedContext/apex/log-8 50000 26441 ns/op +BenchmarkAccumulatedContext/go-kit/kit/log-8 200000 6196 ns/op +BenchmarkAccumulatedContext/inconshreveable/log15-8 100000 17316 ns/op +BenchmarkAccumulatedContext/sirupsen/logrus-8 200000 7227 ns/op +BenchmarkAccumulatedContext/go.pedge.io/lion-8 500000 2353 ns/op +BenchmarkAccumulatedContext/rs/zerolog-8 20000000 113 ns/op +BenchmarkAccumulatedContext/rs/zerolog.Check-8 20000000 114 ns/op +BenchmarkAccumulatedContext/rs/zerolog.Formatting-8 500000 3237 ns/op +BenchmarkAddingFields/Zap-8 1000000 1224 ns/op +BenchmarkAddingFields/Zap.Check-8 1000000 1210 ns/op +BenchmarkAddingFields/Zap.CheckSampled-8 10000000 166 ns/op +BenchmarkAddingFields/Zap.Sugar-8 1000000 1448 ns/op +BenchmarkAddingFields/apex/log-8 50000 26937 ns/op +BenchmarkAddingFields/go-kit/kit/log-8 200000 6039 ns/op +BenchmarkAddingFields/inconshreveable/log15-8 50000 30591 ns/op +BenchmarkAddingFields/sirupsen/logrus-8 200000 7216 ns/op +BenchmarkAddingFields/go.pedge.io/lion-8 200000 5695 ns/op +BenchmarkAddingFields/rs/zerolog-8 300000 5173 ns/op +BenchmarkAddingFields/rs/zerolog.Check-8 300000 5158 ns/op +```` \ No newline at end of file diff --git a/log/doc/zap.md b/log/doc/survey/zap.md similarity index 98% rename from log/doc/zap.md rename to log/doc/survey/zap.md index 19acc05..a2f34f3 100644 --- a/log/doc/zap.md +++ b/log/doc/survey/zap.md @@ -314,6 +314,8 @@ func (f Field) AddTo(enc ObjectEncoder) { Core zapcore/core.go +- when adding fields to logger, it is encoded right away, so when log is called, only those extra new fields will be encoded + ````go // Core is a minimal, fast logger interface. It's designed for library authors // to wrap in a more user-friendly API. @@ -345,6 +347,12 @@ type ioCore struct { out WriteSyncer } +func (c *ioCore) With(fields []Field) Core { + clone := c.clone() + addFields(clone.enc, fields) + return clone +} + func (c *ioCore) Check(ent Entry, ce *CheckedEntry) *CheckedEntry { if c.Enabled(ent.Level) { return ce.AddCore(ent, c) diff --git a/log/doc/zerolog.md b/log/doc/survey/zerolog.md similarity index 54% rename from log/doc/zerolog.md rename to log/doc/survey/zerolog.md index 31880e5..a2ada7f 100644 --- a/log/doc/zerolog.md +++ b/log/doc/survey/zerolog.md @@ -3,6 +3,7 @@ https://github.com/rs/zerolog - mainly focus on JSON logging (not using `encoding/json` as I remember) +- also support CBOR http://cbor.io/ RFC 7049 Concise Binary Object Representation Like JSON but in binary, 123456 is no longer encoded as '123456' and bytes are no longer base64 encoded ````go zerolog.TimestampFieldName = "t" diff --git a/log/field.go b/log/field.go index 02032f3..d7ac5ad 100644 --- a/log/field.go +++ b/log/field.go @@ -4,7 +4,8 @@ import ( "fmt" ) -// FieldType avoids calling reflection +// FieldType avoids doing type assertion or calling reflection +// TODO: difference between the two methods above type FieldType uint8 const ( @@ -16,11 +17,22 @@ const ( // Fields is a slice of Field type Fields []Field +// CopyFields make a copy of the slice so modifying one won't have effect on another, +func CopyFields(fields Fields) Fields { + copied := make([]Field, len(fields)) + for i := 0; i < len(fields); i++ { + copied[i] = fields[i] + } + return copied +} + // Field is based on uber-go/zap https://github.com/uber-go/zap/blob/master/zapcore/field.go // It can be treated as a Union, the value is stored in either Int, Str or Interface type Field struct { - Key string - Type FieldType + Key string + Type FieldType + + // values Int int64 Str string Interface interface{} @@ -35,6 +47,11 @@ func Int(k string, v int) Field { } } +// TODO: reuse same int64 and rely on type? ... +//func Float(k string, v float64) Field { +// +//} + // Str creates a field with string value func Str(k string, v string) Field { return Field{ diff --git a/log/field_test.go b/log/field_test.go new file mode 100644 index 0000000..9397211 --- /dev/null +++ b/log/field_test.go @@ -0,0 +1,23 @@ +package log + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCopyFields(t *testing.T) { + fields := Fields{ + Str("k1", "v1"), + Int("k2", 2), + } + copied := CopyFields(fields) + assert.Equal(t, 2, cap(copied), "capacity is same as length") + fields[0].Key = "k1modified" + copied[0].Str = "v1modified" + + assert.Equal(t, "k1modified", fields[0].Key) + assert.Equal(t, "v1", fields[0].Str) + assert.Equal(t, "k1", copied[0].Key) + assert.Equal(t, "v1modified", copied[0].Str) +} diff --git a/log/generator.go b/log/generator.go index 408e124..aa06557 100644 --- a/log/generator.go +++ b/log/generator.go @@ -26,7 +26,7 @@ func ({{.Receiver}} {{.Struct}}) GetLogger() *dlog.Logger { return {{.Receiver}}.{{.Field}} } -func ({{.Receiver}} {{.Struct}}) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func ({{.Receiver}} {{.Struct}}) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } ` diff --git a/log/generator_test.go b/log/generator_test.go index b8f15cf..51abe3b 100644 --- a/log/generator_test.go +++ b/log/generator_test.go @@ -22,7 +22,7 @@ func (c *YAMLConfig) GetLogger() *dlog.Logger { return c.log } -func (c *YAMLConfig) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func (c *YAMLConfig) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } `}, @@ -35,7 +35,7 @@ func (c *YAMLConfig) GetLogger() *dlog.Logger { return c.logger } -func (c *YAMLConfig) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func (c *YAMLConfig) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } `}, diff --git a/log/gommon.yml b/log/gommon.yml index c09feca..bdc18ba 100644 --- a/log/gommon.yml +++ b/log/gommon.yml @@ -6,5 +6,6 @@ gotmpls: - Trace - Debug - Info + - Print - Warn - Error \ No newline at end of file diff --git a/log/handler.go b/log/handler.go index 38c8f62..1bce54b 100644 --- a/log/handler.go +++ b/log/handler.go @@ -4,10 +4,12 @@ import ( "io" "os" "strconv" - "sync" + "strings" "time" ) +// handler.go contains Handler interface and builtin implementations + const ( defaultTimeStampFormat = time.RFC3339 ) @@ -17,202 +19,127 @@ const ( // There is NO log entry struct in gommon/log, which is used in many logging packages, the reason is // if extra field is added to the interface, compiler would throw error on stale handler implementations. type Handler interface { - // HandleLog accepts level, log time, formatted log message - HandleLog(level Level, time time.Time, msg string) - // HandleLogWithSource accepts formatted source line of log i.e., http.go:13 - // TODO: pass frame instead of string so handler can use trace for error handling? - HandleLogWithSource(level Level, time time.Time, msg string, source string) - // HandleLogWithFields accepts fields with type hint, - // implementation should inspect the type field instead of using reflection - // TODO: pass pointer for fields? - HandleLogWithFields(level Level, time time.Time, msg string, fields Fields) - // HandleLogWithSourceFields accepts both source and fields - HandleLogWithSourceFields(level Level, time time.Time, msg string, source string, fields Fields) + // HandleLog requires level, now, msg, all the others are optional + // source is Caller which contains full file line TODO: pass frame instead of string so handler can use trace for error handling? + // context are fields attached to the logger instance + // fields are ad-hoc fields from logger method like DebugF(msg, fields) + HandleLog(level Level, now time.Time, msg string, source Caller, context Fields, fields Fields) // Flush writes the buffer to underlying storage Flush() } // HandlerFunc is an adapter to allow use of ordinary functions as log entry handlers -//type HandlerFunc func(level Level, msg string) +type HandlerFunc func(level Level, now time.Time, msg string, source string, context Fields, fields Fields) -// TODO: why the receiver is value instead of pointer https://github.com/dyweb/gommon/issues/30 -//func (f HandlerFunc) HandleLog(level Level, msg string) { -// f(level, msg) -//} +// TODO: why the receiver is value instead of pointer https://github.com/dyweb/gommon/issues/30 and what's the overhead +func (f HandlerFunc) HandleLog(level Level, now time.Time, msg string, source string, context Fields, fields Fields) { + f(level, now, msg, source, context, fields) +} var _ Syncer = (*os.File)(nil) // Syncer is implemented by os.File, handler implementation should check this interface and call Sync // if they support using file as sink +// TODO: about sync +// - in go, both os.Stderr and os.Stdout are not (line) buffered +// - what would happen if os.Stderr.Close() +// - os.Stderr.Sync() will there be any different if stderr/stdout is redirected to a file type Syncer interface { Sync() error } -//// TODO: handler for http access log, this should be in extra package -//type HttpAccessLogger struct { -//}` - var defaultHandler = NewIOHandler(os.Stderr) -// DefaultHandler returns the singleton defaultHandler instance, which logs to stdout in text format +// DefaultHandler returns the singleton defaultHandler instance, which logs to stderr in text format func DefaultHandler() Handler { return defaultHandler } -// IOHandler writes log to io.Writer, default handler uses os.Stderr -type IOHandler struct { - w io.Writer +// MultiHandler creates a handler that duplicates the log to all the provided handlers, it runs in +// serial and don't handle panic +func MultiHandler(handlers ...Handler) Handler { + return &multiHandler{handlers: handlers} } -// NewIOHandler -func NewIOHandler(w io.Writer) Handler { - return &IOHandler{w: w} +// TextHandler writes log to io.Writer, default handler is a TextHandler using os.Stderr +type TextHandler struct { + w io.Writer } -// TODO: performance (which is not a major concern now ...) -// - when using raw byte slice, have a correct length, fields can also return length required -// - is calling level.String() faster than %s level -// - use buffer (pool) -// TODO: correctness -// - in go, both os.Stderr and os.Stdout are not (line) buffered -// - what would happen if os.Stderr.Close() -// - os.Stderr.Sync() will there be any different if stderr/stdout is redirected to a file - -// HandleLog implements Handler interface -func (h *IOHandler) HandleLog(level Level, time time.Time, msg string) { - b := formatHead(level, time, msg) - b = append(b, '\n') - h.w.Write(b) +// NewIOHandler returns the default text handler, the name is for backward compatibility +func NewIOHandler(w io.Writer) Handler { + return &TextHandler{w: w} } -// HandleLogWithSource implements Handler interface -func (h *IOHandler) HandleLogWithSource(level Level, time time.Time, msg string, source string) { - b := formatHeadWithSource(level, time, msg, source) - b = append(b, '\n') - h.w.Write(b) +// NewTextHandler formats log in human readable format without any escape, thus it is NOT machine readable +func NewTextHandler(w io.Writer) Handler { + return &TextHandler{w: w} } -// HandleLogWithFields implements Handler interface -func (h *IOHandler) HandleLogWithFields(level Level, time time.Time, msg string, fields Fields) { - // we use raw slice instead of bytes buffer because we need to use strconv.Append*, which requires raw slice - b := formatHead(level, time, msg) +func (h *TextHandler) HandleLog(level Level, time time.Time, msg string, source Caller, context Fields, fields Fields) { + b := make([]byte, 0, 50+len(msg)+len(source.File)+30*len(context)+30*len(fields)) + // level + b = append(b, level.AlignedUpperString()...) + // time b = append(b, ' ') - b = formatFields(b, fields) - b[len(b)-1] = '\n' - h.w.Write(b) -} - -// HandleLogWithSourceFields implements Handler interface -func (h *IOHandler) HandleLogWithSourceFields(level Level, time time.Time, msg string, source string, fields Fields) { - b := formatHeadWithSource(level, time, msg, source) + b = time.AppendFormat(b, defaultTimeStampFormat) + // source, optional + if source.Line != 0 { + b = append(b, ' ') + last := strings.LastIndex(source.File, "/") + b = append(b, source.File[last+1:]...) + b = append(b, ':') + b = strconv.AppendInt(b, int64(source.Line), 10) + } + // message b = append(b, ' ') - b = formatFields(b, fields) - b[len(b)-1] = '\n' + b = append(b, msg...) + // context + if len(context) > 0 { + b = append(b, ' ') + b = formatFields(b, context) + } + // fields + if len(fields) > 0 { + b = formatFields(b, fields) + } + b = append(b, '\n') h.w.Write(b) } // Flush implements Handler interface -func (h *IOHandler) Flush() { +func (h *TextHandler) Flush() { if s, ok := h.w.(Syncer); ok { s.Sync() } } -// entry is only used for test, it is not passed around like other loging packages -type entry struct { - level Level - time time.Time - msg string - fields Fields - source string -} - -var _ Handler = (*TestHandler)(nil) - -// TestHandler stores log as entry, its slice is protected by a RWMutex and safe for concurrent use -type TestHandler struct { - mu sync.RWMutex - entries []entry -} - -// NewTestHandler returns a test handler, it should only be used in test, -// a concrete type instead of Handler interface is returned to reduce unnecessary type cast in test -func NewTestHandler() *TestHandler { - return &TestHandler{} -} - -// HandleLog implements Handler interface -func (h *TestHandler) HandleLog(level Level, time time.Time, msg string) { - h.mu.Lock() - h.entries = append(h.entries, entry{level: level, time: time, msg: msg}) - h.mu.Unlock() -} - -// HandleLogWithSource implements Handler interface -func (h *TestHandler) HandleLogWithSource(level Level, time time.Time, msg string, source string) { - h.mu.Lock() - h.entries = append(h.entries, entry{level: level, time: time, msg: msg, source: source}) - h.mu.Unlock() -} +// ----------------- start of multi handler --------------------------- -// HandleLogWithFields implements Handler interface -func (h *TestHandler) HandleLogWithFields(level Level, time time.Time, msg string, fields Fields) { - h.mu.Lock() - h.entries = append(h.entries, entry{level: level, time: time, msg: msg, fields: fields}) - h.mu.Unlock() -} +var _ Handler = (*multiHandler)(nil) -// HandleLogWithSourceFields implements Handler interface -func (h *TestHandler) HandleLogWithSourceFields(level Level, time time.Time, msg string, source string, fields Fields) { - h.mu.Lock() - h.entries = append(h.entries, entry{level: level, time: time, msg: msg, source: source, fields: fields}) - h.mu.Unlock() +// https://github.com/dyweb/gommon/issues/87 +type multiHandler struct { + handlers []Handler } -// Flush implements Handler interface -func (h *TestHandler) Flush() { - // nop +func (m *multiHandler) HandleLog(level Level, now time.Time, msg string, source Caller, context Fields, fields Fields) { + for _, h := range m.handlers { + h.HandleLog(level, now, msg, source, context, fields) + } } -// HasLog checks if a log with specified level and message exists in slice -// TODO: support field, source etc. -func (h *TestHandler) HasLog(level Level, msg string) bool { - h.mu.RLock() - defer h.mu.RUnlock() - for _, e := range h.entries { - if e.level == level && e.msg == msg { - return true - } +func (m *multiHandler) Flush() { + for _, h := range m.handlers { + h.Flush() } - return false } -// no need to use fmt.Printf since we don't need any format -func formatHead(level Level, time time.Time, msg string) []byte { - b := make([]byte, 0, 5+4+len(defaultTimeStampFormat)+len(msg)) - b = append(b, level.String()...) - b = append(b, ' ') - b = time.AppendFormat(b, defaultTimeStampFormat) - b = append(b, ' ') - b = append(b, msg...) - return b -} +// ----------------- end of multi handler --------------------------- -// we have a new function because source sits between time and msg in output, instead of after msg -// i.e. info 2018-02-04T21:03:20-08:00 main.go:18 show me the line -func formatHeadWithSource(level Level, time time.Time, msg string, source string) []byte { - b := make([]byte, 0, 5+4+len(defaultTimeStampFormat)+len(msg)+len(source)) - b = append(b, level.String()...) - b = append(b, ' ') - b = time.AppendFormat(b, defaultTimeStampFormat) - b = append(b, ' ') - b = append(b, source...) - b = append(b, ' ') - b = append(b, msg...) - return b -} +// ----------------- start of text format util --------------------------- -// it has an extra tailing space, which can be updated inplace to a \n +// it has an extra tailing space, which can be updated in place to a \n func formatFields(b []byte, fields Fields) []byte { for _, f := range fields { b = append(b, f.Key...) @@ -225,5 +152,8 @@ func formatFields(b []byte, fields Fields) []byte { } b = append(b, ' ') } + b = b[:len(b)-1] // remove trailing space return b } + +// ----------------- end of text format util --------------------------- diff --git a/log/handler_test.go b/log/handler_test.go new file mode 100644 index 0000000..d9c9f3f --- /dev/null +++ b/log/handler_test.go @@ -0,0 +1,33 @@ +package log + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMultiHandler(t *testing.T) { + f1, err := os.Create("testdata/f1.log") + require.Nil(t, err) + f2, err := os.Create("testdata/f2.log") + require.Nil(t, err) + + mh := MultiHandler(NewIOHandler(f1), NewIOHandler(f2)) + logger := NewTestLogger(InfoLevel) + logger.AddFields(Str("s1", "v1"), Int("i1", 1)) + logger.SetHandler(mh) + + logger.Info("should write to both files") + logger.Warn("this is a warning") + assert.NoError(t, f1.Close()) + assert.NoError(t, f2.Close()) + + b1, err := ioutil.ReadFile("testdata/f1.log") + require.Nil(t, err) + b2, err := ioutil.ReadFile("testdata/f2.log") + require.Nil(t, err) + assert.Equal(t, b1, b2) +} diff --git a/log/handlers/cli/handler.go b/log/handlers/cli/handler.go index 59ca84f..bba320d 100644 --- a/log/handlers/cli/handler.go +++ b/log/handlers/cli/handler.go @@ -1,13 +1,12 @@ -/* -Package cli writes is same as builtin IOHandler except color and delta time. -It is used by go.ice as default handler -TODO: color can't be disabled and we don't detect tty like logrus -*/ -package cli // import "github.com/dyweb/gommon/log/handlers/cli" +// Package cli writes is same as builtin TextHandler except color and delta time. +// It is used by go.ice as default handler +// TODO: color can't be disabled and we don't detect tty like logrus +package cli import ( "io" "strconv" + "strings" "time" "github.com/dyweb/gommon/log" @@ -18,6 +17,8 @@ const ( defaultTimeStampFormat = time.RFC3339 ) +var _ log.Handler = (*Handler)(nil) + type Handler struct { w io.Writer start time.Time @@ -32,32 +33,41 @@ func New(w io.Writer, delta bool) *Handler { } } -func (h *Handler) HandleLog(level log.Level, time time.Time, msg string) { - b := h.formatHead(level, time, msg) - b = append(b, '\n') - h.w.Write(b) -} - -func (h *Handler) HandleLogWithSource(level log.Level, time time.Time, msg string, source string) { - b := h.formatHeadWithSource(level, time, msg, source) - b = append(b, '\n') - h.w.Write(b) -} - -func (h *Handler) HandleLogWithFields(level log.Level, time time.Time, msg string, fields log.Fields) { - // we use raw slice instead of bytes buffer because we need to use strconv.Append*, which requires raw slice - b := h.formatHead(level, time, msg) +func (h *Handler) HandleLog(level log.Level, now time.Time, msg string, source log.Caller, context log.Fields, fields log.Fields) { + b := make([]byte, 0, 50+len(msg)+len(source.File)+30*len(context)+30*len(fields)) + // level + b = append(b, level.ColoredAlignedUpperString()...) + // time b = append(b, ' ') - b = formatFields(b, fields) - b[len(b)-1] = '\n' - h.w.Write(b) -} - -func (h *Handler) HandleLogWithSourceFields(level log.Level, time time.Time, msg string, source string, fields log.Fields) { - b := h.formatHeadWithSource(level, time, msg, source) + if h.delta { + b = append(b, formatNum(uint(now.Sub(h.start)/time.Second), 4)...) + } else { + b = now.AppendFormat(b, defaultTimeStampFormat) + } + // source + if source.Line != 0 { + b = append(b, ' ') + b = append(b, color.CyanStart...) + last := strings.LastIndex(source.File, "/") + b = append(b, source.File[last+1:]...) + b = append(b, ':') + b = strconv.AppendInt(b, int64(source.Line), 10) + b = append(b, color.End...) + } + // message b = append(b, ' ') - b = formatFields(b, fields) - b[len(b)-1] = '\n' + b = append(b, msg...) + // context + if len(context) > 0 { + b = append(b, ' ') + b = formatFields(b, context) + } + // field + if len(fields) > 0 { + b = append(b, ' ') + b = formatFields(b, fields) + } + b = append(b, '\n') h.w.Write(b) } @@ -87,41 +97,6 @@ func formatNum(u uint, digits int) []byte { return b } -// no need to use fmt.Printf since we don't need any format -func (h *Handler) formatHead(level log.Level, tm time.Time, msg string) []byte { - b := make([]byte, 0, 18+4+len(defaultTimeStampFormat)+len(msg)) - b = append(b, level.ColoredAlignedUpperString()...) - b = append(b, ' ') - if h.delta { - b = append(b, formatNum(uint(tm.Sub(h.start)/time.Second), 4)...) - } else { - b = tm.AppendFormat(b, defaultTimeStampFormat) - } - b = append(b, ' ') - b = append(b, msg...) - return b -} - -// we have a new function because source sits between time and msg in output, instead of after msg -// i.e. info 2018-02-04T21:03:20-08:00 main.go:18 show me the line -func (h *Handler) formatHeadWithSource(level log.Level, tm time.Time, msg string, source string) []byte { - b := make([]byte, 0, 18+4+len(defaultTimeStampFormat)+len(msg)) - b = append(b, level.ColoredAlignedUpperString()...) - b = append(b, ' ') - if h.delta { - b = append(b, formatNum(uint(tm.Sub(h.start)/time.Second), 4)...) - } else { - b = tm.AppendFormat(b, defaultTimeStampFormat) - } - b = append(b, ' ') - b = append(b, color.CyanStart...) - b = append(b, source...) - b = append(b, color.End...) - b = append(b, ' ') - b = append(b, msg...) - return b -} - // it has an extra tailing space, which can be updated inplace to a \n func formatFields(b []byte, fields log.Fields) []byte { for _, f := range fields { @@ -137,5 +112,6 @@ func formatFields(b []byte, fields log.Fields) []byte { } b = append(b, ' ') } + b = b[:len(b)-1] // remove trailing space return b } diff --git a/log/handlers/cli/handler_test.go b/log/handlers/cli/handler_test.go index a6f26a9..3b03853 100644 --- a/log/handlers/cli/handler_test.go +++ b/log/handlers/cli/handler_test.go @@ -1,14 +1,14 @@ package cli import ( + "fmt" "testing" asst "github.com/stretchr/testify/assert" ) -func TestHandler_HandleLog(t *testing.T) { - // FIXME: the test seems to be broken, it has no output and when run in GoLand it shows terminated instead of pass - //fmt.Printf("%04d", 2) +func TestFmt_FormatNum(t *testing.T) { + fmt.Printf("%04d\n", 2) } func Test_FormatNum(t *testing.T) { diff --git a/log/handlers/json/handler.go b/log/handlers/json/handler.go index 3930a56..172407f 100644 --- a/log/handlers/json/handler.go +++ b/log/handlers/json/handler.go @@ -1,14 +1,17 @@ /* Package json writes log in JSON format, it concatenates string directly and does not use encoding/json. -TODO: support escape +TODO: compare with standard json encoding */ -package json // import "github.com/dyweb/gommon/log/handlers/json" +package json import ( - "github.com/dyweb/gommon/log" "io" "strconv" + "strings" "time" + "unicode/utf8" + + "github.com/dyweb/gommon/log" ) var _ log.Handler = (*Handler)(nil) @@ -23,34 +26,37 @@ func New(w io.Writer) *Handler { } } -func (h *Handler) HandleLog(level log.Level, time time.Time, msg string) { - b := formatHead(level, time, msg) - b = append(b, "}\n"...) - h.w.Write(b) -} - -func (h *Handler) HandleLogWithSource(level log.Level, time time.Time, msg string, source string) { - b := formatHead(level, time, msg) - b = append(b, `,"s":"`...) - b = append(b, source...) - b = append(b, "\"}\n"...) - h.w.Write(b) -} - -func (h *Handler) HandleLogWithFields(level log.Level, time time.Time, msg string, fields log.Fields) { - b := formatHead(level, time, msg) - b = append(b, ',') - b = formatFields(b, fields) - b = append(b, "}\n"...) - h.w.Write(b) -} - -func (h *Handler) HandleLogWithSourceFields(level log.Level, time time.Time, msg string, source string, fields log.Fields) { - b := formatHead(level, time, msg) - b = append(b, `,"s":"`...) - b = append(b, source...) - b = append(b, `",`...) - b = formatFields(b, fields) +func (h *Handler) HandleLog(level log.Level, time time.Time, msg string, source log.Caller, context log.Fields, fields log.Fields) { + b := make([]byte, 0, 50+len(msg)+len(source.File)+30*len(context)+30*len(fields)) + // level + b = append(b, `{"l":"`...) + b = append(b, level.String()...) + // time + b = append(b, `","t":`...) + b = strconv.AppendInt(b, time.Unix(), 10) + // message + b = append(b, `,"m":`...) + b = encodeString(b, msg) + // source + if source.Line != 0 { + b = append(b, `,"s":"`...) + // TODO: can file path contains character that need escape in json? + last := strings.LastIndex(source.File, "/") + b = append(b, source.File[last+1:]...) + b = append(b, ':') + b = strconv.AppendInt(b, int64(source.Line), 10) + b = append(b, '"') + } + // context + if len(context) > 0 { + b = append(b, `,`...) + b = formatFields(b, context) + } + // fields + if len(fields) > 0 { + b = append(b, `,`...) + b = formatFields(b, fields) + } b = append(b, "}\n"...) h.w.Write(b) } @@ -61,20 +67,9 @@ func (h *Handler) Flush() { } } -func formatHead(level log.Level, time time.Time, msg string) []byte { - b := make([]byte, 0, 5+4+10+len(msg)) - b = append(b, `{"l":"`...) - b = append(b, level.String()...) - b = append(b, `","t":`...) - b = strconv.AppendInt(b, time.Unix(), 10) - b = append(b, `,"m":"`...) - b = append(b, msg...) - b = append(b, '"') - return b -} - func formatFields(b []byte, fields log.Fields) []byte { for _, f := range fields { + // TODO: should we also escape key? ... b = append(b, '"') b = append(b, f.Key...) b = append(b, "\":"...) @@ -82,12 +77,68 @@ func formatFields(b []byte, fields log.Fields) []byte { case log.IntType: b = strconv.AppendInt(b, f.Int, 10) case log.StringType: - b = append(b, '"') - b = append(b, f.Str...) - b = append(b, '"') + b = encodeString(b, f.Str) } b = append(b, ',') } b = b[:len(b)-1] // remove trailing comma return b } + +// encodeString escape character like " \n, it does not handle jsonp or html like standard library does +// it is based on encoding/json/encode.go func (e *encodeState) string(s string, escapeHTML bool) w/ some comment +func encodeString(buf []byte, s string) []byte { + buf = append(buf, '"') + start := 0 + for i := 0; i < len(s); { + b := s[i] + if b < utf8.RuneSelf { // characters below RuneSelf are represented as themselves in a single byte. + if safeSet[b] { + i++ + // we don't call append right away for this byte + // because if the entire string is safe, we only need to call append once + continue + } + // append previous processed bytes + if start < i { + buf = append(buf, s[start:i]...) + } + // some special ascii characters need escape + switch b { + case '\\', '"': + buf = append(buf, '\\', b) + case '\n': + buf = append(buf, '\\', 'n') + case '\r': + buf = append(buf, '\\', 'r') + case '\t': + buf = append(buf, '\\', 't') + default: + // TODO: I don't get what this section does ... + buf = append(buf, '\\', 'u', '0', '0', hex[b>>4], hex[b&0xF]) + } + i++ + start = i + continue + } + // it's utf8 rune + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + // when error, first append previous processed bytes + if start < i { + buf = append(buf, s[start:i]...) + } + buf = append(buf, `\ufffd`...) + i += size + start = i + continue + } + i += size // only move the cursor, append happens when need to escape or there is error or at last + } + if start < len(s) { + // NOTE: it's s[start:] not buf[star:] .... + buf = append(buf, s[start:]...) + } + buf = append(buf, '"') + return buf +} diff --git a/log/handlers/json/handler_test.go b/log/handlers/json/handler_test.go index 892afdc..865ca31 100644 --- a/log/handlers/json/handler_test.go +++ b/log/handlers/json/handler_test.go @@ -2,12 +2,13 @@ package json import ( "bytes" - "github.com/dyweb/gommon/log" + "encoding/json" "testing" "time" - "encoding/json" - asst "github.com/stretchr/testify/assert" + "github.com/dyweb/gommon/log" + + "github.com/stretchr/testify/assert" ) var tm = time.Unix(1517861935, 0) @@ -20,83 +21,100 @@ type entry struct { } func TestHandler_HandleLog(t *testing.T) { - assert := asst.New(t) b := &bytes.Buffer{} h := New(b) - h.HandleLog(log.DebugLevel, tm, "hi") - h.HandleLog(log.InfoLevel, tm, "hello") + h.HandleLog(log.DebugLevel, tm, "hi", log.EmptyCaller(), nil, nil) + h.HandleLog(log.InfoLevel, tm, "hello", log.EmptyCaller(), nil, nil) //t.Log(b.String()) expected := `{"l":"debug","t":1517861935,"m":"hi"} {"l":"info","t":1517861935,"m":"hello"} ` - assert.Equal(expected, b.String()) + assert.Equal(t, expected, b.String()) dec := json.NewDecoder(b) e1 := &entry{} if err := dec.Decode(e1); err != nil { t.Fatalf("can't decode using encoding/json %s", err) } - assert.Equal("debug", e1.Level) - assert.Equal(tm.Unix(), e1.Time) + assert.Equal(t, "debug", e1.Level) + assert.Equal(t, tm.Unix(), e1.Time) e2 := &entry{} if err := dec.Decode(e2); err != nil { t.Fatalf("can't decode using encoding/json %s", err) } - assert.Equal("info", e2.Level) - assert.Equal(tm.Unix(), e2.Time) + assert.Equal(t, "info", e2.Level) + assert.Equal(t, tm.Unix(), e2.Time) } func TestHandler_HandleLogWithSource(t *testing.T) { - assert := asst.New(t) b := &bytes.Buffer{} h := New(b) - h.HandleLogWithSource(log.DebugLevel, tm, "hi", "abc.go:12") + h.HandleLog(log.DebugLevel, tm, "hi", log.Caller{File: "abc.go", Line: 12}, nil, nil) //t.Log(b.String()) - assert.Equal(`{"l":"debug","t":1517861935,"m":"hi","s":"abc.go:12"} + assert.Equal(t, `{"l":"debug","t":1517861935,"m":"hi","s":"abc.go:12"} `, b.String()) validJSON(t, b.Bytes()) } func TestHandler_HandleLogWithFields(t *testing.T) { - assert := asst.New(t) b := &bytes.Buffer{} h := New(b) - h.HandleLogWithFields(log.DebugLevel, tm, "hi", log.Fields{ + h.HandleLog(log.DebugLevel, tm, "hi", log.EmptyCaller(), log.Fields{ log.Int("num", 1), log.Str("str", "rts"), - }) - assert.Equal(`{"l":"debug","t":1517861935,"m":"hi","num":1,"str":"rts"} + }, nil) + assert.Equal(t, `{"l":"debug","t":1517861935,"m":"hi","num":1,"str":"rts"} `, b.String()) validJSON(t, b.Bytes()) } func TestHandler_HandleLogWithSourceFields(t *testing.T) { - assert := asst.New(t) b := &bytes.Buffer{} h := New(b) - h.HandleLogWithSourceFields(log.DebugLevel, tm, "hi", "abc.go:12", log.Fields{ + h.HandleLog(log.DebugLevel, tm, "hi", log.Caller{File: "abc.go", Line: 12}, log.Fields{ log.Int("num", 1), log.Str("str", "rts"), - }) - assert.Equal(`{"l":"debug","t":1517861935,"m":"hi","s":"abc.go:12","num":1,"str":"rts"} + }, nil) + assert.Equal(t, `{"l":"debug","t":1517861935,"m":"hi","s":"abc.go:12","num":1,"str":"rts"} `, b.String()) validJSON(t, b.Bytes()) } -func Test_FormatHead(t *testing.T) { - assert := asst.New(t) - assert.Equal(`{"l":"debug","t":1517861935,"m":"hi"`, string(formatHead(log.DebugLevel, tm, "hi"))) +func TestJsonEscape(t *testing.T) { + var buf bytes.Buffer + h := New(&buf) + h.HandleLog(log.DebugLevel, tm, `I have "quote"`, log.EmptyCaller(), nil, nil) + validJSON(t, buf.Bytes()) } func Test_FormatFields(t *testing.T) { - assert := asst.New(t) - b := make([]byte, 0) - assert.Equal(`"num":1,"str":"rts"`, string(formatFields(b, log.Fields{ + var b []byte + assert.Equal(t, `"num":1,"str":"rts"`, string(formatFields(b, log.Fields{ log.Int("num", 1), log.Str("str", "rts"), }))) } +func TestEncodeString(t *testing.T) { + strs := []string{ + "normal string", + `has "quote"`, + "support 中文 me?", + } + // TODO: add assert here + for _, s := range strs { + var b []byte + b = encodeString(b, s) + t.Log(string(b)) + } + + t.Run("no escape", func(t *testing.T) { + var b []byte + b = encodeString(b, "nothing") + t.Log(string(b)) + }) +} + func validJSON(t *testing.T, b []byte) { e := &entry{} if err := json.Unmarshal(b, e); err != nil { diff --git a/log/handlers/json/tables.go b/log/handlers/json/tables.go new file mode 100644 index 0000000..7038472 --- /dev/null +++ b/log/handlers/json/tables.go @@ -0,0 +1,115 @@ +package json + +import "unicode/utf8" + +// tables.go is copy and pasted from encoding/json/tables.go +// I think the reason they didn't export it is go can't have array as constant + +//const a = [2]int{1, 2} // won't work + +var hex = "0123456789abcdef" + +// safeSet holds the value true if the ASCII character with the given array +// position can be represented inside a JSON string without any further +// escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), and the backslash character ("\"). +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} diff --git a/log/level.go b/log/level.go index 29155e3..2de9d86 100644 --- a/log/level.go +++ b/log/level.go @@ -26,26 +26,8 @@ const ( TraceLevel ) -// based on https://github.com/apex/log/blob/master/levels.go -var levelStrings = []string{ - FatalLevel: "fatal", - PanicLevel: "panic", - ErrorLevel: "error", - WarnLevel: "warn", - InfoLevel: "info", - DebugLevel: "debug", - TraceLevel: "trace", -} - -var levelAlignedUpperStrings = []string{ - FatalLevel: "FATA", - PanicLevel: "PANI", - ErrorLevel: "ERRO", - WarnLevel: "WARN", - InfoLevel: "INFO", - DebugLevel: "DEBU", - TraceLevel: "TRAC", -} +// PrintLevel is for library/application that requires a Printf based logger interface +const PrintLevel = InfoLevel var levelColoredStrings = []string{ FatalLevel: color.RedStart + "fatal" + color.End, @@ -67,15 +49,51 @@ var levelColoredAlignedUpperStrings = []string{ TraceLevel: color.GrayStart + "TRAC" + color.End, } +// String returns log level in lower case and not aligned in length, i.e. fatal, warn func (level Level) String() string { - return levelStrings[level] + switch level { + case FatalLevel: + return "fatal" + case PanicLevel: + return "panic" + case ErrorLevel: + return "error" + case WarnLevel: + return "warn" + case InfoLevel: + return "info" + case DebugLevel: + return "debug" + case TraceLevel: + return "trace" + default: + return "unknown" + } } -// AlignedUpperString returns fixed length level string in uppercase +// AlignedUpperString returns log level with fixed length of 4 in uppercase, i.e. FATA, WARN func (level Level) AlignedUpperString() string { - return levelAlignedUpperStrings[level] + switch level { + case FatalLevel: + return "FATA" + case PanicLevel: + return "PANI" + case ErrorLevel: + return "ERRO" + case WarnLevel: + return "WARN" + case InfoLevel: + return "INFO" + case DebugLevel: + return "DEBG" // TODO: or DEBU + case TraceLevel: + return "TRAC" + default: + return "UNKN" + } } +// TODO: use switch and generate the function ... or just generate it manually // ColoredString returns level string wrapped by terminal color characters, only works on *nix func (level Level) ColoredString() string { return levelColoredStrings[level] diff --git a/log/logger.go b/log/logger.go index 0d0cf05..da3f3dd 100644 --- a/log/logger.go +++ b/log/logger.go @@ -3,8 +3,6 @@ package log import ( "fmt" "os" - "runtime" - "strings" "sync" "time" ) @@ -14,65 +12,151 @@ import ( // Lock is used when updating logger attributes like Level. // // For Printf style logging (Levelf), Logger formats string using fmt.Sprintf before passing it to handlers. +// // logger.Debugf("id is %d", id) -// For structual logging (LevelF), Logger passes fields to handlers without any processing. +// +// For structural logging (LevelF), Logger passes fields to handlers without any processing. +// // logger.DebugF("hi", log.Fields{log.Str("foo", "bar")}) +// // If you want to mix two styles, call fmt.Sprintf before calling DebugF, +// // logger.DebugF(fmt.Sprintf("id is %d", id), log.Fields{log.Str("foo", "bar")}) type Logger struct { - mu sync.RWMutex - h Handler - level Level - // TODO: Fields in logger are never used, we are using DebugF to pass temporary fields - // which does not allow inherit fields from parent logger - //fields Fields - children map[string][]*Logger - source bool - id *Identity + // mu is a Mutex instead of RWMutex because it's only for avoid concurrent write, + // for performance reason and the natural of logging, reading stale config is not a big problem, + // so we don't check mutex on read operation (i.e. log message) and allow race condition + mu sync.Mutex + h Handler + level Level + source bool + skip int + // fields contains common context, i.e. the struct is created for a specific task and it has "taskId": 0ac-123 + fields Fields + + id *Identity // use nil so we can have logger without identity +} + +// Copy create a new logger with different identity, the identity is based on where Copy is called +// Normally you should call Copy inside func or method on a package/strcut logger +func (l *Logger) Copy() *Logger { + id := NewIdentityFromCaller(1) + c := &Logger{ + id: &id, + } + return newLogger(l, c) +} + +// AddField add field to current logger in place, it does NOT create a copy of logger +// Use Copy if you want a copy +// It does NOT check duplication +func (l *Logger) AddField(f Field) *Logger { + l.mu.Lock() + // TODO: check dup or not? or may it optional + l.fields = append(l.fields, f) + l.mu.Unlock() + return l +} + +// AddFields add fields to current logger in place, it does NOT create a copy of logger +// Use Copy if you want a copy +// It does NOT check duplication +func (l *Logger) AddFields(fields ...Field) *Logger { + l.mu.Lock() + // TODO: check dup or not? or may it optional + l.fields = append(l.fields, fields...) + l.mu.Unlock() + return l +} + +// Flush calls Flush of its handler +func (l *Logger) Flush() { + l.h.Flush() } func (l *Logger) Level() Level { - // TODO: might use the mutex here? return l.level } -func (l *Logger) SetLevel(level Level) { +func (l *Logger) SetLevel(level Level) *Logger { l.mu.Lock() l.level = level l.mu.Unlock() + return l } -func (l *Logger) SetHandler(h Handler) { +func (l *Logger) SetHandler(h Handler) *Logger { l.mu.Lock() l.h = h l.mu.Unlock() + return l } -func (l *Logger) EnableSource() { +func (l *Logger) EnableSource() *Logger { l.mu.Lock() l.source = true l.mu.Unlock() + return l } -func (l *Logger) DisableSource() { +func (l *Logger) DisableSource() *Logger { l.mu.Lock() l.source = false l.mu.Unlock() + return l +} + +// SetCallerSkip is used for util function to log using its caller's location instead of its own +// Without extra skip, some common util function will keep logging same line and make the real +// source hard to track. +// +// func echo(w http.ResponseWriter, r *http.Request) { +// if r.Query().Get("word") == "" { +// writeError(w, errors.New("word is required") +// return +// } +// w.Write([]byte(r.Query().Get("word"))) +// } +// +// func writeError(w http.ResponseWriter, err error) { +// l := pkgLogger.Copy().SetCallerSkip(1) +// l.Error(err) +// w.Write([]byte(err.String())) +// } +func (l *Logger) SetCallerSkip(skip int) *Logger { + l.mu.Lock() + // ignore invalid skip, most time it should just be one + if skip > 0 && skip < 5 { + l.skip = skip + } + l.mu.Unlock() + return l +} + +// ResetCallerSkip set skip to 0, the default value +func (l *Logger) ResetCallerSkip() *Logger { + l.mu.Lock() + l.skip = 0 + l.mu.Unlock() + return l } // Identity returns the identity set when the logger is created. // NOTE: caller can modify the identity because all fields are public, but they should NOT do this -func (l *Logger) Identity() *Identity { - return l.id +func (l *Logger) Identity() Identity { + if l.id == nil { + return UnknownIdentity + } + return *l.id } // Panic calls panic after it writes and flushes the log func (l *Logger) Panic(args ...interface{}) { s := fmt.Sprint(args...) - if !l.source { - l.h.HandleLog(PanicLevel, time.Now(), s) + if len(l.fields) == 0 { + l.h.HandleLog(PanicLevel, time.Now(), s, caller(l.skip), nil, nil) } else { - l.h.HandleLogWithSource(PanicLevel, time.Now(), s, caller()) + l.h.HandleLog(PanicLevel, time.Now(), s, caller(l.skip), l.fields, nil) } l.h.Flush() panic(s) @@ -81,21 +165,21 @@ func (l *Logger) Panic(args ...interface{}) { // Panicf duplicates instead of calling Panic to keep source line correct func (l *Logger) Panicf(format string, args ...interface{}) { s := fmt.Sprintf(format, args...) - if !l.source { - l.h.HandleLog(PanicLevel, time.Now(), s) + if len(l.fields) == 0 { + l.h.HandleLog(PanicLevel, time.Now(), s, caller(l.skip), nil, nil) } else { - l.h.HandleLogWithSource(PanicLevel, time.Now(), s, caller()) + l.h.HandleLog(PanicLevel, time.Now(), s, caller(l.skip), l.fields, nil) } l.h.Flush() panic(s) } // PanicF duplicates instead of calling Panic to keep source line correct -func (l *Logger) PanicF(msg string, fields Fields) { - if !l.source { - l.h.HandleLogWithFields(PanicLevel, time.Now(), msg, fields) +func (l *Logger) PanicF(msg string, fields ...Field) { + if len(l.fields) == 0 { + l.h.HandleLog(PanicLevel, time.Now(), msg, caller(l.skip), nil, fields) } else { - l.h.HandleLogWithSourceFields(PanicLevel, time.Now(), msg, caller(), fields) + l.h.HandleLog(PanicLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } l.h.Flush() panic(msg) @@ -104,10 +188,10 @@ func (l *Logger) PanicF(msg string, fields Fields) { // Fatal calls os.Exit(1) after it writes and flushes the log func (l *Logger) Fatal(args ...interface{}) { s := fmt.Sprint(args...) - if !l.source { - l.h.HandleLog(FatalLevel, time.Now(), s) + if len(l.fields) == 0 { + l.h.HandleLog(FatalLevel, time.Now(), s, caller(l.skip), nil, nil) } else { - l.h.HandleLogWithSource(FatalLevel, time.Now(), s, caller()) + l.h.HandleLog(FatalLevel, time.Now(), s, caller(l.skip), l.fields, nil) } l.h.Flush() // TODO: allow user to register hook to do cleanup before exit directly @@ -117,37 +201,27 @@ func (l *Logger) Fatal(args ...interface{}) { // Fatalf duplicates instead of calling Fatal to keep source line correct func (l *Logger) Fatalf(format string, args ...interface{}) { s := fmt.Sprintf(format, args...) - if !l.source { - l.h.HandleLog(FatalLevel, time.Now(), s) + if len(l.fields) == 0 { + l.h.HandleLog(FatalLevel, time.Now(), s, caller(l.skip), nil, nil) } else { - l.h.HandleLogWithSource(FatalLevel, time.Now(), s, caller()) + l.h.HandleLog(FatalLevel, time.Now(), s, caller(l.skip), l.fields, nil) } l.h.Flush() os.Exit(1) } // FatalF duplicates instead of calling Fatal to keep source line correct -func (l *Logger) FatalF(msg string, fields Fields) { - if !l.source { - l.h.HandleLogWithFields(FatalLevel, time.Now(), msg, fields) +func (l *Logger) FatalF(msg string, fields ...Field) { + if len(l.fields) == 0 { + l.h.HandleLog(FatalLevel, time.Now(), msg, caller(l.skip), nil, fields) } else { - l.h.HandleLogWithSourceFields(FatalLevel, time.Now(), msg, caller(), fields) + l.h.HandleLog(FatalLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } l.h.Flush() os.Exit(1) } -// caller gets source location at runtime, in the future we may generate it at compile time to reduce the -// overhead, though I am not sure what the overhead is without actual benchmark and profiling -// TODO: https://github.com/dyweb/gommon/issues/43 -func caller() string { - _, file, line, ok := runtime.Caller(2) - if !ok { - file = "" - line = 1 - } else { - last := strings.LastIndex(file, "/") - file = file[last+1:] - } - return fmt.Sprintf("%s:%d", file, line) +// Noop is only for test escape analysis +func (l *Logger) NoopF(msg string, fields ...Field) { + // noop } diff --git a/log/logger_factory.go b/log/logger_factory.go index f77c6ff..1779d73 100644 --- a/log/logger_factory.go +++ b/log/logger_factory.go @@ -1,14 +1,10 @@ package log -func NewApplicationLogger() *Logger { - l := NewPackageLoggerWithSkip(1) - l.id.Type = ApplicationLogger - return l -} - -func NewLibraryLogger() *Logger { - l := NewPackageLoggerWithSkip(1) - l.id.Type = LibraryLogger +// NewTestLogger does not have identity and handler, it is mainly used for benchmark test +func NewTestLogger(level Level) *Logger { + l := &Logger{ + level: level, + } return l } @@ -17,43 +13,53 @@ func NewPackageLogger() *Logger { } func NewPackageLoggerWithSkip(skip int) *Logger { + id := NewIdentityFromCaller(skip + 1) l := &Logger{ - id: NewIdentityFromCaller(skip + 1), + id: &id, } return newLogger(nil, l) } +// Deprecated: use Copy method on package logger func NewFunctionLogger(packageLogger *Logger) *Logger { + id := NewIdentityFromCaller(1) l := &Logger{ - id: NewIdentityFromCaller(1), + id: &id, } return newLogger(packageLogger, l) } func NewStructLogger(packageLogger *Logger, loggable LoggableStruct) *Logger { + id := loggable.LoggerIdentity(func() Identity { + return NewIdentityFromCaller(1) + }) l := &Logger{ - id: loggable.LoggerIdentity(func() *Identity { - return NewIdentityFromCaller(1) - }), + id: &id, } l = newLogger(packageLogger, l) loggable.SetLogger(l) return l } +// Deprecated: use Copy method on struct logger func NewMethodLogger(structLogger *Logger) *Logger { + id := NewIdentityFromCaller(1) l := &Logger{ - id: NewIdentityFromCaller(1), + id: &id, } return newLogger(structLogger, l) } func newLogger(parent *Logger, child *Logger) *Logger { if parent != nil { - parent.AddChild(child) child.h = parent.h child.level = parent.level child.source = parent.source + if len(parent.fields) != 0 { + fields := make([]Field, len(parent.fields)) + copy(fields, parent.fields) + child.fields = fields + } } else { child.h = DefaultHandler() child.level = InfoLevel diff --git a/log/logger_generated.go b/log/logger_generated.go index 8bb059b..c645397 100644 --- a/log/logger_generated.go +++ b/log/logger_generated.go @@ -12,32 +12,35 @@ func (l *Logger) IsTraceEnabled() bool { } func (l *Logger) Trace(args ...interface{}) { - if l.level >= TraceLevel { - if !l.source { - l.h.HandleLog(TraceLevel, time.Now(), fmt.Sprint(args...)) - } else { - l.h.HandleLogWithSource(TraceLevel, time.Now(), fmt.Sprint(args...), caller()) - } + if l.level < TraceLevel { + return + } + if !l.source { + l.h.HandleLog(TraceLevel, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(TraceLevel, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) } } func (l *Logger) Tracef(format string, args ...interface{}) { - if l.level >= TraceLevel { - if !l.source { - l.h.HandleLog(TraceLevel, time.Now(), fmt.Sprintf(format, args...)) - } else { - l.h.HandleLogWithSource(TraceLevel, time.Now(), fmt.Sprintf(format, args...), caller()) - } + if l.level < TraceLevel { + return + } + if !l.source { + l.h.HandleLog(TraceLevel, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(TraceLevel, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) } } -func (l *Logger) TraceF(msg string, fields Fields) { - if l.level >= TraceLevel { - if !l.source { - l.h.HandleLogWithFields(TraceLevel, time.Now(), msg, fields) - } else { - l.h.HandleLogWithSourceFields(TraceLevel, time.Now(), msg, caller(), fields) - } +func (l *Logger) TraceF(msg string, fields ...Field) { + if l.level < TraceLevel { + return + } + if !l.source { + l.h.HandleLog(TraceLevel, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog(TraceLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } } @@ -46,32 +49,35 @@ func (l *Logger) IsDebugEnabled() bool { } func (l *Logger) Debug(args ...interface{}) { - if l.level >= DebugLevel { - if !l.source { - l.h.HandleLog(DebugLevel, time.Now(), fmt.Sprint(args...)) - } else { - l.h.HandleLogWithSource(DebugLevel, time.Now(), fmt.Sprint(args...), caller()) - } + if l.level < DebugLevel { + return + } + if !l.source { + l.h.HandleLog(DebugLevel, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(DebugLevel, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) } } func (l *Logger) Debugf(format string, args ...interface{}) { - if l.level >= DebugLevel { - if !l.source { - l.h.HandleLog(DebugLevel, time.Now(), fmt.Sprintf(format, args...)) - } else { - l.h.HandleLogWithSource(DebugLevel, time.Now(), fmt.Sprintf(format, args...), caller()) - } + if l.level < DebugLevel { + return + } + if !l.source { + l.h.HandleLog(DebugLevel, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(DebugLevel, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) } } -func (l *Logger) DebugF(msg string, fields Fields) { - if l.level >= DebugLevel { - if !l.source { - l.h.HandleLogWithFields(DebugLevel, time.Now(), msg, fields) - } else { - l.h.HandleLogWithSourceFields(DebugLevel, time.Now(), msg, caller(), fields) - } +func (l *Logger) DebugF(msg string, fields ...Field) { + if l.level < DebugLevel { + return + } + if !l.source { + l.h.HandleLog(DebugLevel, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog(DebugLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } } @@ -80,32 +86,72 @@ func (l *Logger) IsInfoEnabled() bool { } func (l *Logger) Info(args ...interface{}) { - if l.level >= InfoLevel { - if !l.source { - l.h.HandleLog(InfoLevel, time.Now(), fmt.Sprint(args...)) - } else { - l.h.HandleLogWithSource(InfoLevel, time.Now(), fmt.Sprint(args...), caller()) - } + if l.level < InfoLevel { + return + } + if !l.source { + l.h.HandleLog(InfoLevel, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(InfoLevel, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) } } func (l *Logger) Infof(format string, args ...interface{}) { - if l.level >= InfoLevel { - if !l.source { - l.h.HandleLog(InfoLevel, time.Now(), fmt.Sprintf(format, args...)) - } else { - l.h.HandleLogWithSource(InfoLevel, time.Now(), fmt.Sprintf(format, args...), caller()) - } + if l.level < InfoLevel { + return + } + if !l.source { + l.h.HandleLog(InfoLevel, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(InfoLevel, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) + } +} + +func (l *Logger) InfoF(msg string, fields ...Field) { + if l.level < InfoLevel { + return + } + if !l.source { + l.h.HandleLog(InfoLevel, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog(InfoLevel, time.Now(), msg, caller(l.skip), l.fields, fields) + } +} + +func (l *Logger) IsPrintEnabled() bool { + return l.level >= PrintLevel +} + +func (l *Logger) Print(args ...interface{}) { + if l.level < PrintLevel { + return + } + if !l.source { + l.h.HandleLog(PrintLevel, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(PrintLevel, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) + } +} + +func (l *Logger) Printf(format string, args ...interface{}) { + if l.level < PrintLevel { + return + } + if !l.source { + l.h.HandleLog(PrintLevel, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(PrintLevel, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) } } -func (l *Logger) InfoF(msg string, fields Fields) { - if l.level >= InfoLevel { - if !l.source { - l.h.HandleLogWithFields(InfoLevel, time.Now(), msg, fields) - } else { - l.h.HandleLogWithSourceFields(InfoLevel, time.Now(), msg, caller(), fields) - } +func (l *Logger) PrintF(msg string, fields ...Field) { + if l.level < PrintLevel { + return + } + if !l.source { + l.h.HandleLog(PrintLevel, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog(PrintLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } } @@ -114,32 +160,35 @@ func (l *Logger) IsWarnEnabled() bool { } func (l *Logger) Warn(args ...interface{}) { - if l.level >= WarnLevel { - if !l.source { - l.h.HandleLog(WarnLevel, time.Now(), fmt.Sprint(args...)) - } else { - l.h.HandleLogWithSource(WarnLevel, time.Now(), fmt.Sprint(args...), caller()) - } + if l.level < WarnLevel { + return + } + if !l.source { + l.h.HandleLog(WarnLevel, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(WarnLevel, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) } } func (l *Logger) Warnf(format string, args ...interface{}) { - if l.level >= WarnLevel { - if !l.source { - l.h.HandleLog(WarnLevel, time.Now(), fmt.Sprintf(format, args...)) - } else { - l.h.HandleLogWithSource(WarnLevel, time.Now(), fmt.Sprintf(format, args...), caller()) - } + if l.level < WarnLevel { + return + } + if !l.source { + l.h.HandleLog(WarnLevel, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(WarnLevel, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) } } -func (l *Logger) WarnF(msg string, fields Fields) { - if l.level >= WarnLevel { - if !l.source { - l.h.HandleLogWithFields(WarnLevel, time.Now(), msg, fields) - } else { - l.h.HandleLogWithSourceFields(WarnLevel, time.Now(), msg, caller(), fields) - } +func (l *Logger) WarnF(msg string, fields ...Field) { + if l.level < WarnLevel { + return + } + if !l.source { + l.h.HandleLog(WarnLevel, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog(WarnLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } } @@ -148,31 +197,34 @@ func (l *Logger) IsErrorEnabled() bool { } func (l *Logger) Error(args ...interface{}) { - if l.level >= ErrorLevel { - if !l.source { - l.h.HandleLog(ErrorLevel, time.Now(), fmt.Sprint(args...)) - } else { - l.h.HandleLogWithSource(ErrorLevel, time.Now(), fmt.Sprint(args...), caller()) - } + if l.level < ErrorLevel { + return + } + if !l.source { + l.h.HandleLog(ErrorLevel, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(ErrorLevel, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) } } func (l *Logger) Errorf(format string, args ...interface{}) { - if l.level >= ErrorLevel { - if !l.source { - l.h.HandleLog(ErrorLevel, time.Now(), fmt.Sprintf(format, args...)) - } else { - l.h.HandleLogWithSource(ErrorLevel, time.Now(), fmt.Sprintf(format, args...), caller()) - } - } -} - -func (l *Logger) ErrorF(msg string, fields Fields) { - if l.level >= ErrorLevel { - if !l.source { - l.h.HandleLogWithFields(ErrorLevel, time.Now(), msg, fields) - } else { - l.h.HandleLogWithSourceFields(ErrorLevel, time.Now(), msg, caller(), fields) - } + if l.level < ErrorLevel { + return + } + if !l.source { + l.h.HandleLog(ErrorLevel, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog(ErrorLevel, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) + } +} + +func (l *Logger) ErrorF(msg string, fields ...Field) { + if l.level < ErrorLevel { + return + } + if !l.source { + l.h.HandleLog(ErrorLevel, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog(ErrorLevel, time.Now(), msg, caller(l.skip), l.fields, fields) } } diff --git a/log/logger_generated.go.tmpl b/log/logger_generated.go.tmpl index 414a187..aafd2aa 100644 --- a/log/logger_generated.go.tmpl +++ b/log/logger_generated.go.tmpl @@ -11,32 +11,35 @@ func (l *Logger) Is{{.}}Enabled() bool { } func (l *Logger) {{.}}(args ...interface{}) { - if l.level >= {{.}}Level { - if !l.source { - l.h.HandleLog({{.}}Level, time.Now(), fmt.Sprint(args...)) - } else { - l.h.HandleLogWithSource({{.}}Level, time.Now(), fmt.Sprint(args...), caller()) - } + if l.level < {{.}}Level { + return + } + if !l.source { + l.h.HandleLog({{.}}Level, time.Now(), fmt.Sprint(args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog({{.}}Level, time.Now(), fmt.Sprint(args...), caller(l.skip), l.fields, nil) } } func (l *Logger) {{.}}f(format string, args ...interface{}) { - if l.level >= {{.}}Level { - if !l.source { - l.h.HandleLog({{.}}Level, time.Now(), fmt.Sprintf(format, args...)) - } else { - l.h.HandleLogWithSource({{.}}Level, time.Now(), fmt.Sprintf(format, args...), caller()) - } + if l.level < {{.}}Level { + return + } + if !l.source { + l.h.HandleLog({{.}}Level, time.Now(), fmt.Sprintf(format, args...), emptyCaller, l.fields, nil) + } else { + l.h.HandleLog({{.}}Level, time.Now(), fmt.Sprintf(format, args...), caller(l.skip), l.fields, nil) } } -func (l *Logger) {{.}}F(msg string, fields Fields) { - if l.level >= {{.}}Level { - if !l.source { - l.h.HandleLogWithFields({{.}}Level, time.Now(), msg, fields) - } else { - l.h.HandleLogWithSourceFields({{.}}Level, time.Now(), msg, caller(), fields) - } - } +func (l *Logger) {{.}}F(msg string, fields ...Field) { + if l.level < {{.}}Level { + return + } + if !l.source { + l.h.HandleLog({{.}}Level, time.Now(), msg, emptyCaller, l.fields, fields) + } else { + l.h.HandleLog({{.}}Level, time.Now(), msg, caller(l.skip), l.fields, fields) + } } {{ end }} \ No newline at end of file diff --git a/log/logger_identity.go b/log/logger_identity.go index d5907c0..ac286fd 100644 --- a/log/logger_identity.go +++ b/log/logger_identity.go @@ -12,8 +12,8 @@ type LoggerType uint8 const ( UnknownLogger LoggerType = iota - ApplicationLogger - LibraryLogger + // PackageLogger is normally singleton in entire package + // We used to have application and library logger but they are replaced by registry PackageLogger FunctionLogger StructLogger @@ -21,13 +21,11 @@ const ( ) var loggerTypeStrings = []string{ - UnknownLogger: "unk", - ApplicationLogger: "app", - LibraryLogger: "lib", - PackageLogger: "pkg", - FunctionLogger: "func", - StructLogger: "struct", - MethodLogger: "method", + UnknownLogger: "unk", + PackageLogger: "pkg", + FunctionLogger: "func", + StructLogger: "struct", + MethodLogger: "method", } func (tpe LoggerType) String() string { @@ -53,7 +51,7 @@ const MagicPackageLoggerFunctionName = "init" // TODO: document all the black magic here ... // https://github.com/dyweb/gommon/issues/32 -func NewIdentityFromCaller(skip int) *Identity { +func NewIdentityFromCaller(skip int) Identity { frame := runtimeutil.GetCallerFrame(skip + 1) var ( pkg string @@ -61,7 +59,7 @@ func NewIdentityFromCaller(skip int) *Identity { st string ) tpe := UnknownLogger - // TODO: does it handle vendor correctly + // TODO: does it handle vendor correctly, and what about vgo ... pkg, function = runtimeutil.SplitPackageFunc(frame.Function) tpe = FunctionLogger // NOTE: we distinguish a struct logger and method logger using the magic name, @@ -77,7 +75,7 @@ func NewIdentityFromCaller(skip int) *Identity { tpe = PackageLogger } - return &Identity{ + return Identity{ Package: pkg, Function: function, Struct: st, diff --git a/log/logger_identity_test.go b/log/logger_identity_test.go index 920f96e..e7821cf 100644 --- a/log/logger_identity_test.go +++ b/log/logger_identity_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/dyweb/gommon/util/testutil" - asst "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/assert" ) var lg = NewPackageLogger() @@ -15,6 +15,10 @@ func foo() *Logger { return funcLog } +func fooUseCopy() *Logger { + return lg.Copy() +} + type Foo struct { log *Logger } @@ -27,7 +31,7 @@ func (f *Foo) SetLogger(logger *Logger) { f.log = logger } -func (f *Foo) LoggerIdentity(justCallMe func() *Identity) *Identity { +func (f *Foo) LoggerIdentity(justCallMe func() Identity) Identity { return justCallMe() } @@ -36,56 +40,57 @@ func (f *Foo) method() *Logger { return mlog } +func (f *Foo) methodUseCopy() *Logger { + return f.log.Copy() +} + +func (f *Foo) methodOrphanCopy() *Logger { + return lg.Copy() +} + var dummyFoo = &Foo{} // used for get struct logger identity func TestNewPackageLogger(t *testing.T) { - assert := asst.New(t) id := lg.id - assert.Equal(PackageLogger, id.Type) - assert.Equal("pkg", id.Type.String()) - assert.Equal("init", id.Function) - assert.Equal(testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:11", + assert.Equal(t, PackageLogger, id.Type) + assert.Equal(t, "pkg", id.Type.String()) + assert.Equal(t, "init", id.Function) + assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:11", fmt.Sprintf("%s:%d", id.File, id.Line)) } func TestNewFunctionLogger(t *testing.T) { - assert := asst.New(t) - flog := foo() - id := flog.id - assert.Equal(FunctionLogger, id.Type) + assert.Equal(t, FunctionLogger, foo().id.Type) + assert.Equal(t, FunctionLogger, fooUseCopy().id.Type) } func TestNewStructLogger(t *testing.T) { - assert := asst.New(t) slog := NewStructLogger(lg, dummyFoo) id := slog.id - assert.Equal(StructLogger, id.Type) - assert.Equal("struct", id.Type.String()) - assert.Equal("Foo", id.Struct) - assert.Equal(MagicStructLoggerFunctionName, id.Function) - assert.Equal(testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:31", + assert.Equal(t, StructLogger, id.Type) + assert.Equal(t, "struct", id.Type.String()) + assert.Equal(t, "Foo", id.Struct) + assert.Equal(t, MagicStructLoggerFunctionName, id.Function) + assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:35", fmt.Sprintf("%s:%d", id.File, id.Line)) } func TestNewMethodLogger(t *testing.T) { - assert := asst.New(t) slog := NewStructLogger(lg, dummyFoo) dummyFoo.log = slog mlog := dummyFoo.method() id := mlog.id - assert.Equal(MethodLogger, id.Type) - assert.Equal("method", id.Type.String()) - assert.Equal("Foo", id.Struct) - assert.Equal("method", id.Function) - assert.Equal(testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:35", + assert.Equal(t, MethodLogger, id.Type) + assert.Equal(t, "method", id.Type.String()) + assert.Equal(t, "Foo", id.Struct) + assert.Equal(t, "method", id.Function) + assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:39", fmt.Sprintf("%s:%d", id.File, id.Line)) -} -func ExampleNewApplicationLogger() { - logger := NewApplicationLogger() - fmt.Println(logger.Identity().Package) - fmt.Println(logger.Identity().Type) - // Output: - // github.com/dyweb/gommon/log - // app + assert.Equal(t, MethodLogger, dummyFoo.methodUseCopy().id.Type) + + orphanLog := dummyFoo.methodOrphanCopy() + assert.Equal(t, MethodLogger, orphanLog.id.Type) + assert.Equal(t, "Foo", orphanLog.id.Struct) + assert.Equal(t, "methodOrphanCopy", orphanLog.id.Function) } diff --git a/log/logger_tree.go b/log/logger_tree.go index 0fe5f28..423e8ef 100644 --- a/log/logger_tree.go +++ b/log/logger_tree.go @@ -1,3 +1,6 @@ +// +build ignore + +// TODO: enable this file after refactor on tree of logger is finished package log import ( @@ -7,77 +10,6 @@ import ( "github.com/dyweb/gommon/structure" ) -// -// TODO: allow release a child logger, this will be a trouble if we created 1,000 Client struct with its own logger... -func (l *Logger) AddChild(child *Logger) { - l.mu.Lock() - if l.children == nil { - l.children = make(map[string][]*Logger, 1) - } - // children are group by their identity, i.e a package logger may have many struct logger of same struct because - // that struct is used in multiple goroutines, those loggers have different pointer for identity, but they should - // have same source line, so we use SourceLocation as key - k := child.id.SourceLocation() - children := l.children[k] - // avoid putting same pointer twice, though it should never happen if AddChild is called correctly - exists := false - for _, c := range children { - if c == child { - exists = true - break - } - } - if !exists { - l.children[k] = append(children, child) - } - l.mu.Unlock() -} - -func SetLevelRecursive(root *Logger, level Level) { - visited := make(map[*Logger]bool) - PreOrderDFS(root, visited, func(l *Logger) { - // TODO: remove it after we have tested it .... - //fmt.Println(l.Identity().String()) - l.SetLevel(level) - }) -} - -func SetHandlerRecursive(root *Logger, handler Handler) { - visited := make(map[*Logger]bool) - PreOrderDFS(root, visited, func(l *Logger) { - l.SetHandler(handler) - }) -} - -// FIXME: this fixed typo requires update in go.ice -func EnableSourceRecursive(root *Logger) { - visited := make(map[*Logger]bool) - PreOrderDFS(root, visited, func(l *Logger) { - l.EnableSource() - }) -} - -func DisableSourceRecursive(root *Logger) { - visited := make(map[*Logger]bool) - PreOrderDFS(root, visited, func(l *Logger) { - l.DisableSource() - }) -} - -// TODO: test it .... map traverse order is random, we need radix tree, it is need for pretty print as well -func PreOrderDFS(root *Logger, visited map[*Logger]bool, cb func(l *Logger)) { - if visited[root] { - return - } - cb(root) - visited[root] = true - for _, group := range root.children { - for _, l := range group { - PreOrderDFS(l, visited, cb) - } - } -} - func ToStringTree(root *Logger) *structure.StringTreeNode { visited := make(map[*Logger]bool) return toStringTreeHelper(root, visited) diff --git a/log/logger_tree_test.go b/log/logger_tree_test.go deleted file mode 100644 index 0c7e666..0000000 --- a/log/logger_tree_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -var tPkgLogger = NewPackageLogger() // lv0 - -func TestPreOrderDFS(t *testing.T) { - t.Skip("FIXME: map does not guarantee order, the traverse result is unstable") - assert := asst.New(t) - lv1a := NewFunctionLogger(tPkgLogger) - lv1b := NewFunctionLogger(tPkgLogger) - lv2a := NewFunctionLogger(lv1a) - var ids []string - // FIXME: the result it unstable ... we should use a radix tree, map does not guarantee the order ... - var expected = []string{ - tPkgLogger.id.SourceLocation(), // 9 - lv1a.id.SourceLocation(), // 13 - lv2a.id.SourceLocation(), // 14 - lv1b.id.SourceLocation(), - } - PreOrderDFS(tPkgLogger, make(map[*Logger]bool), func(l *Logger) { - ids = append(ids, l.id.SourceLocation()) - }) - assert.Equal(expected, ids) -} diff --git a/log/pkg.go b/log/pkg.go index 3491c0d..d1016fe 100644 --- a/log/pkg.go +++ b/log/pkg.go @@ -1,4 +1,4 @@ -package log // import "github.com/dyweb/gommon/log" +package log // LoggableStruct is used to inject a logger into the struct, // the methods for the interface can and should be generated using gommon. @@ -7,5 +7,5 @@ package log // import "github.com/dyweb/gommon/log" type LoggableStruct interface { GetLogger() *Logger SetLogger(logger *Logger) - LoggerIdentity(justCallMe func() *Identity) *Identity + LoggerIdentity(justCallMe func() Identity) Identity } diff --git a/log/registry.go b/log/registry.go new file mode 100644 index 0000000..1822501 --- /dev/null +++ b/log/registry.go @@ -0,0 +1,206 @@ +package log + +import ( + "sync" + + "github.com/dyweb/gommon/util/runtimeutil" +) + +// registry.go is used for maintain relationship between loggers across packages and projects +// it also contains util func for traverse registry and logger + +// Registry contains child registry and loggers +type Registry struct { + mu sync.Mutex + children []*Registry + loggers []*Logger + + // immutable + identity RegistryIdentity +} + +type RegistryType uint8 + +const ( + UnknownRegistry RegistryType = iota + ApplicationRegistry + LibraryRegistry + PackageRegistry +) + +func (r RegistryType) String() string { + switch r { + case UnknownRegistry: + return "unk" + case ApplicationRegistry: + return "app" + case LibraryRegistry: + return "lib" + case PackageRegistry: + return "pkg" + default: + return "unk" + } +} + +type RegistryIdentity struct { + // Project is specified by user, i.e. for all the packages under gommon, they would have github.com/dyweb/gommon + Project string + // Package is detected base on runtime, i.e. github.com/dyweb/gommon/noodle + Package string + // Type is specified by user when creating registry + Type RegistryType + // File is where create registry is called + File string + // Line is where create registry is called + Line int +} + +func NewLibraryRegistry(project string) Registry { + return Registry{ + identity: newRegistryId(project, LibraryRegistry, 0), + } +} + +// TODO: validate skip +func NewApplicationLoggerAndRegistry(project string) (*Logger, *Registry) { + reg := Registry{ + identity: newRegistryId(project, ApplicationRegistry, 1), + } + logger := NewPackageLoggerWithSkip(1) + reg.AddLogger(logger) + return logger, ® +} + +func NewPackageLoggerAndRegistryWithSkip(project string, skip int) (*Logger, *Registry) { + reg := Registry{ + identity: newRegistryId(project, PackageRegistry, skip+1), + } + logger := NewPackageLoggerWithSkip(skip + 1) + reg.AddLogger(logger) + return logger, ® +} + +func newRegistryId(proj string, tpe RegistryType, skip int) RegistryIdentity { + // TODO: check if the skip works .... we need another package for testing that + frame := runtimeutil.GetCallerFrame(skip + 1) + pkg, _ := runtimeutil.SplitPackageFunc(frame.Function) + return RegistryIdentity{ + Project: proj, + Package: pkg, + Type: tpe, + File: frame.File, + Line: frame.Line, + } +} + +// AddRegistry is for adding a package level log registry to a library/application level log registry +// It skips add if child registry already there +func (r *Registry) AddRegistry(child *Registry) { + r.mu.Lock() + defer r.mu.Unlock() + for _, c := range r.children { + if c == child { + return + } + } + r.children = append(r.children, child) +} + +// AddLogger is used for registering a logger to package level log registry +// It skips add if the logger is already there +func (r *Registry) AddLogger(l *Logger) { + r.mu.Lock() + defer r.mu.Unlock() + for _, ol := range r.loggers { + if ol == l { + return + } + } + r.loggers = append(r.loggers, l) +} + +func (r *Registry) Identity() RegistryIdentity { + return r.identity +} + +func SetLevel(root *Registry, level Level) { + WalkLogger(root, func(l *Logger) { + l.SetLevel(level) + }) +} + +func SetHandler(root *Registry, handler Handler) { + WalkLogger(root, func(l *Logger) { + l.SetHandler(handler) + }) +} + +func EnableSource(root *Registry) { + WalkLogger(root, func(l *Logger) { + l.EnableSource() + }) +} + +func DisableSource(root *Registry) { + WalkLogger(root, func(l *Logger) { + l.DisableSource() + }) +} + +// WalkLogger is PreOrderDfs +func WalkLogger(root *Registry, cb func(l *Logger)) { + walkLogger(root, nil, nil, cb) +} + +// walkLogger loops loggers in current registry first, then visist its children in DFS +// It will create the map if it is nil, so caller don't need to do the bootstrap, +// However, caller can provide a map so they can use it to get all the loggers and registry +func walkLogger(root *Registry, loggers map[*Logger]bool, registries map[*Registry]bool, cb func(l *Logger)) { + // first call + if loggers == nil { + loggers = make(map[*Logger]bool) + } + if registries == nil { + registries = make(map[*Registry]bool) + } + // pre order + registries[root] = true + for _, l := range root.loggers { + // avoid dup + if loggers[l] { + continue + } + loggers[l] = true + cb(l) // visit + } + // dfs + for _, r := range root.children { + // avoid cycle + if registries[r] { + continue + } + walkLogger(r, loggers, registries, cb) + } +} + +func WalkRegistry(root *Registry, cb func(r *Registry)) { + walkRegistry(root, nil, cb) +} + +func walkRegistry(root *Registry, registries map[*Registry]bool, cb func(r *Registry)) { + // first call + if registries == nil { + registries = make(map[*Registry]bool) + } + registries[root] = true + cb(root) // visit + // dfs + for _, r := range root.children { + // avoid dup + if registries[r] { + continue + } + walkRegistry(r, registries, cb) + } +} diff --git a/log/registry_test.go b/log/registry_test.go new file mode 100644 index 0000000..2b77ee7 --- /dev/null +++ b/log/registry_test.go @@ -0,0 +1,25 @@ +package log + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRegistry_AddRegistry(t *testing.T) { + rTop := Registry{} + r1 := Registry{} + rTop.AddRegistry(&r1) + assert.Equal(t, 1, len(rTop.children)) + rTop.AddRegistry(&r1) + assert.Equal(t, 1, len(rTop.children), "don't add registry if it already exists") +} + +func TestRegistry_AddLogger(t *testing.T) { + rTop := Registry{} + l1 := NewTestLogger(InfoLevel) + rTop.AddLogger(l1) + assert.Equal(t, 1, len(rTop.loggers)) + rTop.AddLogger(l1) + assert.Equal(t, 1, len(rTop.loggers), "don't add logger if it already exists") +} diff --git a/log/testdata/.gitignore b/log/testdata/.gitignore new file mode 100644 index 0000000..9c23e5a --- /dev/null +++ b/log/testdata/.gitignore @@ -0,0 +1,2 @@ +f1.log +f2.log \ No newline at end of file diff --git a/log/testing.go b/log/testing.go new file mode 100644 index 0000000..dfc7442 --- /dev/null +++ b/log/testing.go @@ -0,0 +1,68 @@ +package log + +// test.go contains helpers for both internal and external testing + +import ( + "sync" + "time" +) + +// entry is only used for test, we do not use it as contract for interface +type entry struct { + level Level + time time.Time + msg string + context Fields + fields Fields + source Caller +} + +var _ Handler = (*TestHandler)(nil) + +// TestHandler stores log as entry, its slice is protected by a RWMutex and safe for concurrent use +type TestHandler struct { + mu sync.RWMutex + entries []entry +} + +// NewTestHandler returns a test handler, it should only be used in test, +// a concrete type instead of Handler interface is returned to reduce unnecessary type cast in test +func NewTestHandler() *TestHandler { + return &TestHandler{} +} + +func (h *TestHandler) HandleLog(level Level, time time.Time, msg string, source Caller, context Fields, fields Fields) { + h.mu.Lock() + h.entries = append(h.entries, entry{level: level, time: time, msg: msg, source: source, context: CopyFields(context), fields: CopyFields(fields)}) + h.mu.Unlock() +} + +// Flush implements Handler interface +func (h *TestHandler) Flush() { + // nop +} + +// HasLog checks if a log with specified level and message exists in slice +// TODO: support field, source etc. +func (h *TestHandler) HasLog(level Level, msg string) bool { + h.mu.RLock() + defer h.mu.RUnlock() + for _, e := range h.entries { + if e.level == level && e.msg == msg { + return true + } + } + return false +} + +func (h *TestHandler) getLogByMessage(msg string) (entry, bool) { + h.mu.RLock() + defer h.mu.RUnlock() + + for _, e := range h.entries { + if e.msg == msg { + return e, true + } + } + return entry{}, false +} diff --git a/noodle/_examples/embed/gen/noodle.go b/noodle/_examples/embed/gen/noodle.go index fcd8287..621191a 100644 --- a/noodle/_examples/embed/gen/noodle.go +++ b/noodle/_examples/embed/gen/noodle.go @@ -1,4 +1,4 @@ -// Code generated by gommon/noodle from assets,third_party DO NOT EDIT. +// Code generated by gommon/noodle from noodle/_examples/embed/assets,noodle/_examples/embed/third_party DO NOT EDIT. package gen @@ -8,7 +8,7 @@ import ( "github.com/dyweb/gommon/noodle" ) -// GetNoodleAssets returns an extracted EmbedBowl generated from assets +// GetNoodleAssets returns an extracted EmbedBowl generated from noodle/_examples/embed/assets func GetNoodleAssets() (noodle.EmbedBowel, error) { dirs := map[string]noodle.EmbedDir{"": { @@ -58,13 +58,7 @@ func GetNoodleAssets() (noodle.EmbedBowel, error) { FileModTime: time.Unix(1523142440, 0), FileIsDir: true, }, - Entries: []noodle.FileInfo{{ - FileName: "index.html", - FileSize: 119, - FileMode: 0664, - FileModTime: time.Unix(1523142440, 0), - FileIsDir: false, - }}, + Entries: []noodle.FileInfo{}, }, "/idx": { FileInfo: noodle.FileInfo{ FileName: "idx", @@ -79,12 +73,6 @@ func GetNoodleAssets() (noodle.EmbedBowel, error) { FileMode: 020000000775, FileModTime: time.Unix(1523142440, 0), FileIsDir: true, - }, { - FileName: "a.txt", - FileSize: 5, - FileMode: 0664, - FileModTime: time.Unix(1523142440, 0), - FileIsDir: false, }, { FileName: "index.html", FileSize: 180, @@ -148,7 +136,7 @@ func GetNoodleAssets() (noodle.EmbedBowel, error) { }}, }} - data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x54, 0xcb, 0xb1, 0xaa, 0x85, 0x30, 0x10, 0x84, 0xe1, 0x7e, 0x9f, 0x62, 0xc0, 0x2e, 0x85, 0xde, 0xc2, 0xf7, 0xb9, 0x4, 0x77, 0x34, 0x81, 0x4d, 0x2, 0xc9, 0x86, 0xe3, 0xe3, 0x1f, 0xac, 0xe4, 0x94, 0xc3, 0x37, 0xff, 0x2, 0x4f, 0x79, 0x20, 0xe2, 0x68, 0xa5, 0xb0, 0xba, 0xd4, 0xa6, 0xfc, 0x2f, 0x4d, 0xa7, 0x71, 0x60, 0x41, 0xbe, 0x6a, 0xeb, 0x44, 0x8d, 0x85, 0x22, 0x61, 0x4d, 0xf3, 0xe2, 0x99, 0x8d, 0xbf, 0x82, 0x4f, 0x36, 0x3d, 0x62, 0x57, 0x91, 0xfd, 0x6f, 0xdf, 0xc2, 0xab, 0xd1, 0xc, 0x9e, 0x88, 0xa7, 0x19, 0x98, 0x55, 0xd9, 0x11, 0x71, 0x36, 0x53, 0x76, 0xc9, 0x7a, 0x6f, 0x61, 0xf5, 0xdb, 0xdf, 0xff, 0xb3, 0xbe, 0x1, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0x28, 0xca, 0xcf, 0x2f, 0xb1, 0xd1, 0x87, 0x8, 0x70, 0xd9, 0xe8, 0x43, 0x14, 0xd9, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0xb4, 0x18, 0x22, 0x14, 0xa9, 0x17, 0x2b, 0x64, 0xe6, 0xa5, 0xa4, 0x56, 0xe8, 0x81, 0x8c, 0xb3, 0xd1, 0xcf, 0x30, 0x4, 0x29, 0x87, 0xaa, 0xd3, 0x7, 0x5b, 0x1, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x0, 0x9, 0x0, 0x2f, 0x34, 0x30, 0x34, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x34, 0xcd, 0xc1, 0x9, 0x3, 0x31, 0xc, 0x44, 0xd1, 0xbb, 0xab, 0x50, 0x7, 0x6e, 0x40, 0xa8, 0x8f, 0x1c, 0x13, 0x34, 0x89, 0xc, 0x8e, 0x67, 0xb1, 0xb5, 0x87, 0x74, 0x1f, 0x58, 0xbc, 0xc7, 0x61, 0x78, 0x7c, 0x8d, 0xfc, 0x76, 0x2b, 0x1a, 0x78, 0xba, 0x3d, 0x78, 0xca, 0xa, 0x9e, 0xdd, 0x65, 0x30, 0x65, 0x1, 0x92, 0xd1, 0x96, 0xd6, 0xeb, 0x2d, 0xfa, 0xa2, 0xff, 0xac, 0xe8, 0x61, 0x83, 0xf4, 0xe, 0x69, 0x9f, 0xc1, 0x89, 0x9b, 0xec, 0x95, 0x1, 0xc1, 0xc8, 0x36, 0x21, 0x6f, 0x76, 0xc7, 0xd4, 0x7a, 0x58, 0xd1, 0xba, 0x71, 0xbd, 0x82, 0xff, 0x0, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xe6, 0x35, 0x5b, 0xf5, 0x60, 0x0, 0x0, 0x0, 0x77, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x61, 0x2e, 0x74, 0x78, 0x74, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x4a, 0xd4, 0x2b, 0xa9, 0x28, 0x1, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xba, 0xf7, 0xeb, 0xc1, 0xb, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x0, 0x9, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x64, 0xce, 0x31, 0xae, 0x3, 0x21, 0xc, 0x4, 0xd0, 0x9e, 0x53, 0x58, 0x1c, 0x60, 0xd1, 0xf6, 0x5e, 0xf7, 0xff, 0x18, 0x7c, 0x70, 0x4, 0x89, 0x21, 0x11, 0x76, 0x91, 0xbd, 0x7d, 0x44, 0xd8, 0x2e, 0xed, 0x68, 0xfc, 0x3c, 0x58, 0xac, 0x9, 0x39, 0x2c, 0x1c, 0x33, 0x39, 0x0, 0x0, 0xb4, 0x6a, 0xc2, 0xf4, 0x7, 0xb1, 0x41, 0xed, 0x99, 0xdf, 0xdb, 0xac, 0x60, 0x58, 0xf1, 0xaa, 0x48, 0xed, 0xf, 0x18, 0x2c, 0x87, 0x57, 0x3b, 0x85, 0xb5, 0x30, 0x9b, 0x87, 0x32, 0xf8, 0x76, 0xf8, 0x16, 0x6b, 0xdf, 0x92, 0xaa, 0x27, 0x87, 0x61, 0xb1, 0xf8, 0xff, 0xcc, 0xe7, 0x7c, 0xb2, 0xff, 0xb2, 0x65, 0x27, 0x87, 0x9a, 0x46, 0x7d, 0x19, 0xe8, 0x48, 0xd7, 0xfd, 0x5d, 0x3d, 0x61, 0x58, 0xf1, 0x74, 0x2e, 0x20, 0x7c, 0xd7, 0x7e, 0x2, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xb7, 0x3b, 0x1b, 0x4d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6a, 0xa8, 0x83, 0x5b, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0xb7, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0x57, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x13, 0x0, 0x9, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0xc8, 0xcc, 0x4b, 0x49, 0xad, 0xd0, 0x3, 0x29, 0xb1, 0xd1, 0x87, 0x8, 0x73, 0xd9, 0xe8, 0x43, 0x94, 0xda, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0x34, 0x1a, 0xa2, 0x2b, 0x55, 0xc8, 0xcc, 0x2b, 0xce, 0x4c, 0x49, 0x55, 0x28, 0x2e, 0x4d, 0xb2, 0xd1, 0xcf, 0x30, 0x4, 0x69, 0x81, 0xaa, 0xd5, 0x7, 0x5b, 0x6, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf, 0x0, 0x9, 0x0, 0x2f, 0x6e, 0x6f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0xb3, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x2f, 0x6e, 0x6f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0x53, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x13, 0x0, 0x9, 0x0, 0x2f, 0x6e, 0x6f, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x6c, 0xce, 0x31, 0xe, 0xc3, 0x20, 0xc, 0x40, 0xd1, 0x9d, 0x53, 0x58, 0x1c, 0x20, 0x28, 0xbb, 0xe3, 0xbd, 0xc7, 0xa0, 0xe0, 0xca, 0xb4, 0x40, 0x2a, 0xec, 0xa1, 0xb9, 0x7d, 0x95, 0x92, 0xb1, 0xab, 0xfd, 0xfd, 0x64, 0x14, 0x6b, 0x95, 0x1c, 0xa, 0xc7, 0x4c, 0xe, 0x0, 0x0, 0xad, 0x58, 0x65, 0xba, 0x41, 0x6c, 0xd0, 0x77, 0x83, 0xd2, 0x33, 0x7f, 0x96, 0x33, 0xc3, 0x30, 0x57, 0x33, 0xab, 0xa5, 0xbf, 0x60, 0x70, 0xdd, 0xbc, 0xda, 0x51, 0x59, 0x85, 0xd9, 0x3c, 0xc8, 0xe0, 0xc7, 0xe6, 0x5b, 0x2c, 0x7d, 0x49, 0xaa, 0x9e, 0x1c, 0x86, 0x49, 0xe3, 0x7d, 0xcf, 0xc7, 0x75, 0x2a, 0xeb, 0x7f, 0x5e, 0x56, 0x72, 0xa8, 0x69, 0x94, 0xb7, 0x81, 0x8e, 0x74, 0x39, 0x4f, 0xf5, 0x84, 0x61, 0x8e, 0x4f, 0x6f, 0x42, 0x18, 0x7e, 0x9f, 0x7f, 0x3, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xb1, 0x0, 0x0, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0xe6, 0x35, 0x5b, 0xf5, 0x60, 0x0, 0x0, 0x0, 0x77, 0x0, 0x0, 0x0, 0xf, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x43, 0x1, 0x0, 0x0, 0x2f, 0x34, 0x30, 0x34, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0xba, 0xf7, 0xeb, 0xc1, 0xb, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xe9, 0x1, 0x0, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x61, 0x2e, 0x74, 0x78, 0x74, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0xf, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x35, 0x2, 0x0, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xb7, 0x3b, 0x1b, 0x4d, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xf7, 0x2, 0x0, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6a, 0xa8, 0x83, 0x5b, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x67, 0x3, 0x0, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x13, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xe3, 0x3, 0x0, 0x0, 0x2f, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0xf, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x82, 0x4, 0x0, 0x0, 0x2f, 0x6e, 0x6f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xf3, 0x4, 0x0, 0x0, 0x2f, 0x6e, 0x6f, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x13, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x70, 0x5, 0x0, 0x0, 0x2f, 0x6e, 0x6f, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0xb, 0x0, 0xf8, 0x2, 0x0, 0x0, 0x38, 0x6, 0x0, 0x0, 0x0, 0x0} + data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x54, 0xcb, 0xb1, 0xaa, 0x85, 0x30, 0x10, 0x84, 0xe1, 0x7e, 0x9f, 0x62, 0xc0, 0x2e, 0x85, 0xde, 0xc2, 0xf7, 0xb9, 0x4, 0x77, 0x34, 0x81, 0x4d, 0x2, 0xc9, 0x86, 0xe3, 0xe3, 0x1f, 0xac, 0xe4, 0x94, 0xc3, 0x37, 0xff, 0x2, 0x4f, 0x79, 0x20, 0xe2, 0x68, 0xa5, 0xb0, 0xba, 0xd4, 0xa6, 0xfc, 0x2f, 0x4d, 0xa7, 0x71, 0x60, 0x41, 0xbe, 0x6a, 0xeb, 0x44, 0x8d, 0x85, 0x22, 0x61, 0x4d, 0xf3, 0xe2, 0x99, 0x8d, 0xbf, 0x82, 0x4f, 0x36, 0x3d, 0x62, 0x57, 0x91, 0xfd, 0x6f, 0xdf, 0xc2, 0xab, 0xd1, 0xc, 0x9e, 0x88, 0xa7, 0x19, 0x98, 0x55, 0xd9, 0x11, 0x71, 0x36, 0x53, 0x76, 0xc9, 0x7a, 0x6f, 0x61, 0xf5, 0xdb, 0xdf, 0xff, 0xb3, 0xbe, 0x1, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0x28, 0xca, 0xcf, 0x2f, 0xb1, 0xd1, 0x87, 0x8, 0x70, 0xd9, 0xe8, 0x43, 0x14, 0xd9, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0xb4, 0x18, 0x22, 0x14, 0xa9, 0x17, 0x2b, 0x64, 0xe6, 0xa5, 0xa4, 0x56, 0xe8, 0x81, 0x8c, 0xb3, 0xd1, 0xcf, 0x30, 0x4, 0x29, 0x87, 0xaa, 0xd3, 0x7, 0x5b, 0x1, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x64, 0xce, 0x31, 0xae, 0x3, 0x21, 0xc, 0x4, 0xd0, 0x9e, 0x53, 0x58, 0x1c, 0x60, 0xd1, 0xf6, 0x5e, 0xf7, 0xff, 0x18, 0x7c, 0x70, 0x4, 0x89, 0x21, 0x11, 0x76, 0x91, 0xbd, 0x7d, 0x44, 0xd8, 0x2e, 0xed, 0x68, 0xfc, 0x3c, 0x58, 0xac, 0x9, 0x39, 0x2c, 0x1c, 0x33, 0x39, 0x0, 0x0, 0xb4, 0x6a, 0xc2, 0xf4, 0x7, 0xb1, 0x41, 0xed, 0x99, 0xdf, 0xdb, 0xac, 0x60, 0x58, 0xf1, 0xaa, 0x48, 0xed, 0xf, 0x18, 0x2c, 0x87, 0x57, 0x3b, 0x85, 0xb5, 0x30, 0x9b, 0x87, 0x32, 0xf8, 0x76, 0xf8, 0x16, 0x6b, 0xdf, 0x92, 0xaa, 0x27, 0x87, 0x61, 0xb1, 0xf8, 0xff, 0xcc, 0xe7, 0x7c, 0xb2, 0xff, 0xb2, 0x65, 0x27, 0x87, 0x9a, 0x46, 0x7d, 0x19, 0xe8, 0x48, 0xd7, 0xfd, 0x5d, 0x3d, 0x61, 0x58, 0xf1, 0x74, 0x2e, 0x20, 0x7c, 0xd7, 0x7e, 0x2, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xb7, 0x3b, 0x1b, 0x4d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6a, 0xa8, 0x83, 0x5b, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0xb7, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0x57, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x12, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0xc8, 0xcc, 0x4b, 0x49, 0xad, 0xd0, 0x3, 0x29, 0xb1, 0xd1, 0x87, 0x8, 0x73, 0xd9, 0xe8, 0x43, 0x94, 0xda, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0x34, 0x1a, 0xa2, 0x2b, 0x55, 0xc8, 0xcc, 0x2b, 0xce, 0x4c, 0x49, 0x55, 0x28, 0x2e, 0x4d, 0xb2, 0xd1, 0xcf, 0x30, 0x4, 0x69, 0x81, 0xaa, 0xd5, 0x7, 0x5b, 0x6, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0xb3, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0x53, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x6c, 0xce, 0x31, 0xe, 0xc3, 0x20, 0xc, 0x40, 0xd1, 0x9d, 0x53, 0x58, 0x1c, 0x20, 0x28, 0xbb, 0xe3, 0xbd, 0xc7, 0xa0, 0xe0, 0xca, 0xb4, 0x40, 0x2a, 0xec, 0xa1, 0xb9, 0x7d, 0x95, 0x92, 0xb1, 0xab, 0xfd, 0xfd, 0x64, 0x14, 0x6b, 0x95, 0x1c, 0xa, 0xc7, 0x4c, 0xe, 0x0, 0x0, 0xad, 0x58, 0x65, 0xba, 0x41, 0x6c, 0xd0, 0x77, 0x83, 0xd2, 0x33, 0x7f, 0x96, 0x33, 0xc3, 0x30, 0x57, 0x33, 0xab, 0xa5, 0xbf, 0x60, 0x70, 0xdd, 0xbc, 0xda, 0x51, 0x59, 0x85, 0xd9, 0x3c, 0xc8, 0xe0, 0xc7, 0xe6, 0x5b, 0x2c, 0x7d, 0x49, 0xaa, 0x9e, 0x1c, 0x86, 0x49, 0xe3, 0x7d, 0xcf, 0xc7, 0x75, 0x2a, 0xeb, 0x7f, 0x5e, 0x56, 0x72, 0xa8, 0x69, 0x94, 0xb7, 0x81, 0x8e, 0x74, 0x39, 0x4f, 0xf5, 0x84, 0x61, 0x8e, 0x4f, 0x6f, 0x42, 0x18, 0x7e, 0x9f, 0x7f, 0x3, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xb1, 0x0, 0x0, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x43, 0x1, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xb7, 0x3b, 0x1b, 0x4d, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x4, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6a, 0xa8, 0x83, 0x5b, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x73, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x12, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xee, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x8c, 0x3, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xfa, 0x3, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xea, 0xb8, 0x87, 0x4c, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x74, 0x4, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x28, 0x4f, 0xc9, 0x5a, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x9, 0x0, 0x9, 0x0, 0x64, 0x2, 0x0, 0x0, 0x39, 0x5, 0x0, 0x0, 0x0, 0x0} bowl := noodle.EmbedBowel{ Dirs: dirs, Data: data, @@ -159,7 +147,7 @@ func GetNoodleAssets() (noodle.EmbedBowel, error) { return bowl, nil } -// GetNoodleThirdParty returns an extracted EmbedBowl generated from third_party +// GetNoodleThirdParty returns an extracted EmbedBowl generated from noodle/_examples/embed/third_party func GetNoodleThirdParty() (noodle.EmbedBowel, error) { dirs := map[string]noodle.EmbedDir{"": { @@ -167,19 +155,19 @@ func GetNoodleThirdParty() (noodle.EmbedBowel, error) { FileName: "third_party", FileSize: 4096, FileMode: 020000000775, - FileModTime: time.Unix(1535353229, 0), + FileModTime: time.Unix(1536205156, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{{ FileName: "huge.js", FileSize: 47, FileMode: 0664, - FileModTime: time.Unix(1535353229, 0), + FileModTime: time.Unix(1536205156, 0), FileIsDir: false, }}, }} - data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xe, 0x38, 0x1b, 0x4d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x8d, 0xa1, 0x83, 0x5b, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0xf7, 0x54, 0x48, 0xcc, 0x55, 0x48, 0x54, 0xc8, 0x28, 0x4d, 0x4f, 0x55, 0x28, 0xc9, 0xc8, 0x2c, 0x4a, 0x51, 0x28, 0x48, 0x2c, 0x2a, 0xa9, 0x54, 0xc8, 0xc9, 0x4c, 0x2a, 0x4a, 0x2c, 0xaa, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xe, 0x38, 0x1b, 0x4d, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x8d, 0xa1, 0x83, 0x5b, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x3e, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x0, 0x0} + data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xe8, 0x1c, 0x26, 0x4d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x64, 0xa1, 0x90, 0x5b, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0xf7, 0x54, 0x48, 0xcc, 0x55, 0x48, 0x54, 0xc8, 0x28, 0x4d, 0x4f, 0x55, 0x28, 0xc9, 0xc8, 0x2c, 0x4a, 0x51, 0x28, 0x48, 0x2c, 0x2a, 0xa9, 0x54, 0xc8, 0xc9, 0x4c, 0x2a, 0x4a, 0x2c, 0xaa, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xe8, 0x1c, 0x26, 0x4d, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x64, 0xa1, 0x90, 0x5b, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x3e, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x0, 0x0} bowl := noodle.EmbedBowel{ Dirs: dirs, Data: data, diff --git a/noodle/pkg.go b/noodle/pkg.go index b8b22d5..f56977a 100644 --- a/noodle/pkg.go +++ b/noodle/pkg.go @@ -2,8 +2,9 @@ package noodle import ( - "github.com/dyweb/gommon/util/logutil" "net/http" + + "github.com/dyweb/gommon/util/logutil" ) const ( @@ -11,7 +12,7 @@ const ( DefaultName = "Bowel" ) -var log = logutil.NewPackageLogger() +var log, _ = logutil.NewPackageLoggerAndRegistry() // Bowel is the container for different types of noodles type Bowel interface { diff --git a/playground/stdlib/stdlib_default_test.go b/playground/stdlib/stdlib_default_test.go index 8a44751..65b23ec 100644 --- a/playground/stdlib/stdlib_default_test.go +++ b/playground/stdlib/stdlib_default_test.go @@ -15,4 +15,10 @@ func TestDefault_StructString(t *testing.T) { f := &fooStr{} assert.True(f.str == "") +} + +var ga, gb = newDouble() + +func newDouble() (int, int) { + return 1, 2 } \ No newline at end of file diff --git a/playground/stdlib/stdlib_fmt_test.go b/playground/stdlib/stdlib_fmt_test.go index 52f70e6..9bed8ad 100644 --- a/playground/stdlib/stdlib_fmt_test.go +++ b/playground/stdlib/stdlib_fmt_test.go @@ -1,16 +1,16 @@ package stdlib import ( - "testing" "fmt" "os" + "testing" ) // For #31 [log] extra square brackets when using fmt.Sprint // https://github.com/dyweb/gommon/issues/31 func printArgs(args ...interface{}) { - fmt.Println(args) // WRONG, it will have [arg1 arg2 arg3] + // fmt.Println(args) // WRONG, it will have [arg1 arg2 arg3] NOTE: this is already disabled when go test is used fmt.Println(args...) } diff --git a/playground/stdlib/stdlib_map_test.go b/playground/stdlib/stdlib_map_test.go new file mode 100644 index 0000000..0a16978 --- /dev/null +++ b/playground/stdlib/stdlib_map_test.go @@ -0,0 +1,51 @@ +package stdlib + +import ( + "testing" +) + +// Test in a map of struct value, if the struct also contains map/slice, what would happen when the value +// is modified after it is added to the map, get the added value from map should not have the updated change +// I suppose, but due to the nature of slice, it might cause problem when append + +type registry struct { + childRegistries map[string]registry + childLoggers []int +} + +type registryPtr struct { + childRegistries map[string]*registryPtr + childLoggers []int +} + +func TestMapReference(t *testing.T) { + rTop := registry{ + childRegistries: make(map[string]registry), + } + r1 := registry{ + childRegistries: make(map[string]registry), + childLoggers: []int{0}, + } + t.Logf("r1 registry len %d loggers len %d", len(r1.childRegistries), len(r1.childLoggers)) + rTop.childRegistries["r1"] = r1 + r1.childRegistries["r2"] = registry{} + r1.childLoggers = append(r1.childLoggers, 1) + t.Logf("r1 registry len %d loggers len %d", len(r1.childRegistries), len(r1.childLoggers)) + r1Cp := rTop.childRegistries["r1"] + t.Logf("r1Cp registry len %d loggers len %d", len(r1Cp.childRegistries), len(r1Cp.childLoggers)) + + // We can see map is a bit different than slice, + // + // for slice even if you change the underlying array by append + // the copy of slice will still see the old range and backing array, + // + // for map change it in one place will will impact all the copy of that map + // I think the underlying representation of map is pointer to a struct + // https://github.com/golang/go/blob/master/src/runtime/map.go#L305 + // func makemap(t *maptype, hint int, h *hmap) *hmap { + // } + // + // stdlib_map_test.go:29: r1 registry len 0 loggers len 1 + // stdlib_map_test.go:33: r1 registry len 1 loggers len 2 + // stdlib_map_test.go:35: r1Cp registry len 1 loggers len 1 +} diff --git a/playground/stdlib/stdlib_slice_test.go b/playground/stdlib/stdlib_slice_test.go index 74ace79..28016cd 100644 --- a/playground/stdlib/stdlib_slice_test.go +++ b/playground/stdlib/stdlib_slice_test.go @@ -46,3 +46,19 @@ func TestSlice_ReSliceStructSlice(t *testing.T) { s = append(s, foo{"2r", mr, ss}) assert.Equal("1+", sCopy[0].m["a"]) } + +func acceptVariadicArgs(nums ...int) int { + s := 0 + for _, v := range nums { + s += v + } + return s +} + +func TestVariadicArgs(t *testing.T) { + v1 := []int{1, 2, 3} + v2 := []int{4, 5, 6} + acceptVariadicArgs(v1...) + acceptVariadicArgs(v2...) + //acceptVariadicArgs(v2..., v1...) +} diff --git a/requests/builder_test.go b/requests/builder_test.go index 491e781..96d4a35 100644 --- a/requests/builder_test.go +++ b/requests/builder_test.go @@ -1,8 +1,9 @@ package requests import ( - asst "github.com/stretchr/testify/assert" "testing" + + asst "github.com/stretchr/testify/assert" ) func TestTransportBuilder_UseSocks5(t *testing.T) { diff --git a/util/cast/cast.go b/util/cast/cast.go index f1c99e0..9710473 100644 --- a/util/cast/cast.go +++ b/util/cast/cast.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/dyweb/gommon/errors" - "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v2" ) // ToStringMap converts a map to use string key, non string key will be ignored diff --git a/util/cast/cast_test.go b/util/cast/cast_test.go index dc75332..c479497 100644 --- a/util/cast/cast_test.go +++ b/util/cast/cast_test.go @@ -1,8 +1,9 @@ package cast import ( - asst "github.com/stretchr/testify/assert" "testing" + + asst "github.com/stretchr/testify/assert" ) type userConfig struct { diff --git a/util/fsutil/pkg.go b/util/fsutil/pkg.go index b75ba2f..fc2f7fb 100644 --- a/util/fsutil/pkg.go +++ b/util/fsutil/pkg.go @@ -5,4 +5,4 @@ import ( "github.com/dyweb/gommon/util/logutil" ) -var log = logutil.NewPackageLogger() +var log, _ = logutil.NewPackageLoggerAndRegistry() diff --git a/util/genutil/pkg.go b/util/genutil/pkg.go index 83807ef..a9d79a2 100644 --- a/util/genutil/pkg.go +++ b/util/genutil/pkg.go @@ -1,9 +1,9 @@ -// Package genutil contains helper when generating files, it is used break dependency cycle between generator package +// Package genutil contains helper when generating files, +// it is used to break dependency cycle between generator package // and packages that contain generator logic like log, noodle package genutil -// DefaultHeader returns the standard go header for generated files with two trailing \n, -// the second \n is to avoid this header becomes documentation of the package +// DefaultHeader calls Header and set generator to gommon func DefaultHeader(templateSrc string) string { return "// Code generated by gommon from " + templateSrc + " DO NOT EDIT.\n\n" } diff --git a/util/httputil/pkg.go b/util/httputil/pkg.go new file mode 100644 index 0000000..8819151 --- /dev/null +++ b/util/httputil/pkg.go @@ -0,0 +1,2 @@ +// Package httputil provides helper for net/http, i.e. unix domain socket client, http request logger +package httputil diff --git a/util/httputil/unix.go b/util/httputil/unix.go new file mode 100644 index 0000000..eb8dc96 --- /dev/null +++ b/util/httputil/unix.go @@ -0,0 +1,34 @@ +package httputil + +import ( + "context" + "net" + "net/http" + "time" +) + +func NewPooledUnixTransport(sockFile string) *http.Transport { + return &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + d := net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } + return d.DialContext(ctx, "unix", sockFile) + }, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} + +func ListenAndServeUnix(srv *http.Server, addr string) error { + ln, err := net.Listen("unix", addr) + if err != nil { + return err + } + // TODO: do we need tcpKeepAliveListener like the default tcp ListenAndServe? + return srv.Serve(ln) +} diff --git a/util/httputil/unix_test.go b/util/httputil/unix_test.go new file mode 100644 index 0000000..a48cb44 --- /dev/null +++ b/util/httputil/unix_test.go @@ -0,0 +1,84 @@ +package httputil_test + +import ( + "context" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os" + "strings" + "testing" + "time" + + requir "github.com/stretchr/testify/require" + + "github.com/dyweb/gommon/util/fsutil" + dhttputil "github.com/dyweb/gommon/util/httputil" + "github.com/dyweb/gommon/util/testutil" +) + +func proxyDocker(prefix string) http.Handler { + proxy := httputil.ReverseProxy{ + Transport: dhttputil.NewPooledUnixTransport("/var/run/docker.sock"), + Director: func(r *http.Request) { + r.URL.Scheme = "http" + r.URL.Host = "api" + r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix) + r.Host = "api" + }, + } + return &proxy +} + +func TestNewPooledUnixTransport(t *testing.T) { + t.Skip("onl/y runs on node with docker") + + t.Run("docker client", func(t *testing.T) { + tr := dhttputil.NewPooledUnixTransport("/var/run/docker.sock") + c := &http.Client{Transport: tr} + t.Log(string(testutil.GetBody(t, c, "http://api/version"))) + }) + t.Run("docker proxy", func(t *testing.T) { + mux := http.NewServeMux() + //mux.Handle("/docker/proxy/", proxyDocker("/docker/proxy/")) 400 Bad Request + mux.Handle("/docker/proxy/", proxyDocker("/docker/proxy")) + srv := httptest.NewServer(mux) + c := srv.Client() + t.Log(string(testutil.GetBody(t, c, srv.URL+"/docker/proxy/version"))) + t.Log(string(testutil.GetBody(t, c, srv.URL+"/docker/proxy/info"))) + }) +} + +func TestListenAndServeUnix(t *testing.T) { + t.Skip("fail on travis") + + require := requir.New(t) + + mux := http.NewServeMux() + mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("pong")) + }) + srv := http.Server{ + Handler: mux, + } + if fsutil.FileExists("/tmp/gommon.sock") { + require.Nil(os.Remove("/tmp/gommon.sock")) + } + go func() { + // curl --unix-socket /tmp/gommon.sock http://abc/ping + err := dhttputil.ListenAndServeUnix(&srv, "/tmp/gommon.sock") + if err != nil { + // TODO: race in go tip https://travis-ci.org/dyweb/gommon/jobs/459629689 + t.Logf("error start server: %s", err) + } + // https://travis-ci.org/dyweb/gommon/jobs/431772857 + //require.Nil(err) + }() + time.Sleep(1 * time.Millisecond) + tr := dhttputil.NewPooledUnixTransport("/tmp/gommon.sock") + c := &http.Client{Transport: tr} + s := string(testutil.GetBody(t, c, "http://api/ping")) + require.Equal("pong", s) + srv.Shutdown(context.Background()) + os.Remove("/tmp/gommon.sock") +} diff --git a/util/logutil/pkg.go b/util/logutil/pkg.go index 89971b3..72f01bc 100644 --- a/util/logutil/pkg.go +++ b/util/logutil/pkg.go @@ -1,16 +1,22 @@ // Package logutil is a registry of loggers, it is required for all lib and app that use gommon/log. // You should add the registry as child of your library/application's child if you want to control gommon libraries // logging behavior -package logutil // import "github.com/dyweb/gommon/util/logutil" +package logutil import ( "github.com/dyweb/gommon/log" ) -var Registry = log.NewLibraryLogger() +const Project = "github.com/dyweb/gommon" -func NewPackageLogger() *log.Logger { - l := log.NewPackageLoggerWithSkip(1) - Registry.AddChild(l) - return l +var registry = log.NewLibraryRegistry(Project) + +func Registry() *log.Registry { + return ®istry +} + +func NewPackageLoggerAndRegistry() (*log.Logger, *log.Registry) { + logger, child := log.NewPackageLoggerAndRegistryWithSkip(Project, 1) + registry.AddRegistry(child) + return logger, child } diff --git a/util/logutil/pkg_test.go b/util/logutil/pkg_test.go new file mode 100644 index 0000000..212a405 --- /dev/null +++ b/util/logutil/pkg_test.go @@ -0,0 +1,9 @@ +package logutil + +import "testing" + +func TestNewPackageLoggerAndRegistry(t *testing.T) { + l, reg := NewPackageLoggerAndRegistry() + t.Log(l.Identity()) + t.Log(reg.Identity()) +} diff --git a/util/runtimeutil/caller.go b/util/runtimeutil/caller.go index fcd3c72..b0f3ce8 100644 --- a/util/runtimeutil/caller.go +++ b/util/runtimeutil/caller.go @@ -1,6 +1,7 @@ package runtimeutil import ( + "os" "runtime" "strings" ) @@ -29,8 +30,8 @@ func SplitPackageFunc(f string) (pkg string, function string) { // the first dot splits package (w/ struct) and function, the second dot split package and struct (if any) // we put struct (if any) and function together, so we just need to dot closest to last / for i := len(f) - 1; i >= 0; i-- { - // TODO: it might not work on windows - if f[i] == '/' { + // TODO: validate if this works on windows + if f[i] == os.PathSeparator { break } if f[i] == '.' { diff --git a/util/testutil/data.go b/util/testutil/data.go index 6c81678..481fd30 100644 --- a/util/testutil/data.go +++ b/util/testutil/data.go @@ -1,11 +1,15 @@ package testutil import ( + "bytes" "encoding/json" + "fmt" + "io" "io/ioutil" + "os" "testing" - "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v2" ) func ReadFixture(t *testing.T, path string) []byte { @@ -16,6 +20,15 @@ func ReadFixture(t *testing.T, path string) []byte { return b } +func WriteFixture(t *testing.T, path string, data []byte) { + err := ioutil.WriteFile(path, data, 0664) + if err != nil { + t.Fatalf("can't write fixture %s: %v", path, err) + } +} + +// -------------------- start of json ----------------------- + func ReadJsonTo(t *testing.T, path string, v interface{}) { b := ReadFixture(t, path) if err := json.Unmarshal(b, v); err != nil { @@ -23,6 +36,96 @@ func ReadJsonTo(t *testing.T, path string, v interface{}) { } } +func FormatJson(t *testing.T, src []byte) []byte { + var buf bytes.Buffer + if err := json.Indent(&buf, src, "", " "); err != nil { + t.Fatalf("error ident json: %s", err) + return nil + } + return buf.Bytes() +} + +// TODO: it's dump w/ Print in pretty.go ... wrote too much and forgot ... +func DumpAsJson(t *testing.T, v interface{}) { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatalf("failed to encode as json %v", v) + return + } + if _, err := os.Stdout.Write(b); err != nil { + t.Fatalf("failed to write encoded json to stdout: %s", err) + return + } +} + +func DumpAsJsonTo(t *testing.T, v interface{}, w io.Writer) { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatalf("failed to encode as json %v", v) + return + } + if _, err := w.Write(b); err != nil { + t.Fatalf("failed to write encoded json to writer: %s", err) + return + } +} + +func SaveAsJson(t *testing.T, v interface{}, file string) { + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("failed to encode as json: %s %v", err, v) + return + } + if err := ioutil.WriteFile(file, b, 0664); err != nil { + t.Fatalf("failed to save file %s: %v", file, err) + return + } +} + +func SaveAsJsonf(t *testing.T, v interface{}, format string, args ...interface{}) { + file := fmt.Sprintf(format, args...) + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("failed to encode as json: %s %v", err, v) + return + } + if err := ioutil.WriteFile(file, b, 0664); err != nil { + t.Fatalf("failed to save file %s: %v", file, err) + return + } +} + +func SaveAsPrettyJson(t *testing.T, v interface{}, file string) { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatalf("failed to encode as json: %s %v", err, v) + return + } + if err := ioutil.WriteFile(file, b, 0664); err != nil { + t.Fatalf("failed to save file %s: %v", file, err) + return + } + t.Logf("saved json to %s", file) +} + +func SaveAsPrettyJsonf(t *testing.T, v interface{}, format string, args ...interface{}) { + file := fmt.Sprintf(format, args...) + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatalf("failed to encode as json %v", v) + return + } + if err := ioutil.WriteFile(file, b, 0664); err != nil { + t.Fatalf("failed to save file %s: %v", file, err) + return + } + t.Logf("saved json to %s", file) +} + +// -------------------- end of json ----------------------- + +// -------------------- start of yaml ----------------------- + func ReadYAMLTo(t *testing.T, path string, v interface{}) { b := ReadFixture(t, path) if err := yaml.Unmarshal(b, v); err != nil { @@ -30,6 +133,7 @@ func ReadYAMLTo(t *testing.T, path string, v interface{}) { } } +// ReadYAMLToStrict uses strict mode when decoding, if unknown fields shows up in YAML but not in struct it will error func ReadYAMLToStrict(t *testing.T, path string, v interface{}) { b := ReadFixture(t, path) if err := yaml.UnmarshalStrict(b, v); err != nil { @@ -37,9 +141,31 @@ func ReadYAMLToStrict(t *testing.T, path string, v interface{}) { } } -func WriteFixture(t *testing.T, path string, data []byte) { - err := ioutil.WriteFile(path, data, 0664) +func SaveAsYAML(t *testing.T, v interface{}, file string) { + b, err := yaml.Marshal(v) if err != nil { - t.Fatalf("can't write fixture %s: %v", path, err) + t.Fatalf("failed to encode as YAML: %s %v", err, v) + return + } + if err := ioutil.WriteFile(file, b, 0664); err != nil { + t.Fatalf("failed to save file %s: %v", file, err) + return } + t.Logf("saved YAML to %s", file) } + +func SaveAsYAMLf(t *testing.T, v interface{}, format string, args ...interface{}) { + file := fmt.Sprintf(format, args...) + b, err := yaml.Marshal(v) + if err != nil { + t.Fatalf("failed to encode as YAML: %s %v", err, v) + return + } + if err := ioutil.WriteFile(file, b, 0664); err != nil { + t.Fatalf("failed to save file %s: %v", file, err) + return + } + t.Logf("saved YAML to %s", file) +} + +// -------------------- end of yaml ----------------------- diff --git a/util/testutil/http.go b/util/testutil/http.go index 9b80c2d..bb9af96 100644 --- a/util/testutil/http.go +++ b/util/testutil/http.go @@ -9,12 +9,12 @@ import ( func GetBody(t *testing.T, c *http.Client, url string) []byte { res, err := c.Get(url) if err != nil { - t.Fatalf("error GET %s", url) + t.Fatalf("error GET %s: %s", url, err) return nil } b, err := ioutil.ReadAll(res.Body) if err != nil { - t.Fatalf("error read body of %s", url) + t.Fatalf("error read body of %s: %s", url, err) } res.Body.Close() return b