From 4edf995f81ec258d645842c6881ba1e2082deae5 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Mon, 6 Nov 2023 19:43:38 +0300 Subject: [PATCH] feat: add multiwriter to add a file log sink --- CHANGELOG.md | 5 +- cmd/togomak/main.go | 39 +++++++++- go.mod | 6 +- go.sum | 5 ++ internal/ci/conductor.go | 17 ++++- internal/ci/conductor_config.go | 3 + internal/ci/conductor_logging.go | 48 ------------- internal/logging/google_cloud.go | 80 +++++++++++++++++++++ internal/logging/logging.go | 118 +++++++++++++++++++++++++++++++ internal/x/error.go | 7 ++ 10 files changed, 274 insertions(+), 54 deletions(-) delete mode 100644 internal/ci/conductor_logging.go create mode 100644 internal/logging/google_cloud.go create mode 100644 internal/logging/logging.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 308894c..9216436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [v2.0.0-alpha.11] +- Add `--logging.local.file` and `--logging.local.file.path` for writing logs to file. + +## [v2.0.0-alpha.10] - Fix `path.module` incorrectly being populated for togomak modules ## [v2.0.0-alpha.9] diff --git a/cmd/togomak/main.go b/cmd/togomak/main.go index cf9d25c..c5c6afd 100644 --- a/cmd/togomak/main.go +++ b/cmd/togomak/main.go @@ -9,6 +9,7 @@ import ( "github.com/srevinsaju/togomak/v1/internal/ci" "github.com/srevinsaju/togomak/v1/internal/filter" "github.com/srevinsaju/togomak/v1/internal/global" + "github.com/srevinsaju/togomak/v1/internal/logging" "github.com/srevinsaju/togomak/v1/internal/meta" "github.com/srevinsaju/togomak/v1/internal/orchestra" "github.com/srevinsaju/togomak/v1/internal/path" @@ -157,6 +158,29 @@ func main() { "Variables set this way take precedence over variables set in the pipeline file.", Aliases: []string{"variable"}, }, + &cli.BoolFlag{ + Name: "logging.remote.google-cloud", + Usage: "Enable remote logging to Google Cloud", + EnvVars: []string{"TOGOMAK_LOGGING_REMOTE_GCLOUD"}, + Value: false, + }, + &cli.StringFlag{ + Name: "logging.remote.google-cloud.project", + Usage: "Google Cloud project ID where logs are ingested", + EnvVars: []string{"TOGOMAK_LOGGING_REMOTE_GCLOUD_PROJECT", "GOOGLE_CLOUD_PROJECT"}, + }, + &cli.BoolFlag{ + Name: "logging.local.file", + Usage: "Enable local logging to a file", + EnvVars: []string{"TOGOMAK_LOGGING_LOCAL_FILE"}, + Value: false, + }, + &cli.StringFlag{ + Name: "logging.local.file.path", + Usage: "Path to the file where logs are written", + EnvVars: []string{"TOGOMAK_LOGGING_LOCAL_FILE_PATH"}, + Value: "togomak.log", + }, } cli.VersionFlag = &cli.BoolFlag{ Name: "version", @@ -261,13 +285,26 @@ func newConfigFromCliContext(ctx *cli.Context) ci.ConductorConfig { DryRun: ctx.Bool("dry-run"), }, Variables: variables, + + Logging: logging.Config{ + Verbosity: verboseCount, + Child: ctx.Bool("child"), + IsCI: ctx.Bool("ci"), + JSON: ctx.Bool("json"), + CorrelationID: "", + Sinks: logging.ParseSinksFromCLI(ctx), + }, } return cfg } func run(ctx *cli.Context) error { cfg := newConfigFromCliContext(ctx) - logger := ci.NewLogger(cfg) + logger, err := logging.New(cfg.Logging) + if err != nil { + panic(err) + } + global.SetLogger(logger) t := ci.NewConductor(cfg) diff --git a/go.mod b/go.mod index 2d5e55f..92bf126 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/srevinsaju/togomak/v1 go 1.20 require ( + cloud.google.com/go/logging v1.7.0 code.gitea.io/gitea v1.19.3 github.com/AlecAivazis/survey/v2 v2.3.6 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 @@ -25,6 +26,7 @@ require ( github.com/mattn/go-isatty v0.0.17 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/sys/mountinfo v0.6.2 + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/sirupsen/logrus v1.9.2 github.com/spf13/afero v1.9.5 github.com/stretchr/testify v1.8.2 @@ -33,6 +35,7 @@ require ( github.com/zclconf/go-cty-yaml v1.0.3 golang.org/x/crypto v0.14.0 golang.org/x/text v0.13.0 + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 ) require ( @@ -40,6 +43,7 @@ require ( cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/longrunning v0.4.1 // indirect cloud.google.com/go/storage v1.28.1 // indirect dario.cat/mergo v1.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect @@ -57,10 +61,10 @@ require ( github.com/ulikunitz/xz v0.5.11 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.56.3 // indirect google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index e8c7d23..5ef146d 100644 --- a/go.sum +++ b/go.sum @@ -117,7 +117,10 @@ cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= @@ -562,6 +565,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= diff --git a/internal/ci/conductor.go b/internal/ci/conductor.go index c600a5b..6aefbc7 100644 --- a/internal/ci/conductor.go +++ b/internal/ci/conductor.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/hcl/v2/hclparse" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/internal/conductor" + "github.com/srevinsaju/togomak/v1/internal/logging" "github.com/srevinsaju/togomak/v1/internal/meta" "github.com/srevinsaju/togomak/v1/internal/x" "os" @@ -259,7 +260,19 @@ func NewConductor(cfg ConductorConfig, opts ...ConductorOption) *Conductor { diagWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true) - logger := NewLogger(cfg) + process := NewProcess(cfg) + // create a new logger derived from conductor configurations + logger, err := logging.New(logging.Config{ + Verbosity: cfg.Logging.Verbosity, + Child: cfg.Logging.Child, + IsCI: cfg.Logging.IsCI, + JSON: cfg.Logging.JSON, + CorrelationID: process.Id.String(), + Sinks: cfg.Logging.Sinks, + }) + if err != nil { + panic(err) + } dir := Chdir(cfg, logger) if dir != cfg.Paths.Cwd { @@ -269,8 +282,6 @@ func NewConductor(cfg ConductorConfig, opts ...ConductorOption) *Conductor { cfg.Paths.Module = cfg.Paths.Cwd } - process := NewProcess(cfg) - c := &Conductor{ Parser: &Parser{ parser: parser, diff --git a/internal/ci/conductor_config.go b/internal/ci/conductor_config.go index 071915f..2783a61 100644 --- a/internal/ci/conductor_config.go +++ b/internal/ci/conductor_config.go @@ -2,6 +2,7 @@ package ci import ( "github.com/srevinsaju/togomak/v1/internal/behavior" + "github.com/srevinsaju/togomak/v1/internal/logging" "github.com/srevinsaju/togomak/v1/internal/path" "github.com/srevinsaju/togomak/v1/internal/rules" ) @@ -33,4 +34,6 @@ type ConductorConfig struct { Behavior *behavior.Behavior Variables Variables + + Logging logging.Config } diff --git a/internal/ci/conductor_logging.go b/internal/ci/conductor_logging.go deleted file mode 100644 index 8ad86ba..0000000 --- a/internal/ci/conductor_logging.go +++ /dev/null @@ -1,48 +0,0 @@ -package ci - -import ( - "github.com/sirupsen/logrus" - "os" -) - -func NewLogger(cfg ConductorConfig) *logrus.Logger { - logger := logrus.New() - logger.SetOutput(os.Stdout) - logger.SetFormatter(&logrus.TextFormatter{ - FullTimestamp: false, - DisableTimestamp: cfg.Behavior.Child.Enabled, - }) - switch cfg.Interface.Verbosity { - case -1: - case 0: - logger.SetLevel(logrus.InfoLevel) - break - case 1: - logger.SetLevel(logrus.DebugLevel) - break - default: - logger.SetLevel(logrus.TraceLevel) - break - } - if cfg.Behavior.Ci { - logger.SetFormatter(&logrus.TextFormatter{ - DisableColors: false, - EnvironmentOverrideColors: false, - ForceColors: true, - ForceQuote: false, - }) - } - if cfg.Behavior.Child.Enabled { - logger.SetFormatter(&logrus.TextFormatter{ - DisableTimestamp: true, - DisableColors: false, - EnvironmentOverrideColors: false, - ForceColors: true, - ForceQuote: false, - }) - } - if cfg.Interface.JSONLogging { - logger.SetFormatter(&logrus.JSONFormatter{}) - } - return logger -} diff --git a/internal/logging/google_cloud.go b/internal/logging/google_cloud.go new file mode 100644 index 0000000..38b4391 --- /dev/null +++ b/internal/logging/google_cloud.go @@ -0,0 +1,80 @@ +package logging + +import ( + "context" + "github.com/acarl005/stripansi" + "github.com/sirupsen/logrus" + "github.com/srevinsaju/togomak/v1/internal/meta" + "github.com/srevinsaju/togomak/v1/internal/x" + "google.golang.org/genproto/googleapis/api/monitoredres" + "os" +) +import "cloud.google.com/go/logging" + +var hostname string + +func googleCloudLoggingClient(project string) (*logging.Client, error) { + loggerContext := context.Background() + hostname = x.MustReturn(os.Hostname()).(string) + + // initialize the client + client, err := logging.NewClient(loggerContext, project) + return client, err +} + +type GoogleCloudLoggerHook struct { + client *logging.Client + cfg Config + project string +} + +func NewGoogleCloudLoggerHook(cfg Config, project string) (*GoogleCloudLoggerHook, error) { + client, err := googleCloudLoggingClient(project) + return &GoogleCloudLoggerHook{cfg: cfg, client: client, project: project}, err +} + +func (h *GoogleCloudLoggerHook) Fire(entry *logrus.Entry) error { + // upload to google cloud logging + // using google cloud API + // https://cloud.google.com/logging/docs/reference/libraries#client-libraries-install-gow + client := h.client + logger := client.Logger(meta.AppName) + severityLevel := logging.Default + switch entry.Level { + case logrus.DebugLevel: + severityLevel = logging.Debug + case logrus.InfoLevel: + severityLevel = logging.Info + case logrus.WarnLevel: + severityLevel = logging.Warning + case logrus.ErrorLevel: + severityLevel = logging.Error + case logrus.FatalLevel: + severityLevel = logging.Critical + case logrus.PanicLevel: + severityLevel = logging.Alert + } + logger.Log(logging.Entry{ + Payload: map[string]interface{}{ + "message": stripansi.Strip(entry.Message), + "labels": entry.Data, + "app": meta.AppName, + "version": meta.AppVersion, + "host": hostname, + }, + Resource: &monitoredres.MonitoredResource{Type: "global"}, + Trace: "togomak", + Severity: severityLevel, + Labels: map[string]string{ + "app": meta.AppName, + "version": meta.AppVersion, + "instanceName": meta.AppName, + "instanceId": h.cfg.CorrelationID, + }, + }) + return nil +} + +func (h *GoogleCloudLoggerHook) Levels() []logrus.Level { + return []logrus.Level{logrus.InfoLevel, logrus.WarnLevel, logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel} +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 0000000..8b02975 --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,118 @@ +package logging + +import ( + "errors" + "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" + "os" +) + +type Sink struct { + Name string + Level logrus.Level + + Options map[string]string +} + +type Config struct { + Verbosity int + Child bool + IsCI bool + JSON bool + CorrelationID string + + Sinks []Sink +} + +func ParseSinksFromCLI(ctx *cli.Context) []Sink { + var sinks []Sink + file := ctx.Bool("logging.local.file") + if file { + sinks = append(sinks, Sink{ + Name: "file", + Level: logrus.DebugLevel, + Options: map[string]string{ + "path": ctx.String("logging.local.file.path"), + }, + }) + } + gcloud := ctx.Bool("logging.remote.google-cloud") + if gcloud { + sinks = append(sinks, Sink{ + Name: "google-cloud", + Level: logrus.DebugLevel, + Options: map[string]string{ + "project": ctx.String("logging.remote.google-cloud.project"), + }, + }) + } + return sinks +} + +func New(cfg Config) (*logrus.Logger, error) { + logger := logrus.New() + logger.SetOutput(os.Stdout) + logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: false, + DisableTimestamp: cfg.Child, + }) + switch cfg.Verbosity { + case -1: + case 0: + logger.SetLevel(logrus.InfoLevel) + break + case 1: + logger.SetLevel(logrus.DebugLevel) + break + default: + logger.SetLevel(logrus.TraceLevel) + break + } + if cfg.IsCI { + logger.SetFormatter(&logrus.TextFormatter{ + DisableColors: false, + EnvironmentOverrideColors: false, + ForceColors: true, + ForceQuote: false, + }) + } + if cfg.Child { + logger.SetFormatter(&logrus.TextFormatter{ + DisableTimestamp: true, + DisableColors: false, + EnvironmentOverrideColors: false, + ForceColors: true, + ForceQuote: false, + }) + } + if cfg.JSON { + logger.SetFormatter(&logrus.JSONFormatter{}) + } + + for _, sink := range cfg.Sinks { + switch sink.Name { + case "file": + path, ok := sink.Options["path"] + if !ok { + path = "togomak.log" + } + hook := lfshook.NewHook(path, &logrus.JSONFormatter{}) + logger.AddHook(hook) + case "google-cloud": + project, ok := sink.Options["project"] + if !ok { + return nil, errors.New("google-cloud sink requires project option") + } + hook, err := NewGoogleCloudLoggerHook(cfg, project) + if err != nil { + return nil, err + } + logger.AddHook(hook) + default: + return nil, errors.New("unknown sink: " + sink.Name) + } + } + + return logger, nil +} diff --git a/internal/x/error.go b/internal/x/error.go index e425386..4d40750 100644 --- a/internal/x/error.go +++ b/internal/x/error.go @@ -5,3 +5,10 @@ func Must(err error) { panic(err) } } + +func MustReturn(v any, err error) any { + if err != nil { + panic(err) + } + return v +}