Skip to content

Commit

Permalink
Merge pull request #62 from sorcero/feature/log-multiwriter
Browse files Browse the repository at this point in the history
feat: add multiwriter to add a file log sink
  • Loading branch information
srevinsaju authored Nov 6, 2023
2 parents 572d8f8 + 4edf995 commit 6cddc8e
Show file tree
Hide file tree
Showing 10 changed files with 274 additions and 54 deletions.
5 changes: 4 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
## [v2.0.0-alpha.11]
- Add `--logging.local.file` and `--logging.local.file.path` for writing logs to file.

## [v2.0.0-alpha.10]
- Fix `path.module` incorrectly being populated for togomak modules

## [v2.0.0-alpha.9]
Expand Down
39 changes: 38 additions & 1 deletion cmd/togomak/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/srevinsaju/togomak/v1/internal/ci"
"github.com/srevinsaju/togomak/v1/internal/filter"
"github.com/srevinsaju/togomak/v1/internal/global"
"github.com/srevinsaju/togomak/v1/internal/logging"
"github.com/srevinsaju/togomak/v1/internal/meta"
"github.com/srevinsaju/togomak/v1/internal/orchestra"
"github.com/srevinsaju/togomak/v1/internal/path"
Expand Down Expand Up @@ -157,6 +158,29 @@ func main() {
"Variables set this way take precedence over variables set in the pipeline file.",
Aliases: []string{"variable"},
},
&cli.BoolFlag{
Name: "logging.remote.google-cloud",
Usage: "Enable remote logging to Google Cloud",
EnvVars: []string{"TOGOMAK_LOGGING_REMOTE_GCLOUD"},
Value: false,
},
&cli.StringFlag{
Name: "logging.remote.google-cloud.project",
Usage: "Google Cloud project ID where logs are ingested",
EnvVars: []string{"TOGOMAK_LOGGING_REMOTE_GCLOUD_PROJECT", "GOOGLE_CLOUD_PROJECT"},
},
&cli.BoolFlag{
Name: "logging.local.file",
Usage: "Enable local logging to a file",
EnvVars: []string{"TOGOMAK_LOGGING_LOCAL_FILE"},
Value: false,
},
&cli.StringFlag{
Name: "logging.local.file.path",
Usage: "Path to the file where logs are written",
EnvVars: []string{"TOGOMAK_LOGGING_LOCAL_FILE_PATH"},
Value: "togomak.log",
},
}
cli.VersionFlag = &cli.BoolFlag{
Name: "version",
Expand Down Expand Up @@ -261,13 +285,26 @@ func newConfigFromCliContext(ctx *cli.Context) ci.ConductorConfig {
DryRun: ctx.Bool("dry-run"),
},
Variables: variables,

Logging: logging.Config{
Verbosity: verboseCount,
Child: ctx.Bool("child"),
IsCI: ctx.Bool("ci"),
JSON: ctx.Bool("json"),
CorrelationID: "",
Sinks: logging.ParseSinksFromCLI(ctx),
},
}
return cfg
}

func run(ctx *cli.Context) error {
cfg := newConfigFromCliContext(ctx)
logger := ci.NewLogger(cfg)
logger, err := logging.New(cfg.Logging)
if err != nil {
panic(err)
}

global.SetLogger(logger)

t := ci.NewConductor(cfg)
Expand Down
6 changes: 5 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/srevinsaju/togomak/v1
go 1.20

require (
cloud.google.com/go/logging v1.7.0
code.gitea.io/gitea v1.19.3
github.com/AlecAivazis/survey/v2 v2.3.6
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2
Expand All @@ -25,6 +26,7 @@ require (
github.com/mattn/go-isatty v0.0.17
github.com/mitchellh/go-homedir v1.1.0
github.com/moby/sys/mountinfo v0.6.2
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
github.com/sirupsen/logrus v1.9.2
github.com/spf13/afero v1.9.5
github.com/stretchr/testify v1.8.2
Expand All @@ -33,13 +35,15 @@ require (
github.com/zclconf/go-cty-yaml v1.0.3
golang.org/x/crypto v0.14.0
golang.org/x/text v0.13.0
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
)

require (
cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go/compute v1.19.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.13.0 // indirect
cloud.google.com/go/longrunning v0.4.1 // indirect
cloud.google.com/go/storage v1.28.1 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
Expand All @@ -57,10 +61,10 @@ require (
github.com/ulikunitz/xz v0.5.11 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/oauth2 v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.114.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/grpc v1.56.3 // indirect
google.golang.org/protobuf v1.30.0 // indirect
)
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,10 @@ cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8=
cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08=
cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I=
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4=
cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w=
cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE=
Expand Down Expand Up @@ -562,6 +565,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k=
github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
Expand Down
17 changes: 14 additions & 3 deletions internal/ci/conductor.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/hashicorp/hcl/v2/hclparse"
"github.com/sirupsen/logrus"
"github.com/srevinsaju/togomak/v1/internal/conductor"
"github.com/srevinsaju/togomak/v1/internal/logging"
"github.com/srevinsaju/togomak/v1/internal/meta"
"github.com/srevinsaju/togomak/v1/internal/x"
"os"
Expand Down Expand Up @@ -259,7 +260,19 @@ func NewConductor(cfg ConductorConfig, opts ...ConductorOption) *Conductor {

diagWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true)

logger := NewLogger(cfg)
process := NewProcess(cfg)
// create a new logger derived from conductor configurations
logger, err := logging.New(logging.Config{
Verbosity: cfg.Logging.Verbosity,
Child: cfg.Logging.Child,
IsCI: cfg.Logging.IsCI,
JSON: cfg.Logging.JSON,
CorrelationID: process.Id.String(),
Sinks: cfg.Logging.Sinks,
})
if err != nil {
panic(err)
}

dir := Chdir(cfg, logger)
if dir != cfg.Paths.Cwd {
Expand All @@ -269,8 +282,6 @@ func NewConductor(cfg ConductorConfig, opts ...ConductorOption) *Conductor {
cfg.Paths.Module = cfg.Paths.Cwd
}

process := NewProcess(cfg)

c := &Conductor{
Parser: &Parser{
parser: parser,
Expand Down
3 changes: 3 additions & 0 deletions internal/ci/conductor_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package ci

import (
"github.com/srevinsaju/togomak/v1/internal/behavior"
"github.com/srevinsaju/togomak/v1/internal/logging"
"github.com/srevinsaju/togomak/v1/internal/path"
"github.com/srevinsaju/togomak/v1/internal/rules"
)
Expand Down Expand Up @@ -33,4 +34,6 @@ type ConductorConfig struct {
Behavior *behavior.Behavior

Variables Variables

Logging logging.Config
}
48 changes: 0 additions & 48 deletions internal/ci/conductor_logging.go

This file was deleted.

80 changes: 80 additions & 0 deletions internal/logging/google_cloud.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package logging

import (
"context"
"github.com/acarl005/stripansi"
"github.com/sirupsen/logrus"
"github.com/srevinsaju/togomak/v1/internal/meta"
"github.com/srevinsaju/togomak/v1/internal/x"
"google.golang.org/genproto/googleapis/api/monitoredres"
"os"
)
import "cloud.google.com/go/logging"

var hostname string

func googleCloudLoggingClient(project string) (*logging.Client, error) {
loggerContext := context.Background()
hostname = x.MustReturn(os.Hostname()).(string)

// initialize the client
client, err := logging.NewClient(loggerContext, project)
return client, err
}

type GoogleCloudLoggerHook struct {
client *logging.Client
cfg Config
project string
}

func NewGoogleCloudLoggerHook(cfg Config, project string) (*GoogleCloudLoggerHook, error) {
client, err := googleCloudLoggingClient(project)
return &GoogleCloudLoggerHook{cfg: cfg, client: client, project: project}, err
}

func (h *GoogleCloudLoggerHook) Fire(entry *logrus.Entry) error {
// upload to google cloud logging
// using google cloud API
// https://cloud.google.com/logging/docs/reference/libraries#client-libraries-install-gow
client := h.client
logger := client.Logger(meta.AppName)
severityLevel := logging.Default
switch entry.Level {
case logrus.DebugLevel:
severityLevel = logging.Debug
case logrus.InfoLevel:
severityLevel = logging.Info
case logrus.WarnLevel:
severityLevel = logging.Warning
case logrus.ErrorLevel:
severityLevel = logging.Error
case logrus.FatalLevel:
severityLevel = logging.Critical
case logrus.PanicLevel:
severityLevel = logging.Alert
}
logger.Log(logging.Entry{
Payload: map[string]interface{}{
"message": stripansi.Strip(entry.Message),
"labels": entry.Data,
"app": meta.AppName,
"version": meta.AppVersion,
"host": hostname,
},
Resource: &monitoredres.MonitoredResource{Type: "global"},
Trace: "togomak",
Severity: severityLevel,
Labels: map[string]string{
"app": meta.AppName,
"version": meta.AppVersion,
"instanceName": meta.AppName,
"instanceId": h.cfg.CorrelationID,
},
})
return nil
}

func (h *GoogleCloudLoggerHook) Levels() []logrus.Level {
return []logrus.Level{logrus.InfoLevel, logrus.WarnLevel, logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel}
}
Loading

0 comments on commit 6cddc8e

Please sign in to comment.