feat(formatter): Handle error in NewJsonFormatter and improve methods

cmd/prettylogs/root.go

@@ -42,7 +42,7 @@ func NewRootCmd() *cobra.Command {
 			requestKey, _ := cmd.Flags().GetString("requestKey")
 
 			// Create a new json formatter
-			formatter := formatter.NewJsonFormatter(&formatter.FormatterOptions{
+			formatter, err := formatter.NewJsonFormatter(&formatter.FormatterOptions{
 				FormatTemplateFile: formatTemplateFile,
 				OutputFormat:       outputFormat,
 				TimestampFormat:    timestampFormat,
@@ -56,9 +56,17 @@ func NewRootCmd() *cobra.Command {
 				HostnameKey:        hostnameKey,
 				RequestKey:         requestKey,
 			})
+			if err != nil {
+				cmd.PrintErr(err)
+				os.Exit(1)
+			}
 
 			// Process the log
-			formatter.Process(os.Stdin)
+			err = formatter.Process(os.Stdin)
+			if err != nil {
+				cmd.PrintErr(err)
+				os.Exit(1)
+			}
 		},
 	}
 

go.mod

@@ -4,7 +4,14 @@ go 1.23.3
 
 require github.com/spf13/cobra v1.8.1
 
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
 require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.10.0
 )

go.sum

@@ -1,10 +1,17 @@
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/formatter/fixtures/sample.tpl

@@ -0,0 +1,25 @@
+{{- define "levelDisplay" -}}
+    {{- if . -}}
+        {{- . | printf "%-5s" -}}
+    {{- end -}}
+{{- end -}}
+
+{{- define "metadata" -}}
+    {{- if . -}}
+        {{- if or .Name .Context -}}[{{- if .Name }}{{ .Name }}{{- end }}
+            {{- if and .Name .Context }}/{{- end }}
+            {{- if .Context }}{{ .Context }}{{- end }}]{{- end -}}
+    {{- end -}}
+{{- end -}}
+
+{{- with . }}
+{{ .Time }} {{ template "levelDisplay" .Level }}{{- if .Pid }} ({{ .Pid }}) {{- end -}}
+{{- if or .Name .Context }} --- {{ template "metadata" . }} --- {{- end -}}
+{{- if .Msg }} {{ .Msg }}{{- end -}}
+{{- if .Req }}
+  Request: {{ .Req }}
+{{- end -}}
+{{- if .Error }}
+  Error: {{ .Error }}
+{{- end -}}
+{{ end }}

internal/formatter/fixtures/with_error.log

@@ -0,0 +1,28 @@
+{"level":"INFO","time":1738590002314,"pid":31235,"hostname":"192.168.1.119","context":"NestFactory","name":"SigmaApi","msg":"Starting Nest application..."}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"PrometheusModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"ConfigHostModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"AppModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"LoggerModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"TerminusModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"CommonModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"ConfigModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"HealthModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"SigmaModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"UnembedModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"InstanceLoader","name":"SigmaApi","msg":"WorkbooksModule dependencies initialized"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RoutesResolver","name":"SigmaApi","msg":"AppController {/sigma}:"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RoutesResolver","name":"SigmaApi","msg":"PrometheusController {/sigma/metrics}:"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/metrics, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RoutesResolver","name":"SigmaApi","msg":"HealthController {/sigma/health}:"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/health/live, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/health/ready, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RoutesResolver","name":"SigmaApi","msg":"UnembedController {/sigma/unembed}:"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/unembed/:id, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RoutesResolver","name":"SigmaApi","msg":"WorkbooksController {/sigma/workbooks}:"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/workbooks, POST} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/workbooks/:id/embed, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"RouterExplorer","name":"SigmaApi","msg":"Mapped {/sigma/workbooks, GET} route"}
+{"level":"INFO","time":1738590002315,"pid":31235,"hostname":"192.168.1.119","context":"NestApplication","name":"SigmaApi","msg":"Nest application successfully started"}
+{"level":"ERROR","time":1738590008668,"pid":31235,"hostname":"192.168.1.119","req":{"id":1,"method":"GET","url":"/sigma/workbooks?tenant_id=7&user_id=24e7279c-e7c0-4406-b87a-b7a06150404689","query":{"tenant_id":"7","user_id":"24e7279c-e7c0-4406-b87a-b7a06150404689"},"params":{"0":"workbooks"},"headers":{"host":"localhost:8080","connection":"keep-alive","cache-control":"max-age=0","sec-ch-ua":"\"Not A(Brand\";v=\"8\", \"Chromium\";v=\"132\", \"Google Chrome\";v=\"132\"","sec-ch-ua-mobile":"?0","sec-ch-ua-platform":"\"macOS\"","dnt":"1","upgrade-insecure-requests":"1","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7","sec-fetch-site":"none","sec-fetch-mode":"navigate","sec-fetch-user":"?1","sec-fetch-dest":"document","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-GB,en-US;q=0.9,en;q=0.8","cookie":"[Redacted]","if-none-match":"W/\"2-l9Fw4VUO7kr8CvBlt4zaMCqXZ0w\""},"remoteAddress":"::1","remotePort":51543},"context":"SigmaService","error":{"type":"SigmaComputingApiError","status":401,"message":"client secret provided is invalid","stack":"AxiosError: Request failed with status code 401\n    at settle (/Users/patrickfoh/projects/firstshift.ai/sigma-api/node_modules/axios/dist/node/axios.cjs:2026:12)\n    at IncomingMessage.handleStreamEnd (/Users/patrickfoh/projects/firstshift.ai/sigma-api/node_modules/axios/dist/node/axios.cjs:3142:11)\n    at IncomingMessage.emit (node:events:530:35)\n    at endReadableNT (node:internal/streams/readable:1698:12)\n    at process.processTicksAndRejections (node:internal/process/task_queues:90:21)\n    at Axios.request (/Users/patrickfoh/projects/firstshift.ai/sigma-api/node_modules/axios/dist/node/axios.cjs:4252:41)\n    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)\n    at async SigmaService.authenticate (/Users/patrickfoh/projects/firstshift.ai/sigma-api/dist/sigma/sigma.service.js:99:30)\n    at async SigmaService.handleAuthentication (/Users/patrickfoh/projects/firstshift.ai/sigma-api/dist/sigma/sigma.service.js:148:13)\n    at async SigmaService.getMemberId (/Users/patrickfoh/projects/firstshift.ai/sigma-api/dist/sigma/sigma.service.js:239:9)\n    at async WorkbooksService.getWorkbookList (/Users/patrickfoh/projects/firstshift.ai/sigma-api/dist/workbooks/workbooks.service.js:89:26)\n    at async /Users/patrickfoh/projects/firstshift.ai/sigma-api/node_modules/@nestjs/core/router/router-execution-context.js:46:28\n    at async /Users/patrickfoh/projects/firstshift.ai/sigma-api/node_modules/@nestjs/core/router/router-proxy.js:9:17"},"name":"SigmaApi","msg":"Error authenticating with provider API"}
+{"level":"INFO","time":1738590008670,"pid":31235,"hostname":"192.168.1.119","req":{"id":1,"method":"GET","url":"/sigma/workbooks?tenant_id=7&user_id=24e7279c-e7c0-4406-b87a-b7a06150404689","query":{"tenant_id":"7","user_id":"24e7279c-e7c0-4406-b87a-b7a06150404689"},"params":{"0":"workbooks"},"headers":{"host":"localhost:8080","connection":"keep-alive","cache-control":"max-age=0","sec-ch-ua":"\"Not A(Brand\";v=\"8\", \"Chromium\";v=\"132\", \"Google Chrome\";v=\"132\"","sec-ch-ua-mobile":"?0","sec-ch-ua-platform":"\"macOS\"","dnt":"1","upgrade-insecure-requests":"1","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7","sec-fetch-site":"none","sec-fetch-mode":"navigate","sec-fetch-user":"?1","sec-fetch-dest":"document","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-GB,en-US;q=0.9,en;q=0.8","cookie":"[Redacted]","if-none-match":"W/\"2-l9Fw4VUO7kr8CvBlt4zaMCqXZ0w\""},"remoteAddress":"::1","remotePort":51543},"res":{"statusCode":401,"headers":{"x-powered-by":"Express","access-control-allow-origin":"*","content-type":"application/json; charset=utf-8","content-length":"98","etag":"W/\"62-foAQph7G/AotuyUqDn0Ly6Su4hQ\""}},"responseTime":436,"name":"SigmaApi","msg":"request completed"}

internal/formatter/formatter_interface.go

@@ -25,7 +25,7 @@ type LogLine struct {
 	Name     string
 	Context  string
 	Msg      string
-	Error    string
+	Error    interface{}
 	Req      string
 	Res      string
 	Hostname string

internal/formatter/json_formatter.go

@@ -9,52 +9,80 @@ import (
 )
 
 type JsonFormatter struct {
-	Options *FormatterOptions
+	Options  *FormatterOptions
+	Template *template.Template
 }
 
-func NewJsonFormatter(options *FormatterOptions) Formatter {
-	// Set default error keys if not provided
+// Create a new JsonFormatter with the provided options.
+func NewJsonFormatter(options *FormatterOptions) (Formatter, error) {
+	// Set default error keys if not provided.
 	if options.ErrorObjectKeys == nil {
 		options.ErrorObjectKeys = []string{"err", "error"}
 	}
-	// Set default timestamp format if not provided
+	// Set default timestamp format if not provided.
 	if options.TimestampFormat == "" {
 		options.TimestampFormat = "2006-01-02 15:04:05.000"
 	}
-	return &JsonFormatter{
+
+	jf := &JsonFormatter{
 		Options: options,
 	}
-}
 
-func (f *JsonFormatter) PrintLogLineTemplate(line map[string]interface{}) error {
-	tmpl, err := template.New(f.Options.FormatTemplateFile).ParseFiles(f.Options.FormatTemplateFile)
-	if err != nil {
-		return fmt.Errorf("error parsing template file: %v", err)
+	// Parse the template file once if provided.
+	if options.FormatTemplateFile != "" {
+		// Check if the template file exists
+		if _, err := os.Stat(options.FormatTemplateFile); err != nil {
+			return nil, fmt.Errorf("template file does not exist: %s", options.FormatTemplateFile)
+		}
+
+		tmpl, err := template.New(options.FormatTemplateFile).ParseFiles(options.FormatTemplateFile)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing template file: %w", err)
+		}
+
+		jf.Template = tmpl
 	}
 
+	return jf, nil
+}
+
+// Print the log line using the provided template.
+func (f *JsonFormatter) PrintLogLineTemplate(line map[string]interface{}) error {
 	input := LogLineMapToStruct(line, f.Options)
 
-	err = tmpl.Execute(os.Stdout, input)
-	if err != nil {
-		panic(err)
+	if f.Template == nil {
+		return fmt.Errorf("no template available")
 	}
 
+	if err := f.Template.Execute(os.Stdout, input); err != nil {
+		return fmt.Errorf("error executing template: %w", err)
+	}
 	return nil
 }
 
+// Print the log line to stdout using the provided template or raw JSON.
 func (f *JsonFormatter) PrintLogLine(line map[string]interface{}) {
-	// Check if the template file exists
-	if _, err := os.Stat(f.Options.FormatTemplateFile); err == nil {
-		// Use the template file
-		if err := f.PrintLogLineTemplate(line); err != nil {
-			fmt.Fprintf(os.Stderr, "Error printing log line: %v\n", err)
+	var err error
+	if f.Template != nil {
+		err = f.PrintLogLineTemplate(line)
+	} else {
+		// Fallback: print the raw JSON log line.
+		var out []byte
+		out, err = json.Marshal(line)
+		if err == nil {
+			fmt.Println(string(out))
 		}
 	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error printing log line: %v\n", err)
+	}
 }
 
+// Process the input file and print the formatted log lines.
 func (f *JsonFormatter) Process(input *os.File) error {
 	scanner := bufio.NewScanner(input)
-	scanner.Buffer(make([]byte, 1024*1024), 1024*1024) // Increase buffer size for large lines
+	// Increase buffer size for large lines.
+	scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
 
 	for scanner.Scan() {
 		var entry map[string]interface{}
@@ -65,7 +93,7 @@ func (f *JsonFormatter) Process(input *os.File) error {
 
 		logLine := make(map[string]interface{})
 
-		// Extract standard fields
+		// Extract standard fields.
 		logLine[f.Options.TimeKey] = ExtractValue(entry, f.Options.TimeKey)
 		logLine[f.Options.LevelKey] = ExtractValue(entry, f.Options.LevelKey)
 		logLine[f.Options.PidKey] = ExtractValue(entry, f.Options.PidKey)
@@ -75,11 +103,13 @@ func (f *JsonFormatter) Process(input *os.File) error {
 		logLine[f.Options.MsgKey] = ExtractValue(entry, f.Options.MsgKey)
 		logLine[f.Options.RequestKey] = ExtractValue(entry, f.Options.RequestKey)
 
-		// Handle error if present
-		if HasKeys(&entry, f.Options.ErrorObjectKeys) {
+		// Handle error if present.
+		if HasAnyKey(entry, f.Options.ErrorObjectKeys) {
+			// Find the first error key and extract the error message.
 			for _, key := range f.Options.ErrorObjectKeys {
-				if err, ok := entry[key]; ok {
-					logLine["error"] = FormatError(err)
+				// Check if the key exists in the log entry.
+				if HasKey(entry, key) {
+					logLine[key] = ExtractError(entry[key])
 					break
 				}
 			}
@@ -89,7 +119,7 @@ func (f *JsonFormatter) Process(input *os.File) error {
 	}
 
 	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("error reading input: %v", err)
+		return fmt.Errorf("error reading input: %w", err)
 	}
 
 	return nil

internal/formatter/json_formatter_test.go

@@ -0,0 +1,59 @@
+package formatter
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func OpenTestLogFile(path string) *os.File {
+	file, err := os.Open(path)
+	if err != nil {
+		panic(err)
+	}
+	return file
+}
+
+func CaptureOutput(f func()) string {
+	// Save the original os.Stdout.
+	old := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	// Run the function whose output we wish to capture.
+	f()
+
+	// Close writer and restore os.Stdout.
+	w.Close()
+	os.Stdout = old
+
+	var buf bytes.Buffer
+	_, _ = io.Copy(&buf, r)
+	r.Close()
+	return buf.String()
+}
+
+func TestJsonFormatter(t *testing.T) {
+	t.Run("throws an error if template file is not found", func(t *testing.T) {
+		formatter, err := NewJsonFormatter(&FormatterOptions{
+			FormatTemplateFile: "fixtures/some_file.tpl",
+			ErrorObjectKeys:    []string{"err", "error"},
+			TimeKey:            "time",
+			LevelKey:           "level",
+			PidKey:             "pid",
+			NameKey:            "name",
+			ContextKey:         "context",
+			MsgKey:             "msg",
+			TimestampFormat:    "2006-01-02 15:04:05.000",
+			RequestKey:         "req",
+			ResponseKey:        "res",
+		})
+
+		assert.NotNil(t, err)
+		assert.Error(t, err, "template file does not exist: fixtures/some_file.tpl")
+		assert.Nil(t, formatter)
+	})
+}