From c1305ee304f69bc5ee79324852352d1158fde22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Thu, 19 Jan 2023 11:06:43 +0100 Subject: [PATCH] feat(update-collection-v3): add migration for log format --- src/go/cmd/update-collection-v3/main.go | 5 + .../migrations/logformat/migrate.go | 77 ++++++++++++ .../migrations/logformat/migrate_test.go | 113 ++++++++++++++++++ .../migrations/logformat/values.go | 40 +++++++ .../testdata/simple.input.yaml | 2 + .../testdata/simple.output.yaml | 3 + 6 files changed, 240 insertions(+) create mode 100644 src/go/cmd/update-collection-v3/migrations/logformat/migrate.go create mode 100644 src/go/cmd/update-collection-v3/migrations/logformat/migrate_test.go create mode 100644 src/go/cmd/update-collection-v3/migrations/logformat/values.go diff --git a/src/go/cmd/update-collection-v3/main.go b/src/go/cmd/update-collection-v3/main.go index eb854398..d942e5a6 100644 --- a/src/go/cmd/update-collection-v3/main.go +++ b/src/go/cmd/update-collection-v3/main.go @@ -16,6 +16,7 @@ import ( fluentdlogsconfigs "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/fluentd-logs-configs" kubeprometheusstackrepository "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/kube-prometheus-stack-repository" kubestatemetricscollectors "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/kube-state-metrics-collectors" + "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/logformat" "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/logsmetadataconfig" metricsmetadataconfig "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/metrics-metadata-config" metricsserverupgrade "github.com/SumoLogic/sumologic-kubernetes-collection/tools/cmd/update-collection-v3/migrations/metrics-server-upgrade" @@ -141,6 +142,10 @@ var migrations = []Migration{ directory: "fluentd-autoscaling", action: fluentdautoscaling.Migrate, }, + { + directory: "logformat", + action: logformat.Migrate, + }, } func migrateYaml(input string) (string, error) { diff --git a/src/go/cmd/update-collection-v3/migrations/logformat/migrate.go b/src/go/cmd/update-collection-v3/migrations/logformat/migrate.go new file mode 100644 index 00000000..a11940a3 --- /dev/null +++ b/src/go/cmd/update-collection-v3/migrations/logformat/migrate.go @@ -0,0 +1,77 @@ +package logformat + +import ( + "bytes" + "fmt" + + "gopkg.in/yaml.v3" +) + +func Migrate(inputYaml string) (outputYaml string, err error) { + inputValues, err := parseValues(inputYaml) + if err != nil { + return "", fmt.Errorf("error parsing input yaml: %v", err) + } + + outputValues := migrate(&inputValues) + if err != nil { + return "", fmt.Errorf("error migrating: %v", err) + } + + buffer := bytes.Buffer{} + encoder := yaml.NewEncoder(&buffer) + encoder.SetIndent(2) + err = encoder.Encode(outputValues) + return buffer.String(), err +} + +func parseValues(inputYaml string) (InputValues, error) { + var inputValues InputValues + err := yaml.Unmarshal([]byte(inputYaml), &inputValues) + return inputValues, err +} + +func migrate(inputValues *InputValues) OutputValues { + outputValues := *inputValues + + if outputValues.Fluentd == nil || + outputValues.Fluentd.Logs == nil || + outputValues.Fluentd.Logs.Output == nil || + outputValues.Fluentd.Logs.Output.LogFormat == nil { + // not set for Fluentd, leave things as is + return outputValues + } + + logFormat := *outputValues.Fluentd.Logs.Output.LogFormat + + if outputValues.Sumologic == nil { + outputValues.Sumologic = &Sumologic{} + } + + if outputValues.Sumologic.Logs == nil { + outputValues.Sumologic.Logs = &SumologicLogs{} + } + + if outputValues.Sumologic.Logs.Container == nil { + outputValues.Sumologic.Logs.Container = &SumologicLogsContainer{} + } + + if outputValues.Sumologic.Logs.Container.Format == nil { + outputValues.Sumologic.Logs.Container.Format = new(string) + *outputValues.Sumologic.Logs.Container.Format = logFormat + } + + // remove log format for Fluentd and clean up empty structs + outputValues.Fluentd.Logs.Output.LogFormat = nil + if len(outputValues.Fluentd.Logs.Output.Rest) == 0 { + outputValues.Fluentd.Logs.Output = nil + } + if len(outputValues.Fluentd.Logs.Rest) == 0 { + outputValues.Fluentd.Logs = nil + } + if len(outputValues.Fluentd.Rest) == 0 { + outputValues.Fluentd = nil + } + + return outputValues +} diff --git a/src/go/cmd/update-collection-v3/migrations/logformat/migrate_test.go b/src/go/cmd/update-collection-v3/migrations/logformat/migrate_test.go new file mode 100644 index 00000000..a74e253c --- /dev/null +++ b/src/go/cmd/update-collection-v3/migrations/logformat/migrate_test.go @@ -0,0 +1,113 @@ +package logformat + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type TestCase struct { + inputYaml string + outputYaml string + err error + description string +} + +func runYamlTest(t *testing.T, testCase TestCase) { + actualOutput, err := Migrate(testCase.inputYaml) + if testCase.err == nil { + require.NoError(t, err, testCase.description) + } else { + require.Equal(t, err, testCase.err, testCase.description) + } + require.Equal(t, strings.Trim(testCase.outputYaml, "\n "), strings.Trim(actualOutput, "\n "), testCase.description) +} + +func Test_EventsEnabledProvider(t *testing.T) { + testCases := []TestCase{ + { + inputYaml: `{}`, + outputYaml: `{}`, + description: "No config, no change", + }, + { + inputYaml: ` +fluentd: + key: value +`, + outputYaml: ` +fluentd: + key: value +`, + description: "No config, no change", + }, + { + inputYaml: ` +fluentd: + logs: + key: value +`, + outputYaml: ` +fluentd: + logs: + key: value +`, + description: "No config, no change", + }, + { + inputYaml: ` +fluentd: + logs: + output: + key: value +`, + outputYaml: ` +fluentd: + logs: + output: + key: value +`, + description: "No config, no change", + }, + { + inputYaml: ` +fluentd: + logs: + output: + logFormat: text +`, + outputYaml: ` +sumologic: + logs: + container: + format: text +`, + description: "Moved", + }, + { + inputYaml: ` +fluentd: + logs: + output: + logFormat: text +sumologic: + logs: + container: + format: json_merge +`, + outputYaml: ` +sumologic: + logs: + container: + format: json_merge +`, + description: "Don't overwrite sumologic format if already set", + }, + } + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + runYamlTest(t, testCase) + }) + } +} diff --git a/src/go/cmd/update-collection-v3/migrations/logformat/values.go b/src/go/cmd/update-collection-v3/migrations/logformat/values.go new file mode 100644 index 00000000..d51cb1c7 --- /dev/null +++ b/src/go/cmd/update-collection-v3/migrations/logformat/values.go @@ -0,0 +1,40 @@ +package logformat + +type Values struct { + Sumologic *Sumologic `yaml:"sumologic,omitempty"` + Fluentd *Fluentd `yaml:"fluentd,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type Sumologic struct { + Logs *SumologicLogs `yaml:"logs,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type SumologicLogs struct { + Container *SumologicLogsContainer `yaml:"container,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type SumologicLogsContainer struct { + Format *string `yaml:"format,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type Fluentd struct { + Logs *FluentdLogs `yaml:"logs,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type FluentdLogs struct { + Output *FluentdLogsOutput `yaml:"output,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type FluentdLogsOutput struct { + LogFormat *string `yaml:"logFormat,omitempty"` + Rest map[string]interface{} `yaml:",inline"` +} + +type InputValues = Values +type OutputValues = Values diff --git a/src/go/cmd/update-collection-v3/testdata/simple.input.yaml b/src/go/cmd/update-collection-v3/testdata/simple.input.yaml index 0ab94785..e29c0d99 100644 --- a/src/go/cmd/update-collection-v3/testdata/simple.input.yaml +++ b/src/go/cmd/update-collection-v3/testdata/simple.input.yaml @@ -7,6 +7,8 @@ fluentd: logs: autoscaling: enabled: true + output: + logFormat: text metrics: autoscaling: enabled: true diff --git a/src/go/cmd/update-collection-v3/testdata/simple.output.yaml b/src/go/cmd/update-collection-v3/testdata/simple.output.yaml index 8039bb23..0c52a4ef 100644 --- a/src/go/cmd/update-collection-v3/testdata/simple.output.yaml +++ b/src/go/cmd/update-collection-v3/testdata/simple.output.yaml @@ -3,6 +3,9 @@ sumologic: accessKey: yyy events: enabled: false + logs: + container: + format: text fluentd: logs: autoscaling: