diff --git a/cmd/es-rollover/app/flags.go b/cmd/es-rollover/app/flags.go index d3c54080f43..dae5afd94af 100644 --- a/cmd/es-rollover/app/flags.go +++ b/cmd/es-rollover/app/flags.go @@ -10,65 +10,52 @@ import ( "go.opentelemetry.io/collector/config/configtls" "github.com/jaegertracing/jaeger/pkg/config/tlscfg" + "github.com/jaegertracing/jaeger/pkg/es/config" ) -var tlsFlagsCfg = tlscfg.ClientFlagsConfig{Prefix: "es"} +var ( + tlsFlagsCfg = tlscfg.ClientFlagsConfig{Prefix: "es"} + esrolloverCfg = config.EsRolloverFlagConfig{} +) const ( - indexPrefix = "index-prefix" - archive = "archive" - username = "es.username" - password = "es.password" - useILM = "es.use-ilm" - ilmPolicyName = "es.ilm-policy-name" - timeout = "timeout" - skipDependencies = "skip-dependencies" - adaptiveSampling = "adaptive-sampling" + indexPrefix = "index-prefix" + username = "es.username" + password = "es.password" + useILM = "es.use-ilm" ) // Config holds the global configurations for the es rollover, common to all actions type Config struct { - IndexPrefix string - Archive bool - Username string - Password string - TLSEnabled bool - ILMPolicyName string - UseILM bool - Timeout int - SkipDependencies bool - AdaptiveSampling bool - TLSConfig configtls.ClientConfig + config.RolloverOptions + IndexPrefix string + Username string + Password string + TLSEnabled bool + UseILM bool + TLSConfig configtls.ClientConfig } // AddFlags adds flags func AddFlags(flags *flag.FlagSet) { + esrolloverCfg.AddFlagsForRolloverOptions(flags) flags.String(indexPrefix, "", "Index prefix") - flags.Bool(archive, false, "Handle archive indices") flags.String(username, "", "The username required by storage") flags.String(password, "", "The password required by storage") flags.Bool(useILM, false, "Use ILM to manage jaeger indices") - flags.String(ilmPolicyName, "jaeger-ilm-policy", "The name of the ILM policy to use if ILM is active") - flags.Int(timeout, 120, "Number of seconds to wait for master node response") - flags.Bool(skipDependencies, false, "Disable rollover for dependencies index") - flags.Bool(adaptiveSampling, false, "Enable rollover for adaptive sampling index") tlsFlagsCfg.AddFlags(flags) } // InitFromViper initializes config from viper.Viper. func (c *Config) InitFromViper(v *viper.Viper) { + c.RolloverOptions = esrolloverCfg.InitRolloverOptionsFromViper(v) c.IndexPrefix = v.GetString(indexPrefix) if c.IndexPrefix != "" { c.IndexPrefix += "-" } - c.Archive = v.GetBool(archive) c.Username = v.GetString(username) c.Password = v.GetString(password) - c.ILMPolicyName = v.GetString(ilmPolicyName) c.UseILM = v.GetBool(useILM) - c.Timeout = v.GetInt(timeout) - c.SkipDependencies = v.GetBool(skipDependencies) - c.AdaptiveSampling = v.GetBool(adaptiveSampling) tlsCfg, err := tlsFlagsCfg.InitFromViper(v) if err != nil { panic(err) diff --git a/cmd/es-rollover/app/init/action_test.go b/cmd/es-rollover/app/init/action_test.go index 0f87f579ea2..9c5a4cc2010 100644 --- a/cmd/es-rollover/app/init/action_test.go +++ b/cmd/es-rollover/app/init/action_test.go @@ -15,6 +15,7 @@ import ( "github.com/jaegertracing/jaeger/cmd/es-rollover/app" "github.com/jaegertracing/jaeger/pkg/es/client" "github.com/jaegertracing/jaeger/pkg/es/client/mocks" + "github.com/jaegertracing/jaeger/pkg/es/config" ) func TestIndexCreateIfNotExist(t *testing.T) { @@ -90,8 +91,8 @@ func TestRolloverAction(t *testing.T) { }, config: Config{ Config: app.Config{ - Archive: true, - UseILM: true, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: true, }, }, expectedErr: errors.New("ILM is supported only for ES version 7+"), @@ -104,8 +105,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: errors.New("version error"), config: Config{ Config: app.Config{ - Archive: true, - UseILM: true, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: true, }, }, }, @@ -118,9 +119,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: errors.New("ILM policy myilmpolicy doesn't exist in Elasticsearch. Please create it and re-run init"), config: Config{ Config: app.Config{ - Archive: true, - UseILM: true, - ILMPolicyName: "myilmpolicy", + RolloverOptions: config.RolloverOptions{Archive: true, ILMPolicyName: "myilmpolicy"}, + UseILM: true, }, }, }, @@ -133,9 +133,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: errors.New("error getting ilm policy"), config: Config{ Config: app.Config{ - Archive: true, - UseILM: true, - ILMPolicyName: "myilmpolicy", + RolloverOptions: config.RolloverOptions{Archive: true, ILMPolicyName: "myilmpolicy"}, + UseILM: true, }, }, }, @@ -148,8 +147,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: errors.New("error creating template"), config: Config{ Config: app.Config{ - Archive: true, - UseILM: false, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: false, }, }, }, @@ -164,8 +163,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: errors.New("error getting jaeger indices"), config: Config{ Config: app.Config{ - Archive: true, - UseILM: false, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: false, }, }, }, @@ -184,8 +183,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: errors.New("error creating aliases"), config: Config{ Config: app.Config{ - Archive: true, - UseILM: false, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: false, }, }, }, @@ -204,8 +203,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: nil, config: Config{ Config: app.Config{ - Archive: true, - UseILM: false, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: false, }, }, }, @@ -225,9 +224,8 @@ func TestRolloverAction(t *testing.T) { expectedErr: nil, config: Config{ Config: app.Config{ - Archive: true, - UseILM: true, - ILMPolicyName: "jaeger-ilm", + RolloverOptions: config.RolloverOptions{Archive: true, ILMPolicyName: "jaeger-ilm"}, + UseILM: true, }, }, }, diff --git a/cmd/es-rollover/app/lookback/action_test.go b/cmd/es-rollover/app/lookback/action_test.go index 22f0a8c912c..6f07ab5aec5 100644 --- a/cmd/es-rollover/app/lookback/action_test.go +++ b/cmd/es-rollover/app/lookback/action_test.go @@ -15,6 +15,7 @@ import ( "github.com/jaegertracing/jaeger/cmd/es-rollover/app" "github.com/jaegertracing/jaeger/pkg/es/client" "github.com/jaegertracing/jaeger/pkg/es/client/mocks" + "github.com/jaegertracing/jaeger/pkg/es/config" ) func TestLookBackAction(t *testing.T) { @@ -81,11 +82,13 @@ func TestLookBackAction(t *testing.T) { }).Return(nil) }, config: Config{ - Unit: "days", - UnitCount: 1, + LookBackOptions: config.LookBackOptions{ + Unit: "days", + UnitCount: 1, + }, Config: app.Config{ - Archive: true, - UseILM: true, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: true, }, }, expectedErr: nil, @@ -96,11 +99,13 @@ func TestLookBackAction(t *testing.T) { indexClient.On("GetJaegerIndices", "").Return(indices, errors.New("get indices error")) }, config: Config{ - Unit: "days", - UnitCount: 1, + LookBackOptions: config.LookBackOptions{ + Unit: "days", + UnitCount: 1, + }, Config: app.Config{ - Archive: true, - UseILM: true, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: true, }, }, expectedErr: errors.New("get indices error"), @@ -111,11 +116,13 @@ func TestLookBackAction(t *testing.T) { indexClient.On("GetJaegerIndices", "").Return([]client.Index{}, nil) }, config: Config{ - Unit: "days", - UnitCount: 1, + LookBackOptions: config.LookBackOptions{ + Unit: "days", + UnitCount: 1, + }, Config: app.Config{ - Archive: true, - UseILM: true, + RolloverOptions: config.RolloverOptions{Archive: true}, + UseILM: true, }, }, expectedErr: nil, diff --git a/cmd/es-rollover/app/lookback/flags.go b/cmd/es-rollover/app/lookback/flags.go index 87259792db2..bbe7620396e 100644 --- a/cmd/es-rollover/app/lookback/flags.go +++ b/cmd/es-rollover/app/lookback/flags.go @@ -9,30 +9,23 @@ import ( "github.com/spf13/viper" "github.com/jaegertracing/jaeger/cmd/es-rollover/app" + "github.com/jaegertracing/jaeger/pkg/es/config" ) -const ( - unit = "unit" - unitCount = "unit-count" - defaultUnit = "days" - defaultUnitCount = 1 -) +var esrolloverCfg = config.EsRolloverFlagConfig{} // Config holds configuration for index cleaner binary. type Config struct { app.Config - Unit string - UnitCount int + config.LookBackOptions } // AddFlags adds flags for TLS to the FlagSet. func (*Config) AddFlags(flags *flag.FlagSet) { - flags.String(unit, defaultUnit, "used with lookback to remove indices from read alias e.g, days, weeks, months, years") - flags.Int(unitCount, defaultUnitCount, "count of UNITs") + esrolloverCfg.AddFlagsForLookBackOptions(flags) } // InitFromViper initializes config from viper.Viper. func (c *Config) InitFromViper(v *viper.Viper) { - c.Unit = v.GetString(unit) - c.UnitCount = v.GetInt(unitCount) + c.LookBackOptions = esrolloverCfg.InitLookBackFromViper(v) } diff --git a/cmd/es-rollover/app/rollover/action_test.go b/cmd/es-rollover/app/rollover/action_test.go index 723b4409433..97bb111ee8c 100644 --- a/cmd/es-rollover/app/rollover/action_test.go +++ b/cmd/es-rollover/app/rollover/action_test.go @@ -12,6 +12,7 @@ import ( "github.com/jaegertracing/jaeger/cmd/es-rollover/app" "github.com/jaegertracing/jaeger/pkg/es/client" "github.com/jaegertracing/jaeger/pkg/es/client/mocks" + "github.com/jaegertracing/jaeger/pkg/es/config" ) func TestRolloverAction(t *testing.T) { @@ -115,9 +116,9 @@ func TestRolloverAction(t *testing.T) { rolloverAction := Action{ Config: Config{ - Conditions: test.conditions, + RollOverConditions: config.RollOverConditions{Conditions: test.conditions}, Config: app.Config{ - Archive: true, + RolloverOptions: config.RolloverOptions{Archive: true}, }, }, IndicesClient: indexClient, diff --git a/cmd/es-rollover/app/rollover/flags.go b/cmd/es-rollover/app/rollover/flags.go index 258b83874cf..a8fde414442 100644 --- a/cmd/es-rollover/app/rollover/flags.go +++ b/cmd/es-rollover/app/rollover/flags.go @@ -9,25 +9,23 @@ import ( "github.com/spf13/viper" "github.com/jaegertracing/jaeger/cmd/es-rollover/app" + "github.com/jaegertracing/jaeger/pkg/es/config" ) -const ( - conditions = "conditions" - defaultRollbackCondition = "{\"max_age\": \"2d\"}" -) +var esrolloverCfg = config.EsRolloverFlagConfig{} // Config holds configuration for index cleaner binary. type Config struct { app.Config - Conditions string + config.RollOverConditions } // AddFlags adds flags for TLS to the FlagSet. func (*Config) AddFlags(flags *flag.FlagSet) { - flags.String(conditions, defaultRollbackCondition, "conditions used to rollover to a new write index") + esrolloverCfg.AddFlagsForRollBackOptions(flags) } // InitFromViper initializes config from viper.Viper. func (c *Config) InitFromViper(v *viper.Viper) { - c.Conditions = v.GetString(conditions) + c.RollOverConditions = esrolloverCfg.InitRollBackFromViper(v) } diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index d0ae9c6ad3d..fea2faa9d7c 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -202,6 +202,34 @@ type BearerTokenAuthentication struct { AllowFromContext bool `mapstructure:"from_context"` } +// RolloverOptions store the options for the working of auto-rollover (cron job) in the jaeger. +type RolloverOptions struct { + // Archive if set to true will handle archive indices also + Archive bool `mapstructure:"archive"` + // The name of the ILM policy to use if ILM is active + ILMPolicyName string `mapstructure:"ilm_policy_name"` + // This stores number of seconds to wait for master node response. By default, it is set to 120 + Timeout int `mapstructure:"timeout"` + // SkipDependencies if set to true will disable rollover for dependencies index + SkipDependencies bool `mapstructure:"skip_dependencies"` + // AdaptiveSampling if set to true will enable rollover for adaptive sampling index + AdaptiveSampling bool `mapstructure:"include_adaptive_sampling"` +} + +// LookBackOptions store the options for a sub-process which is similar to index cleaner of auto rollover of jaeger. +type LookBackOptions struct { + // Unit is used with lookback to remove indices from read alias e.g, days, weeks, months, years + Unit string `mapstructure:"unit"` + // UnitCount is the count of units for which look-up is performed + UnitCount int `mapstructure:"unit-count"` +} + +// RollOverConditions store the conditions required for the rollover operation in the auto-rollover cron job of jaeger. +type RollOverConditions struct { + // Conditions stores the conditions on which index writing should be performed, for example: "{\"max_age\": \"2d\"}" + Conditions string `mapstructure:"conditions"` +} + // NewClient creates a new ElasticSearch client func NewClient(c *Configuration, logger *zap.Logger, metricsFactory metrics.Factory) (es.Client, error) { if len(c.Servers) < 1 { diff --git a/pkg/es/config/esrollovercfg.go b/pkg/es/config/esrollovercfg.go new file mode 100644 index 00000000000..d9511efc42c --- /dev/null +++ b/pkg/es/config/esrollovercfg.go @@ -0,0 +1,74 @@ +// Copyright (c) 2025 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "flag" + + "github.com/spf13/viper" +) + +const ( + flagLookBackUnit = "unit" + flagLookBackUnitCount = "unit-count" + flagRolloverConditions = "conditions" + flagRolloverArchive = "archive" + flagRolloverIlmPolicyName = "es.ilm-policy-name" + flagRolloverTimeout = "timeout" + flagRolloverSkipDependencies = "skip-dependencies" + flagRolloverAdaptiveSampling = "adaptive-sampling" + + defaultArchiveValue = false + defaultIlmPolicyName = "jaeger-ilm-policy" + defaultTimeout = 120 + defaultSkipDependencies = false + defaultAdaptiveSampling = false + defaultUnit = "days" + defaultUnitCount = 1 + defaultRollBackCondition = "{\"max_age\": \"2d\"}" +) + +type EsRolloverFlagConfig struct { + Prefix string +} + +func (e EsRolloverFlagConfig) AddFlagsForRolloverOptions(flags *flag.FlagSet) { + flags.Bool(e.Prefix+flagRolloverArchive, defaultArchiveValue, "Handle archive indices") + flags.String(e.Prefix+flagRolloverIlmPolicyName, defaultIlmPolicyName, "The name of the ILM policy to use if ILM is active") + flags.Int(e.Prefix+flagRolloverTimeout, defaultTimeout, "Number of seconds to wait for master node response") + flags.Bool(e.Prefix+flagRolloverSkipDependencies, defaultSkipDependencies, "Disable rollover for dependencies index") + flags.Bool(e.Prefix+flagRolloverAdaptiveSampling, defaultAdaptiveSampling, "Enable rollover for adaptive sampling index") +} + +func (e EsRolloverFlagConfig) InitRolloverOptionsFromViper(v *viper.Viper) RolloverOptions { + r := &RolloverOptions{} + r.Archive = v.GetBool(e.Prefix + flagRolloverArchive) + r.ILMPolicyName = v.GetString(e.Prefix + flagRolloverIlmPolicyName) + r.Timeout = v.GetInt(e.Prefix + flagRolloverTimeout) + r.SkipDependencies = v.GetBool(e.Prefix + flagRolloverSkipDependencies) + r.AdaptiveSampling = v.GetBool(e.Prefix + flagRolloverAdaptiveSampling) + return *r +} + +func (e EsRolloverFlagConfig) AddFlagsForLookBackOptions(flags *flag.FlagSet) { + flags.String(e.Prefix+flagLookBackUnit, defaultUnit, "used with lookback to remove indices from read alias e.g, days, weeks, months, years") + flags.Int(e.Prefix+flagLookBackUnitCount, defaultUnitCount, "count of UNITs") +} + +func (e EsRolloverFlagConfig) InitLookBackFromViper(v *viper.Viper) LookBackOptions { + l := &LookBackOptions{} + l.Unit = v.GetString(e.Prefix + flagLookBackUnit) + l.UnitCount = v.GetInt(e.Prefix + flagLookBackUnitCount) + return *l +} + +func (e EsRolloverFlagConfig) AddFlagsForRollBackOptions(flags *flag.FlagSet) { + flags.String(e.Prefix+flagRolloverConditions, defaultRollBackCondition, "conditions used to rollover to a new write index") +} + +func (e EsRolloverFlagConfig) InitRollBackFromViper(v *viper.Viper) RollOverConditions { + r := &RollOverConditions{} + r.Conditions = v.GetString(e.Prefix + flagRolloverConditions) + return *r +} diff --git a/pkg/es/config/esrollovercfg_test.go b/pkg/es/config/esrollovercfg_test.go new file mode 100644 index 00000000000..e89b27d43aa --- /dev/null +++ b/pkg/es/config/esrollovercfg_test.go @@ -0,0 +1,50 @@ +// Copyright (c) 2025 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "flag" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBindFlags(t *testing.T) { + v := viper.New() + fcfg := EsRolloverFlagConfig{} + command := cobra.Command{} + flags := &flag.FlagSet{} + fcfg.AddFlagsForRolloverOptions(flags) + fcfg.AddFlagsForLookBackOptions(flags) + fcfg.AddFlagsForRollBackOptions(flags) + command.PersistentFlags().AddGoFlagSet(flags) + v.BindPFlags(command.PersistentFlags()) + + err := command.ParseFlags([]string{ + "--archive=true", + "--timeout=150", + "--es.ilm-policy-name=jaeger-ilm", + "--skip-dependencies=true", + "--adaptive-sampling=true", + "--unit=days", + "--unit-count=16", + "--conditions={\"max_age\": \"20000d\"}", + }) + require.NoError(t, err) + + c := fcfg.InitRolloverOptionsFromViper(v) + l := fcfg.InitLookBackFromViper(v) + r := fcfg.InitRollBackFromViper(v) + assert.True(t, c.Archive) + assert.Equal(t, 150, c.Timeout) + assert.Equal(t, "jaeger-ilm", c.ILMPolicyName) + assert.True(t, c.SkipDependencies) + assert.True(t, c.AdaptiveSampling) + assert.Equal(t, "days", l.Unit) + assert.Equal(t, 16, l.UnitCount) + assert.JSONEq(t, `{"max_age": "20000d"}`, r.Conditions) +}