Skip to content

Commit 563f3b0

Browse files
mahadzaryab1JaredTan95yurishkuro
authored
[jaeger-v2] Refactor ElasticSearch/OpenSearch Storage Configurations (jaegertracing#6060)
## Which problem is this PR solving? - Part of jaegertracing#6059 ## Description of the changes - Continuation of jaegertracing#5790 - Refactors the configurations for ElasticSearch and OpenSearch to simplify the configuration by having more logical groupings ## How was this change tested? - Unit Tests and E2E Integration Tests ## Checklist - [x] I have read https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md - [x] I have signed all commits - [x] I have added unit tests for the new functionality - [x] I have run lint and test steps successfully - for `jaeger`: `make lint test` - for `jaeger-ui`: `yarn lint` and `yarn test` --------- Signed-off-by: Jared Tan <[email protected]> Signed-off-by: Mahad Zaryab <[email protected]> Signed-off-by: Yuri Shkuro <[email protected]> Co-authored-by: Jared Tan <[email protected]> Co-authored-by: Yuri Shkuro <[email protected]>
1 parent b2f8be2 commit 563f3b0

27 files changed

+815
-614
lines changed

‎cmd/es-rollover/app/init/action.go

+10-13
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
1414
"github.com/jaegertracing/jaeger/pkg/es"
1515
"github.com/jaegertracing/jaeger/pkg/es/client"
16+
"github.com/jaegertracing/jaeger/pkg/es/config"
1617
"github.com/jaegertracing/jaeger/pkg/es/filter"
1718
"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
1819
)
@@ -28,19 +29,15 @@ type Action struct {
2829
}
2930

3031
func (c Action) getMapping(version uint, templateName string) (string, error) {
32+
c.Config.Indices.IndexPrefix = config.IndexPrefix(c.Config.Config.IndexPrefix)
3133
mappingBuilder := mappings.MappingBuilder{
32-
TemplateBuilder: es.TextTemplateBuilder{},
33-
PrioritySpanTemplate: int64(c.Config.PrioritySpanTemplate),
34-
PriorityServiceTemplate: int64(c.Config.PriorityServiceTemplate),
35-
PriorityDependenciesTemplate: int64(c.Config.PriorityDependenciesTemplate),
36-
PrioritySamplingTemplate: int64(c.Config.PrioritySamplingTemplate),
37-
Shards: int64(c.Config.Shards),
38-
Replicas: int64(c.Config.Replicas),
39-
IndexPrefix: c.Config.IndexPrefix,
40-
UseILM: c.Config.UseILM,
41-
ILMPolicyName: c.Config.ILMPolicyName,
42-
EsVersion: version,
34+
TemplateBuilder: es.TextTemplateBuilder{},
35+
Indices: c.Config.Indices,
36+
UseILM: c.Config.UseILM,
37+
ILMPolicyName: c.Config.ILMPolicyName,
38+
EsVersion: version,
4339
}
40+
4441
return mappingBuilder.GetMapping(templateName)
4542
}
4643

@@ -62,7 +59,7 @@ func (c Action) Do() error {
6259
return fmt.Errorf("ILM policy %s doesn't exist in Elasticsearch. Please create it and re-run init", c.Config.ILMPolicyName)
6360
}
6461
}
65-
rolloverIndices := app.RolloverIndices(c.Config.Archive, c.Config.SkipDependencies, c.Config.AdaptiveSampling, c.Config.IndexPrefix)
62+
rolloverIndices := app.RolloverIndices(c.Config.Archive, c.Config.SkipDependencies, c.Config.AdaptiveSampling, c.Config.Config.IndexPrefix)
6663
for _, indexName := range rolloverIndices {
6764
if err := c.init(version, indexName); err != nil {
6865
return err
@@ -114,7 +111,7 @@ func (c Action) init(version uint, indexopt app.IndexOption) error {
114111
return err
115112
}
116113

117-
jaegerIndices, err := c.IndicesClient.GetJaegerIndices(c.Config.IndexPrefix)
114+
jaegerIndices, err := c.IndicesClient.GetJaegerIndices(c.Config.Config.IndexPrefix)
118115
if err != nil {
119116
return err
120117
}

‎cmd/es-rollover/app/init/flags.go

+17-12
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/spf13/viper"
1010

1111
"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
12+
cfg "github.com/jaegertracing/jaeger/pkg/es/config"
1213
)
1314

1415
const (
@@ -21,14 +22,10 @@ const (
2122
)
2223

2324
// Config holds configuration for index cleaner binary.
25+
// Config.IndexPrefix supersedes Indices.IndexPrefix
2426
type Config struct {
2527
app.Config
26-
Shards int
27-
Replicas int
28-
PrioritySpanTemplate int
29-
PriorityServiceTemplate int
30-
PriorityDependenciesTemplate int
31-
PrioritySamplingTemplate int
28+
cfg.Indices
3229
}
3330

3431
// AddFlags adds flags for TLS to the FlagSet.
@@ -43,10 +40,18 @@ func (*Config) AddFlags(flags *flag.FlagSet) {
4340

4441
// InitFromViper initializes config from viper.Viper.
4542
func (c *Config) InitFromViper(v *viper.Viper) {
46-
c.Shards = v.GetInt(shards)
47-
c.Replicas = v.GetInt(replicas)
48-
c.PrioritySpanTemplate = v.GetInt(prioritySpanTemplate)
49-
c.PriorityServiceTemplate = v.GetInt(priorityServiceTemplate)
50-
c.PriorityDependenciesTemplate = v.GetInt(priorityDependenciesTemplate)
51-
c.PrioritySamplingTemplate = v.GetInt(prioritySamplingTemplate)
43+
c.Indices.Spans.Shards = v.GetInt64(shards)
44+
c.Indices.Services.Shards = v.GetInt64(shards)
45+
c.Indices.Dependencies.Shards = v.GetInt64(shards)
46+
c.Indices.Sampling.Shards = v.GetInt64(shards)
47+
48+
c.Indices.Spans.Replicas = v.GetInt64(replicas)
49+
c.Indices.Services.Replicas = v.GetInt64(replicas)
50+
c.Indices.Dependencies.Replicas = v.GetInt64(replicas)
51+
c.Indices.Sampling.Replicas = v.GetInt64(replicas)
52+
53+
c.Indices.Spans.Priority = v.GetInt64(prioritySpanTemplate)
54+
c.Indices.Services.Priority = v.GetInt64(priorityServiceTemplate)
55+
c.Indices.Dependencies.Priority = v.GetInt64(priorityDependenciesTemplate)
56+
c.Indices.Sampling.Priority = v.GetInt64(prioritySamplingTemplate)
5257
}

‎cmd/es-rollover/app/init/flags_test.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ func TestBindFlags(t *testing.T) {
3333
require.NoError(t, err)
3434

3535
c.InitFromViper(v)
36-
assert.Equal(t, 8, c.Shards)
37-
assert.Equal(t, 16, c.Replicas)
38-
assert.Equal(t, 300, c.PrioritySpanTemplate)
39-
assert.Equal(t, 301, c.PriorityServiceTemplate)
40-
assert.Equal(t, 302, c.PriorityDependenciesTemplate)
41-
assert.Equal(t, 303, c.PrioritySamplingTemplate)
36+
assert.EqualValues(t, 8, c.Indices.Spans.Shards)
37+
assert.EqualValues(t, 16, c.Indices.Spans.Replicas)
38+
assert.EqualValues(t, 300, c.Indices.Spans.Priority)
39+
assert.EqualValues(t, 301, c.Indices.Services.Priority)
40+
assert.EqualValues(t, 302, c.Indices.Dependencies.Priority)
41+
assert.EqualValues(t, 303, c.Indices.Sampling.Priority)
4242
}

‎cmd/esmapping-generator/app/flags_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ func TestOptionsWithDefaultFlags(t *testing.T) {
2020

2121
assert.Equal(t, "", o.Mapping)
2222
assert.Equal(t, uint(7), o.EsVersion)
23-
assert.Equal(t, int64(5), o.Shards)
24-
assert.Equal(t, int64(1), o.Replicas)
23+
assert.EqualValues(t, 5, o.Shards)
24+
assert.EqualValues(t, 1, o.Replicas)
25+
2526
assert.Equal(t, "", o.IndexPrefix)
2627
assert.Equal(t, "false", o.UseILM)
2728
assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName)

‎cmd/esmapping-generator/app/renderer/render.go

+15-6
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
1010
"github.com/jaegertracing/jaeger/pkg/es"
11+
cfg "github.com/jaegertracing/jaeger/pkg/es/config"
1112
"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
1213
)
1314

@@ -25,14 +26,22 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e
2526
return "", err
2627
}
2728

29+
indexOpts := cfg.IndexOptions{
30+
Shards: opt.Shards,
31+
Replicas: opt.Replicas,
32+
}
2833
mappingBuilder := mappings.MappingBuilder{
2934
TemplateBuilder: builder,
30-
Shards: opt.Shards,
31-
Replicas: opt.Replicas,
32-
EsVersion: opt.EsVersion,
33-
IndexPrefix: opt.IndexPrefix,
34-
UseILM: enableILM,
35-
ILMPolicyName: opt.ILMPolicyName,
35+
Indices: cfg.Indices{
36+
IndexPrefix: cfg.IndexPrefix(opt.IndexPrefix),
37+
Spans: indexOpts,
38+
Services: indexOpts,
39+
Dependencies: indexOpts,
40+
Sampling: indexOpts,
41+
},
42+
EsVersion: opt.EsVersion,
43+
UseILM: enableILM,
44+
ILMPolicyName: opt.ILMPolicyName,
3645
}
3746
return mappingBuilder.GetMapping(opt.Mapping)
3847
}

‎cmd/jaeger/config-elasticsearch.yaml

+24-2
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,32 @@ extensions:
2222
backends:
2323
some_storage:
2424
elasticsearch:
25-
index_prefix: "jaeger-main"
25+
indices:
26+
index_prefix: "jaeger-main"
27+
spans:
28+
date_layout: "2006-01-02"
29+
rollover_frequency: "day"
30+
shards: 5
31+
replicas: 1
32+
services:
33+
date_layout: "2006-01-02"
34+
rollover_frequency: "day"
35+
shards: 5
36+
replicas: 1
37+
dependencies:
38+
date_layout: "2006-01-02"
39+
rollover_frequency: "day"
40+
shards: 5
41+
replicas: 1
42+
sampling:
43+
date_layout: "2006-01-02"
44+
rollover_frequency: "day"
45+
shards: 5
46+
replicas: 1
2647
another_storage:
2748
elasticsearch:
28-
index_prefix: "jaeger-archive"
49+
indices:
50+
index_prefix: "jaeger-archive"
2951

3052
receivers:
3153
otlp:

‎cmd/jaeger/config-opensearch.yaml

+24-3
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,32 @@ extensions:
2222
backends:
2323
some_storage:
2424
opensearch:
25-
index_prefix: "jaeger-main"
26-
25+
indices:
26+
index_prefix: "jaeger-main"
27+
spans:
28+
date_layout: "2006-01-02"
29+
rollover_frequency: "day"
30+
shards: 5
31+
replicas: 1
32+
services:
33+
date_layout: "2006-01-02"
34+
rollover_frequency: "day"
35+
shards: 5
36+
replicas: 1
37+
dependencies:
38+
date_layout: "2006-01-02"
39+
rollover_frequency: "day"
40+
shards: 5
41+
replicas: 1
42+
sampling:
43+
date_layout: "2006-01-02"
44+
rollover_frequency: "day"
45+
shards: 5
46+
replicas: 1
2747
another_storage:
2848
opensearch:
29-
index_prefix: "jaeger-main"
49+
indices:
50+
index_prefix: "jaeger-archive"
3051

3152
receivers:
3253
otlp:

0 commit comments

Comments
 (0)