From 0e505d53e4632d6fbcb87543f757bcd7bf065b6c Mon Sep 17 00:00:00 2001 From: Paulo Janotti Date: Wed, 2 Oct 2019 17:42:18 -0700 Subject: [PATCH] Refactor config: pipelines now under service (#376) * Refactor config: pipelines now under service This change moves 'pipelines' sections from the top level to under the 'service' section. * Fix typo in old comment --- README.md | 15 ++-- config/config.go | 38 +++++++---- config/config_test.go | 8 +-- config/configmodels/configmodels.go | 4 +- config/testdata/duplicate-exporter.yaml | 11 +-- config/testdata/duplicate-pipeline.yaml | 19 +++--- config/testdata/duplicate-processor.yaml | 11 +-- config/testdata/duplicate-receiver.yaml | 11 +-- config/testdata/invalid-bool-value.yaml | 11 +-- .../testdata/invalid-disabled-bool-value.yaml | 11 +-- .../invalid-disabled-bool-value2.yaml | 11 +-- config/testdata/invalid-exporter-section.yaml | 16 ++--- config/testdata/invalid-extension-name.yaml | 15 ++-- .../testdata/invalid-extension-section.yaml | 16 ++--- config/testdata/invalid-pipeline-section.yaml | 18 ++--- .../invalid-pipeline-type-and-name.yaml | 11 +-- config/testdata/invalid-pipeline-type.yaml | 11 +-- .../testdata/invalid-processor-section.yaml | 16 ++--- config/testdata/invalid-receiver-name.yaml | 11 +-- .../testdata/invalid-receiver-reference.yaml | 7 +- config/testdata/invalid-receiver-section.yaml | 16 ++--- config/testdata/invalid-sequence-value.yaml | 14 ++-- .../invalid-service-extensions-value.yaml | 19 +++--- config/testdata/invalid-service-section.yaml | 16 ++--- .../testdata/invalid-top-level-section.yaml | 16 ++--- ...etric-pipeline-cannot-have-processors.yaml | 11 +-- config/testdata/missing-all-sections.yaml | 3 +- .../testdata/missing-enabled-exporters.yaml | 11 +-- .../testdata/missing-enabled-receivers.yaml | 11 +-- .../missing-exporter-name-after-slash.yaml | 7 +- config/testdata/missing-exporters.yaml | 7 +- config/testdata/missing-pipelines.yaml | 3 +- config/testdata/missing-processor-type.yaml | 7 +- config/testdata/missing-processors.yaml | 7 +- config/testdata/missing-receiver-type.yaml | 7 +- config/testdata/missing-receivers.yaml | 5 +- config/testdata/multiproto-config.yaml | 17 ++--- .../pipeline-exporter-not-exists.yaml | 9 +-- .../testdata/pipeline-must-have-exporter.yaml | 7 +- .../pipeline-must-have-exporter2.yaml | 7 +- .../pipeline-must-have-processors.yaml | 9 +-- .../testdata/pipeline-must-have-receiver.yaml | 7 +- .../pipeline-processor-not-exists.yaml | 17 ++--- .../testdata/simple-config-with-all-env.yaml | 12 ++-- .../testdata/simple-config-with-no-env.yaml | 12 ++-- .../simple-config-with-partial-env.yaml | 12 ++-- config/testdata/unknown-exporter-type.yaml | 11 +-- config/testdata/unknown-processor-type.yaml | 11 +-- config/testdata/unknown-receiver-type.yaml | 17 ++--- config/testdata/valid-config.yaml | 12 ++-- docs/design.md | 68 ++++++++++--------- examples/demo/otel-agent-config.yaml | 19 +++--- examples/demo/otel-collector-config.yaml | 17 +++-- .../jaegergrpcexporter/testdata/config.yaml | 11 +-- .../testdata/config.yaml | 11 +-- exporter/loggingexporter/testdata/config.yaml | 17 ++--- .../opencensusexporter/testdata/config.yaml | 11 +-- .../prometheusexporter/testdata/config.yaml | 11 +-- exporter/zipkinexporter/testdata/config.yaml | 11 +-- .../healthcheckextension/testdata/config.yaml | 10 +-- extension/pprofextension/testdata/config.yaml | 10 +-- .../zpagesextension/testdata/config.yaml | 10 +-- .../attributesprocessor/testdata/config.yaml | 11 +-- .../nodebatcherprocessor/testdata/config.yaml | 11 +-- .../testdata/config.yaml | 11 +-- .../testdata/empty.yaml | 11 +-- .../queuedprocessor/testdata/config.yaml | 11 +-- processor/spanprocessor/testdata/config.yaml | 11 +-- .../testdata/tail_sampling_config.yaml | 11 +-- receiver/jaegerreceiver/testdata/config.yaml | 11 +-- .../opencensusreceiver/testdata/config.yaml | 17 ++--- .../prometheusreceiver/testdata/config.yaml | 11 +-- .../testdata/invalid-config-section.yaml | 11 +-- .../vmmetricsreceiver/testdata/config.yaml | 11 +-- receiver/zipkinreceiver/testdata/config.yaml | 11 +-- service/builder/exporters_builder.go | 2 +- service/builder/exporters_builder_test.go | 14 ++-- service/builder/pipelines_builder.go | 2 +- service/builder/pipelines_builder_test.go | 4 +- service/builder/receivers_builder.go | 2 +- .../builder/testdata/pipelines_builder.yaml | 37 +++++----- service/testdata/otelcol-config.yaml | 12 ++-- .../tests/testdata/add-attributes-config.yaml | 11 +-- testbed/tests/testdata/agent-config.yaml | 11 +-- 84 files changed, 555 insertions(+), 486 deletions(-) diff --git a/README.md b/README.md index a5254d0bafc..59a4a4ec0c3 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,8 @@ exporters: ... processors: ... -pipelines: +service: + pipelines: ... ``` @@ -218,12 +219,12 @@ a receiver/exporter is reference by all the pipelines. The following is an example pipeline configuration. For more information, refer to [pipeline documentation](docs/pipelines.md) ```yaml -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] - +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] ``` ### Diagnostics diff --git a/config/config.go b/config/config.go index f7ee87a6cf7..69d9a97037a 100644 --- a/config/config.go +++ b/config/config.go @@ -137,7 +137,6 @@ func Load( Receivers map[string]interface{} `mapstructure:"receivers"` Processors map[string]interface{} `mapstructure:"processors"` Exporters map[string]interface{} `mapstructure:"exporters"` - Pipelines map[string]interface{} `mapstructure:"pipelines"` } if err := v.UnmarshalExact(&topLevelSections); err != nil { @@ -147,7 +146,7 @@ func Load( } } - // Start with extensions and service. + // Start with the service extensions. extensions, err := loadExtensions(v, factories.Extensions) if err != nil { @@ -155,13 +154,7 @@ func Load( } config.Extensions = extensions - service, err := loadService(v) - if err != nil { - return nil, err - } - config.Service = service - - // Load data components (receivers, exporters, processores, and pipelines). + // Load data components (receivers, exporters, and processors). receivers, err := loadReceivers(v, factories.Receivers) if err != nil { @@ -181,11 +174,12 @@ func Load( } config.Processors = processors - pipelines, err := loadPipelines(v) + // Load the service and its data pipelines. + service, err := loadService(v) if err != nil { return nil, err } - config.Pipelines = pipelines + config.Service = service // Config is loaded. Now validate it. @@ -296,13 +290,27 @@ func loadExtensions(v *viper.Viper, factories map[string]extension.Factory) (con func loadService(v *viper.Viper) (configmodels.Service, error) { var service configmodels.Service - if err := v.UnmarshalKey(serviceKeyName, &service, errorOnUnused); err != nil { + serviceSub := getConfigSection(v, serviceKeyName) + + // Process the pipelines first so in case of error on them it can be properly + // reported. + pipelines, err := loadPipelines(serviceSub) + if err != nil { + return service, err + } + + // Do an exact match to find any unused section on config. + if err := serviceSub.UnmarshalExact(&service); err != nil { return service, &configError{ code: errUnmarshalErrorOnService, msg: fmt.Sprintf("error reading settings for %q: %v", serviceKeyName, err), } } + // Unmarshal cannot properly build Pipelines field, set it to the value + // previously loaded. + service.Pipelines = pipelines + return service, nil } @@ -640,12 +648,12 @@ func validateServiceExtensions( func validatePipelines(cfg *configmodels.Config, logger *zap.Logger) error { // Must have at least one pipeline. - if len(cfg.Pipelines) < 1 { + if len(cfg.Service.Pipelines) < 1 { return &configError{code: errMissingPipelines, msg: "must have at least one pipeline"} } // Validate pipelines. - for _, pipeline := range cfg.Pipelines { + for _, pipeline := range cfg.Service.Pipelines { if err := validatePipeline(cfg, pipeline, logger); err != nil { return err } @@ -849,7 +857,7 @@ func validateProcessors(cfg *configmodels.Config) { // getConfigSection returns a sub-config from the viper config that has the corresponding given key. // It also expands all the string values. func getConfigSection(v *viper.Viper, key string) *viper.Viper { - // Unmarsh only the subconfig for this processor. + // Unmarshal only the subconfig for this processor. sv := v.Sub(key) if sv == nil { // When the config for this key is empty Sub returns nil. In order to avoid nil checks diff --git a/config/config_test.go b/config/config_test.go index f13bc6e90cc..c1d6cb8b36d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -111,7 +111,7 @@ func TestDecodeConfig(t *testing.T) { "Did not load processor config correctly") // Verify Pipelines - assert.Equal(t, 1, len(config.Pipelines), "Incorrect pipelines count") + assert.Equal(t, 1, len(config.Service.Pipelines), "Incorrect pipelines count") assert.Equal(t, &configmodels.Pipeline{ @@ -121,7 +121,7 @@ func TestDecodeConfig(t *testing.T) { Processors: []string{"exampleprocessor"}, Exporters: []string{"exampleexporter"}, }, - config.Pipelines["traces"], + config.Service.Pipelines["traces"], "Did not load pipeline config correctly") } @@ -275,7 +275,7 @@ func TestSimpleConfig(t *testing.T) { "TEST[%s] Did not load processor config correctly", test.name) // Verify Pipelines - assert.Equalf(t, 1, len(config.Pipelines), "TEST[%s]", test.name) + assert.Equalf(t, 1, len(config.Service.Pipelines), "TEST[%s]", test.name) assert.Equalf(t, &configmodels.Pipeline{ @@ -285,7 +285,7 @@ func TestSimpleConfig(t *testing.T) { Processors: []string{"exampleprocessor"}, Exporters: []string{"exampleexporter"}, }, - config.Pipelines["traces"], + config.Service.Pipelines["traces"], "TEST[%s] Did not load pipeline config correctly", test.name) } } diff --git a/config/configmodels/configmodels.go b/config/configmodels/configmodels.go index 6947b1ed840..74f60c3f087 100644 --- a/config/configmodels/configmodels.go +++ b/config/configmodels/configmodels.go @@ -36,7 +36,6 @@ type Config struct { Receivers Receivers Exporters Exporters Processors Processors - Pipelines Pipelines Extensions Extensions Service Service } @@ -144,6 +143,9 @@ type Extensions map[string]Extension type Service struct { // Extensions is the ordered list of extensions configured for the service. Extensions []string `mapstructure:"extensions"` + + // Pipelines is the set of data pipelines configured for the service. + Pipelines Pipelines `mapstructure:"pipelines"` } // Below are common setting structs for Receivers, Exporters and Processors. diff --git a/config/testdata/duplicate-exporter.yaml b/config/testdata/duplicate-exporter.yaml index 5e859f6395f..399be0b7bb5 100644 --- a/config/testdata/duplicate-exporter.yaml +++ b/config/testdata/duplicate-exporter.yaml @@ -5,8 +5,9 @@ exporters: exampleexporter/ exp : processors: exampleprocessor: -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/duplicate-pipeline.yaml b/config/testdata/duplicate-pipeline.yaml index 152a800a82c..671991a79d5 100644 --- a/config/testdata/duplicate-pipeline.yaml +++ b/config/testdata/duplicate-pipeline.yaml @@ -4,12 +4,13 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces/default: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] - traces/ default: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces/default: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] + traces/ default: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/duplicate-processor.yaml b/config/testdata/duplicate-processor.yaml index 91abbe7f3ef..5345cec5926 100644 --- a/config/testdata/duplicate-processor.yaml +++ b/config/testdata/duplicate-processor.yaml @@ -5,8 +5,9 @@ exporters: processors: exampleprocessor/ abc: exampleprocessor/abc: -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/duplicate-receiver.yaml b/config/testdata/duplicate-receiver.yaml index 2eff9cf2012..275054e6772 100644 --- a/config/testdata/duplicate-receiver.yaml +++ b/config/testdata/duplicate-receiver.yaml @@ -5,8 +5,9 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/invalid-bool-value.yaml b/config/testdata/invalid-bool-value.yaml index 67b6ef5eec5..9a8b9f40b6b 100644 --- a/config/testdata/invalid-bool-value.yaml +++ b/config/testdata/invalid-bool-value.yaml @@ -5,8 +5,9 @@ exporters: processors: exampleprocessor: disabled: [2,3] -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/invalid-disabled-bool-value.yaml b/config/testdata/invalid-disabled-bool-value.yaml index 152ce90cee8..b55bae0a854 100644 --- a/config/testdata/invalid-disabled-bool-value.yaml +++ b/config/testdata/invalid-disabled-bool-value.yaml @@ -5,8 +5,9 @@ exporters: disabled: "string for bool" processors: exampleprocessor: -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/invalid-disabled-bool-value2.yaml b/config/testdata/invalid-disabled-bool-value2.yaml index 4ed972a0daa..d014a4de1e7 100644 --- a/config/testdata/invalid-disabled-bool-value2.yaml +++ b/config/testdata/invalid-disabled-bool-value2.yaml @@ -5,8 +5,9 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/invalid-exporter-section.yaml b/config/testdata/invalid-exporter-section.yaml index 624c593a774..fa5e0257b9a 100644 --- a/config/testdata/invalid-exporter-section.yaml +++ b/config/testdata/invalid-exporter-section.yaml @@ -10,11 +10,11 @@ extensions: service: extensions: - exampleextension -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter diff --git a/config/testdata/invalid-extension-name.yaml b/config/testdata/invalid-extension-name.yaml index 0023c8ddf92..8ed98292506 100644 --- a/config/testdata/invalid-extension-name.yaml +++ b/config/testdata/invalid-extension-name.yaml @@ -1,16 +1,15 @@ extensions: exampleextension: -service: - extensions: [exampleextension, nosuchextension, and, another, three] - receivers: examplereceiver: processors: exampleprocessor: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + extensions: [exampleextension, nosuchextension, and, another, three] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/config/testdata/invalid-extension-section.yaml b/config/testdata/invalid-extension-section.yaml index c7fcca93e74..1e0a809d9a3 100644 --- a/config/testdata/invalid-extension-section.yaml +++ b/config/testdata/invalid-extension-section.yaml @@ -11,11 +11,11 @@ extensions: service: extensions: - examapleextension -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter diff --git a/config/testdata/invalid-pipeline-section.yaml b/config/testdata/invalid-pipeline-section.yaml index 781634601cf..760edb27db4 100644 --- a/config/testdata/invalid-pipeline-section.yaml +++ b/config/testdata/invalid-pipeline-section.yaml @@ -9,12 +9,12 @@ extensions: service: extensions: - exampleextension -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter - unknown_section: 1 + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter + unknown_section: 1 diff --git a/config/testdata/invalid-pipeline-type-and-name.yaml b/config/testdata/invalid-pipeline-type-and-name.yaml index 15a97ea39de..889dd6b1b51 100644 --- a/config/testdata/invalid-pipeline-type-and-name.yaml +++ b/config/testdata/invalid-pipeline-type-and-name.yaml @@ -5,8 +5,9 @@ processors: exporters: exampleexporter: -pipelines: - /metrics: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + /metrics: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/config/testdata/invalid-pipeline-type.yaml b/config/testdata/invalid-pipeline-type.yaml index a222582f957..7c88405a947 100644 --- a/config/testdata/invalid-pipeline-type.yaml +++ b/config/testdata/invalid-pipeline-type.yaml @@ -5,8 +5,9 @@ processors: exporters: exampleexporter: -pipelines: - wrongdatatype: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + wrongdatatype: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/config/testdata/invalid-processor-section.yaml b/config/testdata/invalid-processor-section.yaml index 488eec7007b..01d2a086ed4 100644 --- a/config/testdata/invalid-processor-section.yaml +++ b/config/testdata/invalid-processor-section.yaml @@ -11,11 +11,11 @@ extensions: service: extensions: - examapleextension -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter diff --git a/config/testdata/invalid-receiver-name.yaml b/config/testdata/invalid-receiver-name.yaml index 10627771b38..ee71200b9ce 100644 --- a/config/testdata/invalid-receiver-name.yaml +++ b/config/testdata/invalid-receiver-name.yaml @@ -4,8 +4,9 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [1,2,3] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [1,2,3] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/invalid-receiver-reference.yaml b/config/testdata/invalid-receiver-reference.yaml index 09a7fc4e03a..89288bae84c 100644 --- a/config/testdata/invalid-receiver-reference.yaml +++ b/config/testdata/invalid-receiver-reference.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [invalidreceivername] +service: + pipelines: + traces: + receivers: [invalidreceivername] diff --git a/config/testdata/invalid-receiver-section.yaml b/config/testdata/invalid-receiver-section.yaml index 01c62e3fd93..558f17a4682 100644 --- a/config/testdata/invalid-receiver-section.yaml +++ b/config/testdata/invalid-receiver-section.yaml @@ -11,11 +11,11 @@ extensions: service: extensions: - examapleextension -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter diff --git a/config/testdata/invalid-sequence-value.yaml b/config/testdata/invalid-sequence-value.yaml index b8b05c16209..7f7a0a385ee 100644 --- a/config/testdata/invalid-sequence-value.yaml +++ b/config/testdata/invalid-sequence-value.yaml @@ -4,9 +4,11 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: - examplereceiver: - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: + examplereceiver: + some: config + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/invalid-service-extensions-value.yaml b/config/testdata/invalid-service-extensions-value.yaml index 7a2155ba9ff..948be467f8f 100644 --- a/config/testdata/invalid-service-extensions-value.yaml +++ b/config/testdata/invalid-service-extensions-value.yaml @@ -1,18 +1,17 @@ extensions: exampleextension: -service: - extensions: - exampleextension: - disabled: true - receivers: examplereceiver: processors: exampleprocessor: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + extensions: + exampleextension: + disabled: true + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/config/testdata/invalid-service-section.yaml b/config/testdata/invalid-service-section.yaml index ef68bcc82b4..6a3b8f92772 100644 --- a/config/testdata/invalid-service-section.yaml +++ b/config/testdata/invalid-service-section.yaml @@ -11,11 +11,11 @@ service: - examapleextension unknown_section: a_num: 2 -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter diff --git a/config/testdata/invalid-top-level-section.yaml b/config/testdata/invalid-top-level-section.yaml index aae1d4ab2f7..0b9819d17de 100644 --- a/config/testdata/invalid-top-level-section.yaml +++ b/config/testdata/invalid-top-level-section.yaml @@ -9,13 +9,13 @@ extensions: service: extenstions: - examapleextension -pipelines: - traces: - receivers: - - examplereceiver - processors: - - exampleprocessor - exporters: - - exampleexporter + pipelines: + traces: + receivers: + - examplereceiver + processors: + - exampleprocessor + exporters: + - exampleexporter unknown_section: a_num: 2 diff --git a/config/testdata/metric-pipeline-cannot-have-processors.yaml b/config/testdata/metric-pipeline-cannot-have-processors.yaml index 5ad00b9870e..28c865a1be9 100644 --- a/config/testdata/metric-pipeline-cannot-have-processors.yaml +++ b/config/testdata/metric-pipeline-cannot-have-processors.yaml @@ -4,8 +4,9 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - metrics: - receivers: [multireceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + metrics: + receivers: [multireceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/missing-all-sections.yaml b/config/testdata/missing-all-sections.yaml index cba2bcffb6e..2eaee087e88 100644 --- a/config/testdata/missing-all-sections.yaml +++ b/config/testdata/missing-all-sections.yaml @@ -1,4 +1,5 @@ receivers: exporters: processors: -pipeline: +service: + pipeline: diff --git a/config/testdata/missing-enabled-exporters.yaml b/config/testdata/missing-enabled-exporters.yaml index 9f377780bfc..5d648f86936 100644 --- a/config/testdata/missing-enabled-exporters.yaml +++ b/config/testdata/missing-enabled-exporters.yaml @@ -8,8 +8,9 @@ exporters: processors: exampleprocessor: -pipelines: - traces: - receivers: [examplereceiver/one] - processors: [exampleprocessor] - exporters: [exampleexporter/disabled] +service: + pipelines: + traces: + receivers: [examplereceiver/one] + processors: [exampleprocessor] + exporters: [exampleexporter/disabled] diff --git a/config/testdata/missing-enabled-receivers.yaml b/config/testdata/missing-enabled-receivers.yaml index f64b188087f..548a6503a22 100644 --- a/config/testdata/missing-enabled-receivers.yaml +++ b/config/testdata/missing-enabled-receivers.yaml @@ -16,8 +16,9 @@ exporters: disabled: true exampleexporter: -pipelines: - traces: - receivers: [examplereceiver, examplereceiver/disabled] - processors: [exampleprocessor, exampleprocessor/disabled] - exporters: [exampleexporter/disabled, exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver, examplereceiver/disabled] + processors: [exampleprocessor, exampleprocessor/disabled] + exporters: [exampleexporter/disabled, exampleexporter] diff --git a/config/testdata/missing-exporter-name-after-slash.yaml b/config/testdata/missing-exporter-name-after-slash.yaml index 6558da24fcb..ecb9517bcaf 100644 --- a/config/testdata/missing-exporter-name-after-slash.yaml +++ b/config/testdata/missing-exporter-name-after-slash.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter/: processors: exampleprocessor: -pipelines: - traces: - receivers: [somereceiver] +service: + pipelines: + traces: + receivers: [somereceiver] diff --git a/config/testdata/missing-exporters.yaml b/config/testdata/missing-exporters.yaml index 7c316118fa7..bcedbc83f27 100644 --- a/config/testdata/missing-exporters.yaml +++ b/config/testdata/missing-exporters.yaml @@ -3,6 +3,7 @@ receivers: exporters: processors: exampleprocessor: -pipelines: - traces: - receivers: [invalidreceivername] +service: + pipelines: + traces: + receivers: [invalidreceivername] diff --git a/config/testdata/missing-pipelines.yaml b/config/testdata/missing-pipelines.yaml index 7d1fb921bb1..cf7f3ae1415 100644 --- a/config/testdata/missing-pipelines.yaml +++ b/config/testdata/missing-pipelines.yaml @@ -4,4 +4,5 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: +service: + pipelines: diff --git a/config/testdata/missing-processor-type.yaml b/config/testdata/missing-processor-type.yaml index e083c5579b4..d594ecdea3b 100644 --- a/config/testdata/missing-processor-type.yaml +++ b/config/testdata/missing-processor-type.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter: processors: /exampleprocessor: -pipelines: - traces: - receivers: [somereceiver] +service: + pipelines: + traces: + receivers: [somereceiver] diff --git a/config/testdata/missing-processors.yaml b/config/testdata/missing-processors.yaml index 68a1954ec4d..c3103ead3ea 100644 --- a/config/testdata/missing-processors.yaml +++ b/config/testdata/missing-processors.yaml @@ -3,6 +3,7 @@ receivers: exporters: exampleexporter: processors: -pipelines: - traces: - receivers: [invalidreceivername] +service: + pipelines: + traces: + receivers: [invalidreceivername] diff --git a/config/testdata/missing-receiver-type.yaml b/config/testdata/missing-receiver-type.yaml index c2d7d1f87a3..b05c26a28f7 100644 --- a/config/testdata/missing-receiver-type.yaml +++ b/config/testdata/missing-receiver-type.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [somereceiver] +service: + pipelines: + traces: + receivers: [somereceiver] diff --git a/config/testdata/missing-receivers.yaml b/config/testdata/missing-receivers.yaml index 8d0d0fc0c49..7a79d41a661 100644 --- a/config/testdata/missing-receivers.yaml +++ b/config/testdata/missing-receivers.yaml @@ -3,5 +3,6 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: +service: + pipelines: + traces: diff --git a/config/testdata/multiproto-config.yaml b/config/testdata/multiproto-config.yaml index f8a0764646d..6c916e3c374 100644 --- a/config/testdata/multiproto-config.yaml +++ b/config/testdata/multiproto-config.yaml @@ -17,11 +17,12 @@ exporters: exampleexporter: extra: "locahost:1010" -pipelines: - traces: - receivers: [multireceiver/myreceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] - metrics: - receivers: [multireceiver/myreceiver] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [multireceiver/myreceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] + metrics: + receivers: [multireceiver/myreceiver] + exporters: [exampleexporter] diff --git a/config/testdata/pipeline-exporter-not-exists.yaml b/config/testdata/pipeline-exporter-not-exists.yaml index 84e06474c06..0a88af150f9 100644 --- a/config/testdata/pipeline-exporter-not-exists.yaml +++ b/config/testdata/pipeline-exporter-not-exists.yaml @@ -4,7 +4,8 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - metrics: - receivers: [multireceiver] - exporters: [nosuchexporter] +service: + pipelines: + metrics: + receivers: [multireceiver] + exporters: [nosuchexporter] diff --git a/config/testdata/pipeline-must-have-exporter.yaml b/config/testdata/pipeline-must-have-exporter.yaml index 19668e02701..9c59bfdd776 100644 --- a/config/testdata/pipeline-must-have-exporter.yaml +++ b/config/testdata/pipeline-must-have-exporter.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [multireceiver] +service: + pipelines: + traces: + receivers: [multireceiver] diff --git a/config/testdata/pipeline-must-have-exporter2.yaml b/config/testdata/pipeline-must-have-exporter2.yaml index 53674eb566f..66400e9b123 100644 --- a/config/testdata/pipeline-must-have-exporter2.yaml +++ b/config/testdata/pipeline-must-have-exporter2.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - metrics: - receivers: [multireceiver] +service: + pipelines: + metrics: + receivers: [multireceiver] diff --git a/config/testdata/pipeline-must-have-processors.yaml b/config/testdata/pipeline-must-have-processors.yaml index 6eca7675e70..3af5c14e820 100644 --- a/config/testdata/pipeline-must-have-processors.yaml +++ b/config/testdata/pipeline-must-have-processors.yaml @@ -4,7 +4,8 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [multireceiver] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [multireceiver] + exporters: [exampleexporter] diff --git a/config/testdata/pipeline-must-have-receiver.yaml b/config/testdata/pipeline-must-have-receiver.yaml index 92acc663942..e93446bd35d 100644 --- a/config/testdata/pipeline-must-have-receiver.yaml +++ b/config/testdata/pipeline-must-have-receiver.yaml @@ -4,6 +4,7 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - metrics: - exporters: [exampleexporter] +service: + pipelines: + metrics: + exporters: [exampleexporter] diff --git a/config/testdata/pipeline-processor-not-exists.yaml b/config/testdata/pipeline-processor-not-exists.yaml index 81aabccacdf..634d9a3f4b4 100644 --- a/config/testdata/pipeline-processor-not-exists.yaml +++ b/config/testdata/pipeline-processor-not-exists.yaml @@ -4,11 +4,12 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: - - multireceiver - processors: - - nosuchprocessor - exporters: - - exampleexporter +service: + pipelines: + traces: + receivers: + - multireceiver + processors: + - nosuchprocessor + exporters: + - exampleexporter diff --git a/config/testdata/simple-config-with-all-env.yaml b/config/testdata/simple-config-with-all-env.yaml index f1e82b67b19..f3428d7eda4 100644 --- a/config/testdata/simple-config-with-all-env.yaml +++ b/config/testdata/simple-config-with-all-env.yaml @@ -31,12 +31,6 @@ exporters: - "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1}" - "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2}" -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] - extensions: exampleextension: extra: "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA}" @@ -49,3 +43,9 @@ extensions: service: extensions: [exampleextension] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] + diff --git a/config/testdata/simple-config-with-no-env.yaml b/config/testdata/simple-config-with-no-env.yaml index 86b841ba11c..f9e1d7708b3 100644 --- a/config/testdata/simple-config-with-no-env.yaml +++ b/config/testdata/simple-config-with-no-env.yaml @@ -31,12 +31,6 @@ exporters: - "some exporter list value_1" - "some exporter list value_2" -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] - extensions: exampleextension: extra: "some extension string" @@ -49,3 +43,9 @@ extensions: service: extensions: [exampleextension] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] + diff --git a/config/testdata/simple-config-with-partial-env.yaml b/config/testdata/simple-config-with-partial-env.yaml index be256990c01..7d3087b4a50 100644 --- a/config/testdata/simple-config-with-partial-env.yaml +++ b/config/testdata/simple-config-with-partial-env.yaml @@ -31,12 +31,6 @@ exporters: - "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1}" - "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2}" -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] - extensions: exampleextension: extra: "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA}" @@ -49,3 +43,9 @@ extensions: service: extensions: [exampleextension] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] + diff --git a/config/testdata/unknown-exporter-type.yaml b/config/testdata/unknown-exporter-type.yaml index f5c50bda17a..deb74322a5d 100644 --- a/config/testdata/unknown-exporter-type.yaml +++ b/config/testdata/unknown-exporter-type.yaml @@ -4,8 +4,9 @@ exporters: nosuchexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: [multireceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [multireceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/unknown-processor-type.yaml b/config/testdata/unknown-processor-type.yaml index 56dec81c121..02230c9d565 100644 --- a/config/testdata/unknown-processor-type.yaml +++ b/config/testdata/unknown-processor-type.yaml @@ -4,8 +4,9 @@ exporters: exampleexporter: processors: nosuchprocessor: -pipelines: - traces: - receivers: [examplereceiver] - exporters: [exampleexporter] - processors: [exampleprocessor] +service: + pipelines: + traces: + receivers: [examplereceiver] + exporters: [exampleexporter] + processors: [exampleprocessor] diff --git a/config/testdata/unknown-receiver-type.yaml b/config/testdata/unknown-receiver-type.yaml index d88682b2a41..eef9a5a7948 100644 --- a/config/testdata/unknown-receiver-type.yaml +++ b/config/testdata/unknown-receiver-type.yaml @@ -4,11 +4,12 @@ exporters: exampleexporter: processors: exampleprocessor: -pipelines: - traces: - receivers: - - multireceiver - exporters: - - exampleexporter - processors: - - exampleprocessor +service: + pipelines: + traces: + receivers: + - multireceiver + exporters: + - exampleexporter + processors: + - exampleprocessor diff --git a/config/testdata/valid-config.yaml b/config/testdata/valid-config.yaml index 5f4d389e9b3..12158fd750d 100644 --- a/config/testdata/valid-config.yaml +++ b/config/testdata/valid-config.yaml @@ -18,12 +18,6 @@ exporters: disabled: true exampleexporter: -pipelines: - traces: - receivers: [examplereceiver, examplereceiver/disabled] - processors: [exampleprocessor, exampleprocessor/disabled] - exporters: [exampleexporter/disabled, exampleexporter] - extensions: exampleextension/0: exampleextension/disabled: @@ -33,3 +27,9 @@ extensions: service: extensions: [exampleextension/0, exampleextension/disabled, exampleextension/1] + pipelines: + traces: + receivers: [examplereceiver, examplereceiver/disabled] + processors: [exampleprocessor, exampleprocessor/disabled] + exporters: [exampleexporter/disabled, exampleexporter] + diff --git a/docs/design.md b/docs/design.md index 8bb5701a1c2..9411172c5de 100644 --- a/docs/design.md +++ b/docs/design.md @@ -26,11 +26,12 @@ The pipeline is constructed during Collector startup based on pipeline definitio A pipeline configuration typically looks like this: ```yaml -pipelines: # section that can contain multiple subsections, one per pipeline - traces: # type of the pipeline - receivers: [opencensus, jaeger, zipkin] - processors: [tags, tail_sampling, batch, queued_retry] - exporters: [opencensus, jaeger, stackdriver, zipkin] +service: + pipelines: # section that can contain multiple subsections, one per pipeline + traces: # type of the pipeline + receivers: [opencensus, jaeger, zipkin] + processors: [tags, tail_sampling, batch, queued_retry] + exporters: [opencensus, jaeger, stackdriver, zipkin] ``` The above example defines a pipeline for “traces” type of telemetry data, with 3 receivers, 4 processors and 4 exporters. @@ -46,15 +47,16 @@ receivers: opencensus: endpoint: "localhost:55678” -pipelines: - traces: # a pipeline of “traces” type - receivers: [opencensus] - processors: [tags, tail_sampling, batch, queued_retry] - exporters: [jaeger] - traces/2: # another pipeline of “traces” type - receivers: [opencensus] - processors: [batch] - exporters: [opencensus] +service: + pipelines: + traces: # a pipeline of “traces” type + receivers: [opencensus] + processors: [tags, tail_sampling, batch, queued_retry] + exporters: [jaeger] + traces/2: # another pipeline of “traces” type + receivers: [opencensus] + processors: [batch] + exporters: [opencensus] ``` In the above example “opencensus” receiver will send the same data to pipeline “traces” and to pipeline “traces/2”. (Note: the configuration uses composite key names in the form of `type[/name]` as defined in this [this document](https://docs.google.com/document/d/1GWOzV0H0RTN1adiwo7fTmkjfCATDDFGuOB4jp3ldCc8/edit#)). @@ -88,15 +90,16 @@ exporters: grpc: endpoint: "localhost:14250” -pipelines: - traces: # a pipeline of “traces” type - receivers: [zipkin] - processors: [tags, tail_sampling, batch, queued_retry] - exporters: [jaeger] - traces/2: # another pipeline of “traces” type - receivers: [opencensus] - processors: [batch] - exporters: [jaeger] +service: + pipelines: + traces: # a pipeline of “traces” type + receivers: [zipkin] + processors: [tags, tail_sampling, batch, queued_retry] + exporters: [jaeger] + traces/2: # another pipeline of “traces” type + receivers: [opencensus] + processors: [batch] + exporters: [jaeger] ``` In the above example “jaeger” exporter will get data from pipeline “traces” and from pipeline “traces/2”. When the Collector loads this config the result will look like this (part of processors and receivers are omitted from the diagram for brevity): @@ -120,15 +123,16 @@ processors: per-exporter: true enabled: true -pipelines: - traces: # a pipeline of “traces” type - receivers: [zipkin] - processors: [queued_retry] - exporters: [jaeger] - traces/2: # another pipeline of “traces” type - receivers: [opencensus] - processors: [queued_retry] - exporters: [opencensus] +service: + pipelines: + traces: # a pipeline of “traces” type + receivers: [zipkin] + processors: [queued_retry] + exporters: [jaeger] + traces/2: # another pipeline of “traces” type + receivers: [opencensus] + processors: [queued_retry] + exporters: [opencensus] ``` When the Collector loads this config the result will look like this: diff --git a/examples/demo/otel-agent-config.yaml b/examples/demo/otel-agent-config.yaml index f8606e29913..7032ec8f7dc 100644 --- a/examples/demo/otel-agent-config.yaml +++ b/examples/demo/otel-agent-config.yaml @@ -1,7 +1,6 @@ receivers: opencensus: endpoint: localhost:55678 - reconnection_delay: 2s jaeger: protocols: thrift-http: @@ -18,15 +17,6 @@ processors: batch: queued_retry: -pipelines: - traces: - receivers: [opencensus, jaeger] - exporters: [opencensus, logging] - processors: [batch, queued_retry] - metrics: - receivers: [opencensus] - exporters: [logging,opencensus] - extensions: pprof: endpoint: localhost:1777 @@ -35,3 +25,12 @@ extensions: service: extensions: [pprof, zpages] + pipelines: + traces: + receivers: [opencensus, jaeger] + exporters: [opencensus, logging] + processors: [batch, queued_retry] + metrics: + receivers: [opencensus] + exporters: [logging,opencensus] + diff --git a/examples/demo/otel-collector-config.yaml b/examples/demo/otel-collector-config.yaml index 0a8eb1df764..da1ff5f94bd 100644 --- a/examples/demo/otel-collector-config.yaml +++ b/examples/demo/otel-collector-config.yaml @@ -26,15 +26,6 @@ processors: batch: queued_retry: -pipelines: - traces: - receivers: [opencensus] - exporters: [logging, zipkin, jaeger_grpc] - processors: [batch, queued_retry] - metrics: - receivers: [opencensus] - exporters: [logging,prometheus] - extensions: health_check: pprof: @@ -44,4 +35,12 @@ extensions: service: extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [opencensus] + exporters: [logging, zipkin, jaeger_grpc] + processors: [batch, queued_retry] + metrics: + receivers: [opencensus] + exporters: [logging,prometheus] diff --git a/exporter/jaeger/jaegergrpcexporter/testdata/config.yaml b/exporter/jaeger/jaegergrpcexporter/testdata/config.yaml index f3683a87348..7d36cebb4cd 100644 --- a/exporter/jaeger/jaegergrpcexporter/testdata/config.yaml +++ b/exporter/jaeger/jaegergrpcexporter/testdata/config.yaml @@ -10,8 +10,9 @@ exporters: jaeger_grpc/2: endpoint: "a.new.target:1234" -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [jaeger_grpc, jaeger_grpc/2] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [jaeger_grpc, jaeger_grpc/2] diff --git a/exporter/jaeger/jaegerthrifthttpexporter/testdata/config.yaml b/exporter/jaeger/jaegerthrifthttpexporter/testdata/config.yaml index e6304bf7c25..1cc840d4a4b 100644 --- a/exporter/jaeger/jaegerthrifthttpexporter/testdata/config.yaml +++ b/exporter/jaeger/jaegerthrifthttpexporter/testdata/config.yaml @@ -14,8 +14,9 @@ exporters: added-entry: "added value" dot.test: test -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [jaeger_thrift_http, jaeger_thrift_http/2] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [jaeger_thrift_http, jaeger_thrift_http/2] diff --git a/exporter/loggingexporter/testdata/config.yaml b/exporter/loggingexporter/testdata/config.yaml index ed67211f1ce..3e1c6004ef6 100644 --- a/exporter/loggingexporter/testdata/config.yaml +++ b/exporter/loggingexporter/testdata/config.yaml @@ -9,11 +9,12 @@ exporters: logging/2: loglevel: debug -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [logging] - metrics: - receivers: [examplereceiver] - exporters: [logging,logging/2] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [logging] + metrics: + receivers: [examplereceiver] + exporters: [logging,logging/2] diff --git a/exporter/opencensusexporter/testdata/config.yaml b/exporter/opencensusexporter/testdata/config.yaml index a64a41c5963..3dfb34dcfce 100644 --- a/exporter/opencensusexporter/testdata/config.yaml +++ b/exporter/opencensusexporter/testdata/config.yaml @@ -22,8 +22,9 @@ exporters: timeout: 30 permit_without_stream: true -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [opencensus] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [opencensus] diff --git a/exporter/prometheusexporter/testdata/config.yaml b/exporter/prometheusexporter/testdata/config.yaml index 026094819d3..3159ea97450 100644 --- a/exporter/prometheusexporter/testdata/config.yaml +++ b/exporter/prometheusexporter/testdata/config.yaml @@ -13,8 +13,9 @@ exporters: label1: value1 "another label": spaced value -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [prometheus] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [prometheus] diff --git a/exporter/zipkinexporter/testdata/config.yaml b/exporter/zipkinexporter/testdata/config.yaml index 58e3e10c967..288633a559e 100644 --- a/exporter/zipkinexporter/testdata/config.yaml +++ b/exporter/zipkinexporter/testdata/config.yaml @@ -10,8 +10,9 @@ exporters: zipkin/2: url: "https://somedest:1234/api/v2/spans" -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [zipkin, zipkin/2] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [zipkin, zipkin/2] diff --git a/extension/healthcheckextension/testdata/config.yaml b/extension/healthcheckextension/testdata/config.yaml index fd0de6544c3..79b7d9758c1 100644 --- a/extension/healthcheckextension/testdata/config.yaml +++ b/extension/healthcheckextension/testdata/config.yaml @@ -5,6 +5,11 @@ extensions: service: extensions: [health_check/1] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] # Data pipeline is required to load the config. receivers: @@ -13,8 +18,3 @@ processors: exampleprocessor: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] diff --git a/extension/pprofextension/testdata/config.yaml b/extension/pprofextension/testdata/config.yaml index 63e757ebf72..9b332d8fa6c 100644 --- a/extension/pprofextension/testdata/config.yaml +++ b/extension/pprofextension/testdata/config.yaml @@ -7,6 +7,11 @@ extensions: service: extensions: [pprof/1] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] # Data pipeline is required to load the config. receivers: @@ -15,8 +20,3 @@ processors: exampleprocessor: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] diff --git a/extension/zpagesextension/testdata/config.yaml b/extension/zpagesextension/testdata/config.yaml index fd71ed1a1cd..6819540571b 100644 --- a/extension/zpagesextension/testdata/config.yaml +++ b/extension/zpagesextension/testdata/config.yaml @@ -5,6 +5,11 @@ extensions: service: extensions: [zpages/1] + pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [exampleexporter] # Data pipeline is required to load the config. receivers: @@ -13,8 +18,3 @@ processors: exampleprocessor: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [exampleprocessor] - exporters: [exampleexporter] diff --git a/processor/attributesprocessor/testdata/config.yaml b/processor/attributesprocessor/testdata/config.yaml index 603be1456b1..e965e799476 100644 --- a/processor/attributesprocessor/testdata/config.yaml +++ b/processor/attributesprocessor/testdata/config.yaml @@ -198,10 +198,11 @@ receivers: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [attributes/insert] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [attributes/insert] + exporters: [exampleexporter] diff --git a/processor/nodebatcherprocessor/testdata/config.yaml b/processor/nodebatcherprocessor/testdata/config.yaml index 2d469b595f4..8ab2f0cc7b7 100644 --- a/processor/nodebatcherprocessor/testdata/config.yaml +++ b/processor/nodebatcherprocessor/testdata/config.yaml @@ -13,8 +13,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [batch/2] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [batch/2] + exporters: [exampleexporter] diff --git a/processor/probabilisticsamplerprocessor/testdata/config.yaml b/processor/probabilisticsamplerprocessor/testdata/config.yaml index 6d1726f1571..32a0043ffc6 100644 --- a/processor/probabilisticsamplerprocessor/testdata/config.yaml +++ b/processor/probabilisticsamplerprocessor/testdata/config.yaml @@ -9,8 +9,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [probabilistic_sampler] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [probabilistic_sampler] + exporters: [exampleexporter] diff --git a/processor/probabilisticsamplerprocessor/testdata/empty.yaml b/processor/probabilisticsamplerprocessor/testdata/empty.yaml index b67c1aaec36..c837bd77f0e 100644 --- a/processor/probabilisticsamplerprocessor/testdata/empty.yaml +++ b/processor/probabilisticsamplerprocessor/testdata/empty.yaml @@ -8,8 +8,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [probabilistic_sampler] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [probabilistic_sampler] + exporters: [exampleexporter] diff --git a/processor/queuedprocessor/testdata/config.yaml b/processor/queuedprocessor/testdata/config.yaml index 52918ad3b90..095948de2a3 100644 --- a/processor/queuedprocessor/testdata/config.yaml +++ b/processor/queuedprocessor/testdata/config.yaml @@ -12,8 +12,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [queued_retry/2] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [queued_retry/2] + exporters: [exampleexporter] diff --git a/processor/spanprocessor/testdata/config.yaml b/processor/spanprocessor/testdata/config.yaml index 8c2d0fda50b..2ac0b5b500c 100644 --- a/processor/spanprocessor/testdata/config.yaml +++ b/processor/spanprocessor/testdata/config.yaml @@ -46,8 +46,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [examplereceiver] - processors: [span/custom] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [span/custom] + exporters: [exampleexporter] diff --git a/processor/tailsamplingprocessor/testdata/tail_sampling_config.yaml b/processor/tailsamplingprocessor/testdata/tail_sampling_config.yaml index 35b5b0c5efa..10dd8dd9395 100644 --- a/processor/tailsamplingprocessor/testdata/tail_sampling_config.yaml +++ b/processor/tailsamplingprocessor/testdata/tail_sampling_config.yaml @@ -32,8 +32,9 @@ processors: } ] -pipelines: - traces: - receivers: [examplereceiver] - processors: [tail_sampling] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver] + processors: [tail_sampling] + exporters: [exampleexporter] diff --git a/receiver/jaegerreceiver/testdata/config.yaml b/receiver/jaegerreceiver/testdata/config.yaml index 5471c09eff4..fb34391528d 100644 --- a/receiver/jaegerreceiver/testdata/config.yaml +++ b/receiver/jaegerreceiver/testdata/config.yaml @@ -53,9 +53,10 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [jaeger] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [jaeger] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/receiver/opencensusreceiver/testdata/config.yaml b/receiver/opencensusreceiver/testdata/config.yaml index 420e8afe626..58c79a123cb 100644 --- a/receiver/opencensusreceiver/testdata/config.yaml +++ b/receiver/opencensusreceiver/testdata/config.yaml @@ -60,11 +60,12 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [opencensus/customname] - processors: [exampleprocessor] - exporters: [exampleexporter] - metrics: - receivers: [opencensus] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [opencensus/customname] + processors: [exampleprocessor] + exporters: [exampleexporter] + metrics: + receivers: [opencensus] + exporters: [exampleexporter] diff --git a/receiver/prometheusreceiver/testdata/config.yaml b/receiver/prometheusreceiver/testdata/config.yaml index fb99f32f7bf..dae24b91c81 100644 --- a/receiver/prometheusreceiver/testdata/config.yaml +++ b/receiver/prometheusreceiver/testdata/config.yaml @@ -19,8 +19,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [prometheus] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [prometheus] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/receiver/prometheusreceiver/testdata/invalid-config-section.yaml b/receiver/prometheusreceiver/testdata/invalid-config-section.yaml index 66d4757a0fe..9dfe6cebab3 100644 --- a/receiver/prometheusreceiver/testdata/invalid-config-section.yaml +++ b/receiver/prometheusreceiver/testdata/invalid-config-section.yaml @@ -12,8 +12,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [prometheus] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [prometheus] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/receiver/vmmetricsreceiver/testdata/config.yaml b/receiver/vmmetricsreceiver/testdata/config.yaml index 35a39afaca3..85f46fd3782 100644 --- a/receiver/vmmetricsreceiver/testdata/config.yaml +++ b/receiver/vmmetricsreceiver/testdata/config.yaml @@ -12,8 +12,9 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [vmmetrics] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [vmmetrics] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/receiver/zipkinreceiver/testdata/config.yaml b/receiver/zipkinreceiver/testdata/config.yaml index 9b45f1c20a2..f17cca708d3 100644 --- a/receiver/zipkinreceiver/testdata/config.yaml +++ b/receiver/zipkinreceiver/testdata/config.yaml @@ -9,9 +9,10 @@ processors: exporters: exampleexporter: -pipelines: - traces: - receivers: [zipkin] - processors: [exampleprocessor] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [zipkin] + processors: [exampleprocessor] + exporters: [exampleexporter] diff --git a/service/builder/exporters_builder.go b/service/builder/exporters_builder.go index e54f5690a55..74e41995f9c 100644 --- a/service/builder/exporters_builder.go +++ b/service/builder/exporters_builder.go @@ -121,7 +121,7 @@ func (eb *ExportersBuilder) calcExportersRequiredDataTypes() exportersRequiredDa result := make(exportersRequiredDataTypes) // Iterate over pipelines. - for _, pipeline := range eb.config.Pipelines { + for _, pipeline := range eb.config.Service.Pipelines { // Iterate over all exporters for this pipeline. for _, expName := range pipeline.Exporters { // Find the exporter config by name. diff --git a/service/builder/exporters_builder_test.go b/service/builder/exporters_builder_test.go index 736c1ecdbe6..51cd6f386b3 100644 --- a/service/builder/exporters_builder_test.go +++ b/service/builder/exporters_builder_test.go @@ -46,11 +46,13 @@ func TestExportersBuilder_Build(t *testing.T) { }, }, - Pipelines: map[string]*configmodels.Pipeline{ - "trace": { - Name: "trace", - InputType: configmodels.TracesDataType, - Exporters: []string{"opencensus"}, + Service: configmodels.Service{ + Pipelines: map[string]*configmodels.Pipeline{ + "trace": { + Name: "trace", + InputType: configmodels.TracesDataType, + Exporters: []string{"opencensus"}, + }, }, }, } @@ -80,7 +82,7 @@ func TestExportersBuilder_Build(t *testing.T) { // Remove the pipeline so that the exporter is not attached to any pipeline. // This should result in creating an exporter that has none of consumption // functions set. - delete(cfg.Pipelines, "trace") + delete(cfg.Service.Pipelines, "trace") exporters, err = NewExportersBuilder(zap.NewNop(), cfg, factories.Exporters).Build() assert.NotNil(t, exporters) assert.Nil(t, err) diff --git a/service/builder/pipelines_builder.go b/service/builder/pipelines_builder.go index 72ed725d750..2dc1489ef92 100644 --- a/service/builder/pipelines_builder.go +++ b/service/builder/pipelines_builder.go @@ -62,7 +62,7 @@ func NewPipelinesBuilder( func (pb *PipelinesBuilder) Build() (BuiltPipelines, error) { pipelineProcessors := make(BuiltPipelines) - for _, pipeline := range pb.config.Pipelines { + for _, pipeline := range pb.config.Service.Pipelines { firstProcessor, err := pb.buildPipeline(pipeline) if err != nil { return nil, err diff --git a/service/builder/pipelines_builder_test.go b/service/builder/pipelines_builder_test.go index 605f5950bfa..a87f2faa7da 100644 --- a/service/builder/pipelines_builder_test.go +++ b/service/builder/pipelines_builder_test.go @@ -94,7 +94,7 @@ func testPipeline(t *testing.T, pipelineName string, exporterNames []string) { assert.NoError(t, err) require.NotNil(t, pipelineProcessors) - processor := pipelineProcessors[cfg.Pipelines[pipelineName]] + processor := pipelineProcessors[cfg.Service.Pipelines[pipelineName]] // Ensure pipeline has its fields correctly populated. require.NotNil(t, processor) @@ -164,7 +164,7 @@ func TestPipelinesBuilder_Error(t *testing.T) { // Corrupt the pipeline, change data type to metrics. We have to forcedly do it here // since there is no way to have such config loaded by LoadConfigFile, it would not // pass validation. We are doing this to test failure mode of PipelinesBuilder. - pipeline := cfg.Pipelines["traces"] + pipeline := cfg.Service.Pipelines["traces"] pipeline.InputType = configmodels.MetricsDataType exporters, err := NewExportersBuilder(zap.NewNop(), cfg, factories.Exporters).Build() diff --git a/service/builder/receivers_builder.go b/service/builder/receivers_builder.go index 4598dd6b63a..1a5aefeb0b9 100644 --- a/service/builder/receivers_builder.go +++ b/service/builder/receivers_builder.go @@ -154,7 +154,7 @@ func (rb *ReceiversBuilder) findPipelinesToAttach(config configmodels.Receiver) pipelinesToAttach[configmodels.MetricsDataType] = make([]*builtPipeline, 0) // Iterate over all pipelines. - for _, pipelineCfg := range rb.config.Pipelines { + for _, pipelineCfg := range rb.config.Service.Pipelines { // Get the first processor of the pipeline. pipelineProcessor := rb.builtPipelines[pipelineCfg] if pipelineProcessor == nil { diff --git a/service/builder/testdata/pipelines_builder.yaml b/service/builder/testdata/pipelines_builder.yaml index f4464ac5406..b303bb9bddd 100644 --- a/service/builder/testdata/pipelines_builder.yaml +++ b/service/builder/testdata/pipelines_builder.yaml @@ -15,25 +15,26 @@ exporters: exampleexporter: exampleexporter/2: -pipelines: - traces: - receivers: [examplereceiver, examplereceiver/multi] - processors: [attributes] - exporters: [exampleexporter] +service: + pipelines: + traces: + receivers: [examplereceiver, examplereceiver/multi] + processors: [attributes] + exporters: [exampleexporter] - traces/2: - receivers: [examplereceiver/2, examplereceiver/multi] - processors: [attributes] - exporters: [exampleexporter, exampleexporter/2] + traces/2: + receivers: [examplereceiver/2, examplereceiver/multi] + processors: [attributes] + exporters: [exampleexporter, exampleexporter/2] - metrics: - receivers: [examplereceiver] - exporters: [exampleexporter] + metrics: + receivers: [examplereceiver] + exporters: [exampleexporter] - metrics/2: - receivers: [examplereceiver/3] - exporters: [exampleexporter] + metrics/2: + receivers: [examplereceiver/3] + exporters: [exampleexporter] - metrics/3: - receivers: [examplereceiver/3] - exporters: [exampleexporter/2] + metrics/3: + receivers: [examplereceiver/3] + exporters: [exampleexporter/2] diff --git a/service/testdata/otelcol-config.yaml b/service/testdata/otelcol-config.yaml index fae64408889..cef8e038643 100644 --- a/service/testdata/otelcol-config.yaml +++ b/service/testdata/otelcol-config.yaml @@ -14,12 +14,6 @@ processors: queued_retry: batch: -pipelines: - traces: - receivers: [jaeger] - processors: [attributes, batch, queued_retry] - exporters: [opencensus] - extensions: health_check: pprof: @@ -27,3 +21,9 @@ extensions: service: extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [jaeger] + processors: [attributes, batch, queued_retry] + exporters: [opencensus] + diff --git a/testbed/tests/testdata/add-attributes-config.yaml b/testbed/tests/testdata/add-attributes-config.yaml index b34c10b4502..c50b8b19057 100644 --- a/testbed/tests/testdata/add-attributes-config.yaml +++ b/testbed/tests/testdata/add-attributes-config.yaml @@ -28,8 +28,9 @@ processors: value: test-datacenter action: insert -pipelines: - traces: - receivers: [jaeger] - processors: [attributes,queued_retry] - exporters: [opencensus] +service: + pipelines: + traces: + receivers: [jaeger] + processors: [attributes,queued_retry] + exporters: [opencensus] diff --git a/testbed/tests/testdata/agent-config.yaml b/testbed/tests/testdata/agent-config.yaml index e5d1bfb4429..e6b7e11cd9e 100644 --- a/testbed/tests/testdata/agent-config.yaml +++ b/testbed/tests/testdata/agent-config.yaml @@ -11,8 +11,9 @@ exporters: processors: queued_retry: -pipelines: - traces: - receivers: [jaeger] - processors: [queued_retry] - exporters: [opencensus] +service: + pipelines: + traces: + receivers: [jaeger] + processors: [queued_retry] + exporters: [opencensus]