[chore] Update default metric_relabel_configs for internal metrics

grpc and http metrics are now only sent in detailed level. So, no need to exclude them. However, there is another redundant metric emitted by default in 0.119.0 after the transition to OTel SDK, `promhttp_metric_handler_errors`. This change updates the default configs and the appropriate converter to exclude this metric.
signalfx · Feb 18, 2025 · 6d1a614 · 6d1a614
1 parent 5971796
commit 6d1a614
Show file tree

Hide file tree

Showing 19 changed files with 79 additions and 67 deletions.
diff --git a/cmd/otelcol/config/collector/agent_config.yaml b/cmd/otelcol/config/collector/agent_config.yaml
@@ -74,10 +74,7 @@ receivers:
         - targets: ["0.0.0.0:8888"]
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/config/collector/ecs_ec2_config.yaml b/cmd/otelcol/config/collector/ecs_ec2_config.yaml
@@ -60,10 +60,7 @@ receivers:
             - targets: ['0.0.0.0:8888']
           metric_relabel_configs:
             - source_labels: [ __name__ ]
-              regex: 'otelcol_rpc_.*'
-              action: drop
-            - source_labels: [ __name__ ]
-              regex: 'otelcol_http_.*'
+              regex: 'promhttp_metric_handler_errors.*'
               action: drop
             - source_labels: [ __name__ ]
               regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/config/collector/fargate_config.yaml b/cmd/otelcol/config/collector/fargate_config.yaml
@@ -35,10 +35,7 @@ receivers:
             - targets: ['0.0.0.0:8888']
           metric_relabel_configs:
             - source_labels: [ __name__ ]
-              regex: 'otelcol_rpc_.*'
-              action: drop
-            - source_labels: [ __name__ ]
-              regex: 'otelcol_http_.*'
+              regex: 'promhttp_metric_handler_errors.*'
               action: drop
             - source_labels: [ __name__ ]
               regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/config/collector/full_config_linux.yaml b/cmd/otelcol/config/collector/full_config_linux.yaml
@@ -136,10 +136,7 @@ receivers:
         - targets: ['0.0.0.0:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/config/collector/gateway_config.yaml b/cmd/otelcol/config/collector/gateway_config.yaml
@@ -40,10 +40,7 @@ receivers:
         - targets: ['0.0.0.0:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/config/collector/otlp_config_linux.yaml b/cmd/otelcol/config/collector/otlp_config_linux.yaml
@@ -29,10 +29,7 @@ receivers:
         - targets: ['0.0.0.0:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/config/collector/upstream_agent_config.yaml b/cmd/otelcol/config/collector/upstream_agent_config.yaml
@@ -72,10 +72,7 @@ receivers:
         - targets: ['0.0.0.0:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/fips/config/agent_config.yaml b/cmd/otelcol/fips/config/agent_config.yaml
@@ -68,10 +68,7 @@ receivers:
         - targets: ["0.0.0.0:8888"]
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/fips/config/ecs_ec2_config.yaml b/cmd/otelcol/fips/config/ecs_ec2_config.yaml
@@ -59,10 +59,7 @@ receivers:
             - targets: ['0.0.0.0:8888']
           metric_relabel_configs:
             - source_labels: [ __name__ ]
-              regex: 'otelcol_rpc_.*'
-              action: drop
-            - source_labels: [ __name__ ]
-              regex: 'otelcol_http_.*'
+              regex: 'promhttp_metric_handler_errors.*'
               action: drop
             - source_labels: [ __name__ ]
               regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/fips/config/fargate_config.yaml b/cmd/otelcol/fips/config/fargate_config.yaml
@@ -35,10 +35,7 @@ receivers:
             - targets: ['0.0.0.0:8888']
           metric_relabel_configs:
             - source_labels: [ __name__ ]
-              regex: 'otelcol_rpc_.*'
-              action: drop
-            - source_labels: [ __name__ ]
-              regex: 'otelcol_http_.*'
+              regex: 'promhttp_metric_handler_errors.*'
               action: drop
             - source_labels: [ __name__ ]
               regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/fips/config/gateway_config.yaml b/cmd/otelcol/fips/config/gateway_config.yaml
@@ -40,10 +40,7 @@ receivers:
         - targets: ['0.0.0.0:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/cmd/otelcol/fips/config/otlp_config_linux.yaml b/cmd/otelcol/fips/config/otlp_config_linux.yaml
@@ -29,10 +29,7 @@ receivers:
         - targets: ['0.0.0.0:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/internal/configconverter/disable_excessive_internal_metrics.go b/internal/configconverter/disable_excessive_internal_metrics.go
@@ -36,16 +36,17 @@ var promScrapeConfigsKeys = []string{
 	"receivers::prometheus/collector::config::scrape_configs",
 }
 
-// The metric_relabel_configs prometheus config section to replace.
-var metricRelabelConfigsToReplace = []any{
+// metricRelabelConfigsV1 is the first version of prometheus metric_relabel_configs used in the Splunk distribution.
+var metricRelabelConfigsV1 = []any{
 	map[string]any{
 		"source_labels": []any{model.MetricNameLabel},
 		"regex":         ".*grpc_io.*",
 		"action":        string(relabel.Drop),
 	},
 }
 
-var metricRelabelConfigsToSet = []any{
+// metricRelabelConfigsV2 is the second version of prometheus metric_relabel_configs used in the Splunk distribution.
+var metricRelabelConfigsV2 = []any{
 	map[string]any{
 		"source_labels": []any{model.MetricNameLabel},
 		"regex":         "otelcol_rpc_.*",
@@ -63,10 +64,23 @@ var metricRelabelConfigsToSet = []any{
 	},
 }
 
+// metricRelabelConfigsCurrent is the current version of prometheus metric_relabel_configs used in the Splunk distribution.
+var metricRelabelConfigsCurrent = []any{
+	map[string]any{
+		"source_labels": []any{model.MetricNameLabel},
+		"regex":         "promhttp_metric_handler_errors.*",
+		"action":        string(relabel.Drop),
+	},
+	map[string]any{
+		"source_labels": []any{model.MetricNameLabel},
+		"regex":         "otelcol_processor_batch_.*",
+		"action":        string(relabel.Drop),
+	},
+}
+
 // DisableExcessiveInternalMetrics updates config of the prometheus receiver scraping internal
 // collector metrics to drop excessive internal metrics matching the following patterns:
-// - "otelcol_rpc_.*"
-// - "otelcol_http_.*"
+// - "promhttp_metric_handler_errors.*"
 // - "otelcol_processor_batch_.*"
 func DisableExcessiveInternalMetrics(_ context.Context, cfgMap *confmap.Conf) error {
 	if cfgMap == nil {
@@ -103,9 +117,9 @@ func DisableExcessiveInternalMetrics(_ context.Context, cfgMap *confmap.Conf) er
 				continue // Ignore invalid metric_relabel_configs, as they will be caught by the config validation.
 			}
 
-			// Replace the metric_relabel_configs only if it's set to the old default value.
-			if len(mrcs) == 1 && reflect.DeepEqual(mrcs[0], metricRelabelConfigsToReplace[0]) {
-				sc["metric_relabel_configs"] = metricRelabelConfigsToSet
+			// Replace the metric_relabel_configs only if it's set to the old default values.
+			if reflect.DeepEqual(mrcs, metricRelabelConfigsV1) || reflect.DeepEqual(mrcs, metricRelabelConfigsV2) {
+				sc["metric_relabel_configs"] = metricRelabelConfigsCurrent
 			}
 		}
 

diff --git a/internal/configconverter/disable_excessive_internal_metrics_test.go b/internal/configconverter/disable_excessive_internal_metrics_test.go
@@ -55,9 +55,14 @@ func TestDisableExcessiveInternalMetrics(t *testing.T) {
 			wantOutput: "testdata/disable_excessive_internal_metrics/metric_relabel_configs_with_other_actions.yaml",
 		},
 		{
-			name:       "metric_relabel_configs_with_batch_drop_action",
-			input:      "testdata/disable_excessive_internal_metrics/old_metric_relabel_configs_present_input.yaml",
-			wantOutput: "testdata/disable_excessive_internal_metrics/old_metric_relabel_configs_present_output.yaml",
+			name:       "metric_relabel_configs_v1_to_current",
+			input:      "testdata/disable_excessive_internal_metrics/old_v1_metric_relabel_configs_present_input.yaml",
+			wantOutput: "testdata/disable_excessive_internal_metrics/current_metric_relabel_configs_present_output.yaml",
+		},
+		{
+			name:       "metric_relabel_configs_v2_to_current",
+			input:      "testdata/disable_excessive_internal_metrics/old_v2_metric_relabel_configs_present_input.yaml",
+			wantOutput: "testdata/disable_excessive_internal_metrics/current_metric_relabel_configs_present_output.yaml",
 		},
 		{
 			name:       "all_metric_relabel_configs_are_present",

diff --git a/...ata/disable_excessive_internal_metrics/current_metric_relabel_configs_present_output.yaml b/...ata/disable_excessive_internal_metrics/current_metric_relabel_configs_present_output.yaml
@@ -0,0 +1,35 @@
+receivers:
+  prometheus/internal:
+    config:
+      scrape_configs:
+        - job_name: 'otel-collector'
+          scrape_interval: 10s
+          static_configs:
+            - targets: ["${SPLUNK_LISTEN_INTERFACE}:8888"]
+          metric_relabel_configs:
+            - source_labels: [ __name__ ]
+              regex: 'promhttp_metric_handler_errors.*'
+              action: drop
+            - source_labels: [ __name__ ]
+              regex: 'otelcol_processor_batch_.*'
+              action: drop
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 4000
+    spike_limit_mib: 800
+    ballast_size_mib: 64
+exporters:
+  debug:
+    verbosity: normal
+    sampling_initial: 2
+    sampling_thereafter: 500
+service:
+  pipelines:
+    metrics:
+      receivers:
+        - prometheus/internal
+      processors:
+        - memory_limiter
+      exporters:
+        - logging
diff --git a/...metric_relabel_configs_present_input.yaml → ...metric_relabel_configs_present_input.yaml b/...metric_relabel_configs_present_input.yaml → ...metric_relabel_configs_present_input.yaml
diff --git a/...etric_relabel_configs_present_output.yaml → ...metric_relabel_configs_present_input.yaml b/...etric_relabel_configs_present_output.yaml → ...metric_relabel_configs_present_input.yaml
diff --git a/packaging/technical-addon/Splunk_TA_otel/configs/ta-agent-to-gateway-config.yaml b/packaging/technical-addon/Splunk_TA_otel/configs/ta-agent-to-gateway-config.yaml
@@ -73,10 +73,7 @@ receivers:
         - targets: ["${env:SPLUNK_LISTEN_INTERFACE}:8888"]
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'

diff --git a/packaging/technical-addon/Splunk_TA_otel/configs/ta-gateway-config.yaml b/packaging/technical-addon/Splunk_TA_otel/configs/ta-gateway-config.yaml
@@ -40,10 +40,7 @@ receivers:
         - targets: ['${env:SPLUNK_LISTEN_INTERFACE}:8888']
         metric_relabel_configs:
           - source_labels: [ __name__ ]
-            regex: 'otelcol_rpc_.*'
-            action: drop
-          - source_labels: [ __name__ ]
-            regex: 'otelcol_http_.*'
+            regex: 'promhttp_metric_handler_errors.*'
             action: drop
           - source_labels: [ __name__ ]
             regex: 'otelcol_processor_batch_.*'