From 4176a89fd790a55768854ea26acd2f716dc89f33 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 25 Jan 2024 15:27:10 -0700 Subject: [PATCH 1/3] fix(observability): propagate tracing span context in stream sink request building --- src/sinks/util/builder.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sinks/util/builder.rs b/src/sinks/util/builder.rs index bdf98a21df1f4..45c01ce41dcdd 100644 --- a/src/sinks/util/builder.rs +++ b/src/sinks/util/builder.rs @@ -12,6 +12,7 @@ use std::{ use futures_util::{stream::Map, Stream, StreamExt}; use pin_project::pin_project; use tower::Service; +use tracing::Span; use vector_lib::stream::{ batcher::{config::BatchConfig, Batcher}, ConcurrentMap, Driver, DriverResponse, ExpirationQueue, PartitionedBatcher, @@ -115,10 +116,17 @@ pub trait SinkBuilderExt: Stream { { let builder = Arc::new(builder); + // The future passed into the concurrent map is spawned in a tokio thread so we must preserve + // the span context in order to propagate the sink's automatic tags. + let span = Arc::new(Span::current()); + self.concurrent_map(limit, move |input| { let builder = Arc::clone(&builder); + let span = Arc::clone(&span); Box::pin(async move { + let _entered = span.enter(); + // Split the input into metadata and events. let (metadata, request_metadata_builder, events) = builder.split_input(input); From 95a3c723db881d3eb95ebe5d14dd5b4aa9b606f9 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 25 Jan 2024 15:34:12 -0700 Subject: [PATCH 2/3] changelog --- ...gate_tracing_span_context_stream_sink_request_building.fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md diff --git a/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md b/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md new file mode 100644 index 0000000000000..1fd98f59de401 --- /dev/null +++ b/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md @@ -0,0 +1 @@ +Vector logs and internal metrics emitted while in the context of stream sink request building, now correctly contain the component's automatic tags (`component_kind`, `component_type`, `component_id`). From 59c1b40b1362d92a6ea6784d109771bcf76eecf5 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 25 Jan 2024 17:13:39 -0700 Subject: [PATCH 3/3] feedback js --- ...ontext_stream_sink_request_building.fix.md | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md b/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md index 1fd98f59de401..8bc7a2229da0b 100644 --- a/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md +++ b/changelog.d/19712_propagate_tracing_span_context_stream_sink_request_building.fix.md @@ -1 +1,36 @@ -Vector logs and internal metrics emitted while in the context of stream sink request building, now correctly contain the component's automatic tags (`component_kind`, `component_type`, `component_id`). +The following metrics now correctly have the `component_kind`, `component_type`, and `component_id` tags: + - `component_errors_total` + - `component_discarded_events_total` + +For the following sinks: + - `splunk_hec` + - `clickhouse` + - `loki` + - `redis` + - `azure_blob` + - `azure_monitor_logs` + - `webhdfs` + - `appsignal` + - `amqp` + - `aws_kinesis` + - `statsd` + - `honeycomb` + - `gcp_stackdriver_metrics` + - `gcs_chronicle_unstructured` + - `gcp_stackdriver_logs` + - `gcp_pubsub` + - `gcp_cloud_storage` + - `nats` + - `http` + - `kafka` + - `new_relic` + - `datadog_metrics` + - `datadog_traces` + - `datadog_events` + - `databend` + - `prometheus_remote_write` + - `pulsar` + - `aws_s3` + - `aws_sqs` + - `aws_sns` + - `elasticsearch`