fix(log_to_metric transform): set not working in log-to-metric transf…

…orm when all_metrics=true (#20228)

* Fixing bug - set missing in log-to-metric transform

* Adding a test case

* Adding unit test

* Create 20228_fix-log-to-metric-set-supported.fix.md

* Update 20228_fix-log-to-metric-set-supported.fix.md

* Update changelog.d/20228_fix-log-to-metric-set-supported.fix.md

Co-authored-by: Jesse Szwedko <[email protected]>

* Update changelog.d/20228_fix-log-to-metric-set-supported.fix.md

Co-authored-by: Jesse Szwedko <[email protected]>

* cargo fmt

Signed-off-by: Jesse Szwedko <[email protected]>

---------

Signed-off-by: Jesse Szwedko <[email protected]>
Co-authored-by: Jesse Szwedko <[email protected]>
Co-authored-by: Jesse Szwedko <[email protected]>

changelog.d/20228_fix-log-to-metric-set-supported.fix.md

@@ -0,0 +1,3 @@
+Fixed an issue where the log_to_metric transform with all_metrics=true config failed to convert properly-formatted 'set'-type events into metrics.
+
+authors: pabloem

src/transforms/log_to_metric.rs

@@ -493,6 +493,34 @@ fn get_gauge_value(log: &LogEvent) -> Result<MetricValue, TransformError> {
     })
 }
 
+fn get_set_value(log: &LogEvent) -> Result<MetricValue, TransformError> {
+    let set_values = log
+        .get(event_path!("set", "values"))
+        .ok_or_else(|| TransformError::PathNotFound {
+            path: "set.values".to_string(),
+        })?
+        .as_array()
+        .ok_or_else(|| TransformError::ParseError {
+            path: "set.values".to_string(),
+            kind: TransformParseErrorKind::ArrayError,
+        })?;
+
+    let mut values: Vec<String> = Vec::new();
+    for e_value in set_values {
+        let value = e_value
+            .as_bytes()
+            .ok_or_else(|| TransformError::ParseError {
+                path: "set.values".to_string(),
+                kind: TransformParseErrorKind::ArrayError,
+            })?;
+        values.push(String::from_utf8_lossy(value).to_string());
+    }
+
+    Ok(MetricValue::Set {
+        values: values.into_iter().collect(),
+    })
+}
+
 fn get_distribution_value(log: &LogEvent) -> Result<MetricValue, TransformError> {
     let event_samples = log
         .get(event_path!("distribution", "samples"))
@@ -755,6 +783,7 @@ fn to_metrics(event: &Event) -> Result<Metric, TransformError> {
                 "histogram" => Some(get_histogram_value(log)?),
                 "summary" => Some(get_summary_value(log)?),
                 "counter" => Some(get_counter_value(log)?),
+                "set" => Some(get_set_value(log)?),
                 _ => None,
             };
 
@@ -1933,4 +1962,45 @@ mod tests {
             .with_timestamp(Some(ts()))
         );
     }
+
+    #[tokio::test]
+    async fn transform_set() {
+        let config = parse_yaml_config(
+            r#"
+            metrics: []
+            all_metrics: true
+            "#,
+        );
+
+        let json_str = r#"{
+          "set": {
+            "values": ["990.0", "1234"]
+          },
+          "kind": "incremental",
+          "name": "test.transform.set",
+          "tags": {
+            "env": "test_env",
+            "host": "localhost"
+          }
+        }"#;
+        let log = create_log_event(json_str);
+        let metric = do_transform(config, log.clone()).await.unwrap();
+        assert_eq!(
+            *metric.as_metric(),
+            Metric::new_with_metadata(
+                "test.transform.set",
+                MetricKind::Incremental,
+                MetricValue::Set {
+                    values: vec!["990.0".into(), "1234".into()].into_iter().collect()
+                },
+                metric.metadata().clone(),
+            )
+            .with_namespace(Some("test_namespace"))
+            .with_tags(Some(metric_tags!(
+                "env" => "test_env",
+                "host" => "localhost",
+            )))
+            .with_timestamp(Some(ts()))
+        );
+    }
 }

testing/github-20228/config.toml

@@ -0,0 +1,46 @@
+
+[[tests]]
+name = "Test log_to_metric conversion of sets"
+
+# The inputs for the test
+[[tests.inputs]]
+insert_at = "parse_json"
+type = "log"
+
+[tests.inputs.log_fields]
+message = '{"name": "sample.set.metric", "tags": {"host": "my-host", "region": "us-west"}, "kind": "incremental", "values": [1, 2, 3, 4, 5]}'
+
+[[tests.outputs]]
+extract_from = "convert_metrics"
+
+# We just validate that the values are the same
+[[tests.outputs.conditions]]
+type = "vrl"
+source = '''
+assert!(.name == "sample.set.metric")
+assert!(.tags.host == "my-host")
+assert!(.tags.region == "us-west")
+assert!(.kind == "incremental")
+'''
+
+[sources.stdin]
+type = "stdin"
+
+[sinks.stdout]
+inputs = ["convert_metrics"]
+type = "console"
+encoding.codec = "json"
+
+[transforms.parse_json]
+inputs = ["stdin"]
+type = "remap"
+source = '''
+. = parse_json!(.message)
+'''
+
+[transforms.convert_metrics]
+inputs = ["parse_json"]
+type = "log_to_metric"
+all_metrics = true
+metrics = []
+