From cea535af79aa59e9d21afad80fe33aac19364230 Mon Sep 17 00:00:00 2001 From: "josue.agbekodo@datadoghq.com" Date: Wed, 22 Jan 2025 19:55:55 +0100 Subject: [PATCH] feat(datadog_search transforms): support filtering on ddtags in datadog search syntax --- .../datadog_search_syntax_support_ddtags.md | 3 + src/conditions/datadog_search.rs | 344 ++++++++++++++++-- 2 files changed, 308 insertions(+), 39 deletions(-) create mode 100644 changelog.d/datadog_search_syntax_support_ddtags.md diff --git a/changelog.d/datadog_search_syntax_support_ddtags.md b/changelog.d/datadog_search_syntax_support_ddtags.md new file mode 100644 index 0000000000000..8ef9acea705c4 --- /dev/null +++ b/changelog.d/datadog_search_syntax_support_ddtags.md @@ -0,0 +1,3 @@ +In Datadog (Observability Pipelines) transforms, perform a `ddtags` array lookup every time one on `tags` is done for filtering logs + +authors: 20agbekodo diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 87bf97affee38..5f9756941d8ed 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -86,13 +86,13 @@ impl Filter for EventFilter { Field::Tag(tag) => { let starts_with = format!("{}:", tag); - any_string_match("tags", move |value| { + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { value == tag || value.starts_with(&starts_with) }) } // Literal field 'tags' needs to be compared by key. Field::Reserved(field) if field == "tags" => { - any_string_match("tags", move |value| value == field) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| value == field) } Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => { Run::boxed(move |log: &LogEvent| { @@ -121,7 +121,7 @@ impl Filter for EventFilter { Field::Reserved(field) if field == "tags" => { let to_match = to_match.to_owned(); - array_match(field, move |values| { + array_match_multiple(vec!["ddtags", "tags"], move |values| { values.contains(&Value::Bytes(Bytes::copy_from_slice(to_match.as_bytes()))) }) } @@ -129,7 +129,9 @@ impl Filter for EventFilter { Field::Tag(tag) => { let value_bytes = Value::Bytes(format!("{}:{}", tag, to_match).into()); - array_match("tags", move |values| values.contains(&value_bytes)) + array_match_multiple(vec!["ddtags", "tags"], move |values| { + values.contains(&value_bytes) + }) } // Reserved values are matched by string equality. Field::Reserved(field) => { @@ -162,7 +164,9 @@ impl Filter for EventFilter { Field::Tag(tag) => { let starts_with = format!("{}:{}", tag, prefix); - any_string_match("tags", move |value| value.starts_with(&starts_with)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + value.starts_with(&starts_with) + }) } // All other field types are compared by complete value. Field::Reserved(field) | Field::Attribute(field) => { @@ -187,7 +191,7 @@ impl Filter for EventFilter { Field::Tag(tag) => { let re = wildcard_regex(&format!("{}:{}", tag, wildcard)); - any_string_match("tags", move |value| re.is_match(&value)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| re.is_match(&value)) } Field::Reserved(field) | Field::Attribute(field) => { let re = wildcard_regex(wildcard); @@ -277,18 +281,20 @@ impl Filter for EventFilter { }) } // Tag values need extracting by "key:value" to be compared. - Field::Tag(tag) => any_string_match("tags", move |value| match value.split_once(':') { - Some((t, lhs)) if t == tag => { - let lhs = Cow::from(lhs); + Field::Tag(tag) => any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + match value.split_once(':') { + Some((t, lhs)) if t == tag => { + let lhs = Cow::from(lhs); - match comparator { - Comparison::Lt => lhs < rhs, - Comparison::Lte => lhs <= rhs, - Comparison::Gt => lhs > rhs, - Comparison::Gte => lhs >= rhs, + match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + } } + _ => false, } - _ => false, }), // All other tag types are compared by string. Field::Default(field) | Field::Reserved(field) => { @@ -340,43 +346,44 @@ where }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// the vector of `Value`s the array contains matches the provided `func`. -fn array_match(field: S, func: F) -> Box> +/// Returns a `Matcher` that returns true if any provided field of log event resolves to an array of strings, +/// where at least one string matches the provided `func`. +fn any_string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, - F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, + F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - let field = field.into(); - - Run::boxed(move |log: &LogEvent| { - match log.parse_path_and_get_value(field.as_str()).ok().flatten() { - Some(Value::Array(values)) => func(values), - _ => false, - } + any_match_multiple(fields, move |value| { + let bytes = value.coerce_to_bytes(); + func(String::from_utf8_lossy(&bytes)) }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where /// at least one `Value` it contains matches the provided `func`. -fn any_match(field: S, func: F) -> Box> +fn any_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, + S: Into + Clone + Send + Sync + 'static, F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, { - array_match(field, move |values| values.iter().any(&func)) + array_match_multiple(fields, move |values| values.iter().any(&func)) } -/// Returns a `Matcher` that returns true if the log event resolves to an array of strings, -/// where at least one string matches the provided `func`. -fn any_string_match(field: S, func: F) -> Box> +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// the vector of `Value`s the array contains matches the provided `func`. +fn array_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, - F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, { - any_match(field, move |value| { - let bytes = value.coerce_to_bytes(); - func(String::from_utf8_lossy(&bytes)) + Run::boxed(move |log: &LogEvent| { + fields.iter().any(|field| { + let field = field.clone().into(); + match log.parse_path_and_get_value(field.as_str()).ok().flatten() { + Some(Value::Array(values)) => func(values), + _ => false, + } + }) }) } @@ -1187,6 +1194,265 @@ mod test { log_event!["field" => false, "field2" => "value2"], log_event!["field" => true, "field2" => "value2"], ), + // tags checks with 'ddtags' (DD Agent Source naming) + + // Tag exists. + ( + "_exists_:a", // Source + log_event!["ddtags" => vec!["a:foo"]], // Pass + log_event!["ddtags" => vec!["b:foo"]], // Fail + ), + // Tag exists with - in name. + ( + "_exists_:a-b", // Source + log_event!["ddtags" => vec!["a-b:foo"]], // Pass + log_event!["ddtags" => vec!["ab:foo"]], // Fail + ), + // Tag exists (negate). + ( + "NOT _exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!("ddtags" => vec!["a:foo"]), + ), + // Tag exists (negate w/-). + ( + "-_exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist. + ( + "_missing_:a", + log_event![], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist (negate). + ( + "NOT _missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag doesn't exist (negate w/-). + ( + "-_missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag match. + ( + "a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["ddtags" => vec!["b:bla"]], + ), + // Tag match (negate). + ( + "NOT a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Reserved tag match (negate). + ( + "NOT host:foo", + log_event!["ddtags" => vec!["host:fo o"]], + log_event!["host" => "foo"], + ), + // Tag match (negate w/-). + ( + "-a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match. + ( + r#"a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted tag match (negate). + ( + r#"NOT a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match (negate w/-). + ( + r#"-a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match. + ( + "@a:bla", + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match (negate). + ( + "NOT @a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // String attribute match (negate w/-). + ( + "-@a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match. + ( + r#"@a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted attribute match (negate). + ( + r#"NOT @a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match (negate w/-). + ( + r#"-@a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Integer attribute match. + ( + "@a:200", + log_event!["a" => 200], + log_event!["ddtags" => vec!["a:200"]], + ), + // Float attribute match. + ( + "@a:0.75", + log_event!["a" => 0.75], + log_event!["ddtags" => vec!["a:0.75"]], + ), + ( + "a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["ddtags" => vec!["a:blafoo"]], + ), + // Wildcard prefix - tag (negate). + ( + "NOT a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - tag (negate w/-). + ( + "-a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard suffix - tag. + ( + "b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["ddtags" => vec!["b:bopbla"]], + ), + // Wildcard suffix - tag (negate). + ( + "NOT b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - tag (negate w/-). + ( + "-b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Multiple wildcards - tag. + ( + "c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["custom" => r#"{"title" => "foobla"}"#], + ), + // Multiple wildcards - tag (negate). + ( + "NOT c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - tag (negate w/-). + ( + "-c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Wildcard prefix - attribute. + ( + "@a:*bla", + log_event!["a" => "foobla"], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - attribute (negate). + ( + "NOT @a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard prefix - attribute (negate w/-). + ( + "-@a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard suffix - attribute. + ( + "@b:bla*", + log_event!["b" => "blabop"], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - attribute (negate). + ( + "NOT @b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Wildcard suffix - attribute (negate w/-). + ( + "-@b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Multiple wildcards - attribute. + ( + "@c:*b*la*", + log_event!["c" => "foobla"], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - attribute (negate). + ( + "NOT @c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Multiple wildcards - attribute (negate w/-). + ( + "-@c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Special case for tags. + ( + "tags:a", + log_event!["ddtags" => vec!["a", "b", "c"]], + log_event!["ddtags" => vec!["d", "e", "f"]], + ), + // Special case for tags (negate). + ( + "NOT tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case for tags (negate w/-). + ( + "-tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), ] }