From cbcbeb3f36fb22efc7ff856bc967b82ebda641ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20AGBEKODO?= <70853334+20agbekodo@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:09:34 +0100 Subject: [PATCH] feat(datadog service): support filtering on ddtags and ddsource in datadog search syntax (#22281) * feat(datadog service): support filtering on ddtags in datadog search syntax * feat(datadog service):support filtering on ddsource in datadog search syntax * update changelog --- ...earch_syntax_support_ddtags.enhancement.md | 6 + src/conditions/datadog_search.rs | 470 ++++++++++++++++-- 2 files changed, 432 insertions(+), 44 deletions(-) create mode 100644 changelog.d/datadog_search_syntax_support_ddtags.enhancement.md diff --git a/changelog.d/datadog_search_syntax_support_ddtags.enhancement.md b/changelog.d/datadog_search_syntax_support_ddtags.enhancement.md new file mode 100644 index 0000000000000..ce522e072b28a --- /dev/null +++ b/changelog.d/datadog_search_syntax_support_ddtags.enhancement.md @@ -0,0 +1,6 @@ +In datadog search syntax, allow the following queries to match on several fields (OR condition): + +- `tags` will lookup the fields `tags` and `ddtags` +- `source` will lookup the fields `source` and `ddsource` + +authors: 20agbekodo diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 87bf97affee38..83ec3e00923a7 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -86,22 +86,20 @@ impl Filter for EventFilter { Field::Tag(tag) => { let starts_with = format!("{}:", tag); - any_string_match("tags", move |value| { + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { value == tag || value.starts_with(&starts_with) }) } // Literal field 'tags' needs to be compared by key. Field::Reserved(field) if field == "tags" => { - any_string_match("tags", move |value| value == field) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| value == field) } - Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => { - Run::boxed(move |log: &LogEvent| { - log.parse_path_and_get_value(f.as_str()) - .ok() - .flatten() - .is_some() - }) + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + exists_match_multiple(vec!["ddsource", "source"]) } + + Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => exists_match(f), }) } @@ -121,7 +119,7 @@ impl Filter for EventFilter { Field::Reserved(field) if field == "tags" => { let to_match = to_match.to_owned(); - array_match(field, move |values| { + array_match_multiple(vec!["ddtags", "tags"], move |values| { values.contains(&Value::Bytes(Bytes::copy_from_slice(to_match.as_bytes()))) }) } @@ -129,7 +127,15 @@ impl Filter for EventFilter { Field::Tag(tag) => { let value_bytes = Value::Bytes(format!("{}:{}", tag, to_match).into()); - array_match("tags", move |values| values.contains(&value_bytes)) + array_match_multiple(vec!["ddtags", "tags"], move |values| { + values.contains(&value_bytes) + }) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let to_match = to_match.to_owned(); + + string_match_multiple(vec!["ddsource", "source"], move |value| value == to_match) } // Reserved values are matched by string equality. Field::Reserved(field) => { @@ -162,8 +168,19 @@ impl Filter for EventFilter { Field::Tag(tag) => { let starts_with = format!("{}:{}", tag, prefix); - any_string_match("tags", move |value| value.starts_with(&starts_with)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + value.starts_with(&starts_with) + }) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let prefix = prefix.to_owned(); + + string_match_multiple(vec!["ddsource", "source"], move |value| { + value.starts_with(&prefix) + }) } + // All other field types are compared by complete value. Field::Reserved(field) | Field::Attribute(field) => { let prefix = prefix.to_owned(); @@ -187,7 +204,13 @@ impl Filter for EventFilter { Field::Tag(tag) => { let re = wildcard_regex(&format!("{}:{}", tag, wildcard)); - any_string_match("tags", move |value| re.is_match(&value)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| re.is_match(&value)) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let re = wildcard_regex(wildcard); + + string_match_multiple(vec!["ddsource", "source"], move |value| re.is_match(&value)) } Field::Reserved(field) | Field::Attribute(field) => { let re = wildcard_regex(wildcard); @@ -277,19 +300,30 @@ impl Filter for EventFilter { }) } // Tag values need extracting by "key:value" to be compared. - Field::Tag(tag) => any_string_match("tags", move |value| match value.split_once(':') { - Some((t, lhs)) if t == tag => { - let lhs = Cow::from(lhs); + Field::Tag(tag) => any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + match value.split_once(':') { + Some((t, lhs)) if t == tag => { + let lhs = Cow::from(lhs); - match comparator { - Comparison::Lt => lhs < rhs, - Comparison::Lte => lhs <= rhs, - Comparison::Gt => lhs > rhs, - Comparison::Gte => lhs >= rhs, + match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + } } + _ => false, } - _ => false, }), + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + string_match_multiple(vec!["ddsource", "source"], move |lhs| match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + }) + } // All other tag types are compared by string. Field::Default(field) | Field::Reserved(field) => { string_match(field, move |lhs| match comparator { @@ -303,6 +337,21 @@ impl Filter for EventFilter { } } +// Returns a `Matcher` that returns true if the field exists. +fn exists_match(field: S) -> Box> +where + S: Into, +{ + let field = field.into(); + + Run::boxed(move |log: &LogEvent| { + log.parse_path_and_get_value(field.as_str()) + .ok() + .flatten() + .is_some() + }) +} + /// Returns a `Matcher` that returns true if the field resolves to a string, /// numeric, or boolean which matches the provided `func`. fn simple_scalar_match(field: S, func: F) -> Box> @@ -340,46 +389,71 @@ where }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// the vector of `Value`s the array contains matches the provided `func`. -fn array_match(field: S, func: F) -> Box> +// Returns a `Matcher` that returns true if any provided field exists. +fn exists_match_multiple(fields: Vec) -> Box> where - S: Into, - F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, { - let field = field.into(); - Run::boxed(move |log: &LogEvent| { - match log.parse_path_and_get_value(field.as_str()).ok().flatten() { - Some(Value::Array(values)) => func(values), - _ => false, - } + fields + .iter() + .any(|field| exists_match(field.clone()).run(log)) }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// at least one `Value` it contains matches the provided `func`. -fn any_match(field: S, func: F) -> Box> +/// Returns a `Matcher` that returns true if any provided field resolves to a string which +/// matches the provided `func`. +fn string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, - F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, + F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - array_match(field, move |values| values.iter().any(&func)) + Run::boxed(move |log: &LogEvent| { + fields + .iter() + .any(|field| string_match(field.clone(), func.clone()).run(log)) + }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array of strings, -/// where at least one string matches the provided `func`. -fn any_string_match(field: S, func: F) -> Box> +fn any_string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, + S: Into + Clone + Send + Sync + 'static, F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - any_match(field, move |value| { + any_match_multiple(fields, move |value| { let bytes = value.coerce_to_bytes(); func(String::from_utf8_lossy(&bytes)) }) } +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// at least one `Value` it contains matches the provided `func`. +fn any_match_multiple(fields: Vec, func: F) -> Box> +where + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, +{ + array_match_multiple(fields, move |values| values.iter().any(&func)) +} + +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// the vector of `Value`s the array contains matches the provided `func`. +fn array_match_multiple(fields: Vec, func: F) -> Box> +where + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, +{ + Run::boxed(move |log: &LogEvent| { + fields.iter().any(|field| { + let field = field.clone().into(); + match log.parse_path_and_get_value(field.as_str()).ok().flatten() { + Some(Value::Array(values)) => func(values), + _ => false, + } + }) + }) +} + #[cfg(test)] mod test { use super::*; @@ -1187,6 +1261,314 @@ mod test { log_event!["field" => false, "field2" => "value2"], log_event!["field" => true, "field2" => "value2"], ), + // tags checks with 'ddtags' (DD Agent Source naming) + + // Tag exists. + ( + "_exists_:a", // Source + log_event!["ddtags" => vec!["a:foo"]], // Pass + log_event!["ddtags" => vec!["b:foo"]], // Fail + ), + // Tag exists with - in name. + ( + "_exists_:a-b", // Source + log_event!["ddtags" => vec!["a-b:foo"]], // Pass + log_event!["ddtags" => vec!["ab:foo"]], // Fail + ), + // Tag exists (negate). + ( + "NOT _exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!("ddtags" => vec!["a:foo"]), + ), + // Tag exists (negate w/-). + ( + "-_exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist. + ( + "_missing_:a", + log_event![], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist (negate). + ( + "NOT _missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag doesn't exist (negate w/-). + ( + "-_missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag match. + ( + "a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["ddtags" => vec!["b:bla"]], + ), + // Tag match (negate). + ( + "NOT a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Reserved tag match (negate). + ( + "NOT host:foo", + log_event!["ddtags" => vec!["host:fo o"]], + log_event!["host" => "foo"], + ), + // Tag match (negate w/-). + ( + "-a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match. + ( + r#"a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted tag match (negate). + ( + r#"NOT a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match (negate w/-). + ( + r#"-a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match. + ( + "@a:bla", + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match (negate). + ( + "NOT @a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // String attribute match (negate w/-). + ( + "-@a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match. + ( + r#"@a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted attribute match (negate). + ( + r#"NOT @a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match (negate w/-). + ( + r#"-@a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Integer attribute match. + ( + "@a:200", + log_event!["a" => 200], + log_event!["ddtags" => vec!["a:200"]], + ), + // Float attribute match. + ( + "@a:0.75", + log_event!["a" => 0.75], + log_event!["ddtags" => vec!["a:0.75"]], + ), + ( + "a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["ddtags" => vec!["a:blafoo"]], + ), + // Wildcard prefix - tag (negate). + ( + "NOT a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - tag (negate w/-). + ( + "-a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard suffix - tag. + ( + "b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["ddtags" => vec!["b:bopbla"]], + ), + // Wildcard suffix - tag (negate). + ( + "NOT b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - tag (negate w/-). + ( + "-b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Multiple wildcards - tag. + ( + "c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["custom" => r#"{"title" => "foobla"}"#], + ), + // Multiple wildcards - tag (negate). + ( + "NOT c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - tag (negate w/-). + ( + "-c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Wildcard prefix - attribute. + ( + "@a:*bla", + log_event!["a" => "foobla"], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - attribute (negate). + ( + "NOT @a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard prefix - attribute (negate w/-). + ( + "-@a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard suffix - attribute. + ( + "@b:bla*", + log_event!["b" => "blabop"], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - attribute (negate). + ( + "NOT @b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Wildcard suffix - attribute (negate w/-). + ( + "-@b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Multiple wildcards - attribute. + ( + "@c:*b*la*", + log_event!["c" => "foobla"], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - attribute (negate). + ( + "NOT @c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Multiple wildcards - attribute (negate w/-). + ( + "-@c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Special case for tags. + ( + "tags:a", + log_event!["ddtags" => vec!["a", "b", "c"]], + log_event!["ddtags" => vec!["d", "e", "f"]], + ), + // Special case for tags (negate). + ( + "NOT tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case for tags (negate w/-). + ( + "-tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case: 'source' looks up on 'source' and 'ddsource' (OR condition) + // source + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["tags" => vec!["source:foo"]], + ), + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["source" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["source" => r#"{"value": "foo"}"#], + ), + // ddsource + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["tags" => vec!["ddsource:foo"]], + ), + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["ddsource" => r#"{"value": "foo"}"#], + ), + // both source and ddsource + ( + "source:foo", + log_event!["source" => "foo", "ddsource" => "foo"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foo", "ddsource" => "foobar"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foobar", "ddsource" => "foo"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), ] }