From a58ae6f50bd1fb290a9b6ad21221151113e99e9e Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 17 May 2023 16:54:49 -0600 Subject: [PATCH 01/99] start --- Cargo.toml | 10 ++ .../integration/datadog-e2e-logs/compose.yaml | 33 ++++++ .../integration/datadog-e2e-logs/test.yaml | 15 +++ src/topology/mod.rs | 2 +- tests/e2e/datadog/logs/mod.rs | 109 ++++++++++++++++++ tests/e2e/datadog/mod.rs | 1 + tests/e2e/mod.rs | 2 + 7 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 scripts/integration/datadog-e2e-logs/compose.yaml create mode 100644 scripts/integration/datadog-e2e-logs/test.yaml create mode 100644 tests/e2e/datadog/logs/mod.rs create mode 100644 tests/e2e/datadog/mod.rs create mode 100644 tests/e2e/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 49fd1c3fc65f2..579b30d5a2d69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,10 @@ bench = false name = "integration" path = "tests/integration/lib.rs" +[[test]] +name = "e2e" +path = "tests/e2e/mod.rs" + [[bin]] name = "graphql-schema" path = "src/api/schema/gen.rs" @@ -816,6 +820,12 @@ webhdfs-integration-tests = ["sinks-webhdfs"] disable-resolv-conf = [] shutdown-tests = ["api", "sinks-blackhole", "sinks-console", "sinks-prometheus", "sources", "transforms-lua", "transforms-remap", "unix"] cli-tests = ["sinks-blackhole", "sinks-socket", "sources-demo_logs", "sources-file"] + +e2e-tests-datadog = [ + "sources-datadog_agent", + "sinks-datadog_logs" +] + vector-api-tests = [ "sources-demo_logs", "transforms-remap", diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml new file mode 100644 index 0000000000000..60582b1bde1b5 --- /dev/null +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -0,0 +1,33 @@ +version: '3' + +services: + datadog-agent: + image: docker.io/datadog/agent:${CONFIG_VERSION} + environment: + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_DD_URL=http://fakeintake-agent:80 + - DD_USE_DOGSTATSD=false + - DD_HOSTNAME=datadog-agent + - DD_PROCESS_AGENT_ENABLED=false + - DD_APM_ENABLED=false + - DD_METRICS_ENABLED=false + - DD_LOGS_ENABLED=true + - DD_CONTAINER_EXCLUDE="name:.*" + datadog-agent-vector: + #image: datadog-agent-vector-t1:latest + image: docker.io/datadog/agent:${CONFIG_VERSION} + environment: + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector + - DD_PROCESS_AGENT_ENABLED=false + - DD_METRICS_ENABLED=false + - DD_LOGS_ENABLED=true + - DD_CONTAINER_EXCLUDE="name:.*" + fakeintake-agent: + image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + fakeintake-agent-vector: + image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + +networks: + default: + name: ${VECTOR_NETWORK} \ No newline at end of file diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml new file mode 100644 index 0000000000000..3d10d77b6cf50 --- /dev/null +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -0,0 +1,15 @@ +features: +- e2e-tests-datadog + +test: "e2e" + +runner: + env: + VECTOR_RECEIVE_PORT: '8081' + FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' + FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' + #TEST_LOG: ${TEST_LOG} + #TEST_DATADOG_API_KEY: + +matrix: + version: [latest] \ No newline at end of file diff --git a/src/topology/mod.rs b/src/topology/mod.rs index bb5299839d025..d0f56e7f62ee0 100644 --- a/src/topology/mod.rs +++ b/src/topology/mod.rs @@ -27,7 +27,7 @@ use std::{ pub use controller::{ReloadOutcome, SharedTopologyController, TopologyController}; use futures::{Future, FutureExt}; -pub(super) use running::RunningTopology; +pub use running::RunningTopology; use tokio::sync::{mpsc, watch}; use vector_buffers::topology::channel::{BufferReceiverStream, BufferSender}; diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs new file mode 100644 index 0000000000000..5954473e9cb19 --- /dev/null +++ b/tests/e2e/datadog/logs/mod.rs @@ -0,0 +1,109 @@ +use indoc::indoc; +use reqwest::{Client, Method}; +use serde::Deserialize; +use tokio::sync::mpsc; + +use vector::{config::ConfigBuilder, test_util::start_topology, topology::RunningTopology}; + +fn vector_receive_port() -> u16 { + std::env::var("VECTOR_RECEIVE_PORT") + .unwrap_or_else(|_| "8081".to_string()) + .parse::() + .unwrap() +} + +fn fake_intake_vector_endpoint() -> String { + std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()) +} + +fn fake_intake_agent_endpoint() -> String { + std::env::var("FAKE_INTAKE_AGENT_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8083".to_string()) +} + +// Fakeintake response +#[derive(Deserialize, Debug)] +struct Payloads { + payloads: Vec, +} + +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct Payload { + // base64 encoded + data: String, + encoding: String, + timestamp: String, +} + +async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> Payloads { + let url = format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,); + + Client::new() + .request(Method::GET, &url) + .send() + .await + .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) + .json::() + .await + .expect("Parsing fakeintake payloads failed") +} + +async fn get_payloads_agent() -> Payloads { + get_fakeintake_payloads(&fake_intake_agent_endpoint(), "TODO logs endpoint").await +} + +async fn get_payloads_vector() -> Payloads { + get_fakeintake_payloads(&fake_intake_vector_endpoint(), "TODO logs endpoint").await +} + +async fn start_vector() -> ( + RunningTopology, + (mpsc::UnboundedSender<()>, mpsc::UnboundedReceiver<()>), +) { + let dd_agent_address = format!("0.0.0.0:{}", vector_receive_port()); + + let dd_logs_endpoint = fake_intake_vector_endpoint(); + + let builder: ConfigBuilder = toml::from_str(&format!( + indoc! {r#" + [sources.dd_agent] + type = "datadog_agent" + multiple_outputs = true + disable_metrics = true + disable_traces = true + address = "{}" + + [sinks.dd_logs] + type = "datadog_logs" + inputs = ["dd_agent.logs"] + default_api_key = "unused" + endpoint = "{}" + "#}, + dd_agent_address, dd_logs_endpoint, + )) + .expect("toml parsing should not fail"); + + let config = builder.build().expect("building config should not fail"); + + let (topology, shutdown) = start_topology(config, false).await; + + println!("Started vector."); + + (topology, shutdown) +} + +#[tokio::test] +async fn test_logs() { + println!("foo test"); + + // panics if vector errors during startup + let (_topology, _shutdown) = start_vector().await; + + let _agent_payloads = get_payloads_agent().await; + + let _vector_payloads = get_payloads_vector().await; + + assert!(true); +} diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs new file mode 100644 index 0000000000000..af2c2c342fa4c --- /dev/null +++ b/tests/e2e/datadog/mod.rs @@ -0,0 +1 @@ +pub mod logs; diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs new file mode 100644 index 0000000000000..1e3864b6d42fa --- /dev/null +++ b/tests/e2e/mod.rs @@ -0,0 +1,2 @@ +#[cfg(feature = "e2e-tests-datadog")] +mod datadog; From 98b97126c327d43c6df38648e8ec62349a41d469 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 18 May 2023 13:40:16 -0600 Subject: [PATCH 02/99] check spelling allow --- .github/actions/spelling/allow.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 7b0334fa0e905..437f4df888ee6 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -264,6 +264,7 @@ downsides downwardapi emoji esbuild +fakeintake fargate fibonacci fileapi From 62211ab1a791b47d934733c60a4d4af6d22fe3c2 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 18 May 2023 14:18:40 -0600 Subject: [PATCH 03/99] reorg --- src/sinks/datadog/logs/mod.rs | 2 +- tests/e2e/datadog/logs/mod.rs | 68 ++++++----------------------------- tests/e2e/datadog/mod.rs | 58 ++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 58 deletions(-) diff --git a/src/sinks/datadog/logs/mod.rs b/src/sinks/datadog/logs/mod.rs index 02b857a160181..4e142265c0d96 100644 --- a/src/sinks/datadog/logs/mod.rs +++ b/src/sinks/datadog/logs/mod.rs @@ -28,4 +28,4 @@ mod config; mod service; mod sink; -pub(crate) use config::DatadogLogsConfig; +pub use config::DatadogLogsConfig; diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 5954473e9cb19..a4a8ac05b24ee 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,62 +1,13 @@ use indoc::indoc; -use reqwest::{Client, Method}; -use serde::Deserialize; use tokio::sync::mpsc; -use vector::{config::ConfigBuilder, test_util::start_topology, topology::RunningTopology}; +use vector::{ + config::ConfigBuilder, + test_util::{start_topology, trace_init}, + topology::RunningTopology, +}; -fn vector_receive_port() -> u16 { - std::env::var("VECTOR_RECEIVE_PORT") - .unwrap_or_else(|_| "8081".to_string()) - .parse::() - .unwrap() -} - -fn fake_intake_vector_endpoint() -> String { - std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") - .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()) -} - -fn fake_intake_agent_endpoint() -> String { - std::env::var("FAKE_INTAKE_AGENT_ENDPOINT") - .unwrap_or_else(|_| "http://127.0.0.1:8083".to_string()) -} - -// Fakeintake response -#[derive(Deserialize, Debug)] -struct Payloads { - payloads: Vec, -} - -#[allow(dead_code)] -#[derive(Deserialize, Debug)] -struct Payload { - // base64 encoded - data: String, - encoding: String, - timestamp: String, -} - -async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> Payloads { - let url = format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,); - - Client::new() - .request(Method::GET, &url) - .send() - .await - .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) - .json::() - .await - .expect("Parsing fakeintake payloads failed") -} - -async fn get_payloads_agent() -> Payloads { - get_fakeintake_payloads(&fake_intake_agent_endpoint(), "TODO logs endpoint").await -} - -async fn get_payloads_vector() -> Payloads { - get_fakeintake_payloads(&fake_intake_vector_endpoint(), "TODO logs endpoint").await -} +use super::*; async fn start_vector() -> ( RunningTopology, @@ -96,14 +47,17 @@ async fn start_vector() -> ( #[tokio::test] async fn test_logs() { + trace_init(); + println!("foo test"); // panics if vector errors during startup let (_topology, _shutdown) = start_vector().await; - let _agent_payloads = get_payloads_agent().await; + let logs_endpoint = "/api/v2/logs"; + let _agent_payloads = get_payloads_agent(&logs_endpoint).await; - let _vector_payloads = get_payloads_vector().await; + let _vector_payloads = get_payloads_vector(&logs_endpoint).await; assert!(true); } diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index af2c2c342fa4c..93706a85c47ba 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -1 +1,59 @@ pub mod logs; + + +use reqwest::{Client, Method}; +use serde::Deserialize; + + +fn vector_receive_port() -> u16 { + std::env::var("VECTOR_RECEIVE_PORT") + .unwrap_or_else(|_| "8081".to_string()) + .parse::() + .unwrap() +} + +fn fake_intake_vector_endpoint() -> String { + std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()) +} + +fn fake_intake_agent_endpoint() -> String { + std::env::var("FAKE_INTAKE_AGENT_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8083".to_string()) +} + +// Fakeintake response +#[derive(Deserialize, Debug)] +struct Payloads { + payloads: Vec, +} + +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct Payload { + // base64 encoded + data: String, + encoding: String, + timestamp: String, +} + +async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> Payloads { + let url = format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,); + + Client::new() + .request(Method::GET, &url) + .send() + .await + .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) + .json::() + .await + .expect("Parsing fakeintake payloads failed") +} + +async fn get_payloads_agent(endpoint: &str) -> Payloads { + get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint).await +} + +async fn get_payloads_vector(endpoint: &str) -> Payloads { + get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint).await +} From ae1d41a2aabc8faee1847b86126802a2a7fe3b58 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 18 May 2023 17:13:15 -0600 Subject: [PATCH 04/99] try consume a log --- .../integration/datadog-e2e-logs/compose.yaml | 19 ++++++++++++------- .../logs.conf.d/custom_logs.d/conf.yaml | 5 +++++ .../logs.conf.d/custom_logs.d/conf.yaml | 5 +++++ .../agent_vector/logs.conf.d/datadog.yaml | 13 +++++++++++++ tests/data/e2e/datadog/logs/custom_log.log | 1 + tests/e2e/datadog/logs/mod.rs | 13 ++++++++++++- 6 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml create mode 100644 tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml create mode 100644 tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml create mode 100644 tests/data/e2e/datadog/logs/custom_log.log diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 60582b1bde1b5..b25df71659d05 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -5,24 +5,29 @@ services: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_DD_URL=http://fakeintake-agent:80 - DD_USE_DOGSTATSD=false - DD_HOSTNAME=datadog-agent - - DD_PROCESS_AGENT_ENABLED=false - - DD_APM_ENABLED=false - - DD_METRICS_ENABLED=false - DD_LOGS_ENABLED=true + - DD_LOGS_CONFIG_DD_URL=fakeintake-agent:80 + - DD_SKIP_SSL_VALIDATION=true + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" + volumes: + - ../../../tests/data/e2e/datadog/logs/agent_only/logs.conf.d:/conf.d:ro + - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log datadog-agent-vector: - #image: datadog-agent-vector-t1:latest image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent-vector - - DD_PROCESS_AGENT_ENABLED=false - - DD_METRICS_ENABLED=false - DD_LOGS_ENABLED=true + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" + volumes: + - ../../../tests/data/e2e/datadog/logs/agent_vector/logs.conf.d:/conf.d:ro + - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log fakeintake-agent: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} fakeintake-agent-vector: diff --git a/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml new file mode 100644 index 0000000000000..9f114edc5ecc9 --- /dev/null +++ b/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml @@ -0,0 +1,5 @@ +logs: + - type: file + path: "/var/log/a_custom_log.log" + service: "an_app" + source: "custom_log" diff --git a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml new file mode 100644 index 0000000000000..9f114edc5ecc9 --- /dev/null +++ b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml @@ -0,0 +1,5 @@ +logs: + - type: file + path: "/var/log/a_custom_log.log" + service: "an_app" + source: "custom_log" diff --git a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml new file mode 100644 index 0000000000000..7faf9e2105954 --- /dev/null +++ b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml @@ -0,0 +1,13 @@ +api_key: deadbeef + +vector: + logs: + enabled: true + url: "http://0.0.0.0:8181" + +intervals: + process_realtime: 1 + +use_dogstatsd: false + +#log_level: 'debug' diff --git a/tests/data/e2e/datadog/logs/custom_log.log b/tests/data/e2e/datadog/logs/custom_log.log new file mode 100644 index 0000000000000..b89597bcbe2a4 --- /dev/null +++ b/tests/data/e2e/datadog/logs/custom_log.log @@ -0,0 +1 @@ +{"host":"01.23.456.789", "user-identifier":"morpheous", "datetime":"31/March/1999/:03:11:11", "score":5, "directors": ["Lana Watchowski", "Lilly Watchowski"], "code-name":"white_rabbit"} diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index a4a8ac05b24ee..e0f3304f3ae4e 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,5 +1,8 @@ use indoc::indoc; -use tokio::sync::mpsc; +use tokio::{ + sync::mpsc, + time::{sleep, Duration}, +}; use vector::{ config::ConfigBuilder, @@ -54,10 +57,18 @@ async fn test_logs() { // panics if vector errors during startup let (_topology, _shutdown) = start_vector().await; + // TODO there hopefully is a way to configure the flushing of metrics such that we don't have + // to wait statically for so long here. + sleep(Duration::from_secs(25)).await; + let logs_endpoint = "/api/v2/logs"; let _agent_payloads = get_payloads_agent(&logs_endpoint).await; + dbg!(&_agent_payloads); + let _vector_payloads = get_payloads_vector(&logs_endpoint).await; + dbg!(&_vector_payloads); + assert!(true); } From 8edd7f41a6976f50588e48b404b687c81faa2d2a Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 22 May 2023 11:26:22 -0600 Subject: [PATCH 05/99] trying on logs --- .../integration/datadog-e2e-logs/compose.yaml | 35 ++++++++++++++++--- .../integration/datadog-e2e-logs/test.yaml | 1 - .../e2e/datadog/logs/agent_only/datadog.yaml | 14 ++++++++ .../datadog/logs/agent_vector/datadog.yaml | 23 ++++++++++++ .../agent_vector/logs.conf.d/datadog.yaml | 13 ------- tests/e2e/datadog/logs/mod.rs | 2 ++ vdev/src/testing/config.rs | 2 +- vdev/src/testing/integration.rs | 6 ++-- 8 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 tests/data/e2e/datadog/logs/agent_only/datadog.yaml create mode 100644 tests/data/e2e/datadog/logs/agent_vector/datadog.yaml delete mode 100644 tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index b25df71659d05..67526d42d1c79 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -1,3 +1,4 @@ +#version: '2.4' version: '3' services: @@ -5,33 +6,57 @@ services: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_USE_DOGSTATSD=false + #- DD_USE_DOGSTATSD=false - DD_HOSTNAME=datadog-agent - - DD_LOGS_ENABLED=true - - DD_LOGS_CONFIG_DD_URL=fakeintake-agent:80 - - DD_SKIP_SSL_VALIDATION=true + #- DD_LOGS_ENABLED=true + #- DD_LOGS_CONFIG_DD_URL=fakeintake-agent:80 + #- DD_LOGS_CONFIG_NO_SSL=true + #- DD_LOGS_CONFIG_FORCE_USE_HTTP=true - DD_ENABLE_PAYLOADS_EVENTS=false - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: + # The Agent config file + - ../../../tests/data/e2e/datadog/logs/agent_only/datadog.yaml:/etc/datadog-agent/datadog.yaml + # The custom logs check - ../../../tests/data/e2e/datadog/logs/agent_only/logs.conf.d:/conf.d:ro + # The custom log to tail - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} + # depends_on: + # waitforit: + # condition: service_healthy environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent-vector - - DD_LOGS_ENABLED=true + #- DD_LOGS_ENABLED=true - DD_ENABLE_PAYLOADS_EVENTS=false - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" + #- VECTOR_URL="http://runner:8081" volumes: + # The Agent config file + - ../../../tests/data/e2e/datadog/logs/agent_vector/datadog.yaml:/etc/datadog-agent/datadog.yaml + # The custom logs check - ../../../tests/data/e2e/datadog/logs/agent_vector/logs.conf.d:/conf.d:ro + # The custom log to tail - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log fakeintake-agent: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} fakeintake-agent-vector: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + # waitforit: + # image: docker.io/debian:buster-slim + # command: tail -f /dev/null + # healthcheck: + # test: curl -f http://runner:8181 || exit 1 + # #test: ["CMD", "curl", "-f", "http://runner:8181", "||", "exit", "1"] + # interval: 15s + # timeout: 10s + # retries: 10 + + #test: ["CMD-SHELL", "curl --fail http://runner:8181 || exit 1"] networks: default: diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml index 3d10d77b6cf50..726e027e0a543 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -9,7 +9,6 @@ runner: FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' #TEST_LOG: ${TEST_LOG} - #TEST_DATADOG_API_KEY: matrix: version: [latest] \ No newline at end of file diff --git a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml new file mode 100644 index 0000000000000..8d14182971b15 --- /dev/null +++ b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml @@ -0,0 +1,14 @@ +api_key: deadbeef + +logs_enabled: true + +logs_config: + logs_dd_url: fakeintake-agent:80 + logs_no_ssl: true + force_use_http: true + +#intervals: +# process_realtime: 1 + +use_dogstatsd: false +#log_level: 'debug' diff --git a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml new file mode 100644 index 0000000000000..82ea8e1e7d0ec --- /dev/null +++ b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml @@ -0,0 +1,23 @@ +api_key: deadbeef + +logs_enabled: true + +connection_reset_interval: 5 + +logs_config: + sender_recovery_reset: true + +vector: + logs: + enabled: true + url: "http://runner:8181" + #url: "http://0.0.0.0:8181" + #url: "http://runner:8081" + +#intervals: +# process_realtime: 1 + +use_dogstatsd: false +log_level: 'debug' + +log_file: './agent.log' diff --git a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml deleted file mode 100644 index 7faf9e2105954..0000000000000 --- a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/datadog.yaml +++ /dev/null @@ -1,13 +0,0 @@ -api_key: deadbeef - -vector: - logs: - enabled: true - url: "http://0.0.0.0:8181" - -intervals: - process_realtime: 1 - -use_dogstatsd: false - -#log_level: 'debug' diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index e0f3304f3ae4e..755c7ce59320b 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -70,5 +70,7 @@ async fn test_logs() { dbg!(&_vector_payloads); + sleep(Duration::from_secs(90)).await; + assert!(true); } diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index e7bedd7093805..6fdd7130cb4f0 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -67,7 +67,7 @@ pub struct ComposeService { #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] - pub depends_on: Option>, + pub depends_on: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub healthcheck: Option, } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index ecac804a98842..5b9b0dcd36515 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -55,9 +55,9 @@ impl IntegrationTest { let active = self.envs_dir.check_active(&self.environment)?; self.config.check_required()?; - if !active { - self.start()?; - } + //if !active { + // self.start()?; + //} let mut env_vars = self.config.env.clone(); // Make sure the test runner has the same config environment vars as the services do. From 7a692907b23ea5267c789597af0c3f2f588ba3d9 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 5 Sep 2023 14:27:08 -0600 Subject: [PATCH 06/99] saving progress on vdev changes --- .../integration/datadog-e2e-logs/compose.yaml | 7 +- scripts/integration/http-client/compose.yaml | 4 + tests/e2e/datadog/logs/mod.rs | 6 +- vdev/src/commands/integration/start.rs | 2 +- vdev/src/commands/integration/stop.rs | 2 +- vdev/src/commands/integration/test.rs | 26 +- vdev/src/testing/config.rs | 8 +- vdev/src/testing/integration.rs | 341 +++++++++++++++--- vdev/src/testing/mod.rs | 7 + vdev/src/testing/runner.rs | 229 +++++++----- 10 files changed, 465 insertions(+), 167 deletions(-) diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 67526d42d1c79..8712d5aa01793 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -24,7 +24,8 @@ services: - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} - # depends_on: + depends_on: + - runner # waitforit: # condition: service_healthy environment: @@ -49,8 +50,10 @@ services: # waitforit: # image: docker.io/debian:buster-slim # command: tail -f /dev/null + # # ports: + # # - "4321:4321" # healthcheck: - # test: curl -f http://runner:8181 || exit 1 + # test: curl -f http://localhost:4321 || exit 1 # #test: ["CMD", "curl", "-f", "http://runner:8181", "||", "exit", "1"] # interval: 15s # timeout: 10s diff --git a/scripts/integration/http-client/compose.yaml b/scripts/integration/http-client/compose.yaml index a1723c93d8d1a..67701b1097b9e 100644 --- a/scripts/integration/http-client/compose.yaml +++ b/scripts/integration/http-client/compose.yaml @@ -1,6 +1,10 @@ version: '3' services: + foo: + image: tomgeorge/sleep-infinity:latest + depends_on: + - runner dufs: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 755c7ce59320b..7d2f6bbb0cd91 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -6,6 +6,7 @@ use tokio::{ use vector::{ config::ConfigBuilder, + signal::ShutdownError, test_util::{start_topology, trace_init}, topology::RunningTopology, }; @@ -14,7 +15,10 @@ use super::*; async fn start_vector() -> ( RunningTopology, - (mpsc::UnboundedSender<()>, mpsc::UnboundedReceiver<()>), + ( + mpsc::UnboundedSender, + mpsc::UnboundedReceiver, + ), ) { let dd_agent_address = format!("0.0.0.0:{}", vector_receive_port()); diff --git a/vdev/src/commands/integration/start.rs b/vdev/src/commands/integration/start.rs index da8947e74997a..0d7d76d7345c8 100644 --- a/vdev/src/commands/integration/start.rs +++ b/vdev/src/commands/integration/start.rs @@ -24,6 +24,6 @@ impl Cli { let env = envs.keys().next().expect("Integration has no environments"); env.clone() }; - IntegrationTest::new(self.integration, environment, false, 0)?.start() + IntegrationTest::new(self.integration, environment, false, 0, vec![])?.start() } } diff --git a/vdev/src/commands/integration/stop.rs b/vdev/src/commands/integration/stop.rs index 22955d5fa7daf..deef3d758e997 100644 --- a/vdev/src/commands/integration/stop.rs +++ b/vdev/src/commands/integration/stop.rs @@ -18,7 +18,7 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { if let Some(active) = EnvsDir::new(&self.integration).active()? { - IntegrationTest::new(self.integration, active, self.all_features, 0)?.stop() + IntegrationTest::new(self.integration, active, self.all_features, 0, vec![])?.stop() } else { println!("No environment for {:?} is active.", self.integration); Ok(()) diff --git a/vdev/src/commands/integration/test.rs b/vdev/src/commands/integration/test.rs index ca57d895bd0c8..de4ed1f2d7591 100644 --- a/vdev/src/commands/integration/test.rs +++ b/vdev/src/commands/integration/test.rs @@ -45,18 +45,28 @@ impl Cli { (Some(environment), Some(active)) if environment != active => { bail!("Requested environment {environment:?} does not match active one {active:?}") } - (Some(environment), _) => { - IntegrationTest::new(self.integration, environment, self.build_all, retries)? - .test(self.args) - } + (Some(environment), _) => IntegrationTest::new( + self.integration, + environment, + self.build_all, + retries, + self.args, + )? + .test(), (None, Some(active)) => { - IntegrationTest::new(self.integration, active, self.build_all, retries)? - .test(self.args) + IntegrationTest::new(self.integration, active, self.build_all, retries, self.args)? + .test() } (None, None) => { for env_name in envs.keys() { - IntegrationTest::new(&self.integration, env_name, self.build_all, retries)? - .test(self.args.clone())?; + IntegrationTest::new( + &self.integration, + env_name, + self.build_all, + retries, + self.args.clone(), + )? + .test()?; } Ok(()) } diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index a04fbe58ccc0e..2ab34e461d57b 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -66,10 +66,16 @@ pub struct ComposeService { pub volumes: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, + // #[serde(default, skip_serializing_if = "Option::is_none")] + // pub depends_on: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub depends_on: Option, + pub depends_on: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub healthcheck: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub working_dir: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, } #[derive(Debug, Deserialize, Serialize)] diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index b68305ef082b5..da6b288cb5103 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,12 +1,14 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; use anyhow::{bail, Context, Result}; +use serde_yaml::{Value, Mapping}; use tempfile::{Builder, NamedTempFile}; -use super::config::ComposeConfig; +use super::config::{ComposeConfig, ComposeService}; use super::config::{Environment, IntegrationTestConfig}; use super::runner::{ - ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, + // ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, + ContainerTestRunner as _, IntegrationTestRunner, CONTAINER_TOOL, DOCKER_SOCKET, }; use super::state::EnvsDir; use crate::app::CommandExt as _; @@ -21,8 +23,8 @@ pub struct IntegrationTest { runner: IntegrationTestRunner, compose: Option, env_config: Environment, - build_all: bool, - retries: u8, + // build_all: bool, + // retries: u8, } impl IntegrationTest { @@ -31,6 +33,7 @@ impl IntegrationTest { environment: impl Into, build_all: bool, retries: u8, + extra_args: Vec, ) -> Result { let integration = integration.into(); let environment = environment.into(); @@ -40,57 +43,29 @@ impl IntegrationTest { bail!("Could not find environment named {environment:?}"); }; let network_name = format!("vector-integration-tests-{integration}"); - let compose = Compose::new(test_dir, env_config.clone(), network_name.clone())?; + let volumes = config.runner + .volumes + .iter() + .map(|(a, b)| format!("{a}:{b}")) + .collect(); + // None if compiling with all integration test feature flag. let runner_name = (!build_all).then(|| integration.clone()); - let runner = IntegrationTestRunner::new( - runner_name, - &config.runner, - compose.is_some().then_some(network_name), - )?; - - Ok(Self { - integration, - environment, - config, - envs_dir, - runner, - compose, - env_config, - build_all, - retries, - }) - } - - pub fn test(self, extra_args: Vec) -> Result<()> { - let active = self.envs_dir.check_active(&self.environment)?; - self.config.check_required()?; - - //if !active { - // self.start()?; - //} - - let mut env_vars = self.config.env.clone(); - // Make sure the test runner has the same config environment vars as the services do. - for (key, value) in config_env(&self.env_config) { - env_vars.insert(key, Some(value)); - } - - env_vars.insert("TEST_LOG".to_string(), Some("info".into())); - let mut args = self.config.args.clone().unwrap_or_default(); + + let mut args = config.args.clone().unwrap_or_default(); args.push("--features".to_string()); - args.push(if self.build_all { + args.push(if build_all { "all-integration-tests".to_string() } else { - self.config.features.join(",") + config.features.join(",") }); // If the test field is not present then use the --lib flag - match self.config.test { + match config.test { Some(ref test_arg) => { args.push("--test".to_string()); args.push(test_arg.to_string()); @@ -99,7 +74,7 @@ impl IntegrationTest { } // Ensure the test_filter args are passed as well - if let Some(ref filter) = self.config.test_filter { + if let Some(ref filter) = config.test_filter { args.push(filter.to_string()); } args.extend(extra_args); @@ -109,13 +84,107 @@ impl IntegrationTest { args.push("--no-capture".to_string()); } - if self.retries > 0 { + if retries > 0 { args.push("--retries".to_string()); - args.push(self.retries.to_string()); + args.push(retries.to_string()); + } + + + let mut env_vars = config.env.clone(); + + env_vars.extend(config.runner.env.clone()); + + // Make sure the test runner has the same config environment vars as the services do. + // for (key, value) in config_env(&self.env_config) { + // env_vars.insert(key, Some(value)); + // } + + env_vars.insert("TEST_LOG".to_string(), Some("info".into())); + + println!("new compose"); + let compose = Compose::new(test_dir, env_config.clone(), network_name.clone(), runner_name.clone(), volumes, &args, env_vars)?; + + println!("int runner"); + let runner = IntegrationTestRunner::new( + runner_name, + &config.runner, + compose.is_some().then_some(network_name), + )?; + + println!("done new"); + + Ok(Self { + integration, + environment, + config, + envs_dir, + runner, + compose, + env_config, + // build_all, + // retries, + }) + } + + pub fn test(self) -> Result<()> { + let active = self.envs_dir.check_active(&self.environment)?; + self.config.check_required()?; + + //if !active { + // self.start()?; + //} + if !active { + self.start()?; } - self.runner - .test(&env_vars, &self.config.runner.env, &args)?; + // let mut env_vars = self.config.env.clone(); + // // Make sure the test runner has the same config environment vars as the services do. + // for (key, value) in config_env(&self.env_config) { + // env_vars.insert(key, Some(value)); + // } + + // env_vars.insert("TEST_LOG".to_string(), Some("info".into())); + // let mut args = self.config.args.clone().unwrap_or_default(); + + // args.push("--features".to_string()); + + // args.push(if self.build_all { + // "all-integration-tests".to_string() + // } else { + // self.config.features.join(",") + // }); + + // // If the test field is not present then use the --lib flag + // match self.config.test { + // Some(ref test_arg) => { + // args.push("--test".to_string()); + // args.push(test_arg.to_string()); + // } + // None => args.push("--lib".to_string()), + // } + + // // Ensure the test_filter args are passed as well + // if let Some(ref filter) = self.config.test_filter { + // args.push(filter.to_string()); + // } + // args.extend(extra_args); + + // // Some arguments are not compatible with the --no-capture arg + // if !args.contains(&"--test-threads".to_string()) { + // args.push("--no-capture".to_string()); + // } + + // if self.retries > 0 { + // args.push("--retries".to_string()); + // args.push(self.retries.to_string()); + // } + + // self.runner + // .test(&env_vars, &self.config.runner.env, &args)?; + + if let Some(compose) = &self.compose { + compose.run(&self.env_config)?; + } if !active { self.runner.remove()?; @@ -149,7 +218,7 @@ impl IntegrationTest { } self.runner.remove()?; - compose.stop()?; + compose.stop(&self.env_config)?; self.envs_dir.remove()?; } @@ -165,10 +234,109 @@ struct Compose { config: ComposeConfig, network: String, temp_file: NamedTempFile, + args: Vec, } +const MOUNT_PATH: &str = "/home/vector"; +const TARGET_PATH: &str = "/home/target"; +const VOLUME_TARGET: &str = "vector_target"; +const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; +const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; + +fn default_runner_block(runner_name: Option, mut other_volumes: Vec, dep_on_services: Option>, _args: &[String], env: Environment) -> ComposeService { + + let command = vec![ + "/bin/sleep".to_string(), + "infinity".to_string(), + ]; + + // let mut command = vec![ + // "cargo".to_string(), + // "nextest".to_string(), + // "run".to_string(), + // "--no-fail-fast".to_string(), + // "--no-default-features".to_string(), + // ]; + + // command.extend_from_slice(args); + + let mut volumes = vec![ + format!("{}:{MOUNT_PATH}", crate::app::path()), + format!("{VOLUME_TARGET}:{TARGET_PATH}"), + format!("{VOLUME_CARGO_GIT}:/usr/local/cargo/git"), + format!("{VOLUME_CARGO_REGISTRY}:/usr/local/cargo/registry"), + ]; + + volumes.append(&mut other_volumes); + + let mut environment = vec![]; + + for (key, value) in env { + match value { + Some(value) => environment.push(format!("{key}={value}")), + None => println!("got inner env key: {key}"), + } + } + let environment = if environment.len() > 0 { + Some(environment) + } else { + None + }; + + ComposeService { + image: None, + hostname: None, + // hostname: Some("runner".to_string()), + container_name: Some(container_name(runner_name)), + build: Some(Value::Mapping(Mapping::from_iter([ + (Value::String("context".to_string()), Value::String("${PWD}".to_string())), + (Value::String("dockerfile".to_string()), Value::String("scripts/integration/Dockerfile".to_string())), + (Value::String("args".to_string()), Value::Sequence(vec![Value::String(format!("RUST_VERSION={}", super::get_rust_version())) ] ) ) + ]))), + command: Some(super::config::Command::Multiple(command)), + ports: None, + env_file: None, + volumes: Some(volumes), + environment: environment, + depends_on: dep_on_services, + healthcheck: None, + working_dir: Some("/home/vector".to_string()), + labels: Some(vec!["vector-test-runner=true".to_string()]) + } + // BTreeMap::from_iter([ + // ( + // "build".to_owned(), + // BTreeMap::from_iter([ + // ("context".to_owned(), "${PWD}".to_owned()), + // ( + // "dockerfile".to_owned(), + // "scripts/integration/Dockerfile".to_owned(), + // )("args".to_owned(), vec![]), + // ]), + // ), + // ("working_dir".to_owned(), "/home/vector".to_owned()), + // ("volumes".to_owned(), vec![]), + // ("command".to_owned(), vec![]), + // ]) +} + +fn container_name(runner_name: Option) -> String { + if let Some(runner_name) = runner_name.as_ref() { + format!( + "vector-test-runner-{}-{}", + runner_name, + super::get_rust_version() + ) + } else { + format!("vector-test-runner-{}", super::get_rust_version()) + } +} + +// fn image_name(runner_name: Option) -> String { +// format!("{}:latest", container_name(runner_name)) +// } impl Compose { - fn new(test_dir: PathBuf, env: Environment, network: String) -> Result> { + fn new(test_dir: PathBuf, env: Environment, network: String, runner_name: Option, volumes: Vec, args: &[String], runner_env: Environment) -> Result> { let original_path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect(); match original_path.try_exists() { @@ -185,6 +353,42 @@ impl Compose { ]), ); + let mut runner_dep_on_services = vec![]; + config.services.iter().for_each(|(name, config)| { + let mut is_runner_dep = false; + if let Some(deps) = &config.depends_on { + for dep in deps { + if dep == "runner" { + is_runner_dep = true; + } + } + + } + if !is_runner_dep { + runner_dep_on_services.push(name.to_string()); + } + }); + + println!("deps: {:?}", runner_dep_on_services); + + let runner_dep_on_services = if runner_dep_on_services.is_empty() { + None + } else { + Some(runner_dep_on_services) + }; + + + // Inject the runner block to the services + config + .services + .insert("runner".to_string(), default_runner_block(runner_name, volumes, runner_dep_on_services, args, runner_env)); + + // Inject the volumes + config.volumes.insert("vector_target".to_string(), Value::Mapping(Mapping::new())); + + config.volumes.insert("vector_cargo_git".to_string(), Value::Mapping(Mapping::new())); + config.volumes.insert("vector_cargo_registry".to_string(), Value::Mapping(Mapping::new())); + // Create a named tempfile, there may be resource leakage here in case of SIGINT // Tried tempfile::tempfile() but this returns a File object without a usable path // https://docs.rs/tempfile/latest/tempfile/#resource-leaking @@ -194,12 +398,20 @@ impl Compose { .tempfile_in(&test_dir) .with_context(|| "Failed to create temporary compose file")?; + fs::write( + "/tmp/foo.compose.yaml", + serde_yaml::to_string(&config) + .with_context(|| "Failed to serialize modified compose.yaml")?, + )?; + fs::write( temp_file.path(), serde_yaml::to_string(&config) .with_context(|| "Failed to serialize modified compose.yaml")?, )?; + let args = Vec::from(args); + Ok(Some(Self { original_path, test_dir, @@ -207,6 +419,7 @@ impl Compose { config, network, temp_file, + args, })) } } @@ -214,15 +427,31 @@ impl Compose { fn start(&self, config: &Environment) -> Result<()> { self.prepare()?; - self.run("Starting", &["up", "--detach"], Some(config)) + self.command("Building", &vec!["build".to_string()], Some(config))?; + self.command("Starting", &vec!["up".to_string(), "--detach".to_string()], Some(config)) + } + + fn run(&self, config: &Environment) -> Result<()> { + + let mut args = vec!["run".to_string(), "runner".to_string(), + "cargo".to_string(), + "nextest".to_string(), + "run".to_string(), + "--no-fail-fast".to_string(), + "--no-default-features".to_string(), + ]; + args.extend(self.args.clone()); + + self.command("Starting", &args, Some(config)) } - fn stop(&self) -> Result<()> { + fn stop(&self, config: &Environment) -> Result<()> { // The config settings are not needed when stopping a compose setup. - self.run("Stopping", &["down", "--timeout", "0", "--volumes"], None) + self.command("Stopping", &vec!["down".to_string(), "--timeout".to_string(), "0".to_string(), "--volumes".to_string()], Some(config)) } - fn run(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { + // fn command(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { + fn command(&self, action: &str, args: &Vec, config: Option<&Environment>) -> Result<()> { let mut command = CONTAINER_TOOL.clone(); command.push("-compose"); let mut command = Command::new(command); @@ -239,6 +468,10 @@ impl Compose { command.arg(self.temp_file.path()); } + // command.arg("run"); + // command.arg("--rm"); + // command.arg("runner"); + command.args(args); command.current_dir(&self.test_dir); diff --git a/vdev/src/testing/mod.rs b/vdev/src/testing/mod.rs index 1bec37be971f7..221c64b6c6af9 100644 --- a/vdev/src/testing/mod.rs +++ b/vdev/src/testing/mod.rs @@ -2,3 +2,10 @@ pub mod config; pub mod integration; pub mod runner; pub mod state; + +pub(self) fn get_rust_version() -> String { + match config::RustToolchainConfig::parse() { + Ok(config) => config.channel, + Err(error) => fatal!("Could not read `rust-toolchain.toml` file: {error}"), + } +} diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index e4fd18cdd015b..be7497d5e7696 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -1,20 +1,22 @@ -use std::collections::HashSet; +//use std::collections::HashSet; use std::process::{Command, Stdio}; use std::{env, ffi::OsStr, ffi::OsString, path::PathBuf}; use anyhow::Result; use once_cell::sync::Lazy; -use super::config::{Environment, IntegrationRunnerConfig, RustToolchainConfig}; -use crate::app::{self, CommandExt as _}; -use crate::util::{ChainArgs as _, IS_A_TTY}; +use super::config::{Environment, IntegrationRunnerConfig}; +// use crate::app::{self, CommandExt as _}; +use crate::app::CommandExt as _; +// use crate::util::{ChainArgs as _, IS_A_TTY}; +use crate::util::IS_A_TTY; -const MOUNT_PATH: &str = "/home/vector"; +// const MOUNT_PATH: &str = "/home/vector"; const TARGET_PATH: &str = "/home/target"; -const VOLUME_TARGET: &str = "vector_target"; -const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; -const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; -const RUNNER_HOSTNAME: &str = "runner"; +// const VOLUME_TARGET: &str = "vector_target"; +// const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; +// const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; +// const RUNNER_HOSTNAME: &str = "runner"; const TEST_COMMAND: &[&str] = &[ "cargo", "nextest", @@ -47,13 +49,6 @@ fn detect_container_tool() -> OsString { fatal!("No container tool could be detected."); } -fn get_rust_version() -> String { - match RustToolchainConfig::parse() { - Ok(config) => config.channel, - Err(error) => fatal!("Could not read `rust-toolchain.toml` file: {error}"), - } -} - fn dockercmd>(args: impl IntoIterator) -> Command { let mut command = Command::new(&*CONTAINER_TOOL); command.args(args); @@ -95,10 +90,10 @@ pub trait ContainerTestRunner: TestRunner { fn volumes(&self) -> Vec; - fn stop(&self) -> Result<()> { - dockercmd(["stop", "--time", "0", &self.container_name()]) - .wait(format!("Stopping container {}", self.container_name())) - } + // fn stop(&self) -> Result<()> { + // dockercmd(["stop", "--time", "0", &self.container_name()]) + // .wait(format!("Stopping container {}", self.container_name())) + // } fn state(&self) -> Result { let mut command = dockercmd(["ps", "-a", "--format", "{{.Names}} {{.State}}"]); @@ -151,52 +146,58 @@ pub trait ContainerTestRunner: TestRunner { } fn ensure_volumes(&self) -> Result<()> { - let mut command = dockercmd(["volume", "ls", "--format", "{{.Name}}"]); - - let mut volumes = HashSet::new(); - volumes.insert(VOLUME_TARGET); - volumes.insert(VOLUME_CARGO_GIT); - volumes.insert(VOLUME_CARGO_REGISTRY); - for volume in command.check_output()?.lines() { - volumes.take(volume); - } + // let mut command = dockercmd(["volume", "ls", "--format", "{{.Name}}"]); - for volume in &volumes { - dockercmd(["volume", "create", volume]).wait(format!("Creating volume {volume}"))?; - } + // let mut volumes = HashSet::new(); + // volumes.insert(VOLUME_TARGET); + // volumes.insert(VOLUME_CARGO_GIT); + // volumes.insert(VOLUME_CARGO_REGISTRY); + // for volume in command.check_output()?.lines() { + // volumes.take(volume); + // } + + // for volume in &volumes { + // dockercmd(["volume", "create", volume]).wait(format!("Creating volume {volume}"))?; + // } Ok(()) } fn build(&self) -> Result<()> { - let dockerfile: PathBuf = [app::path(), "scripts", "integration", "Dockerfile"] - .iter() - .collect(); - let mut command = dockercmd(["build"]); - command.current_dir(app::path()); - if *IS_A_TTY { - command.args(["--progress", "tty"]); - } - command.args([ - "--pull", - "--tag", - &self.image_name(), - "--file", - dockerfile.to_str().unwrap(), - "--label", - "vector-test-runner=true", - "--build-arg", - &format!("RUST_VERSION={}", get_rust_version()), - ".", - ]); - - waiting!("Building image {}", self.image_name()); - command.check_run() + // let dockerfile: PathBuf = [app::path(), "scripts", "integration", "Dockerfile"] + // .iter() + // .collect(); + // let mut command = dockercmd(["build"]); + // command.current_dir(app::path()); + // if *IS_A_TTY { + // command.args(["--progress", "tty"]); + // } + // command.args([ + // "--pull", + // "--tag", + // &self.image_name(), + // "--file", + // dockerfile.to_str().unwrap(), + // "--label", + // "vector-test-runner=true", + // "--build-arg", + // &format!("RUST_VERSION={}", super::get_rust_version()), + // ".", + // ]); + + // waiting!("Building image {}", self.image_name()); + // command.check_run() + Ok(()) } fn start(&self) -> Result<()> { - dockercmd(["start", &self.container_name()]) - .wait(format!("Starting container {}", self.container_name())) + Ok(()) + // let res = dockercmd(["start", &self.container_name()]) + // .wait(format!("Starting container {}", self.container_name())); + // std::thread::spawn(move || { + // run_server(); + // }); + // res } fn remove(&self) -> Result<()> { @@ -214,48 +215,74 @@ pub trait ContainerTestRunner: TestRunner { } fn create(&self) -> Result<()> { - let network_name = self.network_name().unwrap_or("host"); - - let docker_socket = format!("{}:/var/run/docker.sock", DOCKER_SOCKET.display()); - let docker_args = self - .needs_docker_socket() - .then(|| vec!["--volume", &docker_socket]) - .unwrap_or_default(); - - let volumes = self.volumes(); - let volumes: Vec<_> = volumes - .iter() - .flat_map(|volume| ["--volume", volume]) - .collect(); - - dockercmd( - [ - "create", - "--name", - &self.container_name(), - "--network", - network_name, - "--hostname", - RUNNER_HOSTNAME, - "--workdir", - MOUNT_PATH, - "--volume", - &format!("{}:{MOUNT_PATH}", app::path()), - "--volume", - &format!("{VOLUME_TARGET}:{TARGET_PATH}"), - "--volume", - &format!("{VOLUME_CARGO_GIT}:/usr/local/cargo/git"), - "--volume", - &format!("{VOLUME_CARGO_REGISTRY}:/usr/local/cargo/registry"), - ] - .chain_args(volumes) - .chain_args(docker_args) - .chain_args([&self.image_name(), "/bin/sleep", "infinity"]), - ) - .wait(format!("Creating container {}", self.container_name())) + Ok(()) + // let network_name = self.network_name().unwrap_or("host"); + + // let docker_socket = format!("{}:/var/run/docker.sock", DOCKER_SOCKET.display()); + // let docker_args = self + // .needs_docker_socket() + // .then(|| vec!["--volume", &docker_socket]) + // .unwrap_or_default(); + + // let volumes = self.volumes(); + // let volumes: Vec<_> = volumes + // .iter() + // .flat_map(|volume| ["--volume", volume]) + // .collect(); + + // dockercmd( + // [ + // "create", + // "--name", + // &self.container_name(), + // "--network", + // network_name, + // "--hostname", + // RUNNER_HOSTNAME, + // "--workdir", + // MOUNT_PATH, + // "--volume", + // &format!("{}:{MOUNT_PATH}", app::path()), + // "--volume", + // &format!("{VOLUME_TARGET}:{TARGET_PATH}"), + // "--volume", + // &format!("{VOLUME_CARGO_GIT}:/usr/local/cargo/git"), + // "--volume", + // &format!("{VOLUME_CARGO_REGISTRY}:/usr/local/cargo/registry"), + // "--expose", + // "4321", + // ] + // .chain_args(volumes) + // .chain_args(docker_args) + // .chain_args([&self.image_name(), "/bin/sleep", "infinity"]), + // ) + // .wait(format!("Creating container {}", self.container_name())) } } +// fn run_server() { +// let listener = std::net::TcpListener::bind("0.0.0.0:4321").unwrap(); +// // accept connections and process them, spawning a new thread for each one +// println!("Server listening on port 3333"); +// for stream in listener.incoming() { +// match stream { +// Ok(stream) => { +// println!("New connection: {}", stream.peer_addr().unwrap()); +// std::thread::spawn(move || { +// // connection succeeded +// //handle_client(stream) +// }); +// } +// Err(e) => { +// println!("Error: {}", e); +// /* connection failed */ +// } +// } +// } +// // close the socket server +// drop(listener); +// } + impl TestRunner for T where T: ContainerTestRunner, @@ -348,9 +375,13 @@ impl ContainerTestRunner for IntegrationTestRunner { fn container_name(&self) -> String { if let Some(integration) = self.integration.as_ref() { - format!("vector-test-runner-{}-{}", integration, get_rust_version()) + format!( + "vector-test-runner-{}-{}", + integration, + super::get_rust_version() + ) } else { - format!("vector-test-runner-{}", get_rust_version()) + format!("vector-test-runner-{}", super::get_rust_version()) } } @@ -375,7 +406,7 @@ impl ContainerTestRunner for DockerTestRunner { } fn container_name(&self) -> String { - format!("vector-test-runner-{}", get_rust_version()) + format!("vector-test-runner-{}", super::get_rust_version()) } fn image_name(&self) -> String { From 758a06a58931c617c901df91fd2deacacb61f317 Mon Sep 17 00:00:00 2001 From: Kyle Criddle Date: Tue, 5 Sep 2023 22:18:31 +0000 Subject: [PATCH 07/99] more vdev drafts --- .../e2e/datadog/logs/agent_only/datadog.yaml | 2 +- .../datadog/logs/agent_vector/datadog.yaml | 2 +- tests/e2e/datadog/logs/mod.rs | 2 +- vdev/src/testing/integration.rs | 187 ++++++++++++------ 4 files changed, 131 insertions(+), 62 deletions(-) diff --git a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml index 8d14182971b15..82def24728aff 100644 --- a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml @@ -11,4 +11,4 @@ logs_config: # process_realtime: 1 use_dogstatsd: false -#log_level: 'debug' +log_level: 'info' diff --git a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml index 82ea8e1e7d0ec..ab1c85a24cd29 100644 --- a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml @@ -18,6 +18,6 @@ vector: # process_realtime: 1 use_dogstatsd: false -log_level: 'debug' +log_level: 'info' log_file: './agent.log' diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 7d2f6bbb0cd91..7092537730787 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -74,7 +74,7 @@ async fn test_logs() { dbg!(&_vector_payloads); - sleep(Duration::from_secs(90)).await; + sleep(Duration::from_secs(300)).await; assert!(true); } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index da6b288cb5103..cee952ed13d0a 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,14 +1,17 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; use anyhow::{bail, Context, Result}; -use serde_yaml::{Value, Mapping}; +use serde_yaml::{Mapping, Value}; use tempfile::{Builder, NamedTempFile}; use super::config::{ComposeConfig, ComposeService}; use super::config::{Environment, IntegrationTestConfig}; use super::runner::{ // ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, - ContainerTestRunner as _, IntegrationTestRunner, CONTAINER_TOOL, DOCKER_SOCKET, + ContainerTestRunner as _, + IntegrationTestRunner, + CONTAINER_TOOL, + DOCKER_SOCKET, }; use super::state::EnvsDir; use crate::app::CommandExt as _; @@ -44,16 +47,16 @@ impl IntegrationTest { }; let network_name = format!("vector-integration-tests-{integration}"); - let volumes = config.runner + let volumes = config + .runner .volumes .iter() .map(|(a, b)| format!("{a}:{b}")) .collect(); - + // None if compiling with all integration test feature flag. let runner_name = (!build_all).then(|| integration.clone()); - let mut args = config.args.clone().unwrap_or_default(); args.push("--features".to_string()); @@ -89,7 +92,6 @@ impl IntegrationTest { args.push(retries.to_string()); } - let mut env_vars = config.env.clone(); env_vars.extend(config.runner.env.clone()); @@ -102,7 +104,15 @@ impl IntegrationTest { env_vars.insert("TEST_LOG".to_string(), Some("info".into())); println!("new compose"); - let compose = Compose::new(test_dir, env_config.clone(), network_name.clone(), runner_name.clone(), volumes, &args, env_vars)?; + let compose = Compose::new( + test_dir, + env_config.clone(), + network_name.clone(), + runner_name.clone(), + volumes, + &args, + env_vars, + )?; println!("int runner"); let runner = IntegrationTestRunner::new( @@ -234,7 +244,7 @@ struct Compose { config: ComposeConfig, network: String, temp_file: NamedTempFile, - args: Vec, + //args: Vec, } const MOUNT_PATH: &str = "/home/vector"; const TARGET_PATH: &str = "/home/target"; @@ -242,22 +252,24 @@ const VOLUME_TARGET: &str = "vector_target"; const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; -fn default_runner_block(runner_name: Option, mut other_volumes: Vec, dep_on_services: Option>, _args: &[String], env: Environment) -> ComposeService { - - let command = vec![ - "/bin/sleep".to_string(), - "infinity".to_string(), +fn default_runner_block( + runner_name: Option, + mut other_volumes: Vec, + dep_on_services: Option>, + args: &[String], + env: Environment, +) -> ComposeService { + //let command = vec!["/bin/sleep".to_string(), "infinity".to_string()]; + + let mut command = vec![ + "cargo".to_string(), + "nextest".to_string(), + "run".to_string(), + "--no-fail-fast".to_string(), + "--no-default-features".to_string(), ]; - // let mut command = vec![ - // "cargo".to_string(), - // "nextest".to_string(), - // "run".to_string(), - // "--no-fail-fast".to_string(), - // "--no-default-features".to_string(), - // ]; - - // command.extend_from_slice(args); + command.extend_from_slice(args); let mut volumes = vec![ format!("{}:{MOUNT_PATH}", crate::app::path()), @@ -270,12 +282,12 @@ fn default_runner_block(runner_name: Option, mut other_volumes: Vec environment.push(format!("{key}={value}")), - None => println!("got inner env key: {key}"), - } + for (key, value) in env { + match value { + Some(value) => environment.push(format!("{key}={value}")), + None => println!("got inner env key: {key}"), } + } let environment = if environment.len() > 0 { Some(environment) } else { @@ -288,19 +300,31 @@ fn default_runner_block(runner_name: Option, mut other_volumes: Vec) -> String { // } impl Compose { - fn new(test_dir: PathBuf, env: Environment, network: String, runner_name: Option, volumes: Vec, args: &[String], runner_env: Environment) -> Result> { + fn new( + test_dir: PathBuf, + env: Environment, + network: String, + runner_name: Option, + volumes: Vec, + args: &[String], + runner_env: Environment, + ) -> Result> { let original_path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect(); match original_path.try_exists() { @@ -362,7 +394,6 @@ impl Compose { is_runner_dep = true; } } - } if !is_runner_dep { runner_dep_on_services.push(name.to_string()); @@ -376,18 +407,32 @@ impl Compose { } else { Some(runner_dep_on_services) }; - // Inject the runner block to the services - config - .services - .insert("runner".to_string(), default_runner_block(runner_name, volumes, runner_dep_on_services, args, runner_env)); + config.services.insert( + "runner".to_string(), + default_runner_block( + runner_name, + volumes, + runner_dep_on_services, + args, + runner_env, + ), + ); // Inject the volumes - config.volumes.insert("vector_target".to_string(), Value::Mapping(Mapping::new())); + config + .volumes + .insert("vector_target".to_string(), Value::Mapping(Mapping::new())); - config.volumes.insert("vector_cargo_git".to_string(), Value::Mapping(Mapping::new())); - config.volumes.insert("vector_cargo_registry".to_string(), Value::Mapping(Mapping::new())); + config.volumes.insert( + "vector_cargo_git".to_string(), + Value::Mapping(Mapping::new()), + ); + config.volumes.insert( + "vector_cargo_registry".to_string(), + Value::Mapping(Mapping::new()), + ); // Create a named tempfile, there may be resource leakage here in case of SIGINT // Tried tempfile::tempfile() but this returns a File object without a usable path @@ -410,7 +455,7 @@ impl Compose { .with_context(|| "Failed to serialize modified compose.yaml")?, )?; - let args = Vec::from(args); + //let args = Vec::from(args); Ok(Some(Self { original_path, @@ -419,7 +464,7 @@ impl Compose { config, network, temp_file, - args, + //args, })) } } @@ -427,31 +472,55 @@ impl Compose { fn start(&self, config: &Environment) -> Result<()> { self.prepare()?; - self.command("Building", &vec!["build".to_string()], Some(config))?; - self.command("Starting", &vec!["up".to_string(), "--detach".to_string()], Some(config)) + self.command("Building", &vec!["build".to_string()], Some(config)) + //self.command( + // "Starting", + // &vec!["up".to_string(), "--detach".to_string()], + // Some(config), + //) } - - fn run(&self, config: &Environment) -> Result<()> { - let mut args = vec!["run".to_string(), "runner".to_string(), - "cargo".to_string(), - "nextest".to_string(), - "run".to_string(), - "--no-fail-fast".to_string(), - "--no-default-features".to_string(), - ]; - args.extend(self.args.clone()); - - self.command("Starting", &args, Some(config)) + fn run(&self, config: &Environment) -> Result<()> { + self.command( + "Starting", + &vec!["up".to_string(), "--detach".to_string()], + Some(config), + ) + //let mut args = vec![ + // "run".to_string(), + // "runner".to_string(), + // "cargo".to_string(), + // "nextest".to_string(), + // "run".to_string(), + // "--no-fail-fast".to_string(), + // "--no-default-features".to_string(), + //]; + //args.extend(self.args.clone()); + + //self.command("Starting", &args, Some(config)) } fn stop(&self, config: &Environment) -> Result<()> { // The config settings are not needed when stopping a compose setup. - self.command("Stopping", &vec!["down".to_string(), "--timeout".to_string(), "0".to_string(), "--volumes".to_string()], Some(config)) + self.command( + "Stopping", + &vec![ + "down".to_string(), + "--timeout".to_string(), + "0".to_string(), + "--volumes".to_string(), + ], + Some(config), + ) } // fn command(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { - fn command(&self, action: &str, args: &Vec, config: Option<&Environment>) -> Result<()> { + fn command( + &self, + action: &str, + args: &Vec, + config: Option<&Environment>, + ) -> Result<()> { let mut command = CONTAINER_TOOL.clone(); command.push("-compose"); let mut command = Command::new(command); From 939676408eac7cb685973146f4ce849603381f52 Mon Sep 17 00:00:00 2001 From: Kyle Criddle Date: Tue, 5 Sep 2023 22:19:26 +0000 Subject: [PATCH 08/99] Revert "more vdev drafts" This reverts commit 758a06a58931c617c901df91fd2deacacb61f317. --- .../e2e/datadog/logs/agent_only/datadog.yaml | 2 +- .../datadog/logs/agent_vector/datadog.yaml | 2 +- tests/e2e/datadog/logs/mod.rs | 2 +- vdev/src/testing/integration.rs | 187 ++++++------------ 4 files changed, 62 insertions(+), 131 deletions(-) diff --git a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml index 82def24728aff..8d14182971b15 100644 --- a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml @@ -11,4 +11,4 @@ logs_config: # process_realtime: 1 use_dogstatsd: false -log_level: 'info' +#log_level: 'debug' diff --git a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml index ab1c85a24cd29..82ea8e1e7d0ec 100644 --- a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml @@ -18,6 +18,6 @@ vector: # process_realtime: 1 use_dogstatsd: false -log_level: 'info' +log_level: 'debug' log_file: './agent.log' diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 7092537730787..7d2f6bbb0cd91 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -74,7 +74,7 @@ async fn test_logs() { dbg!(&_vector_payloads); - sleep(Duration::from_secs(300)).await; + sleep(Duration::from_secs(90)).await; assert!(true); } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index cee952ed13d0a..da6b288cb5103 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,17 +1,14 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; use anyhow::{bail, Context, Result}; -use serde_yaml::{Mapping, Value}; +use serde_yaml::{Value, Mapping}; use tempfile::{Builder, NamedTempFile}; use super::config::{ComposeConfig, ComposeService}; use super::config::{Environment, IntegrationTestConfig}; use super::runner::{ // ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, - ContainerTestRunner as _, - IntegrationTestRunner, - CONTAINER_TOOL, - DOCKER_SOCKET, + ContainerTestRunner as _, IntegrationTestRunner, CONTAINER_TOOL, DOCKER_SOCKET, }; use super::state::EnvsDir; use crate::app::CommandExt as _; @@ -47,16 +44,16 @@ impl IntegrationTest { }; let network_name = format!("vector-integration-tests-{integration}"); - let volumes = config - .runner + let volumes = config.runner .volumes .iter() .map(|(a, b)| format!("{a}:{b}")) .collect(); - + // None if compiling with all integration test feature flag. let runner_name = (!build_all).then(|| integration.clone()); + let mut args = config.args.clone().unwrap_or_default(); args.push("--features".to_string()); @@ -92,6 +89,7 @@ impl IntegrationTest { args.push(retries.to_string()); } + let mut env_vars = config.env.clone(); env_vars.extend(config.runner.env.clone()); @@ -104,15 +102,7 @@ impl IntegrationTest { env_vars.insert("TEST_LOG".to_string(), Some("info".into())); println!("new compose"); - let compose = Compose::new( - test_dir, - env_config.clone(), - network_name.clone(), - runner_name.clone(), - volumes, - &args, - env_vars, - )?; + let compose = Compose::new(test_dir, env_config.clone(), network_name.clone(), runner_name.clone(), volumes, &args, env_vars)?; println!("int runner"); let runner = IntegrationTestRunner::new( @@ -244,7 +234,7 @@ struct Compose { config: ComposeConfig, network: String, temp_file: NamedTempFile, - //args: Vec, + args: Vec, } const MOUNT_PATH: &str = "/home/vector"; const TARGET_PATH: &str = "/home/target"; @@ -252,24 +242,22 @@ const VOLUME_TARGET: &str = "vector_target"; const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; -fn default_runner_block( - runner_name: Option, - mut other_volumes: Vec, - dep_on_services: Option>, - args: &[String], - env: Environment, -) -> ComposeService { - //let command = vec!["/bin/sleep".to_string(), "infinity".to_string()]; - - let mut command = vec![ - "cargo".to_string(), - "nextest".to_string(), - "run".to_string(), - "--no-fail-fast".to_string(), - "--no-default-features".to_string(), +fn default_runner_block(runner_name: Option, mut other_volumes: Vec, dep_on_services: Option>, _args: &[String], env: Environment) -> ComposeService { + + let command = vec![ + "/bin/sleep".to_string(), + "infinity".to_string(), ]; - command.extend_from_slice(args); + // let mut command = vec![ + // "cargo".to_string(), + // "nextest".to_string(), + // "run".to_string(), + // "--no-fail-fast".to_string(), + // "--no-default-features".to_string(), + // ]; + + // command.extend_from_slice(args); let mut volumes = vec![ format!("{}:{MOUNT_PATH}", crate::app::path()), @@ -282,12 +270,12 @@ fn default_runner_block( let mut environment = vec![]; - for (key, value) in env { - match value { - Some(value) => environment.push(format!("{key}={value}")), - None => println!("got inner env key: {key}"), + for (key, value) in env { + match value { + Some(value) => environment.push(format!("{key}={value}")), + None => println!("got inner env key: {key}"), + } } - } let environment = if environment.len() > 0 { Some(environment) } else { @@ -300,31 +288,19 @@ fn default_runner_block( // hostname: Some("runner".to_string()), container_name: Some(container_name(runner_name)), build: Some(Value::Mapping(Mapping::from_iter([ - ( - Value::String("context".to_string()), - Value::String("${PWD}".to_string()), - ), - ( - Value::String("dockerfile".to_string()), - Value::String("scripts/integration/Dockerfile".to_string()), - ), - ( - Value::String("args".to_string()), - Value::Sequence(vec![Value::String(format!( - "RUST_VERSION={}", - super::get_rust_version() - ))]), - ), + (Value::String("context".to_string()), Value::String("${PWD}".to_string())), + (Value::String("dockerfile".to_string()), Value::String("scripts/integration/Dockerfile".to_string())), + (Value::String("args".to_string()), Value::Sequence(vec![Value::String(format!("RUST_VERSION={}", super::get_rust_version())) ] ) ) ]))), command: Some(super::config::Command::Multiple(command)), ports: None, env_file: None, volumes: Some(volumes), - environment, + environment: environment, depends_on: dep_on_services, healthcheck: None, working_dir: Some("/home/vector".to_string()), - labels: Some(vec!["vector-test-runner=true".to_string()]), + labels: Some(vec!["vector-test-runner=true".to_string()]) } // BTreeMap::from_iter([ // ( @@ -360,15 +336,7 @@ fn container_name(runner_name: Option) -> String { // } impl Compose { - fn new( - test_dir: PathBuf, - env: Environment, - network: String, - runner_name: Option, - volumes: Vec, - args: &[String], - runner_env: Environment, - ) -> Result> { + fn new(test_dir: PathBuf, env: Environment, network: String, runner_name: Option, volumes: Vec, args: &[String], runner_env: Environment) -> Result> { let original_path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect(); match original_path.try_exists() { @@ -394,6 +362,7 @@ impl Compose { is_runner_dep = true; } } + } if !is_runner_dep { runner_dep_on_services.push(name.to_string()); @@ -407,32 +376,18 @@ impl Compose { } else { Some(runner_dep_on_services) }; + // Inject the runner block to the services - config.services.insert( - "runner".to_string(), - default_runner_block( - runner_name, - volumes, - runner_dep_on_services, - args, - runner_env, - ), - ); + config + .services + .insert("runner".to_string(), default_runner_block(runner_name, volumes, runner_dep_on_services, args, runner_env)); // Inject the volumes - config - .volumes - .insert("vector_target".to_string(), Value::Mapping(Mapping::new())); + config.volumes.insert("vector_target".to_string(), Value::Mapping(Mapping::new())); - config.volumes.insert( - "vector_cargo_git".to_string(), - Value::Mapping(Mapping::new()), - ); - config.volumes.insert( - "vector_cargo_registry".to_string(), - Value::Mapping(Mapping::new()), - ); + config.volumes.insert("vector_cargo_git".to_string(), Value::Mapping(Mapping::new())); + config.volumes.insert("vector_cargo_registry".to_string(), Value::Mapping(Mapping::new())); // Create a named tempfile, there may be resource leakage here in case of SIGINT // Tried tempfile::tempfile() but this returns a File object without a usable path @@ -455,7 +410,7 @@ impl Compose { .with_context(|| "Failed to serialize modified compose.yaml")?, )?; - //let args = Vec::from(args); + let args = Vec::from(args); Ok(Some(Self { original_path, @@ -464,7 +419,7 @@ impl Compose { config, network, temp_file, - //args, + args, })) } } @@ -472,55 +427,31 @@ impl Compose { fn start(&self, config: &Environment) -> Result<()> { self.prepare()?; - self.command("Building", &vec!["build".to_string()], Some(config)) - //self.command( - // "Starting", - // &vec!["up".to_string(), "--detach".to_string()], - // Some(config), - //) + self.command("Building", &vec!["build".to_string()], Some(config))?; + self.command("Starting", &vec!["up".to_string(), "--detach".to_string()], Some(config)) } - + fn run(&self, config: &Environment) -> Result<()> { - self.command( - "Starting", - &vec!["up".to_string(), "--detach".to_string()], - Some(config), - ) - //let mut args = vec![ - // "run".to_string(), - // "runner".to_string(), - // "cargo".to_string(), - // "nextest".to_string(), - // "run".to_string(), - // "--no-fail-fast".to_string(), - // "--no-default-features".to_string(), - //]; - //args.extend(self.args.clone()); - - //self.command("Starting", &args, Some(config)) + + let mut args = vec!["run".to_string(), "runner".to_string(), + "cargo".to_string(), + "nextest".to_string(), + "run".to_string(), + "--no-fail-fast".to_string(), + "--no-default-features".to_string(), + ]; + args.extend(self.args.clone()); + + self.command("Starting", &args, Some(config)) } fn stop(&self, config: &Environment) -> Result<()> { // The config settings are not needed when stopping a compose setup. - self.command( - "Stopping", - &vec![ - "down".to_string(), - "--timeout".to_string(), - "0".to_string(), - "--volumes".to_string(), - ], - Some(config), - ) + self.command("Stopping", &vec!["down".to_string(), "--timeout".to_string(), "0".to_string(), "--volumes".to_string()], Some(config)) } // fn command(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { - fn command( - &self, - action: &str, - args: &Vec, - config: Option<&Environment>, - ) -> Result<()> { + fn command(&self, action: &str, args: &Vec, config: Option<&Environment>) -> Result<()> { let mut command = CONTAINER_TOOL.clone(); command.push("-compose"); let mut command = Command::new(command); From 7df74ba666ba21d72252cb16bd52d1ac52eb8c83 Mon Sep 17 00:00:00 2001 From: Kyle Criddle Date: Tue, 5 Sep 2023 22:19:40 +0000 Subject: [PATCH 09/99] Revert "saving progress on vdev changes" This reverts commit 7a692907b23ea5267c789597af0c3f2f588ba3d9. --- .../integration/datadog-e2e-logs/compose.yaml | 7 +- scripts/integration/http-client/compose.yaml | 4 - tests/e2e/datadog/logs/mod.rs | 6 +- vdev/src/commands/integration/start.rs | 2 +- vdev/src/commands/integration/stop.rs | 2 +- vdev/src/commands/integration/test.rs | 26 +- vdev/src/testing/config.rs | 8 +- vdev/src/testing/integration.rs | 341 +++--------------- vdev/src/testing/mod.rs | 7 - vdev/src/testing/runner.rs | 229 +++++------- 10 files changed, 167 insertions(+), 465 deletions(-) diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 8712d5aa01793..67526d42d1c79 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -24,8 +24,7 @@ services: - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} - depends_on: - - runner + # depends_on: # waitforit: # condition: service_healthy environment: @@ -50,10 +49,8 @@ services: # waitforit: # image: docker.io/debian:buster-slim # command: tail -f /dev/null - # # ports: - # # - "4321:4321" # healthcheck: - # test: curl -f http://localhost:4321 || exit 1 + # test: curl -f http://runner:8181 || exit 1 # #test: ["CMD", "curl", "-f", "http://runner:8181", "||", "exit", "1"] # interval: 15s # timeout: 10s diff --git a/scripts/integration/http-client/compose.yaml b/scripts/integration/http-client/compose.yaml index 67701b1097b9e..a1723c93d8d1a 100644 --- a/scripts/integration/http-client/compose.yaml +++ b/scripts/integration/http-client/compose.yaml @@ -1,10 +1,6 @@ version: '3' services: - foo: - image: tomgeorge/sleep-infinity:latest - depends_on: - - runner dufs: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 7d2f6bbb0cd91..755c7ce59320b 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -6,7 +6,6 @@ use tokio::{ use vector::{ config::ConfigBuilder, - signal::ShutdownError, test_util::{start_topology, trace_init}, topology::RunningTopology, }; @@ -15,10 +14,7 @@ use super::*; async fn start_vector() -> ( RunningTopology, - ( - mpsc::UnboundedSender, - mpsc::UnboundedReceiver, - ), + (mpsc::UnboundedSender<()>, mpsc::UnboundedReceiver<()>), ) { let dd_agent_address = format!("0.0.0.0:{}", vector_receive_port()); diff --git a/vdev/src/commands/integration/start.rs b/vdev/src/commands/integration/start.rs index 0d7d76d7345c8..da8947e74997a 100644 --- a/vdev/src/commands/integration/start.rs +++ b/vdev/src/commands/integration/start.rs @@ -24,6 +24,6 @@ impl Cli { let env = envs.keys().next().expect("Integration has no environments"); env.clone() }; - IntegrationTest::new(self.integration, environment, false, 0, vec![])?.start() + IntegrationTest::new(self.integration, environment, false, 0)?.start() } } diff --git a/vdev/src/commands/integration/stop.rs b/vdev/src/commands/integration/stop.rs index deef3d758e997..22955d5fa7daf 100644 --- a/vdev/src/commands/integration/stop.rs +++ b/vdev/src/commands/integration/stop.rs @@ -18,7 +18,7 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { if let Some(active) = EnvsDir::new(&self.integration).active()? { - IntegrationTest::new(self.integration, active, self.all_features, 0, vec![])?.stop() + IntegrationTest::new(self.integration, active, self.all_features, 0)?.stop() } else { println!("No environment for {:?} is active.", self.integration); Ok(()) diff --git a/vdev/src/commands/integration/test.rs b/vdev/src/commands/integration/test.rs index de4ed1f2d7591..ca57d895bd0c8 100644 --- a/vdev/src/commands/integration/test.rs +++ b/vdev/src/commands/integration/test.rs @@ -45,28 +45,18 @@ impl Cli { (Some(environment), Some(active)) if environment != active => { bail!("Requested environment {environment:?} does not match active one {active:?}") } - (Some(environment), _) => IntegrationTest::new( - self.integration, - environment, - self.build_all, - retries, - self.args, - )? - .test(), + (Some(environment), _) => { + IntegrationTest::new(self.integration, environment, self.build_all, retries)? + .test(self.args) + } (None, Some(active)) => { - IntegrationTest::new(self.integration, active, self.build_all, retries, self.args)? - .test() + IntegrationTest::new(self.integration, active, self.build_all, retries)? + .test(self.args) } (None, None) => { for env_name in envs.keys() { - IntegrationTest::new( - &self.integration, - env_name, - self.build_all, - retries, - self.args.clone(), - )? - .test()?; + IntegrationTest::new(&self.integration, env_name, self.build_all, retries)? + .test(self.args.clone())?; } Ok(()) } diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 2ab34e461d57b..a04fbe58ccc0e 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -66,16 +66,10 @@ pub struct ComposeService { pub volumes: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, - // #[serde(default, skip_serializing_if = "Option::is_none")] - // pub depends_on: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub depends_on: Option>, + pub depends_on: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub healthcheck: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub working_dir: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub labels: Option>, } #[derive(Debug, Deserialize, Serialize)] diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index da6b288cb5103..b68305ef082b5 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,14 +1,12 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; use anyhow::{bail, Context, Result}; -use serde_yaml::{Value, Mapping}; use tempfile::{Builder, NamedTempFile}; -use super::config::{ComposeConfig, ComposeService}; +use super::config::ComposeConfig; use super::config::{Environment, IntegrationTestConfig}; use super::runner::{ - // ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, - ContainerTestRunner as _, IntegrationTestRunner, CONTAINER_TOOL, DOCKER_SOCKET, + ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, }; use super::state::EnvsDir; use crate::app::CommandExt as _; @@ -23,8 +21,8 @@ pub struct IntegrationTest { runner: IntegrationTestRunner, compose: Option, env_config: Environment, - // build_all: bool, - // retries: u8, + build_all: bool, + retries: u8, } impl IntegrationTest { @@ -33,7 +31,6 @@ impl IntegrationTest { environment: impl Into, build_all: bool, retries: u8, - extra_args: Vec, ) -> Result { let integration = integration.into(); let environment = environment.into(); @@ -43,29 +40,57 @@ impl IntegrationTest { bail!("Could not find environment named {environment:?}"); }; let network_name = format!("vector-integration-tests-{integration}"); + let compose = Compose::new(test_dir, env_config.clone(), network_name.clone())?; - let volumes = config.runner - .volumes - .iter() - .map(|(a, b)| format!("{a}:{b}")) - .collect(); - // None if compiling with all integration test feature flag. let runner_name = (!build_all).then(|| integration.clone()); - - let mut args = config.args.clone().unwrap_or_default(); + let runner = IntegrationTestRunner::new( + runner_name, + &config.runner, + compose.is_some().then_some(network_name), + )?; + + Ok(Self { + integration, + environment, + config, + envs_dir, + runner, + compose, + env_config, + build_all, + retries, + }) + } + + pub fn test(self, extra_args: Vec) -> Result<()> { + let active = self.envs_dir.check_active(&self.environment)?; + self.config.check_required()?; + + //if !active { + // self.start()?; + //} + + let mut env_vars = self.config.env.clone(); + // Make sure the test runner has the same config environment vars as the services do. + for (key, value) in config_env(&self.env_config) { + env_vars.insert(key, Some(value)); + } + + env_vars.insert("TEST_LOG".to_string(), Some("info".into())); + let mut args = self.config.args.clone().unwrap_or_default(); args.push("--features".to_string()); - args.push(if build_all { + args.push(if self.build_all { "all-integration-tests".to_string() } else { - config.features.join(",") + self.config.features.join(",") }); // If the test field is not present then use the --lib flag - match config.test { + match self.config.test { Some(ref test_arg) => { args.push("--test".to_string()); args.push(test_arg.to_string()); @@ -74,7 +99,7 @@ impl IntegrationTest { } // Ensure the test_filter args are passed as well - if let Some(ref filter) = config.test_filter { + if let Some(ref filter) = self.config.test_filter { args.push(filter.to_string()); } args.extend(extra_args); @@ -84,107 +109,13 @@ impl IntegrationTest { args.push("--no-capture".to_string()); } - if retries > 0 { + if self.retries > 0 { args.push("--retries".to_string()); - args.push(retries.to_string()); - } - - - let mut env_vars = config.env.clone(); - - env_vars.extend(config.runner.env.clone()); - - // Make sure the test runner has the same config environment vars as the services do. - // for (key, value) in config_env(&self.env_config) { - // env_vars.insert(key, Some(value)); - // } - - env_vars.insert("TEST_LOG".to_string(), Some("info".into())); - - println!("new compose"); - let compose = Compose::new(test_dir, env_config.clone(), network_name.clone(), runner_name.clone(), volumes, &args, env_vars)?; - - println!("int runner"); - let runner = IntegrationTestRunner::new( - runner_name, - &config.runner, - compose.is_some().then_some(network_name), - )?; - - println!("done new"); - - Ok(Self { - integration, - environment, - config, - envs_dir, - runner, - compose, - env_config, - // build_all, - // retries, - }) - } - - pub fn test(self) -> Result<()> { - let active = self.envs_dir.check_active(&self.environment)?; - self.config.check_required()?; - - //if !active { - // self.start()?; - //} - if !active { - self.start()?; + args.push(self.retries.to_string()); } - // let mut env_vars = self.config.env.clone(); - // // Make sure the test runner has the same config environment vars as the services do. - // for (key, value) in config_env(&self.env_config) { - // env_vars.insert(key, Some(value)); - // } - - // env_vars.insert("TEST_LOG".to_string(), Some("info".into())); - // let mut args = self.config.args.clone().unwrap_or_default(); - - // args.push("--features".to_string()); - - // args.push(if self.build_all { - // "all-integration-tests".to_string() - // } else { - // self.config.features.join(",") - // }); - - // // If the test field is not present then use the --lib flag - // match self.config.test { - // Some(ref test_arg) => { - // args.push("--test".to_string()); - // args.push(test_arg.to_string()); - // } - // None => args.push("--lib".to_string()), - // } - - // // Ensure the test_filter args are passed as well - // if let Some(ref filter) = self.config.test_filter { - // args.push(filter.to_string()); - // } - // args.extend(extra_args); - - // // Some arguments are not compatible with the --no-capture arg - // if !args.contains(&"--test-threads".to_string()) { - // args.push("--no-capture".to_string()); - // } - - // if self.retries > 0 { - // args.push("--retries".to_string()); - // args.push(self.retries.to_string()); - // } - - // self.runner - // .test(&env_vars, &self.config.runner.env, &args)?; - - if let Some(compose) = &self.compose { - compose.run(&self.env_config)?; - } + self.runner + .test(&env_vars, &self.config.runner.env, &args)?; if !active { self.runner.remove()?; @@ -218,7 +149,7 @@ impl IntegrationTest { } self.runner.remove()?; - compose.stop(&self.env_config)?; + compose.stop()?; self.envs_dir.remove()?; } @@ -234,109 +165,10 @@ struct Compose { config: ComposeConfig, network: String, temp_file: NamedTempFile, - args: Vec, } -const MOUNT_PATH: &str = "/home/vector"; -const TARGET_PATH: &str = "/home/target"; -const VOLUME_TARGET: &str = "vector_target"; -const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; -const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; - -fn default_runner_block(runner_name: Option, mut other_volumes: Vec, dep_on_services: Option>, _args: &[String], env: Environment) -> ComposeService { - - let command = vec![ - "/bin/sleep".to_string(), - "infinity".to_string(), - ]; - - // let mut command = vec![ - // "cargo".to_string(), - // "nextest".to_string(), - // "run".to_string(), - // "--no-fail-fast".to_string(), - // "--no-default-features".to_string(), - // ]; - - // command.extend_from_slice(args); - - let mut volumes = vec![ - format!("{}:{MOUNT_PATH}", crate::app::path()), - format!("{VOLUME_TARGET}:{TARGET_PATH}"), - format!("{VOLUME_CARGO_GIT}:/usr/local/cargo/git"), - format!("{VOLUME_CARGO_REGISTRY}:/usr/local/cargo/registry"), - ]; - - volumes.append(&mut other_volumes); - - let mut environment = vec![]; - - for (key, value) in env { - match value { - Some(value) => environment.push(format!("{key}={value}")), - None => println!("got inner env key: {key}"), - } - } - let environment = if environment.len() > 0 { - Some(environment) - } else { - None - }; - - ComposeService { - image: None, - hostname: None, - // hostname: Some("runner".to_string()), - container_name: Some(container_name(runner_name)), - build: Some(Value::Mapping(Mapping::from_iter([ - (Value::String("context".to_string()), Value::String("${PWD}".to_string())), - (Value::String("dockerfile".to_string()), Value::String("scripts/integration/Dockerfile".to_string())), - (Value::String("args".to_string()), Value::Sequence(vec![Value::String(format!("RUST_VERSION={}", super::get_rust_version())) ] ) ) - ]))), - command: Some(super::config::Command::Multiple(command)), - ports: None, - env_file: None, - volumes: Some(volumes), - environment: environment, - depends_on: dep_on_services, - healthcheck: None, - working_dir: Some("/home/vector".to_string()), - labels: Some(vec!["vector-test-runner=true".to_string()]) - } - // BTreeMap::from_iter([ - // ( - // "build".to_owned(), - // BTreeMap::from_iter([ - // ("context".to_owned(), "${PWD}".to_owned()), - // ( - // "dockerfile".to_owned(), - // "scripts/integration/Dockerfile".to_owned(), - // )("args".to_owned(), vec![]), - // ]), - // ), - // ("working_dir".to_owned(), "/home/vector".to_owned()), - // ("volumes".to_owned(), vec![]), - // ("command".to_owned(), vec![]), - // ]) -} - -fn container_name(runner_name: Option) -> String { - if let Some(runner_name) = runner_name.as_ref() { - format!( - "vector-test-runner-{}-{}", - runner_name, - super::get_rust_version() - ) - } else { - format!("vector-test-runner-{}", super::get_rust_version()) - } -} - -// fn image_name(runner_name: Option) -> String { -// format!("{}:latest", container_name(runner_name)) -// } impl Compose { - fn new(test_dir: PathBuf, env: Environment, network: String, runner_name: Option, volumes: Vec, args: &[String], runner_env: Environment) -> Result> { + fn new(test_dir: PathBuf, env: Environment, network: String) -> Result> { let original_path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect(); match original_path.try_exists() { @@ -353,42 +185,6 @@ impl Compose { ]), ); - let mut runner_dep_on_services = vec![]; - config.services.iter().for_each(|(name, config)| { - let mut is_runner_dep = false; - if let Some(deps) = &config.depends_on { - for dep in deps { - if dep == "runner" { - is_runner_dep = true; - } - } - - } - if !is_runner_dep { - runner_dep_on_services.push(name.to_string()); - } - }); - - println!("deps: {:?}", runner_dep_on_services); - - let runner_dep_on_services = if runner_dep_on_services.is_empty() { - None - } else { - Some(runner_dep_on_services) - }; - - - // Inject the runner block to the services - config - .services - .insert("runner".to_string(), default_runner_block(runner_name, volumes, runner_dep_on_services, args, runner_env)); - - // Inject the volumes - config.volumes.insert("vector_target".to_string(), Value::Mapping(Mapping::new())); - - config.volumes.insert("vector_cargo_git".to_string(), Value::Mapping(Mapping::new())); - config.volumes.insert("vector_cargo_registry".to_string(), Value::Mapping(Mapping::new())); - // Create a named tempfile, there may be resource leakage here in case of SIGINT // Tried tempfile::tempfile() but this returns a File object without a usable path // https://docs.rs/tempfile/latest/tempfile/#resource-leaking @@ -398,20 +194,12 @@ impl Compose { .tempfile_in(&test_dir) .with_context(|| "Failed to create temporary compose file")?; - fs::write( - "/tmp/foo.compose.yaml", - serde_yaml::to_string(&config) - .with_context(|| "Failed to serialize modified compose.yaml")?, - )?; - fs::write( temp_file.path(), serde_yaml::to_string(&config) .with_context(|| "Failed to serialize modified compose.yaml")?, )?; - let args = Vec::from(args); - Ok(Some(Self { original_path, test_dir, @@ -419,7 +207,6 @@ impl Compose { config, network, temp_file, - args, })) } } @@ -427,31 +214,15 @@ impl Compose { fn start(&self, config: &Environment) -> Result<()> { self.prepare()?; - self.command("Building", &vec!["build".to_string()], Some(config))?; - self.command("Starting", &vec!["up".to_string(), "--detach".to_string()], Some(config)) - } - - fn run(&self, config: &Environment) -> Result<()> { - - let mut args = vec!["run".to_string(), "runner".to_string(), - "cargo".to_string(), - "nextest".to_string(), - "run".to_string(), - "--no-fail-fast".to_string(), - "--no-default-features".to_string(), - ]; - args.extend(self.args.clone()); - - self.command("Starting", &args, Some(config)) + self.run("Starting", &["up", "--detach"], Some(config)) } - fn stop(&self, config: &Environment) -> Result<()> { + fn stop(&self) -> Result<()> { // The config settings are not needed when stopping a compose setup. - self.command("Stopping", &vec!["down".to_string(), "--timeout".to_string(), "0".to_string(), "--volumes".to_string()], Some(config)) + self.run("Stopping", &["down", "--timeout", "0", "--volumes"], None) } - // fn command(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { - fn command(&self, action: &str, args: &Vec, config: Option<&Environment>) -> Result<()> { + fn run(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { let mut command = CONTAINER_TOOL.clone(); command.push("-compose"); let mut command = Command::new(command); @@ -468,10 +239,6 @@ impl Compose { command.arg(self.temp_file.path()); } - // command.arg("run"); - // command.arg("--rm"); - // command.arg("runner"); - command.args(args); command.current_dir(&self.test_dir); diff --git a/vdev/src/testing/mod.rs b/vdev/src/testing/mod.rs index 221c64b6c6af9..1bec37be971f7 100644 --- a/vdev/src/testing/mod.rs +++ b/vdev/src/testing/mod.rs @@ -2,10 +2,3 @@ pub mod config; pub mod integration; pub mod runner; pub mod state; - -pub(self) fn get_rust_version() -> String { - match config::RustToolchainConfig::parse() { - Ok(config) => config.channel, - Err(error) => fatal!("Could not read `rust-toolchain.toml` file: {error}"), - } -} diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index be7497d5e7696..e4fd18cdd015b 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -1,22 +1,20 @@ -//use std::collections::HashSet; +use std::collections::HashSet; use std::process::{Command, Stdio}; use std::{env, ffi::OsStr, ffi::OsString, path::PathBuf}; use anyhow::Result; use once_cell::sync::Lazy; -use super::config::{Environment, IntegrationRunnerConfig}; -// use crate::app::{self, CommandExt as _}; -use crate::app::CommandExt as _; -// use crate::util::{ChainArgs as _, IS_A_TTY}; -use crate::util::IS_A_TTY; +use super::config::{Environment, IntegrationRunnerConfig, RustToolchainConfig}; +use crate::app::{self, CommandExt as _}; +use crate::util::{ChainArgs as _, IS_A_TTY}; -// const MOUNT_PATH: &str = "/home/vector"; +const MOUNT_PATH: &str = "/home/vector"; const TARGET_PATH: &str = "/home/target"; -// const VOLUME_TARGET: &str = "vector_target"; -// const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; -// const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; -// const RUNNER_HOSTNAME: &str = "runner"; +const VOLUME_TARGET: &str = "vector_target"; +const VOLUME_CARGO_GIT: &str = "vector_cargo_git"; +const VOLUME_CARGO_REGISTRY: &str = "vector_cargo_registry"; +const RUNNER_HOSTNAME: &str = "runner"; const TEST_COMMAND: &[&str] = &[ "cargo", "nextest", @@ -49,6 +47,13 @@ fn detect_container_tool() -> OsString { fatal!("No container tool could be detected."); } +fn get_rust_version() -> String { + match RustToolchainConfig::parse() { + Ok(config) => config.channel, + Err(error) => fatal!("Could not read `rust-toolchain.toml` file: {error}"), + } +} + fn dockercmd>(args: impl IntoIterator) -> Command { let mut command = Command::new(&*CONTAINER_TOOL); command.args(args); @@ -90,10 +95,10 @@ pub trait ContainerTestRunner: TestRunner { fn volumes(&self) -> Vec; - // fn stop(&self) -> Result<()> { - // dockercmd(["stop", "--time", "0", &self.container_name()]) - // .wait(format!("Stopping container {}", self.container_name())) - // } + fn stop(&self) -> Result<()> { + dockercmd(["stop", "--time", "0", &self.container_name()]) + .wait(format!("Stopping container {}", self.container_name())) + } fn state(&self) -> Result { let mut command = dockercmd(["ps", "-a", "--format", "{{.Names}} {{.State}}"]); @@ -146,58 +151,52 @@ pub trait ContainerTestRunner: TestRunner { } fn ensure_volumes(&self) -> Result<()> { - // let mut command = dockercmd(["volume", "ls", "--format", "{{.Name}}"]); - - // let mut volumes = HashSet::new(); - // volumes.insert(VOLUME_TARGET); - // volumes.insert(VOLUME_CARGO_GIT); - // volumes.insert(VOLUME_CARGO_REGISTRY); - // for volume in command.check_output()?.lines() { - // volumes.take(volume); - // } + let mut command = dockercmd(["volume", "ls", "--format", "{{.Name}}"]); + + let mut volumes = HashSet::new(); + volumes.insert(VOLUME_TARGET); + volumes.insert(VOLUME_CARGO_GIT); + volumes.insert(VOLUME_CARGO_REGISTRY); + for volume in command.check_output()?.lines() { + volumes.take(volume); + } - // for volume in &volumes { - // dockercmd(["volume", "create", volume]).wait(format!("Creating volume {volume}"))?; - // } + for volume in &volumes { + dockercmd(["volume", "create", volume]).wait(format!("Creating volume {volume}"))?; + } Ok(()) } fn build(&self) -> Result<()> { - // let dockerfile: PathBuf = [app::path(), "scripts", "integration", "Dockerfile"] - // .iter() - // .collect(); - // let mut command = dockercmd(["build"]); - // command.current_dir(app::path()); - // if *IS_A_TTY { - // command.args(["--progress", "tty"]); - // } - // command.args([ - // "--pull", - // "--tag", - // &self.image_name(), - // "--file", - // dockerfile.to_str().unwrap(), - // "--label", - // "vector-test-runner=true", - // "--build-arg", - // &format!("RUST_VERSION={}", super::get_rust_version()), - // ".", - // ]); - - // waiting!("Building image {}", self.image_name()); - // command.check_run() - Ok(()) + let dockerfile: PathBuf = [app::path(), "scripts", "integration", "Dockerfile"] + .iter() + .collect(); + let mut command = dockercmd(["build"]); + command.current_dir(app::path()); + if *IS_A_TTY { + command.args(["--progress", "tty"]); + } + command.args([ + "--pull", + "--tag", + &self.image_name(), + "--file", + dockerfile.to_str().unwrap(), + "--label", + "vector-test-runner=true", + "--build-arg", + &format!("RUST_VERSION={}", get_rust_version()), + ".", + ]); + + waiting!("Building image {}", self.image_name()); + command.check_run() } fn start(&self) -> Result<()> { - Ok(()) - // let res = dockercmd(["start", &self.container_name()]) - // .wait(format!("Starting container {}", self.container_name())); - // std::thread::spawn(move || { - // run_server(); - // }); - // res + dockercmd(["start", &self.container_name()]) + .wait(format!("Starting container {}", self.container_name())) } fn remove(&self) -> Result<()> { @@ -215,74 +214,48 @@ pub trait ContainerTestRunner: TestRunner { } fn create(&self) -> Result<()> { - Ok(()) - // let network_name = self.network_name().unwrap_or("host"); - - // let docker_socket = format!("{}:/var/run/docker.sock", DOCKER_SOCKET.display()); - // let docker_args = self - // .needs_docker_socket() - // .then(|| vec!["--volume", &docker_socket]) - // .unwrap_or_default(); - - // let volumes = self.volumes(); - // let volumes: Vec<_> = volumes - // .iter() - // .flat_map(|volume| ["--volume", volume]) - // .collect(); - - // dockercmd( - // [ - // "create", - // "--name", - // &self.container_name(), - // "--network", - // network_name, - // "--hostname", - // RUNNER_HOSTNAME, - // "--workdir", - // MOUNT_PATH, - // "--volume", - // &format!("{}:{MOUNT_PATH}", app::path()), - // "--volume", - // &format!("{VOLUME_TARGET}:{TARGET_PATH}"), - // "--volume", - // &format!("{VOLUME_CARGO_GIT}:/usr/local/cargo/git"), - // "--volume", - // &format!("{VOLUME_CARGO_REGISTRY}:/usr/local/cargo/registry"), - // "--expose", - // "4321", - // ] - // .chain_args(volumes) - // .chain_args(docker_args) - // .chain_args([&self.image_name(), "/bin/sleep", "infinity"]), - // ) - // .wait(format!("Creating container {}", self.container_name())) + let network_name = self.network_name().unwrap_or("host"); + + let docker_socket = format!("{}:/var/run/docker.sock", DOCKER_SOCKET.display()); + let docker_args = self + .needs_docker_socket() + .then(|| vec!["--volume", &docker_socket]) + .unwrap_or_default(); + + let volumes = self.volumes(); + let volumes: Vec<_> = volumes + .iter() + .flat_map(|volume| ["--volume", volume]) + .collect(); + + dockercmd( + [ + "create", + "--name", + &self.container_name(), + "--network", + network_name, + "--hostname", + RUNNER_HOSTNAME, + "--workdir", + MOUNT_PATH, + "--volume", + &format!("{}:{MOUNT_PATH}", app::path()), + "--volume", + &format!("{VOLUME_TARGET}:{TARGET_PATH}"), + "--volume", + &format!("{VOLUME_CARGO_GIT}:/usr/local/cargo/git"), + "--volume", + &format!("{VOLUME_CARGO_REGISTRY}:/usr/local/cargo/registry"), + ] + .chain_args(volumes) + .chain_args(docker_args) + .chain_args([&self.image_name(), "/bin/sleep", "infinity"]), + ) + .wait(format!("Creating container {}", self.container_name())) } } -// fn run_server() { -// let listener = std::net::TcpListener::bind("0.0.0.0:4321").unwrap(); -// // accept connections and process them, spawning a new thread for each one -// println!("Server listening on port 3333"); -// for stream in listener.incoming() { -// match stream { -// Ok(stream) => { -// println!("New connection: {}", stream.peer_addr().unwrap()); -// std::thread::spawn(move || { -// // connection succeeded -// //handle_client(stream) -// }); -// } -// Err(e) => { -// println!("Error: {}", e); -// /* connection failed */ -// } -// } -// } -// // close the socket server -// drop(listener); -// } - impl TestRunner for T where T: ContainerTestRunner, @@ -375,13 +348,9 @@ impl ContainerTestRunner for IntegrationTestRunner { fn container_name(&self) -> String { if let Some(integration) = self.integration.as_ref() { - format!( - "vector-test-runner-{}-{}", - integration, - super::get_rust_version() - ) + format!("vector-test-runner-{}-{}", integration, get_rust_version()) } else { - format!("vector-test-runner-{}", super::get_rust_version()) + format!("vector-test-runner-{}", get_rust_version()) } } @@ -406,7 +375,7 @@ impl ContainerTestRunner for DockerTestRunner { } fn container_name(&self) -> String { - format!("vector-test-runner-{}", super::get_rust_version()) + format!("vector-test-runner-{}", get_rust_version()) } fn image_name(&self) -> String { From f4b2f6cc26d2cac1ad4eacabd6fcea6c4f93ddaf Mon Sep 17 00:00:00 2001 From: Kyle Criddle Date: Wed, 6 Sep 2023 23:13:41 +0000 Subject: [PATCH 10/99] extract Vector instance to its own container --- .../integration/datadog-e2e-logs/compose.yaml | 29 ++++++++- tests/data/e2e/datadog/logs/Dockerfile | 41 +++++++++++++ .../e2e/datadog/logs/agent_only/datadog.yaml | 2 +- .../logs.conf.d/custom_logs.d/conf.yaml | 7 ++- .../datadog/logs/agent_vector/datadog.yaml | 11 ++-- .../logs.conf.d/custom_logs.d/conf.yaml | 7 ++- tests/data/e2e/datadog/logs/vector.toml | 22 +++++++ tests/e2e/datadog/logs/mod.rs | 59 +------------------ tests/e2e/datadog/mod.rs | 49 ++++++++++----- vdev/src/testing/config.rs | 2 +- vdev/src/testing/integration.rs | 6 +- 11 files changed, 143 insertions(+), 92 deletions(-) create mode 100644 tests/data/e2e/datadog/logs/Dockerfile create mode 100644 tests/data/e2e/datadog/logs/vector.toml diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 67526d42d1c79..04c27f2bc3998 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -2,6 +2,25 @@ version: '3' services: + vector: + depends_on: + - fakeintake-agent-vector + build: + context: ${PWD} + dockerfile: tests/data/e2e/datadog/logs/Dockerfile + args: + - RUST_VERSION=${RUST_VERSION} + working_dir: /home/vector + network_mode: host + command: + - "-vvv" + - "-c" + - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" + volumes: + - ${PWD}:/home/vector + - target:/home/vector/target + - cargogit:/cargo/git + - cargoregistry:/cargo/registry datadog-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: @@ -24,7 +43,8 @@ services: - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} - # depends_on: + depends_on: + - vector # waitforit: # condition: service_healthy environment: @@ -60,4 +80,9 @@ services: networks: default: - name: ${VECTOR_NETWORK} \ No newline at end of file + name: ${VECTOR_NETWORK} + +volumes: + target: {} + cargogit: {} + cargoregistry: {} diff --git a/tests/data/e2e/datadog/logs/Dockerfile b/tests/data/e2e/datadog/logs/Dockerfile new file mode 100644 index 0000000000000..05ac70d05da30 --- /dev/null +++ b/tests/data/e2e/datadog/logs/Dockerfile @@ -0,0 +1,41 @@ +ARG RUST_VERSION +ARG DEBIAN_RELEASE=bookworm +ARG FEATURES=sources-datadog_agent,sinks-datadog_logs,sinks-console + +# +# VECTOR BUILDER +# +FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} as builder +RUN apt-get update && apt-get -y --no-install-recommends install build-essential git clang cmake libclang-dev libsasl2-dev libstdc++-11-dev libssl-dev libxxhash-dev zlib1g-dev zlib1g +RUN git clone https://github.com/rui314/mold.git \ + && mkdir mold/build \ + && cd mold/build \ + && git checkout v2.0.0 \ + && ../install-build-deps.sh \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ + && cmake --build . -j $(nproc) \ + && cmake --install . + +WORKDIR /vector +COPY . . +ARG FEATURES +RUN scripts/environment/install-protoc.sh +RUN --mount=type=cache,target=/vector/target \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + /usr/local/bin/mold -run cargo build --bin vector \ + --no-default-features --features $FEATURES && \ + cp target/debug/vector . + +# +# TARGET +# +FROM docker.io/debian:${DEBIAN_RELEASE}-slim +RUN apt-get update && apt-get -y --no-install-recommends install zlib1g && rm -rf /var/lib/apt/lists/* +COPY --from=builder /vector/vector /usr/bin/vector +RUN mkdir -p /var/lib/vector + +# Smoke test +RUN ["vector", "--version"] + +ENTRYPOINT ["/usr/bin/vector"] diff --git a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml index 8d14182971b15..20bffd69bf226 100644 --- a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml @@ -11,4 +11,4 @@ logs_config: # process_realtime: 1 use_dogstatsd: false -#log_level: 'debug' +log_level: 'debug' diff --git a/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml index 9f114edc5ecc9..0bb4605289df6 100644 --- a/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml +++ b/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml @@ -1,5 +1,6 @@ logs: - type: file - path: "/var/log/a_custom_log.log" - service: "an_app" - source: "custom_log" + path: /var/log/a_custom_log.log + service: an_app + source: custom_log + start_position: beginning diff --git a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml index 82ea8e1e7d0ec..d22684c83b2f7 100644 --- a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml @@ -2,17 +2,18 @@ api_key: deadbeef logs_enabled: true -connection_reset_interval: 5 + #connection_reset_interval: 5 logs_config: - sender_recovery_reset: true + logs_no_ssl: true + force_use_http: true + #sender_recovery_reset: true vector: logs: enabled: true - url: "http://runner:8181" + url: "http://vector:8181" #url: "http://0.0.0.0:8181" - #url: "http://runner:8081" #intervals: # process_realtime: 1 @@ -20,4 +21,4 @@ vector: use_dogstatsd: false log_level: 'debug' -log_file: './agent.log' + #log_file: './agent.log' diff --git a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml index 9f114edc5ecc9..0bb4605289df6 100644 --- a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml @@ -1,5 +1,6 @@ logs: - type: file - path: "/var/log/a_custom_log.log" - service: "an_app" - source: "custom_log" + path: /var/log/a_custom_log.log + service: an_app + source: custom_log + start_position: beginning diff --git a/tests/data/e2e/datadog/logs/vector.toml b/tests/data/e2e/datadog/logs/vector.toml new file mode 100644 index 0000000000000..a36a075c5efd9 --- /dev/null +++ b/tests/data/e2e/datadog/logs/vector.toml @@ -0,0 +1,22 @@ +data_dir = "/tmp/" + +[sources.agent] +type = "datadog_agent" +address = "0.0.0.0:8181" +multiple_outputs = true +disable_metrics = true +disable_traces = true +store_api_key = false + +[sinks.dd] +inputs = [ "agent.logs" ] +type = "datadog_logs" +default_api_key="unused" +endpoint = "http://fakeintake-agent-vector:80/api/v2/logs" +batch.max_events = 1 +compression = "gzip" + +[sinks.console] +inputs = [ "agent.logs" ] +type = "console" +encoding.codec = "json" diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 755c7ce59320b..da159ff472eeb 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,66 +1,11 @@ -use indoc::indoc; -use tokio::{ - sync::mpsc, - time::{sleep, Duration}, -}; - -use vector::{ - config::ConfigBuilder, - test_util::{start_topology, trace_init}, - topology::RunningTopology, -}; +use vector::test_util::trace_init; use super::*; -async fn start_vector() -> ( - RunningTopology, - (mpsc::UnboundedSender<()>, mpsc::UnboundedReceiver<()>), -) { - let dd_agent_address = format!("0.0.0.0:{}", vector_receive_port()); - - let dd_logs_endpoint = fake_intake_vector_endpoint(); - - let builder: ConfigBuilder = toml::from_str(&format!( - indoc! {r#" - [sources.dd_agent] - type = "datadog_agent" - multiple_outputs = true - disable_metrics = true - disable_traces = true - address = "{}" - - [sinks.dd_logs] - type = "datadog_logs" - inputs = ["dd_agent.logs"] - default_api_key = "unused" - endpoint = "{}" - "#}, - dd_agent_address, dd_logs_endpoint, - )) - .expect("toml parsing should not fail"); - - let config = builder.build().expect("building config should not fail"); - - let (topology, shutdown) = start_topology(config, false).await; - - println!("Started vector."); - - (topology, shutdown) -} - #[tokio::test] async fn test_logs() { trace_init(); - println!("foo test"); - - // panics if vector errors during startup - let (_topology, _shutdown) = start_vector().await; - - // TODO there hopefully is a way to configure the flushing of metrics such that we don't have - // to wait statically for so long here. - sleep(Duration::from_secs(25)).await; - let logs_endpoint = "/api/v2/logs"; let _agent_payloads = get_payloads_agent(&logs_endpoint).await; @@ -70,7 +15,5 @@ async fn test_logs() { dbg!(&_vector_payloads); - sleep(Duration::from_secs(90)).await; - assert!(true); } diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 93706a85c47ba..79ac47cfdc915 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -1,17 +1,8 @@ pub mod logs; - use reqwest::{Client, Method}; use serde::Deserialize; - -fn vector_receive_port() -> u16 { - std::env::var("VECTOR_RECEIVE_PORT") - .unwrap_or_else(|_| "8081".to_string()) - .parse::() - .unwrap() -} - fn fake_intake_vector_endpoint() -> String { std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()) @@ -38,22 +29,48 @@ struct Payload { } async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> Payloads { - let url = format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,); + let url = format!( + "{}/fakeintake/payloads?endpoint={}", + // "{}/fakeintake/payloads?endpoint={}&format=json", + base, + endpoint, + ); - Client::new() + let res = Client::new() .request(Method::GET, &url) .send() .await .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) + //.text() .json::() .await - .expect("Parsing fakeintake payloads failed") + .expect("Parsing fakeintake payloads failed"); + + res + + //println!("body= {:?}", res); + + //Payloads { payloads: vec![] } } -async fn get_payloads_agent(endpoint: &str) -> Payloads { - get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint).await +async fn get_payloads_agent(endpoint: &str) -> Vec { + let mut raw_payloads = get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint) + .await + .payloads; + + // Not sure what this is but the logs endpoint receives some empty payload in the beginning + if raw_payloads.len() > 0 && endpoint == "/api/v2/logs" { + if raw_payloads[0].data == "" && raw_payloads[0].encoding == "" { + raw_payloads.remove(0); + return raw_payloads; + } + } + + raw_payloads } -async fn get_payloads_vector(endpoint: &str) -> Payloads { - get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint).await +async fn get_payloads_vector(endpoint: &str) -> Vec { + get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint) + .await + .payloads } diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index a04fbe58ccc0e..8dab503a493f8 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -67,7 +67,7 @@ pub struct ComposeService { #[serde(default, skip_serializing_if = "Option::is_none")] pub environment: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] - pub depends_on: Option, + pub depends_on: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub healthcheck: Option, } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index b68305ef082b5..8b371853cb37b 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -68,9 +68,9 @@ impl IntegrationTest { let active = self.envs_dir.check_active(&self.environment)?; self.config.check_required()?; - //if !active { - // self.start()?; - //} + if !active { + self.start()?; + } let mut env_vars = self.config.env.clone(); // Make sure the test runner has the same config environment vars as the services do. From 90ac09b17d5fcdeab44dea143d1e9bebd5ebc363 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 6 Sep 2023 17:26:56 -0600 Subject: [PATCH 11/99] cleanup the compose file --- .../integration/datadog-e2e-logs/compose.yaml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 04c27f2bc3998..ace02affbfac1 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -25,12 +25,7 @@ services: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - #- DD_USE_DOGSTATSD=false - DD_HOSTNAME=datadog-agent - #- DD_LOGS_ENABLED=true - #- DD_LOGS_CONFIG_DD_URL=fakeintake-agent:80 - #- DD_LOGS_CONFIG_NO_SSL=true - #- DD_LOGS_CONFIG_FORCE_USE_HTTP=true - DD_ENABLE_PAYLOADS_EVENTS=false - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" @@ -45,16 +40,12 @@ services: image: docker.io/datadog/agent:${CONFIG_VERSION} depends_on: - vector - # waitforit: - # condition: service_healthy environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent-vector - #- DD_LOGS_ENABLED=true - DD_ENABLE_PAYLOADS_EVENTS=false - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" - #- VECTOR_URL="http://runner:8081" volumes: # The Agent config file - ../../../tests/data/e2e/datadog/logs/agent_vector/datadog.yaml:/etc/datadog-agent/datadog.yaml @@ -66,17 +57,6 @@ services: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} fakeintake-agent-vector: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} - # waitforit: - # image: docker.io/debian:buster-slim - # command: tail -f /dev/null - # healthcheck: - # test: curl -f http://runner:8181 || exit 1 - # #test: ["CMD", "curl", "-f", "http://runner:8181", "||", "exit", "1"] - # interval: 15s - # timeout: 10s - # retries: 10 - - #test: ["CMD-SHELL", "curl --fail http://runner:8181 || exit 1"] networks: default: From d95f708ffe6b3653400cecf0eebf84de730760b0 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 7 Sep 2023 16:20:31 -0600 Subject: [PATCH 12/99] Generate logs on fly --- .../integration/datadog-e2e-logs/compose.yaml | 43 +++++++++++++++--- tests/data/e2e/datadog/logs/Dockerfile | 2 +- .../e2e/datadog/logs/agent_only/datadog.yaml | 26 ++++++++--- .../logs.conf.d/custom_logs.d/conf.yaml | 2 +- .../datadog/logs/agent_vector/datadog.yaml | 33 +++++++++----- .../logs.conf.d/custom_logs.d/conf.yaml | 2 +- tests/data/e2e/datadog/logs/custom_log.log | 1 - tests/data/e2e/datadog/logs/vector.toml | 7 +-- tests/e2e/datadog/logs/mod.rs | 44 +++++++++++++++--- tests/e2e/datadog/mod.rs | 45 ++++++++----------- vdev/src/testing/integration.rs | 4 ++ vdev/src/util.rs | 29 ++++++++++++ 12 files changed, 174 insertions(+), 64 deletions(-) delete mode 100644 tests/data/e2e/datadog/logs/custom_log.log diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index ace02affbfac1..7ac17bb3a9366 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -1,7 +1,23 @@ -#version: '2.4' version: '3' services: + # Generates random log data for consumption by the custom Agent check + log_generator: + image: docker.io/mingrammer/flog + command: + - "-f" + - "json" + - "-n" + - "1000" + - "-t" + - "log" + - "-o" + - "/var/log/a_custom.log" + volumes: + - log_path:/var/log/ + + # Receives log data from the `datadog-agent-vector` service and sends + # to the `fakeintake-agent-vector` service. vector: depends_on: - fakeintake-agent-vector @@ -21,8 +37,14 @@ services: - target:/home/vector/target - cargogit:/cargo/git - cargoregistry:/cargo/registry + + # Tails a custom log created by `log_generator` and sends log data to + # the `fakeintake-agent` service datadog-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} + depends_on: + - log_generator + - fakeintake-agent environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent @@ -34,11 +56,15 @@ services: - ../../../tests/data/e2e/datadog/logs/agent_only/datadog.yaml:/etc/datadog-agent/datadog.yaml # The custom logs check - ../../../tests/data/e2e/datadog/logs/agent_only/logs.conf.d:/conf.d:ro - # The custom log to tail - - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log + # The custom log to tail, created by the `log_generator` service + - log_path:/var/log/ + + # Tails a custom log created by `log_generator` and sends log data to + # the `vector` service datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} depends_on: + - log_generator - vector environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} @@ -51,10 +77,16 @@ services: - ../../../tests/data/e2e/datadog/logs/agent_vector/datadog.yaml:/etc/datadog-agent/datadog.yaml # The custom logs check - ../../../tests/data/e2e/datadog/logs/agent_vector/logs.conf.d:/conf.d:ro - # The custom log to tail - - ../../../tests/data/e2e/datadog/logs/custom_log.log:/var/log/a_custom_log.log + # The custom log to tail, created by the `log_generator` service + - log_path:/var/log/ + + # Receives log data from the `datadog-agent` service. Is queried by the test runner + # which does the validation of consistency with the other fakeintake service. fakeintake-agent: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + + # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner + # which does the validation of consistency with the other fakeintake service. fakeintake-agent-vector: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} @@ -63,6 +95,7 @@ networks: name: ${VECTOR_NETWORK} volumes: + log_path: {} target: {} cargogit: {} cargoregistry: {} diff --git a/tests/data/e2e/datadog/logs/Dockerfile b/tests/data/e2e/datadog/logs/Dockerfile index 05ac70d05da30..d427ff1294af8 100644 --- a/tests/data/e2e/datadog/logs/Dockerfile +++ b/tests/data/e2e/datadog/logs/Dockerfile @@ -1,6 +1,6 @@ ARG RUST_VERSION ARG DEBIAN_RELEASE=bookworm -ARG FEATURES=sources-datadog_agent,sinks-datadog_logs,sinks-console +ARG FEATURES=sources-datadog_agent,sinks-datadog_logs # # VECTOR BUILDER diff --git a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml index 20bffd69bf226..69985f3d84496 100644 --- a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_only/datadog.yaml @@ -1,14 +1,28 @@ api_key: deadbeef +log_level: 'debug' + +# disable bunch of stuff we don't need +inventories_configuration_enabled: false +enable_metadata_collection: false +enable_gohai: false + +apm_config: + enabled: false + +process_config: + container_collection: + enabled: false + process_discovery: + enabled: false + disable_realtime_checks: true + +use_dogstatsd: false +# configure logs logs_enabled: true logs_config: logs_dd_url: fakeintake-agent:80 logs_no_ssl: true force_use_http: true - -#intervals: -# process_realtime: 1 - -use_dogstatsd: false -log_level: 'debug' + batch_wait: 1 diff --git a/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml index 0bb4605289df6..716c64baffb35 100644 --- a/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml +++ b/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml @@ -1,6 +1,6 @@ logs: - type: file - path: /var/log/a_custom_log.log + path: /var/log/a_custom.log service: an_app source: custom_log start_position: beginning diff --git a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml index d22684c83b2f7..ac1ed5714055d 100644 --- a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml @@ -1,24 +1,33 @@ api_key: deadbeef +log_level: 'debug' -logs_enabled: true +# disable bunch of stuff we don't need +inventories_configuration_enabled: false +enable_metadata_collection: false +enable_gohai: false + +apm_config: + enabled: false + +process_config: + container_collection: + enabled: false + process_discovery: + enabled: false + disable_realtime_checks: true + +use_dogstatsd: false - #connection_reset_interval: 5 +# configure logs +logs_enabled: true logs_config: logs_no_ssl: true force_use_http: true - #sender_recovery_reset: true + batch_wait: 1 +# send to vector vector: logs: enabled: true url: "http://vector:8181" - #url: "http://0.0.0.0:8181" - -#intervals: -# process_realtime: 1 - -use_dogstatsd: false -log_level: 'debug' - - #log_file: './agent.log' diff --git a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml index 0bb4605289df6..716c64baffb35 100644 --- a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml @@ -1,6 +1,6 @@ logs: - type: file - path: /var/log/a_custom_log.log + path: /var/log/a_custom.log service: an_app source: custom_log start_position: beginning diff --git a/tests/data/e2e/datadog/logs/custom_log.log b/tests/data/e2e/datadog/logs/custom_log.log deleted file mode 100644 index b89597bcbe2a4..0000000000000 --- a/tests/data/e2e/datadog/logs/custom_log.log +++ /dev/null @@ -1 +0,0 @@ -{"host":"01.23.456.789", "user-identifier":"morpheous", "datetime":"31/March/1999/:03:11:11", "score":5, "directors": ["Lana Watchowski", "Lilly Watchowski"], "code-name":"white_rabbit"} diff --git a/tests/data/e2e/datadog/logs/vector.toml b/tests/data/e2e/datadog/logs/vector.toml index a36a075c5efd9..aa78504def09d 100644 --- a/tests/data/e2e/datadog/logs/vector.toml +++ b/tests/data/e2e/datadog/logs/vector.toml @@ -13,10 +13,5 @@ inputs = [ "agent.logs" ] type = "datadog_logs" default_api_key="unused" endpoint = "http://fakeintake-agent-vector:80/api/v2/logs" -batch.max_events = 1 +batch.timeout_secs = 1 compression = "gzip" - -[sinks.console] -inputs = [ "agent.logs" ] -type = "console" -encoding.codec = "json" diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index da159ff472eeb..0afdd605c5e8a 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,19 +1,53 @@ use vector::test_util::trace_init; +use std::{thread::sleep, time::Duration}; + use super::*; +fn assert_timestamp_hostname(payloads: &mut Vec) -> usize { + let mut n_payloads = 0; + payloads.iter_mut().for_each(|payload_array| { + payload_array + .as_array_mut() + .expect("should be array") + .iter_mut() + .for_each(|payload| { + n_payloads += 1; + let obj = payload + .as_object_mut() + .expect("payload entries should be objects"); + assert!(obj.remove("timestamp").is_some()); + assert!(obj.remove("hostname").is_some()); + }) + }); + n_payloads +} + #[tokio::test] async fn test_logs() { trace_init(); + // As it stands, we do still need a small sleep to allow the events to flow through. + // There doesn't seem to be a great way to avoid this. + sleep(Duration::from_secs(5)); + let logs_endpoint = "/api/v2/logs"; - let _agent_payloads = get_payloads_agent(&logs_endpoint).await; + let mut agent_payloads = get_payloads_agent(&logs_endpoint).await; + + //dbg!(&agent_payloads); + + let mut vector_payloads = get_payloads_vector(&logs_endpoint).await; + + //dbg!(&vector_payloads); - dbg!(&_agent_payloads); + assert!(agent_payloads.len() > 0); + assert!(vector_payloads.len() > 0); - let _vector_payloads = get_payloads_vector(&logs_endpoint).await; + let n_agent_payloads = assert_timestamp_hostname(&mut agent_payloads); + let n_vector_payloads = assert_timestamp_hostname(&mut vector_payloads); - dbg!(&_vector_payloads); + println!("n agent payloads: {n_agent_payloads}"); + println!("n vector payloads: {n_vector_payloads}"); - assert!(true); + assert_eq!(agent_payloads, vector_payloads); } diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 79ac47cfdc915..3336d666904ff 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -2,6 +2,7 @@ pub mod logs; use reqwest::{Client, Method}; use serde::Deserialize; +use serde_json::Value; fn fake_intake_vector_endpoint() -> String { std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") @@ -15,62 +16,54 @@ fn fake_intake_agent_endpoint() -> String { // Fakeintake response #[derive(Deserialize, Debug)] -struct Payloads { - payloads: Vec, +struct FakeIntakeResponse { + payloads: Vec, } #[allow(dead_code)] #[derive(Deserialize, Debug)] -struct Payload { - // base64 encoded - data: String, +struct FakeIntakePayload { + data: Value, encoding: String, timestamp: String, } -async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> Payloads { +async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> FakeIntakeResponse { let url = format!( - "{}/fakeintake/payloads?endpoint={}", - // "{}/fakeintake/payloads?endpoint={}&format=json", - base, - endpoint, + "{}/fakeintake/payloads?endpoint={}&format=json", + base, endpoint, ); - let res = Client::new() + Client::new() .request(Method::GET, &url) .send() .await .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) - //.text() - .json::() + .json::() .await - .expect("Parsing fakeintake payloads failed"); - - res - - //println!("body= {:?}", res); - - //Payloads { payloads: vec![] } + .expect("Parsing fakeintake payloads failed") } -async fn get_payloads_agent(endpoint: &str) -> Vec { +async fn get_payloads_agent(endpoint: &str) -> Vec { let mut raw_payloads = get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint) .await .payloads; - // Not sure what this is but the logs endpoint receives some empty payload in the beginning + // Not sure what this is but the logs endpoint receives an empty payload in the beginning if raw_payloads.len() > 0 && endpoint == "/api/v2/logs" { - if raw_payloads[0].data == "" && raw_payloads[0].encoding == "" { + if raw_payloads[0].data.as_array().unwrap().len() == 0 && raw_payloads[0].encoding == "" { raw_payloads.remove(0); - return raw_payloads; } } - raw_payloads + raw_payloads.into_iter().map(|raw| raw.data).collect() } -async fn get_payloads_vector(endpoint: &str) -> Vec { +async fn get_payloads_vector(endpoint: &str) -> Vec { get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint) .await .payloads + .into_iter() + .map(|raw| raw.data) + .collect() } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 8b371853cb37b..78890f479ba1b 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -10,6 +10,7 @@ use super::runner::{ }; use super::state::EnvsDir; use crate::app::CommandExt as _; +use crate::util::export_rust_version; const NETWORK_ENV_VAR: &str = "VECTOR_NETWORK"; @@ -32,6 +33,9 @@ impl IntegrationTest { build_all: bool, retries: u8, ) -> Result { + // some tests require the rust version env var to be set + export_rust_version()?; + let integration = integration.into(); let environment = environment.into(); let (test_dir, config) = IntegrationTestConfig::load(&integration)?; diff --git a/vdev/src/util.rs b/vdev/src/util.rs index 306ccd9c1c1b8..617a75058b90c 100644 --- a/vdev/src/util.rs +++ b/vdev/src/util.rs @@ -51,6 +51,35 @@ pub fn get_channel() -> String { std::env::var("CHANNEL").unwrap_or_else(|_| "custom".to_string()) } +#[derive(Deserialize)] +pub struct RustToolChain { + pub channel: String, +} + +/// The bits of the top-level `rust-toolchain.toml` configuration that `vdev` uses. +#[derive(Deserialize)] +pub struct RustToolChainToml { + pub toolchain: RustToolChain, +} + +impl RustToolChainToml { + pub fn load() -> Result { + let text = fs::read_to_string("rust-toolchain.toml") + .context("Could not read `rust-toolchain.toml`")?; + toml::from_str::(&text) + .context("Invalid contents in `rust-toolchain.toml`") + } +} + +pub fn get_rust_version() -> Result { + RustToolChainToml::load().map(|toml| toml.toolchain.channel) +} + +pub fn export_rust_version() -> Result<()> { + std::env::set_var("RUST_VERSION", get_rust_version()?); + Ok(()) +} + pub fn exists(path: impl AsRef + Debug) -> Result { match fs::metadata(path.as_ref()) { Ok(_) => Ok(true), From 1cc6f8db625397686ebd1a10c4bc2667a94b8f02 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 7 Sep 2023 16:33:17 -0600 Subject: [PATCH 13/99] cleanup --- scripts/integration/datadog-e2e-logs/compose.yaml | 11 ++++++----- scripts/integration/datadog-e2e-logs/test.yaml | 1 - src/topology/mod.rs | 2 +- tests/data/e2e/datadog/{logs => }/Dockerfile | 3 ++- .../logs/{agent_only/datadog.yaml => agent_only.yaml} | 0 .../{agent_vector/datadog.yaml => agent_vector.yaml} | 0 .../agent_vector/logs.conf.d/custom_logs.d/conf.yaml | 6 ------ .../logs.conf.d/custom_logs.d/conf.yaml | 0 8 files changed, 9 insertions(+), 14 deletions(-) rename tests/data/e2e/datadog/{logs => }/Dockerfile (95%) rename tests/data/e2e/datadog/logs/{agent_only/datadog.yaml => agent_only.yaml} (100%) rename tests/data/e2e/datadog/logs/{agent_vector/datadog.yaml => agent_vector.yaml} (100%) delete mode 100644 tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml rename tests/data/e2e/datadog/logs/{agent_only => }/logs.conf.d/custom_logs.d/conf.yaml (100%) diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 7ac17bb3a9366..2f4591c301434 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -23,9 +23,10 @@ services: - fakeintake-agent-vector build: context: ${PWD} - dockerfile: tests/data/e2e/datadog/logs/Dockerfile + dockerfile: tests/data/e2e/datadog/Dockerfile args: - RUST_VERSION=${RUST_VERSION} + - FEATURES=sources-datadog_agent,sinks-datadog_logs working_dir: /home/vector network_mode: host command: @@ -53,9 +54,9 @@ services: - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/logs/agent_only/datadog.yaml:/etc/datadog-agent/datadog.yaml + - ../../../tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml # The custom logs check - - ../../../tests/data/e2e/datadog/logs/agent_only/logs.conf.d:/conf.d:ro + - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro # The custom log to tail, created by the `log_generator` service - log_path:/var/log/ @@ -74,9 +75,9 @@ services: - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/logs/agent_vector/datadog.yaml:/etc/datadog-agent/datadog.yaml + - ../../../tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # The custom logs check - - ../../../tests/data/e2e/datadog/logs/agent_vector/logs.conf.d:/conf.d:ro + - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro # The custom log to tail, created by the `log_generator` service - log_path:/var/log/ diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml index 726e027e0a543..a3ba29e2d8448 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -8,7 +8,6 @@ runner: VECTOR_RECEIVE_PORT: '8081' FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' - #TEST_LOG: ${TEST_LOG} matrix: version: [latest] \ No newline at end of file diff --git a/src/topology/mod.rs b/src/topology/mod.rs index d909803c3280b..2b3991914cd8a 100644 --- a/src/topology/mod.rs +++ b/src/topology/mod.rs @@ -27,7 +27,7 @@ use std::{ pub use controller::{ReloadOutcome, SharedTopologyController, TopologyController}; use futures::{Future, FutureExt}; -pub use running::RunningTopology; +pub(super) use running::RunningTopology; use tokio::sync::{mpsc, watch}; use vector_buffers::topology::channel::{BufferReceiverStream, BufferSender}; diff --git a/tests/data/e2e/datadog/logs/Dockerfile b/tests/data/e2e/datadog/Dockerfile similarity index 95% rename from tests/data/e2e/datadog/logs/Dockerfile rename to tests/data/e2e/datadog/Dockerfile index d427ff1294af8..3bc0ebbda61c7 100644 --- a/tests/data/e2e/datadog/logs/Dockerfile +++ b/tests/data/e2e/datadog/Dockerfile @@ -1,6 +1,7 @@ ARG RUST_VERSION +ARG FEATURES ARG DEBIAN_RELEASE=bookworm -ARG FEATURES=sources-datadog_agent,sinks-datadog_logs +#ARG FEATURES=sources-datadog_agent,sinks-datadog_logs # # VECTOR BUILDER diff --git a/tests/data/e2e/datadog/logs/agent_only/datadog.yaml b/tests/data/e2e/datadog/logs/agent_only.yaml similarity index 100% rename from tests/data/e2e/datadog/logs/agent_only/datadog.yaml rename to tests/data/e2e/datadog/logs/agent_only.yaml diff --git a/tests/data/e2e/datadog/logs/agent_vector/datadog.yaml b/tests/data/e2e/datadog/logs/agent_vector.yaml similarity index 100% rename from tests/data/e2e/datadog/logs/agent_vector/datadog.yaml rename to tests/data/e2e/datadog/logs/agent_vector.yaml diff --git a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml deleted file mode 100644 index 716c64baffb35..0000000000000 --- a/tests/data/e2e/datadog/logs/agent_vector/logs.conf.d/custom_logs.d/conf.yaml +++ /dev/null @@ -1,6 +0,0 @@ -logs: - - type: file - path: /var/log/a_custom.log - service: an_app - source: custom_log - start_position: beginning diff --git a/tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml b/tests/data/e2e/datadog/logs/logs.conf.d/custom_logs.d/conf.yaml similarity index 100% rename from tests/data/e2e/datadog/logs/agent_only/logs.conf.d/custom_logs.d/conf.yaml rename to tests/data/e2e/datadog/logs/logs.conf.d/custom_logs.d/conf.yaml From 69238755fa96e285e3c762cad7289a1b23c4ac5a Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 7 Sep 2023 16:52:50 -0600 Subject: [PATCH 14/99] cleanup --- .../integration/datadog-e2e-logs/test.yaml | 1 + tests/data/e2e/datadog/logs/Dockerfile | 41 +++++++++++ tests/e2e/datadog/logs/mod.rs | 73 ++++++++++++------- 3 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 tests/data/e2e/datadog/logs/Dockerfile diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml index a3ba29e2d8448..6379b6cb8b897 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -5,6 +5,7 @@ test: "e2e" runner: env: + EXPECTED_LOG_EVENTS: '1000' VECTOR_RECEIVE_PORT: '8081' FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' diff --git a/tests/data/e2e/datadog/logs/Dockerfile b/tests/data/e2e/datadog/logs/Dockerfile new file mode 100644 index 0000000000000..a000766d915c3 --- /dev/null +++ b/tests/data/e2e/datadog/logs/Dockerfile @@ -0,0 +1,41 @@ +ARG RUST_VERSION +ARG FEATURES +ARG DEBIAN_RELEASE=bookworm + +# +# VECTOR BUILDER +# +FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} as builder +RUN apt-get update && apt-get -y --no-install-recommends install build-essential git clang cmake libclang-dev libsasl2-dev libstdc++-11-dev libssl-dev libxxhash-dev zlib1g-dev zlib1g +RUN git clone https://github.com/rui314/mold.git \ + && mkdir mold/build \ + && cd mold/build \ + && git checkout v2.0.0 \ + && ../install-build-deps.sh \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ + && cmake --build . -j $(nproc) \ + && cmake --install . + +WORKDIR /vector +COPY . . +ARG FEATURES +RUN scripts/environment/install-protoc.sh +RUN --mount=type=cache,target=/vector/target \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + /usr/local/bin/mold -run cargo build --bin vector \ + --no-default-features --features $FEATURES && \ + cp target/debug/vector . + +# +# TARGET +# +FROM docker.io/debian:${DEBIAN_RELEASE}-slim +RUN apt-get update && apt-get -y --no-install-recommends install zlib1g && rm -rf /var/lib/apt/lists/* +COPY --from=builder /vector/vector /usr/bin/vector +RUN mkdir -p /var/lib/vector + +# Smoke test +RUN ["vector", "--version"] + +ENTRYPOINT ["/usr/bin/vector"] diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 0afdd605c5e8a..c744d745bee13 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,26 +1,57 @@ -use vector::test_util::trace_init; - +use serde_json::Value; use std::{thread::sleep, time::Duration}; +use vector::test_util::trace_init; + use super::*; -fn assert_timestamp_hostname(payloads: &mut Vec) -> usize { - let mut n_payloads = 0; +const LOGS_ENDPOIINT: &str = "/api/v2/logs"; + +fn expected_log_events() -> u32 { + std::env::var("EXPECTED_LOG_EVENTS") + .unwrap_or_else(|_| "1000".to_string()) + .parse::() + .expect("EXPECTED_LOG_EVENTS should be an unsigned int") +} + +// Asserts that each log event has the hostname and timestamp fields, and +// Removes them from the log so that comparison can more easily be made. +// @return the number of log entries in the payload. +fn assert_timestamp_hostname(payloads: &mut Vec) -> u32 { + let mut n_log_events = 0; + payloads.iter_mut().for_each(|payload_array| { payload_array .as_array_mut() .expect("should be array") .iter_mut() - .for_each(|payload| { - n_payloads += 1; - let obj = payload + .for_each(|log_val| { + n_log_events += 1; + + let log = log_val .as_object_mut() - .expect("payload entries should be objects"); - assert!(obj.remove("timestamp").is_some()); - assert!(obj.remove("hostname").is_some()); + .expect("log entries should be objects"); + + assert!(log.remove("timestamp").is_some()); + assert!(log.remove("hostname").is_some()); }) }); - n_payloads + + n_log_events +} + +// runs assertions that each set of payloads should be true to regardless +// of the pipeline +fn common_assertions(payloads: &mut Vec) { + //dbg!(&payloads); + + assert!(payloads.len() > 0); + + let n_log_events = assert_timestamp_hostname(payloads); + + println!("log events received: {n_log_events}"); + + assert!(n_log_events == expected_log_events()); } #[tokio::test] @@ -31,23 +62,15 @@ async fn test_logs() { // There doesn't seem to be a great way to avoid this. sleep(Duration::from_secs(5)); - let logs_endpoint = "/api/v2/logs"; - let mut agent_payloads = get_payloads_agent(&logs_endpoint).await; - - //dbg!(&agent_payloads); - - let mut vector_payloads = get_payloads_vector(&logs_endpoint).await; - - //dbg!(&vector_payloads); + println!("getting log payloads from agent-only pipeline"); + let mut agent_payloads = get_payloads_agent(LOGS_ENDPOIINT).await; - assert!(agent_payloads.len() > 0); - assert!(vector_payloads.len() > 0); + common_assertions(&mut agent_payloads); - let n_agent_payloads = assert_timestamp_hostname(&mut agent_payloads); - let n_vector_payloads = assert_timestamp_hostname(&mut vector_payloads); + println!("getting log payloads from agent-vector pipeline"); + let mut vector_payloads = get_payloads_vector(LOGS_ENDPOIINT).await; - println!("n agent payloads: {n_agent_payloads}"); - println!("n vector payloads: {n_vector_payloads}"); + common_assertions(&mut vector_payloads); assert_eq!(agent_payloads, vector_payloads); } From 7400659564b4224f1dbf859a8809d5474cff1183 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 7 Sep 2023 16:54:21 -0600 Subject: [PATCH 15/99] cleanup --- tests/data/e2e/datadog/Dockerfile | 1 - tests/data/e2e/datadog/logs/Dockerfile | 41 -------------------------- 2 files changed, 42 deletions(-) delete mode 100644 tests/data/e2e/datadog/logs/Dockerfile diff --git a/tests/data/e2e/datadog/Dockerfile b/tests/data/e2e/datadog/Dockerfile index 3bc0ebbda61c7..a000766d915c3 100644 --- a/tests/data/e2e/datadog/Dockerfile +++ b/tests/data/e2e/datadog/Dockerfile @@ -1,7 +1,6 @@ ARG RUST_VERSION ARG FEATURES ARG DEBIAN_RELEASE=bookworm -#ARG FEATURES=sources-datadog_agent,sinks-datadog_logs # # VECTOR BUILDER diff --git a/tests/data/e2e/datadog/logs/Dockerfile b/tests/data/e2e/datadog/logs/Dockerfile deleted file mode 100644 index a000766d915c3..0000000000000 --- a/tests/data/e2e/datadog/logs/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -ARG RUST_VERSION -ARG FEATURES -ARG DEBIAN_RELEASE=bookworm - -# -# VECTOR BUILDER -# -FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} as builder -RUN apt-get update && apt-get -y --no-install-recommends install build-essential git clang cmake libclang-dev libsasl2-dev libstdc++-11-dev libssl-dev libxxhash-dev zlib1g-dev zlib1g -RUN git clone https://github.com/rui314/mold.git \ - && mkdir mold/build \ - && cd mold/build \ - && git checkout v2.0.0 \ - && ../install-build-deps.sh \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ - && cmake --build . -j $(nproc) \ - && cmake --install . - -WORKDIR /vector -COPY . . -ARG FEATURES -RUN scripts/environment/install-protoc.sh -RUN --mount=type=cache,target=/vector/target \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - /usr/local/bin/mold -run cargo build --bin vector \ - --no-default-features --features $FEATURES && \ - cp target/debug/vector . - -# -# TARGET -# -FROM docker.io/debian:${DEBIAN_RELEASE}-slim -RUN apt-get update && apt-get -y --no-install-recommends install zlib1g && rm -rf /var/lib/apt/lists/* -COPY --from=builder /vector/vector /usr/bin/vector -RUN mkdir -p /var/lib/vector - -# Smoke test -RUN ["vector", "--version"] - -ENTRYPOINT ["/usr/bin/vector"] From a2ceff5f79c099506258020922caa6247790e547 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 7 Sep 2023 16:59:35 -0600 Subject: [PATCH 16/99] remove sleep --- tests/e2e/datadog/logs/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index c744d745bee13..a10af253c23b7 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,5 +1,4 @@ use serde_json::Value; -use std::{thread::sleep, time::Duration}; use vector::test_util::trace_init; @@ -58,10 +57,6 @@ fn common_assertions(payloads: &mut Vec) { async fn test_logs() { trace_init(); - // As it stands, we do still need a small sleep to allow the events to flow through. - // There doesn't seem to be a great way to avoid this. - sleep(Duration::from_secs(5)); - println!("getting log payloads from agent-only pipeline"); let mut agent_payloads = get_payloads_agent(LOGS_ENDPOIINT).await; From 070c69eada6593cd8d287bb0645370e3317e1e92 Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 11 Sep 2023 14:26:34 -0600 Subject: [PATCH 17/99] add to CI --- .github/workflows/integration-comment.yml | 8 ++++++++ .github/workflows/integration.yml | 9 +++++++++ scripts/integration/datadog-e2e-logs/test.yaml | 13 ++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index f6ed21bda71b5..228bd4961f741 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -177,6 +177,14 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces + - name: datadog-e2e-logs + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-logs') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-e2e-logs + - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 01a2a7dabca3a..5fe7eddaa5ccb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -202,6 +202,15 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: datadog-e2e-logs + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-e2e-logs + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap uses: nick-fields/retry@v2 diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml index 6379b6cb8b897..4087dd62020dc 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -11,4 +11,15 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' matrix: - version: [latest] \ No newline at end of file + version: [latest] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: +- "src/common/datadog.rs" +- "src/sources/datadog_agent/**" +- "src/internal_events/datadog_*" +- "src/sinks/datadog/logs/**" +- "src/sinks/util/**" +- "scripts/integration/datadog-e2e/logs/**" +- "tests/data/e2e/datadog/logs/**" From eb9f465afb1210a450afa61ca27edf5ed34f658e Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 11 Sep 2023 14:40:01 -0600 Subject: [PATCH 18/99] spell checker --- .github/actions/spelling/expect.txt | 1 + scripts/integration/datadog-e2e-logs/compose.yaml | 4 ---- tests/data/e2e/datadog/logs/agent_only.yaml | 2 +- tests/data/e2e/datadog/logs/agent_vector.yaml | 2 +- tests/e2e/datadog/logs/mod.rs | 8 +++----- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index eeae2da48b292..398c60824aa54 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -442,6 +442,7 @@ gnux gny Godbolt gogoproto +gohai goldberg goldmark GPB diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 2f4591c301434..39d331206eca3 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -36,8 +36,6 @@ services: volumes: - ${PWD}:/home/vector - target:/home/vector/target - - cargogit:/cargo/git - - cargoregistry:/cargo/registry # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service @@ -98,5 +96,3 @@ networks: volumes: log_path: {} target: {} - cargogit: {} - cargoregistry: {} diff --git a/tests/data/e2e/datadog/logs/agent_only.yaml b/tests/data/e2e/datadog/logs/agent_only.yaml index 69985f3d84496..aeebb113c1075 100644 --- a/tests/data/e2e/datadog/logs/agent_only.yaml +++ b/tests/data/e2e/datadog/logs/agent_only.yaml @@ -1,4 +1,4 @@ -api_key: deadbeef +api_key: DEADBEEF log_level: 'debug' # disable bunch of stuff we don't need diff --git a/tests/data/e2e/datadog/logs/agent_vector.yaml b/tests/data/e2e/datadog/logs/agent_vector.yaml index ac1ed5714055d..1686986746971 100644 --- a/tests/data/e2e/datadog/logs/agent_vector.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector.yaml @@ -1,4 +1,4 @@ -api_key: deadbeef +api_key: DEADBEEF log_level: 'debug' # disable bunch of stuff we don't need diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index a10af253c23b7..8bc2edbc30efb 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -4,7 +4,7 @@ use vector::test_util::trace_init; use super::*; -const LOGS_ENDPOIINT: &str = "/api/v2/logs"; +const LOGS_ENDPOINT: &str = "/api/v2/logs"; fn expected_log_events() -> u32 { std::env::var("EXPECTED_LOG_EVENTS") @@ -42,8 +42,6 @@ fn assert_timestamp_hostname(payloads: &mut Vec) -> u32 { // runs assertions that each set of payloads should be true to regardless // of the pipeline fn common_assertions(payloads: &mut Vec) { - //dbg!(&payloads); - assert!(payloads.len() > 0); let n_log_events = assert_timestamp_hostname(payloads); @@ -58,12 +56,12 @@ async fn test_logs() { trace_init(); println!("getting log payloads from agent-only pipeline"); - let mut agent_payloads = get_payloads_agent(LOGS_ENDPOIINT).await; + let mut agent_payloads = get_payloads_agent(LOGS_ENDPOINT).await; common_assertions(&mut agent_payloads); println!("getting log payloads from agent-vector pipeline"); - let mut vector_payloads = get_payloads_vector(LOGS_ENDPOIINT).await; + let mut vector_payloads = get_payloads_vector(LOGS_ENDPOINT).await; common_assertions(&mut vector_payloads); From a1c7f4923f59890fa0c421f668f55554a9ae4acf Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 11 Sep 2023 15:04:12 -0600 Subject: [PATCH 19/99] fix issue with DD changes detection --- .github/workflows/changes.yml | 18 +++++++++++++++--- .github/workflows/integration.yml | 10 +++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 653d3395bdfda..89cd9d7b622dd 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -56,8 +56,16 @@ on: value: ${{ jobs.int_tests.outputs.clickhouse }} databend: value: ${{ jobs.int_tests.outputs.databend }} - datadog: - value: ${{ jobs.int_tests.outputs.datadog }} + datadog-agent: + value: ${{ jobs.int_tests.outputs.datadog-agent }} + datadog-logs: + value: ${{ jobs.int_tests.outputs.datadog-logs }} + datadog-metrics: + value: ${{ jobs.int_tests.outputs.datadog-metrics }} + datadog-traces: + value: ${{ jobs.int_tests.outputs.datadog-traces }} + datadog-e2e-logs: + value: ${{ jobs.int_tests.outputs.datadog-e2e-logs }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: @@ -189,7 +197,11 @@ jobs: azure: ${{ steps.filter.outputs.azure }} clickhouse: ${{ steps.filter.outputs.clickhouse }} databend: ${{ steps.filter.outputs.databend }} - datadog: ${{ steps.filter.outputs.datadog }} + datadog-agent: ${{ steps.filter.outputs.datadog-agent }} + datadog-logs: ${{ steps.filter.outputs.datadog-logs }} + datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} + datadog-traces: ${{ steps.filter.outputs.datadog-traces }} + datadog-e2e-logs: ${{ steps.filter.outputs.datadog-e2e-logs }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5fe7eddaa5ccb..96d1fd88a6924 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -166,7 +166,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh databend - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-agent uses: nick-fields/retry@v2 @@ -175,7 +175,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-agent - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-logs uses: nick-fields/retry@v2 @@ -184,7 +184,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-logs - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-metrics uses: nick-fields/retry@v2 @@ -193,7 +193,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-metrics - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-traces uses: nick-fields/retry@v2 @@ -202,7 +202,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-e2e-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-e2e-logs uses: nick-fields/retry@v2 From 7b70ca1f324eef57772d4874fa2c70a6b91af0ad Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 11 Sep 2023 15:30:58 -0600 Subject: [PATCH 20/99] more workflow fix --- .github/workflows/integration.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 96d1fd88a6924..415b63c8ca4e7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -60,7 +60,11 @@ jobs: || needs.changes.outputs.azure == 'true' || needs.changes.outputs.clickhouse == 'true' || needs.changes.outputs.databend == 'true' - || needs.changes.outputs.datadog == 'true' + || needs.changes.outputs.datadog-agent == 'true' + || needs.changes.outputs.datadog-logs == 'true' + || needs.changes.outputs.datadog-metrics == 'true' + || needs.changes.outputs.datadog-traces == 'true' + || needs.changes.outputs.datadog-e2e-logs == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' || needs.changes.outputs.elasticsearch == 'true' From 7dd26c8f17ed514d5d50116feb2ff5645466bcd8 Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 11 Sep 2023 16:14:47 -0600 Subject: [PATCH 21/99] increase timeout as experiment --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 415b63c8ca4e7..894d181f46a0b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -211,7 +211,7 @@ jobs: name: datadog-e2e-logs uses: nick-fields/retry@v2 with: - timeout_minutes: 30 + timeout_minutes: 60 max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-e2e-logs From 0dff9f9f58a28d89535b14f50a4a5e26c6c79678 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 14 Sep 2023 14:39:55 -0600 Subject: [PATCH 22/99] update vector config for log endpoint field --- tests/data/e2e/datadog/logs/vector.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/e2e/datadog/logs/vector.toml b/tests/data/e2e/datadog/logs/vector.toml index aa78504def09d..bbcf8729fef8e 100644 --- a/tests/data/e2e/datadog/logs/vector.toml +++ b/tests/data/e2e/datadog/logs/vector.toml @@ -12,6 +12,6 @@ store_api_key = false inputs = [ "agent.logs" ] type = "datadog_logs" default_api_key="unused" -endpoint = "http://fakeintake-agent-vector:80/api/v2/logs" +endpoint = "http://fakeintake-agent-vector:80" batch.timeout_secs = 1 compression = "gzip" From c8c38d26b44ac300c3f9cda55d20ecb036902941 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 14 Sep 2023 14:40:50 -0600 Subject: [PATCH 23/99] touch up --- scripts/integration/datadog-e2e-logs/test.yaml | 2 ++ tests/e2e/datadog/logs/mod.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml index 4087dd62020dc..2c2114f58c56e 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -3,6 +3,8 @@ features: test: "e2e" +test_filter: "datadog::logs::" + runner: env: EXPECTED_LOG_EVENTS: '1000' diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 8bc2edbc30efb..d4d7670e63cb9 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -52,7 +52,7 @@ fn common_assertions(payloads: &mut Vec) { } #[tokio::test] -async fn test_logs() { +async fn validate() { trace_init(); println!("getting log payloads from agent-only pipeline"); From e522dedf2b2ae3a7cbeb5f24e0a5c9e5d2786e60 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 14 Sep 2023 16:30:56 -0600 Subject: [PATCH 24/99] start --- Cargo.toml | 3 +- .../datadog-e2e-metrics/compose.yaml | 88 +++++++++++++++++++ .../dogstatsd_client/Dockerfile | 8 ++ .../dogstatsd_client/client.py | 35 ++++++++ .../dogstatsd_client/requirements.txt | 1 + .../integration/datadog-e2e-metrics/test.yaml | 26 ++++++ .../data/e2e/datadog/metrics/agent_only.yaml | 25 ++++++ .../e2e/datadog/metrics/agent_vector.yaml | 28 ++++++ tests/data/e2e/datadog/metrics/vector.toml | 16 ++++ tests/e2e/datadog/metrics/mod.rs | 39 ++++++++ tests/e2e/datadog/mod.rs | 1 + 11 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 scripts/integration/datadog-e2e-metrics/compose.yaml create mode 100644 scripts/integration/datadog-e2e-metrics/dogstatsd_client/Dockerfile create mode 100644 scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py create mode 100644 scripts/integration/datadog-e2e-metrics/dogstatsd_client/requirements.txt create mode 100644 scripts/integration/datadog-e2e-metrics/test.yaml create mode 100644 tests/data/e2e/datadog/metrics/agent_only.yaml create mode 100644 tests/data/e2e/datadog/metrics/agent_vector.yaml create mode 100644 tests/data/e2e/datadog/metrics/vector.toml create mode 100644 tests/e2e/datadog/metrics/mod.rs diff --git a/Cargo.toml b/Cargo.toml index d1f068470de12..09fb0195cc962 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -862,7 +862,8 @@ cli-tests = ["sinks-blackhole", "sinks-socket", "sources-demo_logs", "sources-fi e2e-tests-datadog = [ "sources-datadog_agent", - "sinks-datadog_logs" + "sinks-datadog_logs", + "sinks-datadog_metrics" ] vector-api-tests = [ diff --git a/scripts/integration/datadog-e2e-metrics/compose.yaml b/scripts/integration/datadog-e2e-metrics/compose.yaml new file mode 100644 index 0000000000000..c1db12d26e9a1 --- /dev/null +++ b/scripts/integration/datadog-e2e-metrics/compose.yaml @@ -0,0 +1,88 @@ +version: '3' + +services: + + dogstatsd-client-agent: + build: ./dogstatsd_client + environment: + - STATSD_HOST=datadog-agent + depends_on: + - datadog-agent + + dogstatsd-client-agent-vector: + build: ./dogstatsd_client + environment: + - STATSD_HOST=datadog-agent-vector + depends_on: + - datadog-agent-vector + + # and sends metric data to + # the `fakeintake-agent` service + datadog-agent: + image: docker.io/datadog/agent:${CONFIG_VERSION} + depends_on: + - fakeintake-agent + environment: + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false + - DD_CONTAINER_EXCLUDE="name:.*" + volumes: + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml + + # and sends metric data to + # the `vector` service + datadog-agent-vector: + image: docker.io/datadog/agent:${CONFIG_VERSION} + depends_on: + - vector + environment: + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false + - DD_CONTAINER_EXCLUDE="name:.*" + volumes: + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + + # Receives metric data from the `datadog-agent-vector` service and sends + # to the `fakeintake-agent-vector` service. + vector: + depends_on: + - fakeintake-agent-vector + build: + context: ${PWD} + dockerfile: tests/data/e2e/datadog/Dockerfile + args: + - RUST_VERSION=${RUST_VERSION} + - FEATURES=sources-datadog_agent,sinks-datadog_metrics + working_dir: /home/vector + network_mode: host + command: + - "-vvv" + - "-c" + - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" + volumes: + - ${PWD}:/home/vector + - target:/home/vector/target + + + # Receives metric data from the `datadog-agent` service. Is queried by the test runner + # which does the validation of consistency with the other fakeintake service. + fakeintake-agent: + image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + + # Receives metric data from the `datadog-agent-vector` service. Is queried by the test runner + # which does the validation of consistency with the other fakeintake service. + fakeintake-agent-vector: + image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + +networks: + default: + name: ${VECTOR_NETWORK} + +volumes: + target: {} diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/Dockerfile b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/Dockerfile new file mode 100644 index 0000000000000..718a61a07b953 --- /dev/null +++ b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.7-alpine + +COPY . /app +WORKDIR /app + +RUN pip install -r requirements.txt + +CMD [ "python3", "./client.py"] diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py new file mode 100644 index 0000000000000..23b31d305649a --- /dev/null +++ b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py @@ -0,0 +1,35 @@ +from datadog import initialize, statsd +import time +import os + +STATSD_HOST = os.getenv('STATSD_HOST') + +print("initializing for {STATSD_HOST}") + +options = { + 'statsd_host':STATSD_HOST, + 'statsd_port':8125 +} + +initialize(**options) + +# time.sleep(5) + +for i in range(50): + print("rate") + statsd.increment('foo_metric.rate', tags=['a_tag:1']) + + print("guage") + statsd.gauge('foo_metric.gauge', i, tags=["a_tag:2"]) + + print("set") + statsd.set('foo_metric.set', i, tags=["a_tag:3"]) + + print("histogram") + statsd.histogram('foo_metric.histogram', random.randint(0, 20), tags=["a_tag:4"]) + + print("distribution") + statsd.distribution('foo_metric.distribution', random.randint(0, 20), tags=["a_tag:5"]) + + statsd.flush() + time.sleep(0.01) diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/requirements.txt b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/requirements.txt new file mode 100644 index 0000000000000..43023b823d991 --- /dev/null +++ b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/requirements.txt @@ -0,0 +1 @@ +datadog diff --git a/scripts/integration/datadog-e2e-metrics/test.yaml b/scripts/integration/datadog-e2e-metrics/test.yaml new file mode 100644 index 0000000000000..aba8a7501e635 --- /dev/null +++ b/scripts/integration/datadog-e2e-metrics/test.yaml @@ -0,0 +1,26 @@ +features: +- e2e-tests-datadog + +test: "e2e" + +test_filter: 'datadog::metrics::' + +runner: + env: + VECTOR_RECEIVE_PORT: '8081' + FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' + FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' + +matrix: + version: [latest] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: +- "src/common/datadog.rs" +- "src/sources/datadog_agent/**" +- "src/internal_events/datadog_*" +- "src/sinks/datadog/metrics/**" +- "src/sinks/util/**" +- "scripts/integration/datadog-e2e/metrics/**" +- "tests/data/e2e/datadog/metrics/**" diff --git a/tests/data/e2e/datadog/metrics/agent_only.yaml b/tests/data/e2e/datadog/metrics/agent_only.yaml new file mode 100644 index 0000000000000..a2236419046d3 --- /dev/null +++ b/tests/data/e2e/datadog/metrics/agent_only.yaml @@ -0,0 +1,25 @@ +api_key: DEADBEEF +log_level: 'debug' + +# disable bunch of stuff we don't need +inventories_configuration_enabled: false +enable_metadata_collection: false +enable_gohai: false + +apm_config: + enabled: false + +process_config: + container_collection: + enabled: false + process_discovery: + enabled: false + disable_realtime_checks: true + +logs_enabled: false + +# configure dogstatsd +use_dogstatsd: true + +# configure metrics +dd_url: http://fakeintake-agent:80 diff --git a/tests/data/e2e/datadog/metrics/agent_vector.yaml b/tests/data/e2e/datadog/metrics/agent_vector.yaml new file mode 100644 index 0000000000000..e5c540e87cee1 --- /dev/null +++ b/tests/data/e2e/datadog/metrics/agent_vector.yaml @@ -0,0 +1,28 @@ +api_key: DEADBEEF +log_level: 'debug' + +# disable bunch of stuff we don't need +inventories_configuration_enabled: false +enable_metadata_collection: false +enable_gohai: false + +apm_config: + enabled: false + +process_config: + container_collection: + enabled: false + process_discovery: + enabled: false + disable_realtime_checks: true + +logs_enabled: false + +# configure dogstatsd +use_dogstatsd: true + +# send to vector +vector: + metrics: + enabled: true + url: "http://vector:8181" diff --git a/tests/data/e2e/datadog/metrics/vector.toml b/tests/data/e2e/datadog/metrics/vector.toml new file mode 100644 index 0000000000000..d153b1f8c7ded --- /dev/null +++ b/tests/data/e2e/datadog/metrics/vector.toml @@ -0,0 +1,16 @@ +data_dir = "/tmp/" + +[sources.agent] +type = "datadog_agent" +address = "0.0.0.0:8181" +multiple_outputs = true +disable_logs = true +disable_traces = true +store_api_key = false + +[sinks.dd] +inputs = [ "agent.metrics" ] +type = "datadog_metrics" +default_api_key="unused" +endpoint = "http://fakeintake-agent-vector:80" +batch.timeout_secs = 1 diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs new file mode 100644 index 0000000000000..993c258b964fc --- /dev/null +++ b/tests/e2e/datadog/metrics/mod.rs @@ -0,0 +1,39 @@ +use serde_json::Value; + +use vector::test_util::trace_init; + +use super::*; + +const AGENT_DEFAULT_ENDPOINT: &str = "/api/v2/series"; + +// TODO the v1 endpoint is not compatible with fakeintake parsed +// payloads right now. we might need to change to use v2 +const VECTOR_DEFAULT_ENDPOINT: &str = "/api/v1/series"; + +// runs assertions that each set of payloads should be true to regardless +// of the pipeline +fn common_assertions(payloads: &mut Vec) { + assert!(payloads.len() > 0); + + println!("metric events received: {}", payloads.len()); +} + +#[tokio::test] +async fn validate() { + trace_init(); + + // TODO need to see if can configure the agent flush interval + std::thread::sleep(std::time::Duration::from_secs(5)); + + println!("getting log payloads from agent-only pipeline"); + let mut agent_payloads = get_payloads_agent(AGENT_DEFAULT_ENDPOINT).await; + + common_assertions(&mut agent_payloads); + + println!("getting log payloads from agent-vector pipeline"); + let mut vector_payloads = get_payloads_vector(VECTOR_DEFAULT_ENDPOINT).await; + + common_assertions(&mut vector_payloads); + + assert_eq!(agent_payloads, vector_payloads); +} diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 3336d666904ff..668d841f7b7b5 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -1,4 +1,5 @@ pub mod logs; +pub mod metrics; use reqwest::{Client, Method}; use serde::Deserialize; From d7592c68324cf834e0efe6be13a0380088fc6cec Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 14 Sep 2023 16:35:02 -0600 Subject: [PATCH 25/99] CI --- .github/workflows/changes.yml | 3 +++ .github/workflows/integration-comment.yml | 8 ++++++++ .github/workflows/integration.yml | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 89cd9d7b622dd..2c90b2e47b655 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -66,6 +66,8 @@ on: value: ${{ jobs.int_tests.outputs.datadog-traces }} datadog-e2e-logs: value: ${{ jobs.int_tests.outputs.datadog-e2e-logs }} + datadog-e2e-metrics: + value: ${{ jobs.int_tests.outputs.datadog-e2e-metrics }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: @@ -202,6 +204,7 @@ jobs: datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} datadog-traces: ${{ steps.filter.outputs.datadog-traces }} datadog-e2e-logs: ${{ steps.filter.outputs.datadog-e2e-logs }} + datadog-e2e-metrics: ${{ steps.filter.outputs.datadog-e2e-metrics }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 228bd4961f741..03f8359c37039 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -185,6 +185,14 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-e2e-logs + - name: datadog-e2e-metrics + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-e2e-metrics + - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 894d181f46a0b..75b00c96a74e9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -65,6 +65,7 @@ jobs: || needs.changes.outputs.datadog-metrics == 'true' || needs.changes.outputs.datadog-traces == 'true' || needs.changes.outputs.datadog-e2e-logs == 'true' + || needs.changes.outputs.datadog-e2e-metrics == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' || needs.changes.outputs.elasticsearch == 'true' @@ -215,6 +216,15 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-e2e-logs + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-e2e-metrics == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: datadog-e2e-metrics + uses: nick-fields/retry@v2 + with: + timeout_minutes: 60 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-e2e-metrics + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap uses: nick-fields/retry@v2 From 3631ddbd074af227eb498afa8fb34fae84a7f3cf Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 28 Sep 2023 16:24:14 -0600 Subject: [PATCH 26/99] feedback bg --- Makefile | 2 +- tests/e2e/datadog/logs/mod.rs | 15 +++++++++------ vdev/src/testing/config.rs | 7 +++++++ vdev/src/testing/integration.rs | 8 ++++---- vdev/src/util.rs | 24 ------------------------ 5 files changed, 21 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 7b9c33a22c5dc..12cd025769c0f 100644 --- a/Makefile +++ b/Makefile @@ -355,7 +355,7 @@ test-integration: test-integration-databend test-integration-docker-logs test-in test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar -test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs +test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-datadog-e2e-logs test-integration: test-integration-datadog-traces test-integration-shutdown test-integration-%-cleanup: diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index d4d7670e63cb9..663e243c58682 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -6,17 +6,20 @@ use super::*; const LOGS_ENDPOINT: &str = "/api/v2/logs"; -fn expected_log_events() -> u32 { +fn expected_log_events() -> usize { std::env::var("EXPECTED_LOG_EVENTS") - .unwrap_or_else(|_| "1000".to_string()) - .parse::() - .expect("EXPECTED_LOG_EVENTS should be an unsigned int") + .map(|n_expected| { + n_expected + .parse::() + .expect("EXPECTED_LOG_EVENTS should be an unsigned integer.") + }) + .unwrap_or(1000) } // Asserts that each log event has the hostname and timestamp fields, and // Removes them from the log so that comparison can more easily be made. // @return the number of log entries in the payload. -fn assert_timestamp_hostname(payloads: &mut Vec) -> u32 { +fn assert_timestamp_hostname(payloads: &mut [Value]) -> usize { let mut n_log_events = 0; payloads.iter_mut().for_each(|payload_array| { @@ -41,7 +44,7 @@ fn assert_timestamp_hostname(payloads: &mut Vec) -> u32 { // runs assertions that each set of payloads should be true to regardless // of the pipeline -fn common_assertions(payloads: &mut Vec) { +fn common_assertions(payloads: &mut [Value]) { assert!(payloads.len() > 0); let n_log_events = assert_timestamp_hostname(payloads); diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 2beb3fc909d26..c98c81b3451d2 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -37,6 +37,13 @@ impl RustToolchainConfig { } } +pub fn get_rust_version() -> String { + match RustToolchainConfig::parse() { + Ok(config) => config.channel, + Err(error) => fatal!("Could not read `rust-toolchain.toml` file: {error}"), + } +} + #[derive(Debug, Deserialize, Serialize)] pub struct ComposeConfig { pub services: BTreeMap, diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 78890f479ba1b..3b1f85959818a 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -10,7 +10,7 @@ use super::runner::{ }; use super::state::EnvsDir; use crate::app::CommandExt as _; -use crate::util::export_rust_version; +use crate::testing::config::get_rust_version; const NETWORK_ENV_VAR: &str = "VECTOR_NETWORK"; @@ -33,9 +33,6 @@ impl IntegrationTest { build_all: bool, retries: u8, ) -> Result { - // some tests require the rust version env var to be set - export_rust_version()?; - let integration = integration.into(); let environment = environment.into(); let (test_dir, config) = IntegrationTestConfig::load(&integration)?; @@ -250,6 +247,9 @@ impl Compose { command.env("DOCKER_SOCKET", &*DOCKER_SOCKET); command.env(NETWORK_ENV_VAR, &self.network); + // some services require this in order to build Vector + command.env("RUST_VERSION", get_rust_version()); + for (key, value) in &self.env { if let Some(value) = value { command.env(key, value); diff --git a/vdev/src/util.rs b/vdev/src/util.rs index 617a75058b90c..341ba9c0d25e9 100644 --- a/vdev/src/util.rs +++ b/vdev/src/util.rs @@ -56,30 +56,6 @@ pub struct RustToolChain { pub channel: String, } -/// The bits of the top-level `rust-toolchain.toml` configuration that `vdev` uses. -#[derive(Deserialize)] -pub struct RustToolChainToml { - pub toolchain: RustToolChain, -} - -impl RustToolChainToml { - pub fn load() -> Result { - let text = fs::read_to_string("rust-toolchain.toml") - .context("Could not read `rust-toolchain.toml`")?; - toml::from_str::(&text) - .context("Invalid contents in `rust-toolchain.toml`") - } -} - -pub fn get_rust_version() -> Result { - RustToolChainToml::load().map(|toml| toml.toolchain.channel) -} - -pub fn export_rust_version() -> Result<()> { - std::env::set_var("RUST_VERSION", get_rust_version()?); - Ok(()) -} - pub fn exists(path: impl AsRef + Debug) -> Result { match fs::metadata(path.as_ref()) { Ok(_) => Ok(true), From d8891fd1e65de919bbecd0e0fa5edf1bd46e6e5b Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 2 Oct 2023 11:09:49 -0600 Subject: [PATCH 27/99] add scaffolding for the differentiation of series API versions --- src/sinks/datadog/metrics/config.rs | 45 ++++++++++++--- src/sinks/datadog/metrics/encoder.rs | 60 ++++++++++++-------- src/sinks/datadog/metrics/request_builder.rs | 4 +- src/sinks/datadog/metrics/sink.rs | 8 +-- 4 files changed, 81 insertions(+), 36 deletions(-) diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 9fb8c4cd48137..b839dc5af9470 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -42,12 +42,38 @@ impl SinkBatchSettings for DatadogMetricsDefaultBatchSettings { const TIMEOUT_SECS: f64 = 2.0; } +const SERIES_V2_PATH: &str = "/api/v2/series"; +const SERIES_V1_PATH: &str = "/api/v1/series"; +const SKETCHES_PATH: &str = "/api/beta/sketches"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[allow(dead_code)] +pub enum SeriesApiVersion { + V1, + V2, +} + +impl SeriesApiVersion { + pub fn get_path(&self) -> &'static str { + match self { + Self::V1 => SERIES_V1_PATH, + Self::V2 => SERIES_V2_PATH, + } + } +} + +impl Default for SeriesApiVersion { + fn default() -> Self { + Self::V1 + } +} + /// Various metric type-specific API types. /// /// Each of these corresponds to a specific request path when making a request to the agent API. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum DatadogMetricsEndpoint { - Series, + Series(SeriesApiVersion), Sketches, } @@ -55,14 +81,19 @@ impl DatadogMetricsEndpoint { /// Gets the content type associated with the specific encoder for a given metric endpoint. pub const fn content_type(self) -> &'static str { match self { - DatadogMetricsEndpoint::Series => "application/json", - DatadogMetricsEndpoint::Sketches => "application/x-protobuf", + Self::Series(SeriesApiVersion::V1) => "application/json", + Self::Sketches | Self::Series(SeriesApiVersion::V2) => "application/x-protobuf", } } // Gets whether or not this is a series endpoint. pub const fn is_series(self) -> bool { - matches!(self, Self::Series) + matches!(self, Self::Series { .. }) + } + + // Creates an instance of the `Series` variant with the default API version. + pub fn series() -> Self { + Self::Series(SeriesApiVersion::default()) } } @@ -84,7 +115,7 @@ impl DatadogMetricsEndpointConfiguration { /// Gets the URI for the given Datadog metrics endpoint. pub fn get_uri_for_endpoint(&self, endpoint: DatadogMetricsEndpoint) -> Uri { match endpoint { - DatadogMetricsEndpoint::Series => self.series_endpoint.clone(), + DatadogMetricsEndpoint::Series { .. } => self.series_endpoint.clone(), DatadogMetricsEndpoint::Sketches => self.sketches_endpoint.clone(), } } @@ -169,8 +200,8 @@ impl DatadogMetricsConfig { &self, ) -> crate::Result { let base_uri = self.get_base_agent_endpoint(); - let series_endpoint = build_uri(&base_uri, "/api/v1/series")?; - let sketches_endpoint = build_uri(&base_uri, "/api/beta/sketches")?; + let series_endpoint = build_uri(&base_uri, SeriesApiVersion::default().get_path())?; + let sketches_endpoint = build_uri(&base_uri, SKETCHES_PATH)?; Ok(DatadogMetricsEndpointConfiguration::new( series_endpoint, diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 3ffcf8c5a3dda..4cc13e512be89 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -19,7 +19,7 @@ use vector_core::{ }; use super::config::{ - DatadogMetricsEndpoint, MAXIMUM_PAYLOAD_COMPRESSED_SIZE, MAXIMUM_PAYLOAD_SIZE, + DatadogMetricsEndpoint, SeriesApiVersion, MAXIMUM_PAYLOAD_COMPRESSED_SIZE, MAXIMUM_PAYLOAD_SIZE, }; use crate::{ common::datadog::{ @@ -229,8 +229,8 @@ impl DatadogMetricsEncoder { .add_event(&metric, metric.estimated_json_encoded_size_of()); match self.endpoint { - // Series metrics are encoded via JSON, in an incremental fashion. - DatadogMetricsEndpoint::Series => { + // V1 Series metrics are encoded via JSON, in an incremental fashion. + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => { // A single `Metric` might generate multiple Datadog series metrics. let all_series = generate_series_metrics( &metric, @@ -252,6 +252,9 @@ impl DatadogMetricsEncoder { serde_json::to_writer(&mut self.state.buf, series)?; } } + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2) => { + todo!() + } // Sketches are encoded via ProtoBuf, also in an incremental fashion. DatadogMetricsEndpoint::Sketches => match metric.value() { MetricValue::Sketch { sketch } => match sketch { @@ -823,7 +826,7 @@ fn write_payload_header( writer: &mut dyn io::Write, ) -> io::Result { match endpoint { - DatadogMetricsEndpoint::Series => writer + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => writer .write_all(SERIES_PAYLOAD_HEADER) .map(|_| SERIES_PAYLOAD_HEADER.len()), _ => Ok(0), @@ -835,7 +838,7 @@ fn write_payload_delimiter( writer: &mut dyn io::Write, ) -> io::Result { match endpoint { - DatadogMetricsEndpoint::Series => writer + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => writer .write_all(SERIES_PAYLOAD_DELIMITER) .map(|_| SERIES_PAYLOAD_DELIMITER.len()), _ => Ok(0), @@ -847,7 +850,7 @@ fn write_payload_footer( writer: &mut dyn io::Write, ) -> io::Result { match endpoint { - DatadogMetricsEndpoint::Series => writer + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => writer .write_all(SERIES_PAYLOAD_FOOTER) .map(|_| SERIES_PAYLOAD_FOOTER.len()), _ => Ok(0), @@ -890,7 +893,7 @@ mod tests { use crate::{ common::datadog::DatadogMetricType, sinks::datadog::metrics::{ - config::DatadogMetricsEndpoint, + config::{DatadogMetricsEndpoint, SeriesApiVersion}, encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, }, }; @@ -923,10 +926,16 @@ mod tests { fn get_compressed_empty_series_payload() -> Bytes { let mut compressor = get_compressor(); - _ = write_payload_header(DatadogMetricsEndpoint::Series, &mut compressor) - .expect("should not fail"); - _ = write_payload_footer(DatadogMetricsEndpoint::Series, &mut compressor) - .expect("should not fail"); + _ = write_payload_header( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + &mut compressor, + ) + .expect("should not fail"); + _ = write_payload_footer( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + &mut compressor, + ) + .expect("should not fail"); compressor.finish().expect("should not fail").freeze() } @@ -1025,8 +1034,9 @@ mod tests { // And sketches can't go to the series endpoint. // Series metrics can't go to the sketches endpoint. - let mut series_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series, None) - .expect("default payload size limits should be valid"); + let mut series_encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) + .expect("default payload size limits should be valid"); let sketch_result = series_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1128,8 +1138,9 @@ mod tests { fn encode_single_series_metric_with_default_limits() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series, None) - .expect("default payload size limits should be valid"); + let mut encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) + .expect("default payload size limits should be valid"); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1238,13 +1249,16 @@ mod tests { let header_len = max_uncompressed_header_len(); // This is too small. - let result = - validate_payload_size_limits(DatadogMetricsEndpoint::Series, header_len, usize::MAX); + let result = validate_payload_size_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + header_len, + usize::MAX, + ); assert_eq!(result, None); // This is just right. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), header_len + 1, usize::MAX, ); @@ -1257,7 +1271,7 @@ mod tests { // This is too small. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), usize::MAX, compression_overhead_len, ); @@ -1265,7 +1279,7 @@ mod tests { // This is just right. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), usize::MAX, compression_overhead_len + 1, ); @@ -1307,7 +1321,7 @@ mod tests { // uncompressed payload would exceed the limit. let header_len = max_uncompressed_header_len(); let mut encoder = DatadogMetricsEncoder::with_payload_limits( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None, header_len + 1, usize::MAX, @@ -1383,7 +1397,7 @@ mod tests { let uncompressed_limit = 128; let compressed_limit = 32; let mut encoder = DatadogMetricsEncoder::with_payload_limits( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None, uncompressed_limit, compressed_limit, @@ -1486,7 +1500,7 @@ mod tests { // We check this with targeted unit tests as well but this is some cheap insurance to // show that we're hopefully not missing any particular corner cases. let result = DatadogMetricsEncoder::with_payload_limits( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None, uncompressed_limit, compressed_limit, diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index d217986d6f520..f84c95d482669 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -81,7 +81,7 @@ impl DatadogMetricsRequestBuilder { Ok(Self { endpoint_configuration, series_encoder: DatadogMetricsEncoder::new( - DatadogMetricsEndpoint::Series, + DatadogMetricsEndpoint::series(), default_namespace.clone(), )?, sketches_encoder: DatadogMetricsEncoder::new( @@ -93,7 +93,7 @@ impl DatadogMetricsRequestBuilder { fn get_encoder(&mut self, endpoint: DatadogMetricsEndpoint) -> &mut DatadogMetricsEncoder { match endpoint { - DatadogMetricsEndpoint::Series => &mut self.series_encoder, + DatadogMetricsEndpoint::Series { .. } => &mut self.series_encoder, DatadogMetricsEndpoint::Sketches => &mut self.sketches_encoder, } } diff --git a/src/sinks/datadog/metrics/sink.rs b/src/sinks/datadog/metrics/sink.rs index 5ceefc3c487d2..c2e914560f19e 100644 --- a/src/sinks/datadog/metrics/sink.rs +++ b/src/sinks/datadog/metrics/sink.rs @@ -42,12 +42,12 @@ impl Partitioner for DatadogMetricsTypePartitioner { fn partition(&self, item: &Self::Item) -> Self::Key { let endpoint = match item.data().value() { - MetricValue::Counter { .. } => DatadogMetricsEndpoint::Series, - MetricValue::Gauge { .. } => DatadogMetricsEndpoint::Series, - MetricValue::Set { .. } => DatadogMetricsEndpoint::Series, + MetricValue::Counter { .. } => DatadogMetricsEndpoint::series(), + MetricValue::Gauge { .. } => DatadogMetricsEndpoint::series(), + MetricValue::Set { .. } => DatadogMetricsEndpoint::series(), MetricValue::Distribution { .. } => DatadogMetricsEndpoint::Sketches, MetricValue::AggregatedHistogram { .. } => DatadogMetricsEndpoint::Sketches, - MetricValue::AggregatedSummary { .. } => DatadogMetricsEndpoint::Series, + MetricValue::AggregatedSummary { .. } => DatadogMetricsEndpoint::series(), MetricValue::Sketch { .. } => DatadogMetricsEndpoint::Sketches, }; (item.metadata().datadog_api_key(), endpoint) From 91ca255730041fe961537870017a4bbdaacba4c3 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 12:44:54 -0600 Subject: [PATCH 28/99] fix(datadog_metrics sink): fix the integration tests which weren't actually validating anything --- scripts/integration/datadog-metrics/test.yaml | 2 +- .../datadog/metrics/integration_tests.rs | 124 ++++++++++-------- 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/scripts/integration/datadog-metrics/test.yaml b/scripts/integration/datadog-metrics/test.yaml index 237008a0d2551..a45e55a9797f2 100644 --- a/scripts/integration/datadog-metrics/test.yaml +++ b/scripts/integration/datadog-metrics/test.yaml @@ -1,7 +1,7 @@ features: - datadog-metrics-integration-tests -test_filter: '::datadog::metrics::' +test_filter: '::datadog::metrics::integration_tests' runner: env: diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 458cc5be987f3..0b12b5aefdf0b 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -1,4 +1,5 @@ use bytes::Bytes; +use chrono::{SubsecRound, Utc}; use flate2::read::ZlibDecoder; use futures::{channel::mpsc::Receiver, stream, StreamExt}; use hyper::StatusCode; @@ -22,28 +23,25 @@ use crate::{ }, }; -enum ApiStatus { - OK, - // Forbidden, -} +fn generate_metric_events() -> Vec { + let timestamp = Utc::now().trunc_subsecs(3); + let events: Vec<_> = (0..10) + .map(|index| { + let ts = timestamp + (std::time::Duration::from_secs(2) * index); + Event::Metric( + Metric::new( + format!("counter_{}", thread_rng().gen::()), + MetricKind::Incremental, + MetricValue::Counter { + value: index as f64, + }, + ) + .with_timestamp(Some(ts)), + ) + }) + .collect(); -fn test_server( - addr: std::net::SocketAddr, - api_status: ApiStatus, -) -> ( - futures::channel::mpsc::Receiver<(http::request::Parts, Bytes)>, - stream_cancel::Trigger, - impl std::future::Future>, -) { - let status = match api_status { - ApiStatus::OK => StatusCode::OK, - // ApiStatus::Forbidden => StatusCode::FORBIDDEN, - }; - - // NOTE: we pass `Trigger` out to the caller even though this suite never - // uses it as it's being dropped cancels the stream machinery here, - // indicating failures that might not be valid. - build_test_server_status(addr, status) + events } /// Starts a test sink with random metrics running into it @@ -55,10 +53,7 @@ fn test_server( /// Testers may set `http_status` and `batch_status`. The first controls what /// status code faked HTTP responses will have, the second acts as a check on /// the `Receiver`'s status before being returned to the caller. -async fn start_test( - api_status: ApiStatus, - batch_status: BatchStatus, -) -> (Vec, Receiver<(http::request::Parts, Bytes)>) { +async fn start_test() -> (Vec, Receiver<(http::request::Parts, Bytes)>) { let config = indoc! {r#" default_api_key = "atoken" default_namespace = "foo" @@ -73,25 +68,18 @@ async fn start_test( let (sink, _) = config.build(cx).await.unwrap(); - let (rx, _trigger, server) = test_server(addr, api_status); + let (rx, _trigger, server) = build_test_server_status(addr, StatusCode::OK); tokio::spawn(server); - let (batch, receiver) = BatchNotifier::new_with_receiver(); - let events: Vec<_> = (0..10) - .map(|index| { - Event::Metric(Metric::new( - format!("counter_{}", thread_rng().gen::()), - MetricKind::Absolute, - MetricValue::Counter { - value: index as f64, - }, - )) - }) - .collect(); + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + + let events = generate_metric_events(); + let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); sink.run(stream).await.unwrap(); - assert_eq!(receiver.await, batch_status); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); (events, rx) } @@ -110,11 +98,13 @@ fn decompress_payload(payload: Vec) -> std::io::Result> { /// were delivered and then asserts that every message is able to be /// deserialized. async fn smoke() { - let (expected, rx) = start_test(ApiStatus::OK, BatchStatus::Delivered).await; + let (expected, rx) = start_test().await; let output = rx.take(expected.len()).collect::>().await; - for val in output.iter() { + assert!(output.len() == 1); + + output.first().map(|val| { assert_eq!( val.0.headers.get("Content-Type").unwrap(), "application/json" @@ -155,10 +145,12 @@ async fn smoke() { assert_eq!(metric_names, sorted_names); let entry = series.first().unwrap().as_object().unwrap(); - assert_eq!( - entry.get("metric").unwrap().as_str().unwrap(), - "foo.counter" - ); + assert!(entry + .get("metric") + .unwrap() + .as_str() + .unwrap() + .starts_with("foo.counter_"),); assert_eq!(entry.get("type").unwrap().as_str().unwrap(), "count"); let points = entry .get("points") @@ -170,8 +162,32 @@ async fn smoke() { .as_array() .unwrap(); assert_eq!(points.len(), 2); - assert_eq!(points.get(1).unwrap().as_f64().unwrap(), 1.0); - } + + // validate that all values were received + let all_values: f64 = series + .iter() + .map(|entry| { + entry + .as_object() + .unwrap() + .get("points") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .as_array() + .unwrap() + .get(1) + .unwrap() + .as_f64() + .unwrap() + }) + .sum(); + + // the input values are [0..10) + assert_eq!(all_values, 45.0); + }); } async fn run_sink() { @@ -186,17 +202,9 @@ async fn run_sink() { let (sink, _) = config.build(cx).await.unwrap(); let (batch, receiver) = BatchNotifier::new_with_receiver(); - let events: Vec<_> = (0..10) - .map(|index| { - Event::Metric(Metric::new( - "counter", - MetricKind::Absolute, - MetricValue::Counter { - value: index as f64, - }, - )) - }) - .collect(); + + let events = generate_metric_events(); + let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); sink.run(stream).await.unwrap(); From 9afbd836e36d47d7bea14e52666ed383d974f424 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 12:53:13 -0600 Subject: [PATCH 29/99] fix workflows --- .github/workflows/changes.yml | 15 ++++++++++++--- .github/workflows/integration.yml | 13 ++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 653d3395bdfda..f480bac6265e4 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -56,8 +56,14 @@ on: value: ${{ jobs.int_tests.outputs.clickhouse }} databend: value: ${{ jobs.int_tests.outputs.databend }} - datadog: - value: ${{ jobs.int_tests.outputs.datadog }} + datadog-agent: + value: ${{ jobs.int_tests.outputs.datadog-agent }} + datadog-logs: + value: ${{ jobs.int_tests.outputs.datadog-logs }} + datadog-metrics: + value: ${{ jobs.int_tests.outputs.datadog-metrics }} + datadog-traces: + value: ${{ jobs.int_tests.outputs.datadog-traces }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: @@ -189,7 +195,10 @@ jobs: azure: ${{ steps.filter.outputs.azure }} clickhouse: ${{ steps.filter.outputs.clickhouse }} databend: ${{ steps.filter.outputs.databend }} - datadog: ${{ steps.filter.outputs.datadog }} + datadog-agent: ${{ steps.filter.outputs.datadog-agent }} + datadog-logs: ${{ steps.filter.outputs.datadog-logs }} + datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} + datadog-traces: ${{ steps.filter.outputs.datadog-traces }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 01a2a7dabca3a..709f2091fe0b1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -60,7 +60,10 @@ jobs: || needs.changes.outputs.azure == 'true' || needs.changes.outputs.clickhouse == 'true' || needs.changes.outputs.databend == 'true' - || needs.changes.outputs.datadog == 'true' + || needs.changes.outputs.datadog-agent == 'true' + || needs.changes.outputs.datadog-logs == 'true' + || needs.changes.outputs.datadog-metrics == 'true' + || needs.changes.outputs.datadog-traces == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' || needs.changes.outputs.elasticsearch == 'true' @@ -166,7 +169,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh databend - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-agent uses: nick-fields/retry@v2 @@ -175,7 +178,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-agent - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-logs uses: nick-fields/retry@v2 @@ -184,7 +187,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-logs - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-metrics uses: nick-fields/retry@v2 @@ -193,7 +196,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-metrics - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-traces uses: nick-fields/retry@v2 From cc9cc2d46bd831c8ce62784c6721785a0be512dc Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 12:56:06 -0600 Subject: [PATCH 30/99] clippy --- .../datadog/metrics/integration_tests.rs | 172 +++++++++--------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 0b12b5aefdf0b..99645780b65ca 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -102,92 +102,92 @@ async fn smoke() { let output = rx.take(expected.len()).collect::>().await; - assert!(output.len() == 1); - - output.first().map(|val| { - assert_eq!( - val.0.headers.get("Content-Type").unwrap(), - "application/json" - ); - assert_eq!(val.0.headers.get("DD-API-KEY").unwrap(), "atoken"); - assert!(val.0.headers.contains_key("DD-Agent-Payload")); - - let compressed_payload = val.1.to_vec(); - let payload = decompress_payload(compressed_payload).unwrap(); - let payload = std::str::from_utf8(&payload).unwrap(); - let payload: serde_json::Value = serde_json::from_str(payload).unwrap(); - - let series = payload - .as_object() - .unwrap() - .get("series") - .unwrap() - .as_array() - .unwrap(); - assert!(!series.is_empty()); - - // check metrics are sorted by name, which helps HTTP compression - let metric_names: Vec = series - .iter() - .map(|value| { - value - .as_object() - .unwrap() - .get("metric") - .unwrap() - .as_str() - .unwrap() - .to_string() - }) - .collect(); - let mut sorted_names = metric_names.clone(); - sorted_names.sort(); - assert_eq!(metric_names, sorted_names); - - let entry = series.first().unwrap().as_object().unwrap(); - assert!(entry - .get("metric") - .unwrap() - .as_str() - .unwrap() - .starts_with("foo.counter_"),); - assert_eq!(entry.get("type").unwrap().as_str().unwrap(), "count"); - let points = entry - .get("points") - .unwrap() - .as_array() - .unwrap() - .first() - .unwrap() - .as_array() - .unwrap(); - assert_eq!(points.len(), 2); - - // validate that all values were received - let all_values: f64 = series - .iter() - .map(|entry| { - entry - .as_object() - .unwrap() - .get("points") - .unwrap() - .as_array() - .unwrap() - .first() - .unwrap() - .as_array() - .unwrap() - .get(1) - .unwrap() - .as_f64() - .unwrap() - }) - .sum(); - - // the input values are [0..10) - assert_eq!(all_values, 45.0); - }); + assert!(output.len() == 1, "Should have received a response"); + + let val = output.first().unwrap(); + + assert_eq!( + val.0.headers.get("Content-Type").unwrap(), + "application/json" + ); + assert_eq!(val.0.headers.get("DD-API-KEY").unwrap(), "atoken"); + assert!(val.0.headers.contains_key("DD-Agent-Payload")); + + let compressed_payload = val.1.to_vec(); + let payload = decompress_payload(compressed_payload).unwrap(); + let payload = std::str::from_utf8(&payload).unwrap(); + let payload: serde_json::Value = serde_json::from_str(payload).unwrap(); + + let series = payload + .as_object() + .unwrap() + .get("series") + .unwrap() + .as_array() + .unwrap(); + assert!(!series.is_empty()); + + // check metrics are sorted by name, which helps HTTP compression + let metric_names: Vec = series + .iter() + .map(|value| { + value + .as_object() + .unwrap() + .get("metric") + .unwrap() + .as_str() + .unwrap() + .to_string() + }) + .collect(); + let mut sorted_names = metric_names.clone(); + sorted_names.sort(); + assert_eq!(metric_names, sorted_names); + + let entry = series.first().unwrap().as_object().unwrap(); + assert!(entry + .get("metric") + .unwrap() + .as_str() + .unwrap() + .starts_with("foo.counter_"),); + assert_eq!(entry.get("type").unwrap().as_str().unwrap(), "count"); + let points = entry + .get("points") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .as_array() + .unwrap(); + assert_eq!(points.len(), 2); + + // validate that all values were received + let all_values: f64 = series + .iter() + .map(|entry| { + entry + .as_object() + .unwrap() + .get("points") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .as_array() + .unwrap() + .get(1) + .unwrap() + .as_f64() + .unwrap() + }) + .sum(); + + // the input values are [0..10) + assert_eq!(all_values, 45.0); } async fn run_sink() { From 597305d79da1fc95fd8505a92c72fdb933a97f60 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 13:41:44 -0600 Subject: [PATCH 31/99] fix filter for traces --- scripts/integration/datadog-traces/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/integration/datadog-traces/test.yaml b/scripts/integration/datadog-traces/test.yaml index 31c4c0f97ef11..e518c8d3c1d93 100644 --- a/scripts/integration/datadog-traces/test.yaml +++ b/scripts/integration/datadog-traces/test.yaml @@ -19,6 +19,6 @@ matrix: paths: - "src/common/datadog.rs" - "src/internal_events/datadog_*" -- "src/sinks/datadog/**" +- "src/sinks/datadog/traces/**" - "src/sinks/util/**" - "scripts/integration/datadog-traces/**" From 3f9cd727a34dc637524cb6f73af0712264d43206 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 14:18:35 -0600 Subject: [PATCH 32/99] add first pass --- scripts/integration/datadog-metrics/test.yaml | 2 +- src/sinks/datadog/metrics/config.rs | 2 +- src/sinks/datadog/metrics/encoder.rs | 182 +++++++++++++++++- src/sinks/datadog/metrics/sink.rs | 1 + vdev/src/testing/integration.rs | 3 +- 5 files changed, 184 insertions(+), 6 deletions(-) diff --git a/scripts/integration/datadog-metrics/test.yaml b/scripts/integration/datadog-metrics/test.yaml index 237008a0d2551..a45e55a9797f2 100644 --- a/scripts/integration/datadog-metrics/test.yaml +++ b/scripts/integration/datadog-metrics/test.yaml @@ -1,7 +1,7 @@ features: - datadog-metrics-integration-tests -test_filter: '::datadog::metrics::' +test_filter: '::datadog::metrics::integration_tests' runner: env: diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index b839dc5af9470..6b127724d3f6c 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -42,8 +42,8 @@ impl SinkBatchSettings for DatadogMetricsDefaultBatchSettings { const TIMEOUT_SECS: f64 = 2.0; } -const SERIES_V2_PATH: &str = "/api/v2/series"; const SERIES_V1_PATH: &str = "/api/v1/series"; +const SERIES_V2_PATH: &str = "/api/v2/series"; const SKETCHES_PATH: &str = "/api/beta/sketches"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 4cc13e512be89..2e30755f4e8bd 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -252,9 +252,30 @@ impl DatadogMetricsEncoder { serde_json::to_writer(&mut self.state.buf, series)?; } } - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2) => { - todo!() - } + // V2 Series metrics are encoded via ProtoBuf, in an incremental fashion. + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2) => match metric.value() { + MetricValue::Counter { .. } + | MetricValue::Gauge { .. } + | MetricValue::Set { .. } + | MetricValue::AggregatedSummary { .. } => { + encode_series_v2_incremental( + &metric, + &self.default_namespace, + self.log_schema, + &mut self.state.buf, + self.origin_product_value, + ) + .map_err(|_| EncoderError::ProtoEncodingFailed)?; + } + MetricValue::Distribution { .. } + | MetricValue::AggregatedHistogram { .. } + | MetricValue::Sketch { .. } => { + return Err(EncoderError::InvalidMetric { + expected: "series", + metric_value: metric.value().as_name(), + }) + } + }, // Sketches are encoded via ProtoBuf, also in an incremental fashion. DatadogMetricsEndpoint::Sketches => match metric.value() { MetricValue::Sketch { sketch } => match sketch { @@ -539,6 +560,160 @@ where } } +fn get_series_payload_series_field_number() -> u32 { + static SERIES_PAYLOAD_SERIES_FIELD_NUM: OnceLock = OnceLock::new(); + *SERIES_PAYLOAD_SERIES_FIELD_NUM.get_or_init(|| { + let descriptors = protobuf_descriptors(); + let descriptor = descriptors + .get_message_by_name("datadog.agentpayload.MetricPayload") + .expect("should not fail to find `MetricPayload` message in descriptor pool"); + + descriptor + .get_field_by_name("series") + .map(|field| field.number()) + .expect("`series` field must exist in `MetricPayload` message") + }) +} + +fn series_to_proto_message( + metric: &Metric, + default_namespace: &Option>, + log_schema: &'static LogSchema, + origin_product_value: u32, +) -> Option { + let metric_name = get_namespaced_name(metric, default_namespace); + let mut tags = metric.tags().cloned().unwrap_or_default(); + + let mut resources = vec![]; + + if let Some(host) = log_schema + .host_key() + .map(|key| tags.remove(key.to_string().as_str()).unwrap_or_default()) + { + resources.push(ddmetric_proto::metric_payload::Resource { + r#type: "host".to_string(), + name: host, + }); + } + + if let Some(device) = tags.remove("device") { + resources.push(ddmetric_proto::metric_payload::Resource { + r#type: "device".to_string(), + name: device, + }); + } + + let source_type_name = tags.remove("source_type_name").unwrap_or_default(); + + let tags = encode_tags(&tags); + + let event_metadata = metric.metadata(); + let metadata = generate_sketch_metadata( + event_metadata.datadog_origin_metadata(), + event_metadata.source_type(), + origin_product_value, + ); + trace!(?metadata, "Generated MetricSeries metadata."); + + let timestamp = encode_timestamp(metric.timestamp()); + + let (points, metric_type, interval) = match (metric.value(), metric.interval_ms()) { + (MetricValue::Counter { value }, maybe_interval_ms) => { + let (value, interval, metric_type) = match maybe_interval_ms { + None => (*value, 0, ddmetric_proto::metric_payload::MetricType::Count), + // When an interval is defined, it implies the value should be in a per-second form, + // so we need to get back to seconds from our milliseconds-based interval, and then + // divide our value by that amount as well. + Some(interval_ms) => ( + (*value) * 1000.0 / (interval_ms.get() as f64), + interval_ms.get() as i64 / 1000, + ddmetric_proto::metric_payload::MetricType::Rate, + ), + }; + let points = vec![ddmetric_proto::metric_payload::MetricPoint { value, timestamp }]; + (points, metric_type, interval) + } + (MetricValue::Set { values }, _) => { + let points = vec![ddmetric_proto::metric_payload::MetricPoint { + value: values.len() as f64, + timestamp, + }]; + let metric_type = ddmetric_proto::metric_payload::MetricType::Gauge; + let interval = 0; + (points, metric_type, interval) + } + (MetricValue::Gauge { value }, _) => { + let points = vec![ddmetric_proto::metric_payload::MetricPoint { + value: *value, + timestamp, + }]; + let metric_type = ddmetric_proto::metric_payload::MetricType::Gauge; + let interval = 0; + (points, metric_type, interval) + } + // NOTE: AggregatedSummary will have been previously split into counters and gauges during normalization + (value, _) => { + // this case will have already been surfaced by encode_single_metric() so this should never be reached + let metric_type = value.as_name(); + error!(?metric_type, "Invalid metric. Expected series."); + return None; + } + }; + + Some(ddmetric_proto::metric_payload::MetricSeries { + resources, + metric: metric_name, + tags, + points, + r#type: metric_type.into(), + // unit is omitted + unit: "".to_string(), + source_type_name, + interval, + metadata, + }) +} + +fn encode_series_v2_incremental( + metric: &Metric, + default_namespace: &Option>, + log_schema: &'static LogSchema, + buf: &mut B, + origin_product_value: u32, +) -> Result<(), prost::EncodeError> +where + B: BufMut, +{ + // This encodes a single sketch metric incrementally, which means that we specifically write it + // as if we were writing a single field entry in the overall `SketchPayload` message + // type. + // + // By doing so, we can encode multiple sketches and concatenate all the buffers, and have the + // resulting buffer appear as if it's a normal `SketchPayload` message with a bunch of repeats + // of the `sketches` field. + // + // Crucially, this code works because `SketchPayload` has two fields -- metadata and sketches -- + // and we never actually set the metadata field... so the resulting message generated overall + // for `SketchPayload` with a single sketch looks just like as if we literally wrote out a + // single value for the given field. + + if let Some(series_proto) = + series_to_proto_message(metric, default_namespace, log_schema, origin_product_value) + { + // Manually write the field tag for `sketches` and then encode the sketch payload directly as a + // length-delimited message. + prost::encoding::encode_key( + get_series_payload_series_field_number(), + prost::encoding::WireType::LengthDelimited, + buf, + ); + series_proto.encode_length_delimited(buf) + } else { + // If the sketch was empty, that's fine too + Ok(()) + } +} + fn get_namespaced_name(metric: &Metric, default_namespace: &Option>) -> String { encode_namespace( metric @@ -739,6 +914,7 @@ fn generate_series_metrics( device, metadata, }], + // NOTE: AggregatedSummary will have been previously split into counters and gauges during normalization (value, _) => { return Err(EncoderError::InvalidMetric { expected: "series", diff --git a/src/sinks/datadog/metrics/sink.rs b/src/sinks/datadog/metrics/sink.rs index c2e914560f19e..6dc145e625c40 100644 --- a/src/sinks/datadog/metrics/sink.rs +++ b/src/sinks/datadog/metrics/sink.rs @@ -47,6 +47,7 @@ impl Partitioner for DatadogMetricsTypePartitioner { MetricValue::Set { .. } => DatadogMetricsEndpoint::series(), MetricValue::Distribution { .. } => DatadogMetricsEndpoint::Sketches, MetricValue::AggregatedHistogram { .. } => DatadogMetricsEndpoint::Sketches, + // NOTE: AggregatedSummary will be split into counters and gauges during normalization MetricValue::AggregatedSummary { .. } => DatadogMetricsEndpoint::series(), MetricValue::Sketch { .. } => DatadogMetricsEndpoint::Sketches, }; diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 8b371853cb37b..a434f3923b68a 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -78,7 +78,8 @@ impl IntegrationTest { env_vars.insert(key, Some(value)); } - env_vars.insert("TEST_LOG".to_string(), Some("info".into())); + env_vars.insert("TEST_LOG".to_string(), Some("debug".into())); + let mut args = self.config.args.clone().unwrap_or_default(); args.push("--features".to_string()); From 3a96f184e2c4d421049c611633503eade1b51c05 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 16:40:25 -0600 Subject: [PATCH 33/99] add testing coverage --- Cargo.lock | 4 +- lib/vector-core/src/event/mod.rs | 7 + src/sinks/datadog/metrics/config.rs | 10 +- src/sinks/datadog/metrics/encoder.rs | 14 +- .../datadog/metrics/integration_tests.rs | 124 +++++++++++++++++- 5 files changed, 139 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f5bfb12895ae..005c3c0e0a171 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2296,7 +2296,7 @@ dependencies = [ "futures-core", "prost 0.12.1", "prost-types 0.12.1", - "tonic 0.10.1", + "tonic 0.10.2", "tracing-core 0.1.30", ] @@ -2318,7 +2318,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic 0.10.1", + "tonic 0.10.2", "tracing 0.1.37", "tracing-core 0.1.30", "tracing-subscriber", diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index c96198f27d1bb..5bb7d144f91b2 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -319,6 +319,13 @@ impl Event { self } + /// Sets the `source_type` in the event metadata to the provided value. + #[must_use] + pub fn with_source_type(mut self, source_type: &'static str) -> Self { + self.metadata_mut().set_source_type(source_type); + self + } + /// Sets the `upstream_id` in the event metadata to the provided value. #[must_use] pub fn with_upstream_id(mut self, upstream_id: Arc) -> Self { diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 6b127724d3f6c..2c7a490daf685 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -42,9 +42,9 @@ impl SinkBatchSettings for DatadogMetricsDefaultBatchSettings { const TIMEOUT_SECS: f64 = 2.0; } -const SERIES_V1_PATH: &str = "/api/v1/series"; -const SERIES_V2_PATH: &str = "/api/v2/series"; -const SKETCHES_PATH: &str = "/api/beta/sketches"; +pub(super) const SERIES_V1_PATH: &str = "/api/v1/series"; +pub(super) const SERIES_V2_PATH: &str = "/api/v2/series"; +pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[allow(dead_code)] @@ -54,7 +54,7 @@ pub enum SeriesApiVersion { } impl SeriesApiVersion { - pub fn get_path(&self) -> &'static str { + pub const fn get_path(self) -> &'static str { match self { Self::V1 => SERIES_V1_PATH, Self::V2 => SERIES_V2_PATH, @@ -64,7 +64,7 @@ impl SeriesApiVersion { impl Default for SeriesApiVersion { fn default() -> Self { - Self::V1 + Self::V2 } } diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 2e30755f4e8bd..8b86b0f917a4b 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -33,11 +33,11 @@ const SERIES_PAYLOAD_HEADER: &[u8] = b"{\"series\":["; const SERIES_PAYLOAD_FOOTER: &[u8] = b"]}"; const SERIES_PAYLOAD_DELIMITER: &[u8] = b","; -const ORIGIN_CATEGORY_VALUE: u32 = 11; +pub(super) const ORIGIN_CATEGORY_VALUE: u32 = 11; const DEFAULT_DD_ORIGIN_PRODUCT_VALUE: u32 = 14; -static ORIGIN_PRODUCT_VALUE: Lazy = Lazy::new(|| { +pub(super) static ORIGIN_PRODUCT_VALUE: Lazy = Lazy::new(|| { option_env!("DD_ORIGIN_PRODUCT") .map(|p| { p.parse::() @@ -433,7 +433,7 @@ fn get_sketch_payload_sketches_field_number() -> u32 { }) } -fn generate_sketch_metadata( +fn generate_proto_metadata( maybe_pass_through: Option<&DatadogMetricOriginMetadata>, maybe_source_type: Option<&'static str>, origin_product_value: u32, @@ -488,7 +488,7 @@ fn sketch_to_proto_message( let n = counts.into_iter().map(Into::into).collect(); let event_metadata = metric.metadata(); - let metadata = generate_sketch_metadata( + let metadata = generate_proto_metadata( event_metadata.datadog_origin_metadata(), event_metadata.source_type(), origin_product_value, @@ -596,7 +596,9 @@ fn series_to_proto_message( }); } - if let Some(device) = tags.remove("device") { + // In the `datadog_agent` source, the tag is added as `device` for the V1 endoint + // and `resource.device` for the V2 endpoint. + if let Some(device) = tags.remove("device").or(tags.remove("resource.device")) { resources.push(ddmetric_proto::metric_payload::Resource { r#type: "device".to_string(), name: device, @@ -608,7 +610,7 @@ fn series_to_proto_message( let tags = encode_tags(&tags); let event_metadata = metric.metadata(); - let metadata = generate_sketch_metadata( + let metadata = generate_proto_metadata( event_metadata.datadog_origin_metadata(), event_metadata.source_type(), origin_product_value, diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 99645780b65ca..8f27867d70ce5 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -2,15 +2,18 @@ use bytes::Bytes; use chrono::{SubsecRound, Utc}; use flate2::read::ZlibDecoder; use futures::{channel::mpsc::Receiver, stream, StreamExt}; +use http::request::Parts; use hyper::StatusCode; use indoc::indoc; +use prost::Message; use rand::{thread_rng, Rng}; + use vector_core::{ config::{init_telemetry, Tags, Telemetry}, event::{BatchNotifier, BatchStatus, Event, Metric, MetricKind, MetricValue}, + metric_tags, }; -use super::DatadogMetricsConfig; use crate::{ config::SinkConfig, sinks::util::test::{build_test_server_status, load_sink}, @@ -23,6 +26,17 @@ use crate::{ }, }; +use super::{ + config::{SERIES_V1_PATH, SERIES_V2_PATH}, + encoder::{ORIGIN_CATEGORY_VALUE, ORIGIN_PRODUCT_VALUE}, + DatadogMetricsConfig, +}; + +#[allow(warnings, clippy::pedantic, clippy::nursery)] +mod ddmetric_proto { + include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); +} + fn generate_metric_events() -> Vec { let timestamp = Utc::now().trunc_subsecs(3); let events: Vec<_> = (0..10) @@ -36,8 +50,15 @@ fn generate_metric_events() -> Vec { value: index as f64, }, ) - .with_timestamp(Some(ts)), + .with_timestamp(Some(ts)) + .with_tags(Some(metric_tags!( + "resource.device" => "a_device", + "host" => "a_host", + "source_type_name" => "a_name", + "cool_tag_name" => "ikr", + ))), ) + .with_source_type("a_source_like_none_other") }) .collect(); @@ -104,16 +125,105 @@ async fn smoke() { assert!(output.len() == 1, "Should have received a response"); - let val = output.first().unwrap(); + let request = output.first().unwrap(); + + match request.0.uri.path() { + SERIES_V1_PATH => validate_json(request), + SERIES_V2_PATH => validate_protobuf(request), + _ => panic!("Unexpected request type received!"), + } +} +fn validate_common(request: &(Parts, Bytes)) { + assert_eq!(request.0.headers.get("DD-API-KEY").unwrap(), "atoken"); + assert!(request.0.headers.contains_key("DD-Agent-Payload")); +} + +fn validate_protobuf(request: &(Parts, Bytes)) { assert_eq!( - val.0.headers.get("Content-Type").unwrap(), + request.0.headers.get("Content-Type").unwrap(), + "application/x-protobuf" + ); + + validate_common(request); + + let compressed_payload = request.1.to_vec(); + let payload = decompress_payload(compressed_payload).expect("Could not decompress payload"); + let frame = Bytes::copy_from_slice(&payload); + + let payload = + ddmetric_proto::MetricPayload::decode(frame).expect("Could not decode protobuf frame"); + + let series = payload.series; + + assert!(!series.is_empty()); + + // check metrics are sorted by name, which helps HTTP compression + let metric_names: Vec = series.iter().map(|serie| serie.metric.clone()).collect(); + let mut sorted_names = metric_names.clone(); + sorted_names.sort(); + assert_eq!(metric_names, sorted_names); + + series.iter().for_each(|serie| { + // name + assert!(serie.metric.starts_with("foo.counter_")); + + // type + assert_eq!( + serie.r#type(), + ddmetric_proto::metric_payload::MetricType::Count + ); + + // resources + serie + .resources + .iter() + .for_each(|resource| match resource.r#type.as_str() { + "host" => assert_eq!(resource.name.as_str(), "a_host"), + "device" => assert_eq!(resource.name.as_str(), "a_device"), + _ => panic!("Unexpected resource found!"), + }); + + // source_type_name + assert_eq!(serie.source_type_name, "a_name"); + + // tags + assert_eq!(serie.tags.len(), 1); + assert_eq!(serie.tags.first().unwrap(), "cool_tag_name:ikr"); + + // unit + assert!(serie.unit.is_empty()); + + // interval + assert_eq!(serie.interval, 0); + + // metadata + let origin_metadata = serie.metadata.as_ref().unwrap().origin.as_ref().unwrap(); + assert_eq!(origin_metadata.origin_product, *ORIGIN_PRODUCT_VALUE); + assert_eq!(origin_metadata.origin_category, ORIGIN_CATEGORY_VALUE); + assert_eq!(origin_metadata.origin_service, 0); + }); + + // points + // the input values are [0..10) + assert_eq!( + series + .iter() + .map(|serie| serie.points.iter().map(|point| point.value).sum::()) + .sum::(), + 45.0 + ); +} + +fn validate_json(request: &(Parts, Bytes)) { + assert_eq!( + request.0.headers.get("Content-Type").unwrap(), "application/json" ); - assert_eq!(val.0.headers.get("DD-API-KEY").unwrap(), "atoken"); - assert!(val.0.headers.contains_key("DD-Agent-Payload")); - let compressed_payload = val.1.to_vec(); + validate_common(request); + + let compressed_payload = request.1.to_vec(); let payload = decompress_payload(compressed_payload).unwrap(); let payload = std::str::from_utf8(&payload).unwrap(); let payload: serde_json::Value = serde_json::from_str(payload).unwrap(); From 992321980d82a5cc5afcf532f9beed0142bbd556 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 3 Oct 2023 16:56:32 -0600 Subject: [PATCH 34/99] cargo.lock --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 005c3c0e0a171..1f5bfb12895ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2296,7 +2296,7 @@ dependencies = [ "futures-core", "prost 0.12.1", "prost-types 0.12.1", - "tonic 0.10.2", + "tonic 0.10.1", "tracing-core 0.1.30", ] @@ -2318,7 +2318,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic 0.10.2", + "tonic 0.10.1", "tracing 0.1.37", "tracing-core 0.1.30", "tracing-subscriber", From 4523fcd8f779a864eeb809759351d4ab2a610d2f Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 4 Oct 2023 12:43:55 -0600 Subject: [PATCH 35/99] reduce duplicated code --- src/sinks/datadog/metrics/encoder.rs | 238 +++++++----------- .../datadog/metrics/integration_tests.rs | 5 +- vdev/src/testing/integration.rs | 2 +- 3 files changed, 100 insertions(+), 145 deletions(-) diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 8b86b0f917a4b..41c43c72b5202 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -8,7 +8,6 @@ use std::{ use bytes::{BufMut, Bytes}; use chrono::{DateTime, Utc}; use once_cell::sync::Lazy; -use prost::Message; use snafu::{ResultExt, Snafu}; use vector_common::request_metadata::GroupedCountByteSize; use vector_core::{ @@ -228,6 +227,21 @@ impl DatadogMetricsEncoder { .byte_size .add_event(&metric, metric.estimated_json_encoded_size_of()); + // For V2 Series metrics, and Sketches: We encode a single Series or Sketch metric incrementally, + // which means that we specifically write it as if we were writing a single field entry in the + // overall `SketchPayload` message or `MetricPayload` type. + // + // By doing so, we can encode multiple metrics and concatenate all the buffers, and have the + // resulting buffer appear as if it's a normal `<>Payload` message with a bunch of repeats + // of the `sketches` / `series` field. + // + // Crucially, this code works because `SketchPayload` has two fields -- metadata and sketches -- + // and we never actually set the metadata field... so the resulting message generated overall + // for `SketchPayload` with a single sketch looks just like as if we literally wrote out a + // single value for the given field. + // + // Similary, `MetricPayload` has a single repeated `series` field. + match self.endpoint { // V1 Series metrics are encoded via JSON, in an incremental fashion. DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => { @@ -258,21 +272,23 @@ impl DatadogMetricsEncoder { | MetricValue::Gauge { .. } | MetricValue::Set { .. } | MetricValue::AggregatedSummary { .. } => { - encode_series_v2_incremental( + let series_proto = series_to_proto_message( &metric, &self.default_namespace, self.log_schema, - &mut self.state.buf, self.origin_product_value, - ) - .map_err(|_| EncoderError::ProtoEncodingFailed)?; + )?; + + encode_proto_key_and_message( + series_proto, + get_series_payload_series_field_number(), + &mut self.state.buf, + )?; } - MetricValue::Distribution { .. } - | MetricValue::AggregatedHistogram { .. } - | MetricValue::Sketch { .. } => { + value => { return Err(EncoderError::InvalidMetric { expected: "series", - metric_value: metric.value().as_name(), + metric_value: value.as_name(), }) } }, @@ -280,15 +296,21 @@ impl DatadogMetricsEncoder { DatadogMetricsEndpoint::Sketches => match metric.value() { MetricValue::Sketch { sketch } => match sketch { MetricSketch::AgentDDSketch(ddsketch) => { - encode_sketch_incremental( + if let Some(sketch_proto) = sketch_to_proto_message( &metric, ddsketch, &self.default_namespace, self.log_schema, - &mut self.state.buf, self.origin_product_value, - ) - .map_err(|_| EncoderError::ProtoEncodingFailed)?; + ) { + encode_proto_key_and_message( + sketch_proto, + get_sketch_payload_sketches_field_number(), + &mut self.state.buf, + )?; + } else { + // If the sketch was empty, that's fine too + } } }, value => { @@ -418,6 +440,22 @@ impl DatadogMetricsEncoder { } } +fn generate_proto_metadata( + maybe_pass_through: Option<&DatadogMetricOriginMetadata>, + maybe_source_type: Option<&'static str>, + origin_product_value: u32, +) -> Option { + generate_origin_metadata(maybe_pass_through, maybe_source_type, origin_product_value).map( + |origin| ddmetric_proto::Metadata { + origin: Some(ddmetric_proto::Origin { + origin_product: origin.product().expect("OriginProduct should be set"), + origin_category: origin.category().expect("OriginCategory should be set"), + origin_service: origin.service().expect("OriginService should be set"), + }), + }, + ) +} + fn get_sketch_payload_sketches_field_number() -> u32 { static SKETCH_PAYLOAD_SKETCHES_FIELD_NUM: OnceLock = OnceLock::new(); *SKETCH_PAYLOAD_SKETCHES_FIELD_NUM.get_or_init(|| { @@ -433,20 +471,19 @@ fn get_sketch_payload_sketches_field_number() -> u32 { }) } -fn generate_proto_metadata( - maybe_pass_through: Option<&DatadogMetricOriginMetadata>, - maybe_source_type: Option<&'static str>, - origin_product_value: u32, -) -> Option { - generate_origin_metadata(maybe_pass_through, maybe_source_type, origin_product_value).map( - |origin| ddmetric_proto::Metadata { - origin: Some(ddmetric_proto::Origin { - origin_product: origin.product().expect("OriginProduct should be set"), - origin_category: origin.category().expect("OriginCategory should be set"), - origin_service: origin.service().expect("OriginService should be set"), - }), - }, - ) +fn get_series_payload_series_field_number() -> u32 { + static SERIES_PAYLOAD_SERIES_FIELD_NUM: OnceLock = OnceLock::new(); + *SERIES_PAYLOAD_SERIES_FIELD_NUM.get_or_init(|| { + let descriptors = protobuf_descriptors(); + let descriptor = descriptors + .get_message_by_name("datadog.agentpayload.MetricPayload") + .expect("should not fail to find `MetricPayload` message in descriptor pool"); + + descriptor + .get_field_by_name("series") + .map(|field| field.number()) + .expect("`series` field must exist in `MetricPayload` message") + }) } fn sketch_to_proto_message( @@ -515,72 +552,12 @@ fn sketch_to_proto_message( }) } -fn encode_sketch_incremental( - metric: &Metric, - ddsketch: &AgentDDSketch, - default_namespace: &Option>, - log_schema: &'static LogSchema, - buf: &mut B, - origin_product_value: u32, -) -> Result<(), prost::EncodeError> -where - B: BufMut, -{ - // This encodes a single sketch metric incrementally, which means that we specifically write it - // as if we were writing a single field entry in the overall `SketchPayload` message - // type. - // - // By doing so, we can encode multiple sketches and concatenate all the buffers, and have the - // resulting buffer appear as if it's a normal `SketchPayload` message with a bunch of repeats - // of the `sketches` field. - // - // Crucially, this code works because `SketchPayload` has two fields -- metadata and sketches -- - // and we never actually set the metadata field... so the resulting message generated overall - // for `SketchPayload` with a single sketch looks just like as if we literally wrote out a - // single value for the given field. - - if let Some(sketch_proto) = sketch_to_proto_message( - metric, - ddsketch, - default_namespace, - log_schema, - origin_product_value, - ) { - // Manually write the field tag for `sketches` and then encode the sketch payload directly as a - // length-delimited message. - prost::encoding::encode_key( - get_sketch_payload_sketches_field_number(), - prost::encoding::WireType::LengthDelimited, - buf, - ); - sketch_proto.encode_length_delimited(buf) - } else { - // If the sketch was empty, that's fine too - Ok(()) - } -} - -fn get_series_payload_series_field_number() -> u32 { - static SERIES_PAYLOAD_SERIES_FIELD_NUM: OnceLock = OnceLock::new(); - *SERIES_PAYLOAD_SERIES_FIELD_NUM.get_or_init(|| { - let descriptors = protobuf_descriptors(); - let descriptor = descriptors - .get_message_by_name("datadog.agentpayload.MetricPayload") - .expect("should not fail to find `MetricPayload` message in descriptor pool"); - - descriptor - .get_field_by_name("series") - .map(|field| field.number()) - .expect("`series` field must exist in `MetricPayload` message") - }) -} - fn series_to_proto_message( metric: &Metric, default_namespace: &Option>, log_schema: &'static LogSchema, origin_product_value: u32, -) -> Option { +) -> Result { let metric_name = get_namespaced_name(metric, default_namespace); let mut tags = metric.tags().cloned().unwrap_or_default(); @@ -596,7 +573,7 @@ fn series_to_proto_message( }); } - // In the `datadog_agent` source, the tag is added as `device` for the V1 endoint + // In the `datadog_agent` source, the tag is added as `device` for the V1 endpoint // and `resource.device` for the V2 endpoint. if let Some(device) = tags.remove("device").or(tags.remove("resource.device")) { resources.push(ddmetric_proto::metric_payload::Resource { @@ -655,14 +632,15 @@ fn series_to_proto_message( } // NOTE: AggregatedSummary will have been previously split into counters and gauges during normalization (value, _) => { - // this case will have already been surfaced by encode_single_metric() so this should never be reached - let metric_type = value.as_name(); - error!(?metric_type, "Invalid metric. Expected series."); - return None; + // this case should have already been surfaced by encode_single_metric() so this should never be reached + return Err(EncoderError::InvalidMetric { + expected: "series", + metric_value: value.as_name(), + }); } }; - Some(ddmetric_proto::metric_payload::MetricSeries { + Ok(ddmetric_proto::metric_payload::MetricSeries { resources, metric: metric_name, tags, @@ -676,44 +654,16 @@ fn series_to_proto_message( }) } -fn encode_series_v2_incremental( - metric: &Metric, - default_namespace: &Option>, - log_schema: &'static LogSchema, - buf: &mut B, - origin_product_value: u32, -) -> Result<(), prost::EncodeError> +// Manually write the field tag and then encode the Message payload directly as a length-delimited message. +fn encode_proto_key_and_message(msg: T, tag: u32, buf: &mut B) -> Result<(), EncoderError> where + T: prost::Message, B: BufMut, { - // This encodes a single sketch metric incrementally, which means that we specifically write it - // as if we were writing a single field entry in the overall `SketchPayload` message - // type. - // - // By doing so, we can encode multiple sketches and concatenate all the buffers, and have the - // resulting buffer appear as if it's a normal `SketchPayload` message with a bunch of repeats - // of the `sketches` field. - // - // Crucially, this code works because `SketchPayload` has two fields -- metadata and sketches -- - // and we never actually set the metadata field... so the resulting message generated overall - // for `SketchPayload` with a single sketch looks just like as if we literally wrote out a - // single value for the given field. + prost::encoding::encode_key(tag, prost::encoding::WireType::LengthDelimited, buf); - if let Some(series_proto) = - series_to_proto_message(metric, default_namespace, log_schema, origin_product_value) - { - // Manually write the field tag for `sketches` and then encode the sketch payload directly as a - // length-delimited message. - prost::encoding::encode_key( - get_series_payload_series_field_number(), - prost::encoding::WireType::LengthDelimited, - buf, - ); - series_proto.encode_length_delimited(buf) - } else { - // If the sketch was empty, that's fine too - Ok(()) - } + msg.encode_length_delimited(buf) + .map_err(|_| EncoderError::ProtoEncodingFailed) } fn get_namespaced_name(metric: &Metric, default_namespace: &Option>) -> String { @@ -1063,10 +1013,11 @@ mod tests { }; use super::{ - ddmetric_proto, encode_sketch_incremental, encode_tags, encode_timestamp, - generate_series_metrics, get_compressor, max_compression_overhead_len, - max_uncompressed_header_len, sketch_to_proto_message, validate_payload_size_limits, - write_payload_footer, write_payload_header, DatadogMetricsEncoder, EncoderError, + ddmetric_proto, encode_proto_key_and_message, encode_tags, encode_timestamp, + generate_series_metrics, get_compressor, get_sketch_payload_sketches_field_number, + max_compression_overhead_len, max_uncompressed_header_len, sketch_to_proto_message, + validate_payload_size_limits, write_payload_footer, write_payload_header, + DatadogMetricsEncoder, EncoderError, }; use crate::{ common::datadog::DatadogMetricType, @@ -1404,17 +1355,20 @@ mod tests { for metric in &metrics { match metric.value() { MetricValue::Sketch { sketch } => match sketch { - MetricSketch::AgentDDSketch(ddsketch) => encode_sketch_incremental( - metric, - ddsketch, - &None, - log_schema(), - &mut incremental_buf, - 14, - ) - .unwrap(), + MetricSketch::AgentDDSketch(ddsketch) => { + if let Some(sketch_proto) = + sketch_to_proto_message(&metric, ddsketch, &None, log_schema(), 14) + { + encode_proto_key_and_message( + sketch_proto, + get_sketch_payload_sketches_field_number(), + &mut incremental_buf, + ) + .unwrap(); + } + } }, - _ => panic!("should be a sketch"), + _ => panic!("should be sketch"), } } diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 8f27867d70ce5..542624c906d5d 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -55,9 +55,10 @@ fn generate_metric_events() -> Vec { "resource.device" => "a_device", "host" => "a_host", "source_type_name" => "a_name", - "cool_tag_name" => "ikr", + "cool_tag_name" => "i_know_right", ))), ) + // this ensures we get Origin Metadata, with an undefined service but that's ok. .with_source_type("a_source_like_none_other") }) .collect(); @@ -189,7 +190,7 @@ fn validate_protobuf(request: &(Parts, Bytes)) { // tags assert_eq!(serie.tags.len(), 1); - assert_eq!(serie.tags.first().unwrap(), "cool_tag_name:ikr"); + assert_eq!(serie.tags.first().unwrap(), "cool_tag_name:i_know_right"); // unit assert!(serie.unit.is_empty()); diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index a434f3923b68a..e59ae4eb9446a 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -78,7 +78,7 @@ impl IntegrationTest { env_vars.insert(key, Some(value)); } - env_vars.insert("TEST_LOG".to_string(), Some("debug".into())); + env_vars.insert("TEST_LOG".to_string(), Some("info".into())); let mut args = self.config.args.clone().unwrap_or_default(); From a87bf0c85779139ee5cc446032ece162e46a68ad Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 4 Oct 2023 14:12:43 -0600 Subject: [PATCH 36/99] cleanup --- src/sinks/datadog/metrics/encoder.rs | 2 +- vdev/src/testing/integration.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 41c43c72b5202..1b8c369949ee6 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -1368,7 +1368,7 @@ mod tests { } } }, - _ => panic!("should be sketch"), + _ => panic!("should be a sketch"), } } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index e59ae4eb9446a..8b371853cb37b 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -79,7 +79,6 @@ impl IntegrationTest { } env_vars.insert("TEST_LOG".to_string(), Some("info".into())); - let mut args = self.config.args.clone().unwrap_or_default(); args.push("--features".to_string()); From 376da18a477653370851df3c3c37ede3890f2842 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 4 Oct 2023 14:32:08 -0600 Subject: [PATCH 37/99] clippy --- src/sinks/datadog/metrics/encoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 1b8c369949ee6..32eeafd77c64c 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -1357,7 +1357,7 @@ mod tests { MetricValue::Sketch { sketch } => match sketch { MetricSketch::AgentDDSketch(ddsketch) => { if let Some(sketch_proto) = - sketch_to_proto_message(&metric, ddsketch, &None, log_schema(), 14) + sketch_to_proto_message(metric, ddsketch, &None, log_schema(), 14) { encode_proto_key_and_message( sketch_proto, From c95a326c30a471e660742b6e354ab5d73e1201de Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 11:12:38 -0600 Subject: [PATCH 38/99] feedback ds: remove check for sort by name --- src/sinks/datadog/metrics/integration_tests.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 542624c906d5d..a2c0fd4ee2f81 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -159,12 +159,6 @@ fn validate_protobuf(request: &(Parts, Bytes)) { assert!(!series.is_empty()); - // check metrics are sorted by name, which helps HTTP compression - let metric_names: Vec = series.iter().map(|serie| serie.metric.clone()).collect(); - let mut sorted_names = metric_names.clone(); - sorted_names.sort(); - assert_eq!(metric_names, sorted_names); - series.iter().for_each(|serie| { // name assert!(serie.metric.starts_with("foo.counter_")); From d73cfe076826aee8d21c0de794dbc1320c39b9d1 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 11:41:13 -0600 Subject: [PATCH 39/99] feedback ds: extend unit tests for v2 --- src/sinks/datadog/metrics/encoder.rs | 186 ++++++++++++++++++++------- 1 file changed, 136 insertions(+), 50 deletions(-) diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 32eeafd77c64c..775cbf80fe556 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -1015,9 +1015,9 @@ mod tests { use super::{ ddmetric_proto, encode_proto_key_and_message, encode_tags, encode_timestamp, generate_series_metrics, get_compressor, get_sketch_payload_sketches_field_number, - max_compression_overhead_len, max_uncompressed_header_len, sketch_to_proto_message, - validate_payload_size_limits, write_payload_footer, write_payload_header, - DatadogMetricsEncoder, EncoderError, + max_compression_overhead_len, max_uncompressed_header_len, series_to_proto_message, + sketch_to_proto_message, validate_payload_size_limits, write_payload_footer, + write_payload_header, DatadogMetricsEncoder, EncoderError, }; use crate::{ common::datadog::DatadogMetricType, @@ -1162,11 +1162,19 @@ mod tests { )); // And sketches can't go to the series endpoint. - // Series metrics can't go to the sketches endpoint. - let mut series_encoder = + let mut series_v1_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) .expect("default payload size limits should be valid"); - let sketch_result = series_encoder.try_encode(get_simple_sketch()); + let sketch_result = series_v1_encoder.try_encode(get_simple_sketch()); + assert!(matches!( + sketch_result.err(), + Some(EncoderError::InvalidMetric { .. }) + )); + + let mut series_v2_encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) + .expect("default payload size limits should be valid"); + let sketch_result = series_v2_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), Some(EncoderError::InvalidMetric { .. }) @@ -1186,23 +1194,41 @@ mod tests { let expected_value = value / (interval_ms / 1000) as f64; let expected_interval = interval_ms / 1000; - // Encode the metric and make sure we did the rate conversion correctly. - let result = generate_series_metrics( - &rate_counter, - &None, - log_schema(), - DEFAULT_DD_ORIGIN_PRODUCT_VALUE, - ); - assert!(result.is_ok()); + // series v1 + { + // Encode the metric and make sure we did the rate conversion correctly. + let result = generate_series_metrics( + &rate_counter, + &None, + log_schema(), + DEFAULT_DD_ORIGIN_PRODUCT_VALUE, + ); + assert!(result.is_ok()); + + let metrics = result.unwrap(); + assert_eq!(metrics.len(), 1); - let metrics = result.unwrap(); - assert_eq!(metrics.len(), 1); + let actual = &metrics[0]; + assert_eq!(actual.r#type, DatadogMetricType::Rate); + assert_eq!(actual.interval, Some(expected_interval)); + assert_eq!(actual.points.len(), 1); + assert_eq!(actual.points[0].1, expected_value); + } - let actual = &metrics[0]; - assert_eq!(actual.r#type, DatadogMetricType::Rate); - assert_eq!(actual.interval, Some(expected_interval)); - assert_eq!(actual.points.len(), 1); - assert_eq!(actual.points[0].1, expected_value); + // series v2 + { + let series_proto = series_to_proto_message( + &rate_counter, + &None, + log_schema(), + DEFAULT_DD_ORIGIN_PRODUCT_VALUE, + ) + .unwrap(); + assert_eq!(series_proto.r#type, 2); + assert_eq!(series_proto.interval, expected_interval as i64); + assert_eq!(series_proto.points.len(), 1); + assert_eq!(series_proto.points[0].value, expected_value); + } } #[test] @@ -1219,23 +1245,41 @@ mod tests { ); let counter = get_simple_counter_with_metadata(event_metadata); - let result = generate_series_metrics( - &counter, - &None, - log_schema(), - DEFAULT_DD_ORIGIN_PRODUCT_VALUE, - ); - assert!(result.is_ok()); + // series v1 + { + let result = generate_series_metrics( + &counter, + &None, + log_schema(), + DEFAULT_DD_ORIGIN_PRODUCT_VALUE, + ); + assert!(result.is_ok()); - let metrics = result.unwrap(); - assert_eq!(metrics.len(), 1); + let metrics = result.unwrap(); + assert_eq!(metrics.len(), 1); - let actual = &metrics[0]; - let generated_origin = actual.metadata.as_ref().unwrap().origin.as_ref().unwrap(); + let actual = &metrics[0]; + let generated_origin = actual.metadata.as_ref().unwrap().origin.as_ref().unwrap(); - assert_eq!(generated_origin.product().unwrap(), product); - assert_eq!(generated_origin.category().unwrap(), category); - assert_eq!(generated_origin.service().unwrap(), service); + assert_eq!(generated_origin.product().unwrap(), product); + assert_eq!(generated_origin.category().unwrap(), category); + assert_eq!(generated_origin.service().unwrap(), service); + } + // series v2 + { + let series_proto = series_to_proto_message( + &counter, + &None, + log_schema(), + DEFAULT_DD_ORIGIN_PRODUCT_VALUE, + ) + .unwrap(); + + let generated_origin = series_proto.metadata.unwrap().origin.unwrap(); + assert_eq!(generated_origin.origin_product, product); + assert_eq!(generated_origin.origin_category, category); + assert_eq!(generated_origin.origin_service, service); + } } #[test] @@ -1249,22 +1293,40 @@ mod tests { counter.metadata_mut().set_source_type("statsd"); - let result = generate_series_metrics(&counter, &None, log_schema(), product); - assert!(result.is_ok()); + // series v1 + { + let result = generate_series_metrics(&counter, &None, log_schema(), product); + assert!(result.is_ok()); - let metrics = result.unwrap(); - assert_eq!(metrics.len(), 1); + let metrics = result.unwrap(); + assert_eq!(metrics.len(), 1); - let actual = &metrics[0]; - let generated_origin = actual.metadata.as_ref().unwrap().origin.as_ref().unwrap(); + let actual = &metrics[0]; + let generated_origin = actual.metadata.as_ref().unwrap().origin.as_ref().unwrap(); - assert_eq!(generated_origin.product().unwrap(), product); - assert_eq!(generated_origin.category().unwrap(), category); - assert_eq!(generated_origin.service().unwrap(), service); + assert_eq!(generated_origin.product().unwrap(), product); + assert_eq!(generated_origin.category().unwrap(), category); + assert_eq!(generated_origin.service().unwrap(), service); + } + // series v2 + { + let series_proto = series_to_proto_message( + &counter, + &None, + log_schema(), + DEFAULT_DD_ORIGIN_PRODUCT_VALUE, + ) + .unwrap(); + + let generated_origin = series_proto.metadata.unwrap().origin.unwrap(); + assert_eq!(generated_origin.origin_product, product); + assert_eq!(generated_origin.origin_category, category); + assert_eq!(generated_origin.origin_service, service); + } } #[test] - fn encode_single_series_metric_with_default_limits() { + fn encode_single_series_v1_metric_with_default_limits() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = @@ -1287,6 +1349,30 @@ mod tests { assert_eq!(expected, processed.pop().unwrap()); } + #[test] + fn encode_single_series_v2_metric_with_default_limits() { + // This is a simple test where we ensure that a single metric, with the default limits, can + // be encoded without hitting any errors. + let mut encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) + .expect("default payload size limits should be valid"); + let counter = get_simple_counter(); + let expected = counter.clone(); + + // Encode the counter. + let result = encoder.try_encode(counter); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), None); + + // Finish the payload, make sure we got what we came for. + let result = encoder.finish(); + assert!(result.is_ok()); + + let (_payload, mut processed) = result.unwrap(); + assert_eq!(processed.len(), 1); + assert_eq!(expected, processed.pop().unwrap()); + } + #[test] fn encode_single_sketch_metric_with_default_limits() { // This is a simple test where we ensure that a single metric, with the default limits, can @@ -1382,7 +1468,7 @@ mod tests { // This is too small. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), header_len, usize::MAX, ); @@ -1390,7 +1476,7 @@ mod tests { // This is just right. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), header_len + 1, usize::MAX, ); @@ -1403,7 +1489,7 @@ mod tests { // This is too small. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), usize::MAX, compression_overhead_len, ); @@ -1411,7 +1497,7 @@ mod tests { // This is just right. let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), usize::MAX, compression_overhead_len + 1, ); @@ -1632,7 +1718,7 @@ mod tests { // We check this with targeted unit tests as well but this is some cheap insurance to // show that we're hopefully not missing any particular corner cases. let result = DatadogMetricsEncoder::with_payload_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None, uncompressed_limit, compressed_limit, From c9c0fbc9f9326c563a68023c3f67c387376e8032 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 13:56:34 -0600 Subject: [PATCH 40/99] feedback ds: extend the int test coverage --- .../datadog/metrics/integration_tests.rs | 131 ++++++++++++++++-- 1 file changed, 120 insertions(+), 11 deletions(-) diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index a2c0fd4ee2f81..994ee321b8b00 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -1,3 +1,5 @@ +use std::num::NonZeroU32; + use bytes::Bytes; use chrono::{SubsecRound, Utc}; use flate2::read::ZlibDecoder; @@ -37,7 +39,7 @@ mod ddmetric_proto { include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); } -fn generate_metric_events() -> Vec { +fn generate_counters() -> Vec { let timestamp = Utc::now().trunc_subsecs(3); let events: Vec<_> = (0..10) .map(|index| { @@ -66,6 +68,38 @@ fn generate_metric_events() -> Vec { events } +fn generate_counter_gauge_set() -> Vec { + let ts = Utc::now().trunc_subsecs(3); + let events = vec![ + // gauge + Event::Metric(Metric::new( + "gauge", + MetricKind::Incremental, + MetricValue::Gauge { value: 5678.0 }, + )), + // counter with interval + Event::Metric( + Metric::new( + "counter_with_interval", + MetricKind::Incremental, + MetricValue::Counter { value: 1234.0 }, + ) + .with_interval_ms(NonZeroU32::new(2000)) + .with_timestamp(Some(ts)), + ), + // set + Event::Metric(Metric::new( + "set", + MetricKind::Incremental, + MetricValue::Set { + values: vec!["zorp".into(), "zork".into()].into_iter().collect(), + }, + )), + ]; + + events +} + /// Starts a test sink with random metrics running into it /// /// This function starts a Datadog Metrics sink with a simplistic configuration and @@ -75,7 +109,7 @@ fn generate_metric_events() -> Vec { /// Testers may set `http_status` and `batch_status`. The first controls what /// status code faked HTTP responses will have, the second acts as a check on /// the `Receiver`'s status before being returned to the caller. -async fn start_test() -> (Vec, Receiver<(http::request::Parts, Bytes)>) { +async fn start_test(events: Vec) -> (Vec, Receiver<(http::request::Parts, Bytes)>) { let config = indoc! {r#" default_api_key = "atoken" default_namespace = "foo" @@ -95,8 +129,6 @@ async fn start_test() -> (Vec, Receiver<(http::request::Parts, Bytes)>) { let (batch, mut receiver) = BatchNotifier::new_with_receiver(); - let events = generate_metric_events(); - let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); sink.run(stream).await.unwrap(); @@ -114,13 +146,33 @@ fn decompress_payload(payload: Vec) -> std::io::Result> { } #[tokio::test] -/// Assert the basic functionality of the sink in good conditions +/// Assert proper handling of different metric types +async fn all_series_metric_types() { + let metrics = generate_counter_gauge_set(); + let (expected, rx) = start_test(metrics).await; + + let output = rx.take(expected.len()).collect::>().await; + + assert!(output.len() == 1, "Should have received a response"); + + let request = output.first().unwrap(); + + validate_protobuf_counter_gauge_set(request); +} + +#[tokio::test] +/// Assert the basic functionality of the sink in good conditions with +/// a small batch of counters. /// /// This test rigs the sink to return OK to responses, checks that all batches /// were delivered and then asserts that every message is able to be /// deserialized. +/// +/// In addition to validating the counter values, we also validate the various +/// fields such as the Resources, handling of tags, and the Metadata. async fn smoke() { - let (expected, rx) = start_test().await; + let counters = generate_counters(); + let (expected, rx) = start_test(counters).await; let output = rx.take(expected.len()).collect::>().await; @@ -129,8 +181,8 @@ async fn smoke() { let request = output.first().unwrap(); match request.0.uri.path() { - SERIES_V1_PATH => validate_json(request), - SERIES_V2_PATH => validate_protobuf(request), + SERIES_V1_PATH => validate_json_counters(request), + SERIES_V2_PATH => validate_protobuf_counters(request), _ => panic!("Unexpected request type received!"), } } @@ -140,7 +192,7 @@ fn validate_common(request: &(Parts, Bytes)) { assert!(request.0.headers.contains_key("DD-Agent-Payload")); } -fn validate_protobuf(request: &(Parts, Bytes)) { +fn validate_protobuf_counters(request: &(Parts, Bytes)) { assert_eq!( request.0.headers.get("Content-Type").unwrap(), "application/x-protobuf" @@ -210,7 +262,64 @@ fn validate_protobuf(request: &(Parts, Bytes)) { ); } -fn validate_json(request: &(Parts, Bytes)) { +fn validate_protobuf_counter_gauge_set(request: &(Parts, Bytes)) { + assert_eq!( + request.0.headers.get("Content-Type").unwrap(), + "application/x-protobuf" + ); + + validate_common(request); + + let compressed_payload = request.1.to_vec(); + let payload = decompress_payload(compressed_payload).expect("Could not decompress payload"); + let frame = Bytes::copy_from_slice(&payload); + + let payload = + ddmetric_proto::MetricPayload::decode(frame).expect("Could not decode protobuf frame"); + + let mut series = payload.series; + + assert_eq!(series.len(), 3); + + // Note the below evaluation implies validation of sorting the metrics by name to improve HTTP compression + + // validate set (gauge) + { + let gauge = series.pop().unwrap(); + assert_eq!( + gauge.r#type(), + ddmetric_proto::metric_payload::MetricType::Gauge + ); + assert_eq!(gauge.interval, 0); + assert_eq!(gauge.points[0].value, 2_f64); + } + + // validate gauge + { + let gauge = series.pop().unwrap(); + assert_eq!( + gauge.r#type(), + ddmetric_proto::metric_payload::MetricType::Gauge + ); + assert_eq!(gauge.points[0].value, 5678.0); + assert_eq!(gauge.interval, 0); + } + + // validate counter w interval = rate + { + let count = series.pop().unwrap(); + assert_eq!( + count.r#type(), + ddmetric_proto::metric_payload::MetricType::Rate + ); + assert_eq!(count.interval, 2); + + assert_eq!(count.points.len(), 1); + assert_eq!(count.points[0].value, 1234.0 / count.interval as f64); + } +} + +fn validate_json_counters(request: &(Parts, Bytes)) { assert_eq!( request.0.headers.get("Content-Type").unwrap(), "application/json" @@ -308,7 +417,7 @@ async fn run_sink() { let (sink, _) = config.build(cx).await.unwrap(); let (batch, receiver) = BatchNotifier::new_with_receiver(); - let events = generate_metric_events(); + let events = generate_counters(); let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); From 2ac5bbf177273d1866c26f93361d12a206c72b35 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 13:56:39 -0600 Subject: [PATCH 41/99] Revert "feedback ds: remove check for sort by name" This reverts commit c95a326c30a471e660742b6e354ab5d73e1201de. --- src/sinks/datadog/metrics/integration_tests.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 994ee321b8b00..d46d1d17829e9 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -211,6 +211,12 @@ fn validate_protobuf_counters(request: &(Parts, Bytes)) { assert!(!series.is_empty()); + // check metrics are sorted by name, which helps HTTP compression + let metric_names: Vec = series.iter().map(|serie| serie.metric.clone()).collect(); + let mut sorted_names = metric_names.clone(); + sorted_names.sort(); + assert_eq!(metric_names, sorted_names); + series.iter().for_each(|serie| { // name assert!(serie.metric.starts_with("foo.counter_")); From cb92dc5f4faa4054001ac554b4cb0c93b113efe3 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 14:00:30 -0600 Subject: [PATCH 42/99] add explicit sort check --- src/sinks/datadog/metrics/integration_tests.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index d46d1d17829e9..74cc60c28a1b6 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -287,7 +287,13 @@ fn validate_protobuf_counter_gauge_set(request: &(Parts, Bytes)) { assert_eq!(series.len(), 3); - // Note the below evaluation implies validation of sorting the metrics by name to improve HTTP compression + // The below evaluation of each metric type implies validation of sorting the metrics + // by name to improve HTTP compression due to the order they are defined vs processed. + // However just to be safe we will also validate explicitly. + let metric_names: Vec = series.iter().map(|serie| serie.metric.clone()).collect(); + let mut sorted_names = metric_names.clone(); + sorted_names.sort(); + assert_eq!(metric_names, sorted_names); // validate set (gauge) { From 96ef12230071d5ab0394fa85b1bd874c8d13af89 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 14:36:04 -0600 Subject: [PATCH 43/99] add env var for v1 support --- docs/DEPRECATION.md | 4 +-- src/sinks/datadog/metrics/config.rs | 28 ++++++++++++++----- .../datadog/metrics/integration_tests.rs | 8 ++++-- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/docs/DEPRECATION.md b/docs/DEPRECATION.md index ff1edddde596b..4d8c82faaa256 100644 --- a/docs/DEPRECATION.md +++ b/docs/DEPRECATION.md @@ -78,7 +78,7 @@ When introducing a deprecation into Vector, the pull request introducing the dep the new name will be appended with the text `(formerly OldName)`. - Add a log message to Vector that is logged at the `WARN` level starting with the word `DEPRECATION` if Vector detects the deprecated configuration or feature being used (when possible). -- Add the deprecation to [DEPRECATIONS.md](docs/DEPRECATIONS.md) to track migration (if applicable) and removal +- Add the deprecation to [DEPRECATIONS.md](DEPRECATIONS.md) to track migration (if applicable) and removal When removing a deprecation in a subsequent release, the pull request should: @@ -86,4 +86,4 @@ When removing a deprecation in a subsequent release, the pull request should: - Remove the deprecation from the documentation - Add a note to the Breaking Changes section of the upgrade guide for the next release with a description and directions for transitioning if applicable. -- Remove the deprecation from [DEPRECATIONS.md](docs/DEPRECATIONS.md) +- Remove the deprecation from [DEPRECATIONS.md](DEPRECATIONS.md) diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 2c7a490daf685..6e139f0d5e2f5 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -1,3 +1,5 @@ +use std::sync::OnceLock; + use http::Uri; use snafu::ResultExt; use tower::ServiceBuilder; @@ -46,6 +48,9 @@ pub(super) const SERIES_V1_PATH: &str = "/api/v1/series"; pub(super) const SERIES_V2_PATH: &str = "/api/v2/series"; pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; +// TODO: the series V1 endpoint support is considered deprecated and should be removed in a future release. +// At that time when the V1 support is removed, the SeriesApiVersion stops being useful and can be removed. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[allow(dead_code)] pub enum SeriesApiVersion { @@ -60,11 +65,14 @@ impl SeriesApiVersion { Self::V2 => SERIES_V2_PATH, } } -} - -impl Default for SeriesApiVersion { - fn default() -> Self { - Self::V2 + fn get_api_version_backwards_compatible() -> Self { + static API_VERSION: OnceLock = OnceLock::new(); + *API_VERSION.get_or_init( + || match option_env!("VECTOR_TEMP_USE_DD_METRICS_SERIES_V1_API") { + Some(_) => Self::V1, + None => Self::V2, + }, + ) } } @@ -93,7 +101,7 @@ impl DatadogMetricsEndpoint { // Creates an instance of the `Series` variant with the default API version. pub fn series() -> Self { - Self::Series(SeriesApiVersion::default()) + Self::Series(SeriesApiVersion::get_api_version_backwards_compatible()) } } @@ -200,7 +208,13 @@ impl DatadogMetricsConfig { &self, ) -> crate::Result { let base_uri = self.get_base_agent_endpoint(); - let series_endpoint = build_uri(&base_uri, SeriesApiVersion::default().get_path())?; + + // TODO: the V1 endpoint support is considered deprecated and should be removed in a future release. + // At that time, the get_api_version_backwards_compatible() should be replaced with statically using the v2. + let series_endpoint = build_uri( + &base_uri, + SeriesApiVersion::get_api_version_backwards_compatible().get_path(), + )?; let sketches_endpoint = build_uri(&base_uri, SKETCHES_PATH)?; Ok(DatadogMetricsEndpointConfiguration::new( diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 74cc60c28a1b6..ae7d897b19274 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -157,7 +157,11 @@ async fn all_series_metric_types() { let request = output.first().unwrap(); - validate_protobuf_counter_gauge_set(request); + match request.0.uri.path() { + SERIES_V1_PATH => warn!("Deprecated endpoint used!"), + SERIES_V2_PATH => validate_protobuf_set_gauge_rate(request), + _ => panic!("Unexpected request type received!"), + } } #[tokio::test] @@ -268,7 +272,7 @@ fn validate_protobuf_counters(request: &(Parts, Bytes)) { ); } -fn validate_protobuf_counter_gauge_set(request: &(Parts, Bytes)) { +fn validate_protobuf_set_gauge_rate(request: &(Parts, Bytes)) { assert_eq!( request.0.headers.get("Content-Type").unwrap(), "application/x-protobuf" From 2de3617e8f9e3bd5e32d94062721e4e69d9f6320 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 5 Oct 2023 15:25:56 -0600 Subject: [PATCH 44/99] check events --- src/sinks/datadog/metrics/integration_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index ae7d897b19274..8c993b22bca04 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -158,7 +158,7 @@ async fn all_series_metric_types() { let request = output.first().unwrap(); match request.0.uri.path() { - SERIES_V1_PATH => warn!("Deprecated endpoint used!"), + SERIES_V1_PATH => warn!("Deprecated endpoint used."), SERIES_V2_PATH => validate_protobuf_set_gauge_rate(request), _ => panic!("Unexpected request type received!"), } From b9181e01ae5c568863d3258f9ae917b10442ba7c Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 6 Oct 2023 11:40:31 -0600 Subject: [PATCH 45/99] add note in deprecations --- docs/DEPRECATIONS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index 061e19f36e9ed..2dc2553ecd94d 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -6,6 +6,7 @@ See [DEPRECATION.md](docs/DEPRECATION.md#process) for the process for updating t ## To be removed +* Support for `v1` series endpoint in the `datadog_metrics` sink should be removed. * legacy_openssl_provider v0.34.0 OpenSSL legacy provider flag should be removed * armv7_rpm v0.34.0 The armv7 RPM packages should be removed (replaced by armv7hl) * yaml_migration v0.34.0 Prefer loading `/etc/vector/vector.yaml` first From 69bafd099338a55175da53b43c8da765d0d79205 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 10 Oct 2023 16:09:33 -0600 Subject: [PATCH 46/99] feedback ds --- scripts/integration/datadog-e2e-logs/compose.yaml | 9 ++++----- scripts/integration/datadog-e2e-logs/test.yaml | 2 +- tests/data/e2e/datadog/logs/vector.toml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/integration/datadog-e2e-logs/compose.yaml index 39d331206eca3..3ca07c4d88aba 100644 --- a/scripts/integration/datadog-e2e-logs/compose.yaml +++ b/scripts/integration/datadog-e2e-logs/compose.yaml @@ -17,16 +17,16 @@ services: - log_path:/var/log/ # Receives log data from the `datadog-agent-vector` service and sends - # to the `fakeintake-agent-vector` service. + # to the `fakeintake-vector` service. vector: depends_on: - - fakeintake-agent-vector + - fakeintake-vector build: context: ${PWD} dockerfile: tests/data/e2e/datadog/Dockerfile args: - RUST_VERSION=${RUST_VERSION} - - FEATURES=sources-datadog_agent,sinks-datadog_logs + - FEATURES=e2e-tests-datadog working_dir: /home/vector network_mode: host command: @@ -35,7 +35,6 @@ services: - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" volumes: - ${PWD}:/home/vector - - target:/home/vector/target # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service @@ -86,7 +85,7 @@ services: # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. - fakeintake-agent-vector: + fakeintake-vector: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} networks: diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/integration/datadog-e2e-logs/test.yaml index 2c2114f58c56e..14537d66c2017 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/integration/datadog-e2e-logs/test.yaml @@ -10,7 +10,7 @@ runner: EXPECTED_LOG_EVENTS: '1000' VECTOR_RECEIVE_PORT: '8081' FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' - FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' + FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' matrix: version: [latest] diff --git a/tests/data/e2e/datadog/logs/vector.toml b/tests/data/e2e/datadog/logs/vector.toml index bbcf8729fef8e..6894fbf74d8fb 100644 --- a/tests/data/e2e/datadog/logs/vector.toml +++ b/tests/data/e2e/datadog/logs/vector.toml @@ -11,7 +11,7 @@ store_api_key = false [sinks.dd] inputs = [ "agent.logs" ] type = "datadog_logs" -default_api_key="unused" -endpoint = "http://fakeintake-agent-vector:80" +default_api_key = "unused" +endpoint = "http://fakeintake-vector:80" batch.timeout_secs = 1 compression = "gzip" From 7406730dcf41bdbe98ad0d4e341e1ca3eb368b72 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 10 Oct 2023 16:32:29 -0600 Subject: [PATCH 47/99] ds feedback logs PR --- scripts/integration/datadog-e2e-metrics/compose.yaml | 10 ++++------ scripts/integration/datadog-e2e-metrics/test.yaml | 2 +- tests/data/e2e/datadog/metrics/vector.toml | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/integration/datadog-e2e-metrics/compose.yaml b/scripts/integration/datadog-e2e-metrics/compose.yaml index c1db12d26e9a1..8bbac8abf22bf 100644 --- a/scripts/integration/datadog-e2e-metrics/compose.yaml +++ b/scripts/integration/datadog-e2e-metrics/compose.yaml @@ -49,16 +49,16 @@ services: - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # Receives metric data from the `datadog-agent-vector` service and sends - # to the `fakeintake-agent-vector` service. + # to the `fakeintake-vector` service. vector: depends_on: - - fakeintake-agent-vector + - fakeintake-vector build: context: ${PWD} dockerfile: tests/data/e2e/datadog/Dockerfile args: - RUST_VERSION=${RUST_VERSION} - - FEATURES=sources-datadog_agent,sinks-datadog_metrics + - FEATURES=e2e-tests-datadog working_dir: /home/vector network_mode: host command: @@ -67,8 +67,6 @@ services: - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" volumes: - ${PWD}:/home/vector - - target:/home/vector/target - # Receives metric data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. @@ -77,7 +75,7 @@ services: # Receives metric data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. - fakeintake-agent-vector: + fakeintake-vector: image: docker.io/datadog/fakeintake:${CONFIG_VERSION} networks: diff --git a/scripts/integration/datadog-e2e-metrics/test.yaml b/scripts/integration/datadog-e2e-metrics/test.yaml index aba8a7501e635..43078fafb6ba5 100644 --- a/scripts/integration/datadog-e2e-metrics/test.yaml +++ b/scripts/integration/datadog-e2e-metrics/test.yaml @@ -9,7 +9,7 @@ runner: env: VECTOR_RECEIVE_PORT: '8081' FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' - FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-agent-vector:80' + FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' matrix: version: [latest] diff --git a/tests/data/e2e/datadog/metrics/vector.toml b/tests/data/e2e/datadog/metrics/vector.toml index d153b1f8c7ded..c99e2f60ab9ba 100644 --- a/tests/data/e2e/datadog/metrics/vector.toml +++ b/tests/data/e2e/datadog/metrics/vector.toml @@ -11,6 +11,6 @@ store_api_key = false [sinks.dd] inputs = [ "agent.metrics" ] type = "datadog_metrics" -default_api_key="unused" -endpoint = "http://fakeintake-agent-vector:80" +default_api_key = "unused" +endpoint = "http://fakeintake-vector:80" batch.timeout_secs = 1 From b7451f5f9b7dfa40f6d3ad4c1906c1a0c57d8634 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 11 Oct 2023 17:14:25 -0600 Subject: [PATCH 48/99] config pain; start aggregation --- .../datadog-e2e-metrics/compose.yaml | 18 +- .../dogstatsd_client/client.py | 26 ++- tests/data/e2e/datadog/logs/agent_only.yaml | 1 + tests/data/e2e/datadog/logs/agent_vector.yaml | 1 + .../data/e2e/datadog/metrics/agent_only.yaml | 12 +- .../e2e/datadog/metrics/agent_vector.yaml | 12 +- tests/e2e/datadog/logs/mod.rs | 4 +- tests/e2e/datadog/metrics/mod.rs | 157 ++++++++++++++++-- tests/e2e/datadog/mod.rs | 56 ++++++- 9 files changed, 244 insertions(+), 43 deletions(-) diff --git a/scripts/integration/datadog-e2e-metrics/compose.yaml b/scripts/integration/datadog-e2e-metrics/compose.yaml index 8bbac8abf22bf..a2b6f1527280c 100644 --- a/scripts/integration/datadog-e2e-metrics/compose.yaml +++ b/scripts/integration/datadog-e2e-metrics/compose.yaml @@ -2,6 +2,7 @@ version: '3' services: + # Emits metrics to the Agent only path dogstatsd-client-agent: build: ./dogstatsd_client environment: @@ -9,41 +10,36 @@ services: depends_on: - datadog-agent - dogstatsd-client-agent-vector: + # Emits metrics to the Agent-Vector path + dogstatsd-client-vector: build: ./dogstatsd_client environment: - STATSD_HOST=datadog-agent-vector depends_on: - datadog-agent-vector - # and sends metric data to - # the `fakeintake-agent` service + # Sends metric data received from the Emitter to the `fakeintake-agent` service datadog-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} depends_on: - fakeintake-agent environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + # - DD_API_KEY=DEADBEEF - DD_HOSTNAME=datadog-agent - - DD_ENABLE_PAYLOADS_EVENTS=false - - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - ../../../tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml - # and sends metric data to - # the `vector` service + # Sends metric data received from the Emitter to the `vector` service datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} depends_on: - vector environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + # - DD_API_KEY=DEADBEEF - DD_HOSTNAME=datadog-agent-vector - - DD_ENABLE_PAYLOADS_EVENTS=false - - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py index 23b31d305649a..8f8897c84e1d2 100644 --- a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py +++ b/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py @@ -1,10 +1,11 @@ from datadog import initialize, statsd import time import os +import random STATSD_HOST = os.getenv('STATSD_HOST') -print("initializing for {STATSD_HOST}") +print(f"initializing for {STATSD_HOST}") options = { 'statsd_host':STATSD_HOST, @@ -13,7 +14,23 @@ initialize(**options) -# time.sleep(5) +time.sleep(10) + +hist_data = [ + 9, 5, 0, 2, 16, 17, 8, 16, 10, 13, + 15, 3, 9, 13, 11, 17, 5, 18, 14, 9, + 4, 16, 9, 17, 4, 11, 7, 14, 8, 12, + 10, 9, 11, 3, 18, 12, 17, 12, 3, 19, + 9, 11, 19, 9, 15, 2, 7, 10, 4, 14 +] + +dist_data = [ + 18, 5, 19, 0, 13, 12, 5, 12, 10, 4, + 1, 5, 7, 1, 14, 16, 20, 0, 8, 2, 4, + 20, 8, 4, 20, 6, 20, 3, 10, 11, 12, + 15, 2, 12, 5, 19, 19, 5, 9, 6, 18, + 19, 11, 6, 17, 5, 0, 1, 17, 17 +] for i in range(50): print("rate") @@ -26,10 +43,11 @@ statsd.set('foo_metric.set', i, tags=["a_tag:3"]) print("histogram") - statsd.histogram('foo_metric.histogram', random.randint(0, 20), tags=["a_tag:4"]) + statsd.histogram('foo_metric.histogram', hist_data[i], tags=["a_tag:4"]) print("distribution") - statsd.distribution('foo_metric.distribution', random.randint(0, 20), tags=["a_tag:5"]) + statsd.distribution('foo_metric.distribution', dist_data[i], tags=["a_tag:5"]) statsd.flush() time.sleep(0.01) + diff --git a/tests/data/e2e/datadog/logs/agent_only.yaml b/tests/data/e2e/datadog/logs/agent_only.yaml index aeebb113c1075..dcdfcab9b40dd 100644 --- a/tests/data/e2e/datadog/logs/agent_only.yaml +++ b/tests/data/e2e/datadog/logs/agent_only.yaml @@ -5,6 +5,7 @@ log_level: 'debug' inventories_configuration_enabled: false enable_metadata_collection: false enable_gohai: false +cloud_provider_metadata: [] apm_config: enabled: false diff --git a/tests/data/e2e/datadog/logs/agent_vector.yaml b/tests/data/e2e/datadog/logs/agent_vector.yaml index 1686986746971..411da03025e0c 100644 --- a/tests/data/e2e/datadog/logs/agent_vector.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector.yaml @@ -5,6 +5,7 @@ log_level: 'debug' inventories_configuration_enabled: false enable_metadata_collection: false enable_gohai: false +cloud_provider_metadata: [] apm_config: enabled: false diff --git a/tests/data/e2e/datadog/metrics/agent_only.yaml b/tests/data/e2e/datadog/metrics/agent_only.yaml index a2236419046d3..1d7429e01b900 100644 --- a/tests/data/e2e/datadog/metrics/agent_only.yaml +++ b/tests/data/e2e/datadog/metrics/agent_only.yaml @@ -1,10 +1,17 @@ api_key: DEADBEEF -log_level: 'debug' +log_level: 'warn' # disable bunch of stuff we don't need inventories_configuration_enabled: false -enable_metadata_collection: false +# enable_metadata_collection: false enable_gohai: false +cloud_provider_metadata: [] + +# enable_payloads: + # series: false + # events: false + # service_checks: false + # sketches: false apm_config: enabled: false @@ -20,6 +27,7 @@ logs_enabled: false # configure dogstatsd use_dogstatsd: true +dogstatsd_non_local_traffic: true # configure metrics dd_url: http://fakeintake-agent:80 diff --git a/tests/data/e2e/datadog/metrics/agent_vector.yaml b/tests/data/e2e/datadog/metrics/agent_vector.yaml index e5c540e87cee1..28d43898e6f44 100644 --- a/tests/data/e2e/datadog/metrics/agent_vector.yaml +++ b/tests/data/e2e/datadog/metrics/agent_vector.yaml @@ -1,10 +1,17 @@ api_key: DEADBEEF -log_level: 'debug' +log_level: 'warn' # disable bunch of stuff we don't need inventories_configuration_enabled: false -enable_metadata_collection: false +# enable_metadata_collection: false enable_gohai: false +cloud_provider_metadata: [] + +# enable_payloads: + # series: false + # events: false + # service_checks: false + # sketches: false apm_config: enabled: false @@ -20,6 +27,7 @@ logs_enabled: false # configure dogstatsd use_dogstatsd: true +dogstatsd_non_local_traffic: true # send to vector vector: diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 663e243c58682..8121517671603 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -59,12 +59,12 @@ async fn validate() { trace_init(); println!("getting log payloads from agent-only pipeline"); - let mut agent_payloads = get_payloads_agent(LOGS_ENDPOINT).await; + let mut agent_payloads = get_payloads_agent_json(LOGS_ENDPOINT).await; common_assertions(&mut agent_payloads); println!("getting log payloads from agent-vector pipeline"); - let mut vector_payloads = get_payloads_vector(LOGS_ENDPOINT).await; + let mut vector_payloads = get_payloads_vector_json(LOGS_ENDPOINT).await; common_assertions(&mut vector_payloads); diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index 993c258b964fc..f404477b2f9a1 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -1,21 +1,132 @@ -use serde_json::Value; +use std::collections::BTreeMap; + +use base64::{prelude::BASE64_STANDARD, Engine}; +use bytes::Bytes; +use flate2::read::ZlibDecoder; +use prost::Message; use vector::test_util::trace_init; +#[allow(warnings, clippy::pedantic, clippy::nursery)] +mod ddmetric_proto { + include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); +} + use super::*; -const AGENT_DEFAULT_ENDPOINT: &str = "/api/v2/series"; +const SERIES_ENDPOINT: &str = "/api/v2/series"; + +fn decompress_payload(payload: Vec) -> std::io::Result> { + let mut decompressor = ZlibDecoder::new(&payload[..]); + let mut decompressed = Vec::new(); + let result = std::io::copy(&mut decompressor, &mut decompressed); + result.map(|_| decompressed) +} + +fn unpack_payloads_series_v2( + in_payloads: &Vec, +) -> Vec { + let mut out_payloads = vec![]; + + in_payloads.iter().for_each(|payload| { + // decode base64 + let payload = BASE64_STANDARD + .decode(&payload.data) + .expect("Invalid base64 data"); + + // decompress + let bytes = Bytes::from(decompress_payload(payload).unwrap()); + + let payload = ddmetric_proto::MetricPayload::decode(bytes).unwrap(); + + out_payloads.push(payload); + }); + + out_payloads +} + +// Sums up the metrics in each series by name. +fn aggregate_series_metrics( + payloads: &Vec, +) -> BTreeMap { + let mut aggregate = BTreeMap::new(); + + for metric_payload in payloads { + for serie in &metric_payload.series { + // filter out the metrics we don't care about + if !serie.metric.starts_with("foo_metric") { + continue; + } + + // TODO expand on this -// TODO the v1 endpoint is not compatible with fakeintake parsed -// payloads right now. we might need to change to use v2 -const VECTOR_DEFAULT_ENDPOINT: &str = "/api/v1/series"; + let interval = serie.interval as f64; + match serie.r#type() { + ddmetric_proto::metric_payload::MetricType::Unspecified => { + panic!("unspecified metric type") + } + ddmetric_proto::metric_payload::MetricType::Count => { + if let Some((t, v)) = aggregate.get_mut(&serie.metric) { + for point in &serie.points { + *t = point.timestamp; + *v += point.value; + } + } else { + for point in &serie.points { + aggregate.insert(serie.metric.clone(), (point.timestamp, point.value)); + } + } + } + ddmetric_proto::metric_payload::MetricType::Rate => { + if let Some((t, v)) = aggregate.get_mut(&serie.metric) { + for point in &serie.points { + *v += point.value * interval; + *t = point.timestamp; + } + } else { + for (idx, point) in serie.points.iter().enumerate() { + if idx == 0 { + aggregate.insert( + serie.metric.clone(), + (point.timestamp, point.value * interval), + ); + } else { + if let Some((t, v)) = aggregate.get_mut(&serie.metric) { + *v += point.value * interval; + *t = point.timestamp; + } + } + } + } + } + ddmetric_proto::metric_payload::MetricType::Gauge => { + // last one wins + if let Some(point) = serie.points.last() { + if let Some((t, v)) = aggregate.get_mut(&serie.metric) { + if point.timestamp > *t { + *t = point.timestamp; + *v = point.value; + } + } else { + aggregate.insert(serie.metric.clone(), (point.timestamp, point.value)); + } + } + } + } + } + } + + // remove the timestamps + let aggregate = aggregate.into_iter().map(|(k, v)| (k, v.1)).collect(); + + aggregate +} // runs assertions that each set of payloads should be true to regardless // of the pipeline -fn common_assertions(payloads: &mut Vec) { +fn common_assertions(payloads: &Vec) { assert!(payloads.len() > 0); - - println!("metric events received: {}", payloads.len()); + println!("metric payloads received: {}", payloads.len()); } #[tokio::test] @@ -23,17 +134,35 @@ async fn validate() { trace_init(); // TODO need to see if can configure the agent flush interval - std::thread::sleep(std::time::Duration::from_secs(5)); + std::thread::sleep(std::time::Duration::from_secs(30)); + + println!("getting payloads from agent-only pipeline"); + let agent_payloads = get_payloads_agent(SERIES_ENDPOINT).await; + + println!("unpacking payloads from agent-only pipeline"); + let agent_payloads = unpack_payloads_series_v2(&agent_payloads); + common_assertions(&agent_payloads); - println!("getting log payloads from agent-only pipeline"); - let mut agent_payloads = get_payloads_agent(AGENT_DEFAULT_ENDPOINT).await; + println!("aggregating payloads from agent-only pipeline"); + let agent_payloads = aggregate_series_metrics(&agent_payloads); - common_assertions(&mut agent_payloads); + println!("{:?}", agent_payloads.keys()); + + // let foo_rate_agent = agent_payloads.get("foo_metric.rate"); + // println!("AGENT RATE AGGREGATE: {:?}", foo_rate_agent); println!("getting log payloads from agent-vector pipeline"); - let mut vector_payloads = get_payloads_vector(VECTOR_DEFAULT_ENDPOINT).await; + let vector_payloads = get_payloads_vector(SERIES_ENDPOINT).await; + + println!("unpacking payloads from agent-vector pipeline"); + let vector_payloads = unpack_payloads_series_v2(&vector_payloads); + common_assertions(&vector_payloads); - common_assertions(&mut vector_payloads); + println!("aggregating payloads from agent-vector pipeline"); + let vector_payloads = aggregate_series_metrics(&vector_payloads); + println!("{:?}", vector_payloads.keys()); assert_eq!(agent_payloads, vector_payloads); + + // std::thread::sleep(std::time::Duration::from_secs(90)); } diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 668d841f7b7b5..8d79c5d971da3 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -15,7 +15,21 @@ fn fake_intake_agent_endpoint() -> String { .unwrap_or_else(|_| "http://127.0.0.1:8083".to_string()) } -// Fakeintake response +// Fakeintake JSON response +#[derive(Deserialize, Debug)] +struct FakeIntakeJsonResponse { + payloads: Vec, +} + +#[allow(dead_code)] +#[derive(Deserialize, Debug)] +struct FakeIntakeJsonPayload { + data: Value, + encoding: String, + timestamp: String, +} + +// Fakeintake raw response #[derive(Deserialize, Debug)] struct FakeIntakeResponse { payloads: Vec, @@ -24,12 +38,13 @@ struct FakeIntakeResponse { #[allow(dead_code)] #[derive(Deserialize, Debug)] struct FakeIntakePayload { - data: Value, + // base64 encoded + data: String, encoding: String, timestamp: String, } -async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> FakeIntakeResponse { +async fn get_fakeintake_json_payloads(base: &str, endpoint: &str) -> FakeIntakeJsonResponse { let url = format!( "{}/fakeintake/payloads?endpoint={}&format=json", base, endpoint, @@ -40,13 +55,13 @@ async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> FakeIntakeRespon .send() .await .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) - .json::() + .json::() .await .expect("Parsing fakeintake payloads failed") } -async fn get_payloads_agent(endpoint: &str) -> Vec { - let mut raw_payloads = get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint) +async fn get_payloads_agent_json(endpoint: &str) -> Vec { + let mut raw_payloads = get_fakeintake_json_payloads(&fake_intake_agent_endpoint(), endpoint) .await .payloads; @@ -60,11 +75,36 @@ async fn get_payloads_agent(endpoint: &str) -> Vec { raw_payloads.into_iter().map(|raw| raw.data).collect() } -async fn get_payloads_vector(endpoint: &str) -> Vec { - get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint) +async fn get_payloads_vector_json(endpoint: &str) -> Vec { + get_fakeintake_json_payloads(&fake_intake_vector_endpoint(), endpoint) .await .payloads .into_iter() .map(|raw| raw.data) .collect() } + +async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> FakeIntakeResponse { + let url = format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,); + + Client::new() + .request(Method::GET, &url) + .send() + .await + .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) + .json::() + .await + .expect("Parsing fakeintake payloads failed") +} + +async fn get_payloads_agent(endpoint: &str) -> Vec { + get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint) + .await + .payloads +} + +async fn get_payloads_vector(endpoint: &str) -> Vec { + get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint) + .await + .payloads +} From b3b9cc7bd105d83264ac0d3c4c283890836448ea Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 12 Oct 2023 07:58:55 -0600 Subject: [PATCH 49/99] feedback ds: move to scripts/e2e --- .github/workflows/changes.yml | 6 ++-- .github/workflows/integration-comment.yml | 6 ++-- .github/workflows/integration.yml | 8 ++--- scripts/e2e/README.md | 13 ++++++++ .../e2e-datadog-logs}/compose.yaml | 0 .../e2e-datadog-logs}/test.yaml | 2 +- vdev/src/testing/config.rs | 32 ++++++++++++++++--- 7 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 scripts/e2e/README.md rename scripts/{integration/datadog-e2e-logs => e2e/e2e-datadog-logs}/compose.yaml (100%) rename scripts/{integration/datadog-e2e-logs => e2e/e2e-datadog-logs}/test.yaml (93%) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index a3d79ae11343c..12a252b79a8cc 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -64,8 +64,8 @@ on: value: ${{ jobs.int_tests.outputs.datadog-metrics }} datadog-traces: value: ${{ jobs.int_tests.outputs.datadog-traces }} - datadog-e2e-logs: - value: ${{ jobs.int_tests.outputs.datadog-e2e-logs }} + e2e-datadog-logs: + value: ${{ jobs.int_tests.outputs.e2e-datadog-logs }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: @@ -198,7 +198,7 @@ jobs: datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} datadog-traces: ${{ steps.filter.outputs.datadog-traces }} - datadog-e2e-logs: ${{ steps.filter.outputs.datadog-e2e-logs }} + e2e-datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 4dbae7b674a69..f5484544b9f77 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -177,13 +177,13 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces - - name: datadog-e2e-logs - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-logs') || contains(github.event.comment.body, '/ci-run-all') }} + - name: e2e-datadog-logs + if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-e2e-logs + command: bash scripts/ci-integration-test.sh e2e-datadog-logs - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') || contains(github.event.comment.body, '/ci-run-all') }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 894d181f46a0b..f7708fe5c156e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -64,7 +64,7 @@ jobs: || needs.changes.outputs.datadog-logs == 'true' || needs.changes.outputs.datadog-metrics == 'true' || needs.changes.outputs.datadog-traces == 'true' - || needs.changes.outputs.datadog-e2e-logs == 'true' + || needs.changes.outputs.e2e-datadog-logs == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' || needs.changes.outputs.elasticsearch == 'true' @@ -206,14 +206,14 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-e2e-logs == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') - name: datadog-e2e-logs + name: e2e-datadog-logs uses: nick-fields/retry@v2 with: timeout_minutes: 60 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-e2e-logs + command: bash scripts/ci-integration-test.sh e2e-datadog-logs - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md new file mode 100644 index 0000000000000..8ad7e5f90697c --- /dev/null +++ b/scripts/e2e/README.md @@ -0,0 +1,13 @@ +This directory contains a set of end-to-end test frameworks for vector which are executed by the +`vdev` tool. + +Currently these e2e tests are executed with the same `vdev` subcommand as the integration tests, +`cargo vdev integration`. + +See the README in the `scripts/integration` subdirectory for more information. + +A pending future enhancement is to create a `vdev` subcommand `e2e`, that will separate the +invocation of the end-to-end tests from the integration tests in `vdev`, to correspond to the +code separation and fundamental differences between the two classes of tests. + +See https://github.com/vectordotdev/vector/issues/18829 for more information. diff --git a/scripts/integration/datadog-e2e-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml similarity index 100% rename from scripts/integration/datadog-e2e-logs/compose.yaml rename to scripts/e2e/e2e-datadog-logs/compose.yaml diff --git a/scripts/integration/datadog-e2e-logs/test.yaml b/scripts/e2e/e2e-datadog-logs/test.yaml similarity index 93% rename from scripts/integration/datadog-e2e-logs/test.yaml rename to scripts/e2e/e2e-datadog-logs/test.yaml index 14537d66c2017..662b35af7f64e 100644 --- a/scripts/integration/datadog-e2e-logs/test.yaml +++ b/scripts/e2e/e2e-datadog-logs/test.yaml @@ -23,5 +23,5 @@ paths: - "src/internal_events/datadog_*" - "src/sinks/datadog/logs/**" - "src/sinks/util/**" -- "scripts/integration/datadog-e2e/logs/**" +- "scripts/e2e/e2e-datadog-logs/**" - "tests/data/e2e/datadog/logs/**" diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index c98c81b3451d2..7e1ca36cd6452 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -167,20 +167,28 @@ impl IntegrationTestConfig { } pub fn load(integration: &str) -> Result<(PathBuf, Self)> { - let test_dir: PathBuf = [app::path(), "scripts", "integration", integration] + let mut test_dir: PathBuf = [app::path(), "scripts", "integration", integration] .iter() .collect(); if !test_dir.is_dir() { - bail!("unknown integration: {}", integration); + // try the e2e dir now + + // TODO: This is a temporary solution, looking in both dirs. When this GH issue + // https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor + // to have a separate e2e subcommand that both int and e2e will leverage. + test_dir = [app::path(), "scripts", "e2e", integration] + .iter() + .collect(); + if !test_dir.is_dir() { + bail!("unknown integration: {}", integration); + } } let config = Self::parse_file(&test_dir.join(FILE_NAME))?; Ok((test_dir, config)) } - pub fn collect_all() -> Result> { - let mut configs = BTreeMap::new(); - let tests_dir: PathBuf = [app::path(), "scripts", "integration"].iter().collect(); + fn collect_all_dir(tests_dir: &Path, configs: &mut BTreeMap) -> Result<()> { for entry in tests_dir.read_dir()? { let entry = entry?; if entry.path().is_dir() { @@ -192,6 +200,20 @@ impl IntegrationTestConfig { } } } + Ok(()) + } + + pub fn collect_all() -> Result> { + let mut configs = BTreeMap::new(); + + // TODO: This is a temporary solution, looking in both dirs. When this GH issue + // https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor + // to have a separate e2e subcommand that both int and e2e will leverage. + let int_tests_dir: PathBuf = [app::path(), "scripts", "integration"].iter().collect(); + let e2e_tests_dir: PathBuf = [app::path(), "scripts", "e2e"].iter().collect(); + + Self::collect_all_dir(&int_tests_dir, &mut configs)?; + Self::collect_all_dir(&e2e_tests_dir, &mut configs)?; Ok(configs) } From 106a927351aa6bda11ede07e1739066db05ec091 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 12 Oct 2023 09:07:08 -0600 Subject: [PATCH 50/99] add v6 and v7 agent versions to test against --- scripts/e2e/e2e-datadog-logs/compose.yaml | 4 ++-- scripts/e2e/e2e-datadog-logs/test.yaml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml index 3ca07c4d88aba..7d8d5eba5e535 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/e2e-datadog-logs/compose.yaml @@ -81,12 +81,12 @@ services: # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-agent: - image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + image: docker.io/datadog/fakeintake:latest # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-vector: - image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + image: docker.io/datadog/fakeintake:latest networks: default: diff --git a/scripts/e2e/e2e-datadog-logs/test.yaml b/scripts/e2e/e2e-datadog-logs/test.yaml index 662b35af7f64e..32fc49769a4aa 100644 --- a/scripts/e2e/e2e-datadog-logs/test.yaml +++ b/scripts/e2e/e2e-datadog-logs/test.yaml @@ -13,7 +13,9 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' matrix: - version: [latest] + # validate against the latest nightly and also stable v6 and v7 + version: ['latest', '6', '7'] + # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From 244b9eda722ab6accd9d9f2409669d76719c6beb Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 12 Oct 2023 11:45:12 -0600 Subject: [PATCH 51/99] remove dead code allow --- src/sinks/datadog/metrics/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 6e139f0d5e2f5..bfddadd64d6d9 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -52,7 +52,6 @@ pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; // At that time when the V1 support is removed, the SeriesApiVersion stops being useful and can be removed. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[allow(dead_code)] pub enum SeriesApiVersion { V1, V2, From cc351003bb6db7decb6952378edd770352eba1fe Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 12 Oct 2023 13:24:40 -0600 Subject: [PATCH 52/99] move to e2e dir --- .github/workflows/changes.yml | 15 +++++++++------ .github/workflows/integration.yml | 10 +++++----- .../e2e-datadog-metrics}/compose.yaml | 0 .../dogstatsd_client/Dockerfile | 0 .../dogstatsd_client/client.py | 0 .../dogstatsd_client/requirements.txt | 0 .../e2e-datadog-metrics}/test.yaml | 0 7 files changed, 14 insertions(+), 11 deletions(-) rename scripts/{integration/datadog-e2e-metrics => e2e/e2e-datadog-metrics}/compose.yaml (100%) rename scripts/{integration/datadog-e2e-metrics => e2e/e2e-datadog-metrics}/dogstatsd_client/Dockerfile (100%) rename scripts/{integration/datadog-e2e-metrics => e2e/e2e-datadog-metrics}/dogstatsd_client/client.py (100%) rename scripts/{integration/datadog-e2e-metrics => e2e/e2e-datadog-metrics}/dogstatsd_client/requirements.txt (100%) rename scripts/{integration/datadog-e2e-metrics => e2e/e2e-datadog-metrics}/test.yaml (100%) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index c9ee1e7561752..145e2ab71a4b4 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -64,14 +64,14 @@ on: value: ${{ jobs.int_tests.outputs.datadog-metrics }} datadog-traces: value: ${{ jobs.int_tests.outputs.datadog-traces }} - e2e-datadog-logs: - value: ${{ jobs.int_tests.outputs.e2e-datadog-logs }} - datadog-e2e-metrics: - value: ${{ jobs.int_tests.outputs.datadog-e2e-metrics }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: value: ${{ jobs.int_tests.outputs.docker-logs }} + e2e-datadog-logs: + value: ${{ jobs.int_tests.outputs.e2e-datadog-logs }} + e2e-datadog-metrics: + value: ${{ jobs.int_tests.outputs.e2e-datadog-metrics }} elasticsearch: value: ${{ jobs.int_tests.outputs.elasticsearch }} eventstoredb: @@ -200,10 +200,13 @@ jobs: datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} datadog-traces: ${{ steps.filter.outputs.datadog-traces }} - e2e-datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} - datadog-e2e-metrics: ${{ steps.filter.outputs.datadog-e2e-metrics }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} + + # TODO: when https://github.com/vectordotdev/vector/issues/18829 is undertaken, + # part of that will involve updating this job to also run `cargo vdev e2e ci-paths` + e2e-datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} + e2e-datadog-metrics: ${{ steps.filter.outputs.e2e-datadog-metrics }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} eventstoredb: ${{ steps.filter.outputs.eventstoredb }} fluent: ${{ steps.filter.outputs.fluent }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9c92aadfc1d6d..96e8f0e6b02e0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -64,10 +64,10 @@ jobs: || needs.changes.outputs.datadog-logs == 'true' || needs.changes.outputs.datadog-metrics == 'true' || needs.changes.outputs.datadog-traces == 'true' - || needs.changes.outputs.e2e-datadog-logs == 'true' - || needs.changes.outputs.datadog-e2e-metrics == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' + || needs.changes.outputs.e2e-datadog-logs == 'true' + || needs.changes.outputs.e2e-datadog-metrics == 'true' || needs.changes.outputs.elasticsearch == 'true' || needs.changes.outputs.eventstoredb == 'true' || needs.changes.outputs.fluent == 'true' @@ -216,14 +216,14 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh e2e-datadog-logs - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-e2e-metrics == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') - name: datadog-e2e-metrics + name: e2e-datadog-metrics uses: nick-fields/retry@v2 with: timeout_minutes: 60 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-e2e-metrics + command: bash scripts/ci-integration-test.sh e2e-datadog-metrics - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap diff --git a/scripts/integration/datadog-e2e-metrics/compose.yaml b/scripts/e2e/e2e-datadog-metrics/compose.yaml similarity index 100% rename from scripts/integration/datadog-e2e-metrics/compose.yaml rename to scripts/e2e/e2e-datadog-metrics/compose.yaml diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/Dockerfile b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile similarity index 100% rename from scripts/integration/datadog-e2e-metrics/dogstatsd_client/Dockerfile rename to scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py similarity index 100% rename from scripts/integration/datadog-e2e-metrics/dogstatsd_client/client.py rename to scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py diff --git a/scripts/integration/datadog-e2e-metrics/dogstatsd_client/requirements.txt b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/requirements.txt similarity index 100% rename from scripts/integration/datadog-e2e-metrics/dogstatsd_client/requirements.txt rename to scripts/e2e/e2e-datadog-metrics/dogstatsd_client/requirements.txt diff --git a/scripts/integration/datadog-e2e-metrics/test.yaml b/scripts/e2e/e2e-datadog-metrics/test.yaml similarity index 100% rename from scripts/integration/datadog-e2e-metrics/test.yaml rename to scripts/e2e/e2e-datadog-metrics/test.yaml From 50f29882bcfb8690b6ef03ab58a8cd1063bee6ed Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 12 Oct 2023 15:28:13 -0600 Subject: [PATCH 53/99] improve comparison logic --- scripts/e2e/e2e-datadog-metrics/compose.yaml | 4 +- scripts/e2e/e2e-datadog-metrics/test.yaml | 3 + tests/e2e/datadog/metrics/mod.rs | 103 ++++++++----------- 3 files changed, 49 insertions(+), 61 deletions(-) diff --git a/scripts/e2e/e2e-datadog-metrics/compose.yaml b/scripts/e2e/e2e-datadog-metrics/compose.yaml index a2b6f1527280c..c9c9fbe585dfe 100644 --- a/scripts/e2e/e2e-datadog-metrics/compose.yaml +++ b/scripts/e2e/e2e-datadog-metrics/compose.yaml @@ -67,12 +67,12 @@ services: # Receives metric data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-agent: - image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + image: docker.io/datadog/fakeintake:latest # Receives metric data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-vector: - image: docker.io/datadog/fakeintake:${CONFIG_VERSION} + image: docker.io/datadog/fakeintake:latest networks: default: diff --git a/scripts/e2e/e2e-datadog-metrics/test.yaml b/scripts/e2e/e2e-datadog-metrics/test.yaml index 43078fafb6ba5..0253384160a4c 100644 --- a/scripts/e2e/e2e-datadog-metrics/test.yaml +++ b/scripts/e2e/e2e-datadog-metrics/test.yaml @@ -13,6 +13,9 @@ runner: matrix: version: [latest] + # TODO use below when not debugging + # validate against the latest nightly and also stable v6 and v7 + # version: ['latest', '6', '7'] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index f404477b2f9a1..d995dead67029 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -45,10 +45,18 @@ fn unpack_payloads_series_v2( out_payloads } +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct Context { + metric_name: String, + tags: Vec, + r#type: i32, +} + // Sums up the metrics in each series by name. -fn aggregate_series_metrics( +fn aggregate_normalize_series_metrics( payloads: &Vec, -) -> BTreeMap { + // ) -> Vec { +) -> BTreeMap { let mut aggregate = BTreeMap::new(); for metric_payload in payloads { @@ -58,66 +66,43 @@ fn aggregate_series_metrics( continue; } - // TODO expand on this + let ctx = Context { + metric_name: serie.metric.clone(), + tags: serie.tags.clone(), + r#type: serie.r#type, + }; - let interval = serie.interval as f64; - match serie.r#type() { - ddmetric_proto::metric_payload::MetricType::Unspecified => { - panic!("unspecified metric type") - } - ddmetric_proto::metric_payload::MetricType::Count => { - if let Some((t, v)) = aggregate.get_mut(&serie.metric) { - for point in &serie.points { - *t = point.timestamp; - *v += point.value; - } - } else { - for point in &serie.points { - aggregate.insert(serie.metric.clone(), (point.timestamp, point.value)); - } - } - } - ddmetric_proto::metric_payload::MetricType::Rate => { - if let Some((t, v)) = aggregate.get_mut(&serie.metric) { - for point in &serie.points { - *v += point.value * interval; - *t = point.timestamp; - } - } else { - for (idx, point) in serie.points.iter().enumerate() { - if idx == 0 { - aggregate.insert( - serie.metric.clone(), - (point.timestamp, point.value * interval), - ); - } else { - if let Some((t, v)) = aggregate.get_mut(&serie.metric) { - *v += point.value * interval; - *t = point.timestamp; - } - } - } - } - } - ddmetric_proto::metric_payload::MetricType::Gauge => { - // last one wins - if let Some(point) = serie.points.last() { - if let Some((t, v)) = aggregate.get_mut(&serie.metric) { - if point.timestamp > *t { - *t = point.timestamp; - *v = point.value; - } - } else { - aggregate.insert(serie.metric.clone(), (point.timestamp, point.value)); - } - } - } + if !aggregate.contains_key(&ctx) { + aggregate.insert(ctx, serie.clone()); + continue; } + + let existing = aggregate.get_mut(&ctx).unwrap(); + + existing.points.extend_from_slice(&serie.points); } } - // remove the timestamps - let aggregate = aggregate.into_iter().map(|(k, v)| (k, v.1)).collect(); + // remove the timestamps and sum the points and normalize the other metadata + for (_ctx, series) in &mut aggregate { + let mut value = 0.0; + for point in &mut series.points { + point.timestamp = 0; + value += point.value; + } + series.points[0].value = value; + + for resource in &mut series.resources { + if resource.r#type == "host" { + if resource.name.ends_with("-vector") { + resource + .name + .truncate(resource.name.len() - "-vector".len()); + } + } + } + // println!("{:?} {:?}", _ctx, series); + } aggregate } @@ -144,7 +129,7 @@ async fn validate() { common_assertions(&agent_payloads); println!("aggregating payloads from agent-only pipeline"); - let agent_payloads = aggregate_series_metrics(&agent_payloads); + let agent_payloads = aggregate_normalize_series_metrics(&agent_payloads); println!("{:?}", agent_payloads.keys()); @@ -159,7 +144,7 @@ async fn validate() { common_assertions(&vector_payloads); println!("aggregating payloads from agent-vector pipeline"); - let vector_payloads = aggregate_series_metrics(&vector_payloads); + let vector_payloads = aggregate_normalize_series_metrics(&vector_payloads); println!("{:?}", vector_payloads.keys()); assert_eq!(agent_payloads, vector_payloads); From 9705c12949bf4ef1edbc7f7e663b4c24d562835b Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 13 Oct 2023 12:16:52 -0600 Subject: [PATCH 54/99] code reduction --- tests/e2e/datadog/logs/mod.rs | 36 +++++++++- tests/e2e/datadog/metrics/mod.rs | 68 +++++++++++-------- tests/e2e/datadog/mod.rs | 110 ++++++++++--------------------- 3 files changed, 108 insertions(+), 106 deletions(-) diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 8121517671603..3aacc7fa940bc 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -44,7 +44,7 @@ fn assert_timestamp_hostname(payloads: &mut [Value]) -> usize { // runs assertions that each set of payloads should be true to regardless // of the pipeline -fn common_assertions(payloads: &mut [Value]) { +fn common_assertions(payloads: &mut Vec) { assert!(payloads.len() > 0); let n_log_events = assert_timestamp_hostname(payloads); @@ -54,17 +54,47 @@ fn common_assertions(payloads: &mut [Value]) { assert!(n_log_events == expected_log_events()); } +// reduces the payload down to just the log data +fn reduce_to_data(payloads: &mut Vec>) -> Vec { + payloads + .into_iter() + .map(|payload| payload.data.take()) + .collect() +} + #[tokio::test] async fn validate() { trace_init(); println!("getting log payloads from agent-only pipeline"); - let mut agent_payloads = get_payloads_agent_json(LOGS_ENDPOINT).await; + let mut agent_payloads = get_fakeintake_payloads::( + &fake_intake_agent_address(), + LOGS_ENDPOINT, + ) + .await + .payloads; + + // Not sure what this is but the logs endpoint receives an empty payload in the beginning + if agent_payloads.len() > 0 { + if agent_payloads[0].data.as_array().unwrap().len() == 0 && agent_payloads[0].encoding == "" + { + agent_payloads.remove(0); + } + } + + let mut agent_payloads = reduce_to_data(&mut agent_payloads); common_assertions(&mut agent_payloads); println!("getting log payloads from agent-vector pipeline"); - let mut vector_payloads = get_payloads_vector_json(LOGS_ENDPOINT).await; + let mut vector_payloads = get_fakeintake_payloads::( + &fake_intake_vector_address(), + LOGS_ENDPOINT, + ) + .await + .payloads; + + let mut vector_payloads = reduce_to_data(&mut vector_payloads); common_assertions(&mut vector_payloads); diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index d995dead67029..5cbc9409e8cd7 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -24,11 +24,11 @@ fn decompress_payload(payload: Vec) -> std::io::Result> { } fn unpack_payloads_series_v2( - in_payloads: &Vec, + in_payloads: &FakeIntakeResponseRaw, ) -> Vec { let mut out_payloads = vec![]; - in_payloads.iter().for_each(|payload| { + in_payloads.payloads.iter().for_each(|payload| { // decode base64 let payload = BASE64_STANDARD .decode(&payload.data) @@ -101,6 +101,16 @@ fn aggregate_normalize_series_metrics( } } } + + if series.r#type() != ddmetric_proto::metric_payload::MetricType::Rate + && series.interval != 0 + { + println!( + "serie {:?} non-rate metric has interval set. Setting to zero.", + _ctx + ); + series.interval = 0; + } // println!("{:?} {:?}", _ctx, series); } @@ -109,45 +119,45 @@ fn aggregate_normalize_series_metrics( // runs assertions that each set of payloads should be true to regardless // of the pipeline -fn common_assertions(payloads: &Vec) { - assert!(payloads.len() > 0); - println!("metric payloads received: {}", payloads.len()); +fn common_assertions(series: &BTreeMap) { + assert!(series.len() > 0); + println!("metric series received: {}", series.len()); } -#[tokio::test] -async fn validate() { - trace_init(); +async fn get_series_from_pipeline( + address: String, +) -> BTreeMap { + println!("getting payloads"); + let payloads = + get_fakeintake_payloads::(&address, SERIES_ENDPOINT).await; - // TODO need to see if can configure the agent flush interval - std::thread::sleep(std::time::Duration::from_secs(30)); + println!("unpacking payloads"); + let payloads = unpack_payloads_series_v2(&payloads); - println!("getting payloads from agent-only pipeline"); - let agent_payloads = get_payloads_agent(SERIES_ENDPOINT).await; + println!("aggregating payloads"); + let series = aggregate_normalize_series_metrics(&payloads); - println!("unpacking payloads from agent-only pipeline"); - let agent_payloads = unpack_payloads_series_v2(&agent_payloads); - common_assertions(&agent_payloads); + common_assertions(&series); - println!("aggregating payloads from agent-only pipeline"); - let agent_payloads = aggregate_normalize_series_metrics(&agent_payloads); + println!("{:?}", series.keys()); - println!("{:?}", agent_payloads.keys()); + series +} - // let foo_rate_agent = agent_payloads.get("foo_metric.rate"); - // println!("AGENT RATE AGGREGATE: {:?}", foo_rate_agent); +#[tokio::test] +async fn validate() { + trace_init(); - println!("getting log payloads from agent-vector pipeline"); - let vector_payloads = get_payloads_vector(SERIES_ENDPOINT).await; + // TODO need to see if can configure the agent flush interval + std::thread::sleep(std::time::Duration::from_secs(30)); - println!("unpacking payloads from agent-vector pipeline"); - let vector_payloads = unpack_payloads_series_v2(&vector_payloads); - common_assertions(&vector_payloads); + println!("==== getting series data from agent-only pipeline ==== "); + let agent_series = get_series_from_pipeline(fake_intake_agent_address()).await; - println!("aggregating payloads from agent-vector pipeline"); - let vector_payloads = aggregate_normalize_series_metrics(&vector_payloads); - println!("{:?}", vector_payloads.keys()); + println!("==== getting series data from agent-vector pipeline ===="); + let vector_series = get_series_from_pipeline(fake_intake_vector_address()).await; - assert_eq!(agent_payloads, vector_payloads); + assert_eq!(agent_series, vector_series); // std::thread::sleep(std::time::Duration::from_secs(90)); } diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 8d79c5d971da3..0dbe1a46226a9 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -2,109 +2,71 @@ pub mod logs; pub mod metrics; use reqwest::{Client, Method}; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use serde_json::Value; -fn fake_intake_vector_endpoint() -> String { +fn fake_intake_vector_address() -> String { std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()) } -fn fake_intake_agent_endpoint() -> String { +fn fake_intake_agent_address() -> String { std::env::var("FAKE_INTAKE_AGENT_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8083".to_string()) } -// Fakeintake JSON response #[derive(Deserialize, Debug)] -struct FakeIntakeJsonResponse { - payloads: Vec, -} - -#[allow(dead_code)] -#[derive(Deserialize, Debug)] -struct FakeIntakeJsonPayload { - data: Value, +struct FakeIntakePayload { + // When string, base64 encoded + data: D, encoding: String, - timestamp: String, + #[serde(rename = "timestamp")] + _timestamp: String, } -// Fakeintake raw response -#[derive(Deserialize, Debug)] -struct FakeIntakeResponse { - payloads: Vec, +type FakeIntakePayloadJson = FakeIntakePayload; + +type FakeIntakePayloadRaw = FakeIntakePayload; + +trait FakeIntakeResponseT { + fn build_url(base: &str, endpoint: &str) -> String; } -#[allow(dead_code)] #[derive(Deserialize, Debug)] -struct FakeIntakePayload { - // base64 encoded - data: String, - encoding: String, - timestamp: String, +struct FakeIntakeResponse

{ + payloads: Vec

, } -async fn get_fakeintake_json_payloads(base: &str, endpoint: &str) -> FakeIntakeJsonResponse { - let url = format!( - "{}/fakeintake/payloads?endpoint={}&format=json", - base, endpoint, - ); +type FakeIntakeResponseJson = FakeIntakeResponse; - Client::new() - .request(Method::GET, &url) - .send() - .await - .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) - .json::() - .await - .expect("Parsing fakeintake payloads failed") +impl FakeIntakeResponseT for FakeIntakeResponseJson { + fn build_url(base: &str, endpoint: &str) -> String { + format!( + "{}/fakeintake/payloads?endpoint={}&format=json", + base, endpoint, + ) + } } -async fn get_payloads_agent_json(endpoint: &str) -> Vec { - let mut raw_payloads = get_fakeintake_json_payloads(&fake_intake_agent_endpoint(), endpoint) - .await - .payloads; +type FakeIntakeResponseRaw = FakeIntakeResponse; - // Not sure what this is but the logs endpoint receives an empty payload in the beginning - if raw_payloads.len() > 0 && endpoint == "/api/v2/logs" { - if raw_payloads[0].data.as_array().unwrap().len() == 0 && raw_payloads[0].encoding == "" { - raw_payloads.remove(0); - } +impl FakeIntakeResponseT for FakeIntakeResponseRaw { + fn build_url(base: &str, endpoint: &str) -> String { + format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,) } - - raw_payloads.into_iter().map(|raw| raw.data).collect() -} - -async fn get_payloads_vector_json(endpoint: &str) -> Vec { - get_fakeintake_json_payloads(&fake_intake_vector_endpoint(), endpoint) - .await - .payloads - .into_iter() - .map(|raw| raw.data) - .collect() } -async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> FakeIntakeResponse { - let url = format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,); - +async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> R +where + R: FakeIntakeResponseT + DeserializeOwned, +{ + let url = &R::build_url(base, endpoint); Client::new() - .request(Method::GET, &url) + .request(Method::GET, url) .send() .await - .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) - .json::() + .unwrap_or_else(|_| panic!("Sending GET request to {} failed", url)) + .json::() .await .expect("Parsing fakeintake payloads failed") } - -async fn get_payloads_agent(endpoint: &str) -> Vec { - get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint) - .await - .payloads -} - -async fn get_payloads_vector(endpoint: &str) -> Vec { - get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint) - .await - .payloads -} From 93ca19790eca7590d1ff4d976e77320d883f6951 Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 13 Oct 2023 16:24:08 -0600 Subject: [PATCH 55/99] add sketches, some reorg --- .../dogstatsd_client/client.py | 3 + tests/e2e/datadog/metrics/mod.rs | 131 ++----------- tests/e2e/datadog/metrics/series.rs | 181 ++++++++++++++++++ tests/e2e/datadog/metrics/sketches.rs | 104 ++++++++++ 4 files changed, 299 insertions(+), 120 deletions(-) create mode 100644 tests/e2e/datadog/metrics/series.rs create mode 100644 tests/e2e/datadog/metrics/sketches.rs diff --git a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py index 8f8897c84e1d2..41585f2686d73 100644 --- a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py +++ b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py @@ -14,6 +14,9 @@ initialize(**options) +# Give the Agent time to actually spin up. +# The container may return "ready" but the +# Agent process is still booting. time.sleep(10) hist_data = [ diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index 5cbc9409e8cd7..aac43fc5010ba 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -1,21 +1,14 @@ -use std::collections::BTreeMap; - use base64::{prelude::BASE64_STANDARD, Engine}; use bytes::Bytes; use flate2::read::ZlibDecoder; -use prost::Message; use vector::test_util::trace_init; -#[allow(warnings, clippy::pedantic, clippy::nursery)] -mod ddmetric_proto { - include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); -} +mod series; +mod sketches; use super::*; -const SERIES_ENDPOINT: &str = "/api/v2/series"; - fn decompress_payload(payload: Vec) -> std::io::Result> { let mut decompressor = ZlibDecoder::new(&payload[..]); let mut decompressed = Vec::new(); @@ -23,9 +16,10 @@ fn decompress_payload(payload: Vec) -> std::io::Result> { result.map(|_| decompressed) } -fn unpack_payloads_series_v2( - in_payloads: &FakeIntakeResponseRaw, -) -> Vec { +fn unpack_proto_payloads(in_payloads: &FakeIntakeResponseRaw) -> Vec +where + T: prost::Message + std::default::Default, +{ let mut out_payloads = vec![]; in_payloads.payloads.iter().for_each(|payload| { @@ -37,7 +31,7 @@ fn unpack_payloads_series_v2( // decompress let bytes = Bytes::from(decompress_payload(payload).unwrap()); - let payload = ddmetric_proto::MetricPayload::decode(bytes).unwrap(); + let payload = T::decode(bytes).unwrap(); out_payloads.push(payload); }); @@ -45,119 +39,16 @@ fn unpack_payloads_series_v2( out_payloads } -#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -struct Context { - metric_name: String, - tags: Vec, - r#type: i32, -} - -// Sums up the metrics in each series by name. -fn aggregate_normalize_series_metrics( - payloads: &Vec, - // ) -> Vec { -) -> BTreeMap { - let mut aggregate = BTreeMap::new(); - - for metric_payload in payloads { - for serie in &metric_payload.series { - // filter out the metrics we don't care about - if !serie.metric.starts_with("foo_metric") { - continue; - } - - let ctx = Context { - metric_name: serie.metric.clone(), - tags: serie.tags.clone(), - r#type: serie.r#type, - }; - - if !aggregate.contains_key(&ctx) { - aggregate.insert(ctx, serie.clone()); - continue; - } - - let existing = aggregate.get_mut(&ctx).unwrap(); - - existing.points.extend_from_slice(&serie.points); - } - } - - // remove the timestamps and sum the points and normalize the other metadata - for (_ctx, series) in &mut aggregate { - let mut value = 0.0; - for point in &mut series.points { - point.timestamp = 0; - value += point.value; - } - series.points[0].value = value; - - for resource in &mut series.resources { - if resource.r#type == "host" { - if resource.name.ends_with("-vector") { - resource - .name - .truncate(resource.name.len() - "-vector".len()); - } - } - } - - if series.r#type() != ddmetric_proto::metric_payload::MetricType::Rate - && series.interval != 0 - { - println!( - "serie {:?} non-rate metric has interval set. Setting to zero.", - _ctx - ); - series.interval = 0; - } - // println!("{:?} {:?}", _ctx, series); - } - - aggregate -} - -// runs assertions that each set of payloads should be true to regardless -// of the pipeline -fn common_assertions(series: &BTreeMap) { - assert!(series.len() > 0); - println!("metric series received: {}", series.len()); -} - -async fn get_series_from_pipeline( - address: String, -) -> BTreeMap { - println!("getting payloads"); - let payloads = - get_fakeintake_payloads::(&address, SERIES_ENDPOINT).await; - - println!("unpacking payloads"); - let payloads = unpack_payloads_series_v2(&payloads); - - println!("aggregating payloads"); - let series = aggregate_normalize_series_metrics(&payloads); - - common_assertions(&series); - - println!("{:?}", series.keys()); - - series -} - #[tokio::test] async fn validate() { trace_init(); // TODO need to see if can configure the agent flush interval - std::thread::sleep(std::time::Duration::from_secs(30)); - - println!("==== getting series data from agent-only pipeline ==== "); - let agent_series = get_series_from_pipeline(fake_intake_agent_address()).await; + std::thread::sleep(std::time::Duration::from_secs(20)); - println!("==== getting series data from agent-vector pipeline ===="); - let vector_series = get_series_from_pipeline(fake_intake_vector_address()).await; + series::validate().await; - assert_eq!(agent_series, vector_series); + sketches::validate().await; - // std::thread::sleep(std::time::Duration::from_secs(90)); + // std::thread::sleep(std::time::Duration::from_secs(120)); } diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs new file mode 100644 index 0000000000000..666f2692afea9 --- /dev/null +++ b/tests/e2e/datadog/metrics/series.rs @@ -0,0 +1,181 @@ +use std::collections::BTreeMap; + +#[allow(warnings, clippy::pedantic, clippy::nursery)] +mod ddmetric_proto { + include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); +} + +use ddmetric_proto::{ + metric_payload::{MetricSeries, MetricType}, + MetricPayload, +}; + +use super::*; + +const SERIES_ENDPOINT: &str = "/api/v2/series"; + +// unique identification of a Series +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct SeriesContext { + metric_name: String, + tags: Vec, + r#type: i32, +} + +// 1. filters out the metrics not generated by the Emitter +// 2. Aggregates the metric value across the different payloads. +// This is necessary because we can't guarantee the payloads sent from +// Agent/Vector will contain the exact same sets of data across a time period. +// 3. Normalizes the `host` value, and the `interval` +fn aggregate_normalize_series( + payloads: &Vec, +) -> BTreeMap { + let mut aggregate = BTreeMap::new(); + + for metric_payload in payloads { + for serie in &metric_payload.series { + // filter out the metrics we don't care about + if !serie.metric.starts_with("foo_metric") { + continue; + } + + let ctx = SeriesContext { + metric_name: serie.metric.clone(), + tags: serie.tags.clone(), + r#type: serie.r#type, + }; + + if !aggregate.contains_key(&ctx) { + aggregate.insert(ctx, serie.clone()); + continue; + } + + let existing = aggregate.get_mut(&ctx).unwrap(); + + match serie.r#type() { + MetricType::Unspecified => { + panic!("unspecified metric type") + } + MetricType::Count => { + existing.points.extend_from_slice(&serie.points); + } + MetricType::Rate => { + existing.points.extend_from_slice(&serie.points); + } + MetricType::Gauge => { + // last reported one wins + if let Some(point) = serie.points.last() { + let current = existing.points.last().unwrap(); + if point.timestamp > current.timestamp && point.value > 0.0 { + existing.points = vec![point.clone()]; + } + } + } + } + } + } + + // remove the timestamps and sum the points and normalize the other metadata + for (_ctx, series) in &mut aggregate { + // the Agent sets a non-zero default interval for all metric types. + // Vector on ingest from the Agent, only sets the interval if it is a Rate. + if series.r#type() != MetricType::Rate && series.interval != 0 { + println!( + "serie {:?} non-rate metric has interval set. Setting to zero.", + _ctx.metric_name + ); + series.interval = 0; + } + + let mut value = 0.0; + + series.points.retain(|point| point.value > 0.0); + + for point in &mut series.points { + point.timestamp = 0; + + if series.interval > 0 { + value += point.value * series.interval as f64; + } else { + value += point.value; + } + point.value = 0.0; + } + if series.points.len() > 0 { + series.points[0].value = value; + } + + for resource in &mut series.resources { + if resource.r#type == "host" { + if resource.name.ends_with("-vector") { + resource + .name + .truncate(resource.name.len() - "-vector".len()); + } + } + } + + println!("{:?}", series); + } + + aggregate +} + +// runs assertions that each set of payloads should be true to regardless +// of the pipeline +fn common_series_assertions(series: &BTreeMap) { + // we should have received some metrics from the emitter + assert!(series.len() > 0); + println!("metric series received: {}", series.len()); + + // specifically we should have received each of these + let mut found = vec![ + (false, "rate"), + (false, "gauge"), + (false, "set"), + (false, "histogram"), + ]; + series.keys().for_each(|ctx| { + found.iter_mut().for_each(|found| { + if ctx + .metric_name + .starts_with(&format!("foo_metric.{}", found.1)) + { + println!("received {}", found.1); + found.0 = true; + } + }); + }); + + found + .iter() + .for_each(|(found, mtype)| assert!(found, "Didn't receive metric type {}", *mtype)); +} + +async fn get_series_from_pipeline(address: String) -> BTreeMap { + println!("getting series payloads"); + let payloads = + get_fakeintake_payloads::(&address, SERIES_ENDPOINT).await; + + println!("unpacking payloads"); + let payloads = unpack_proto_payloads::(&payloads); + + println!("aggregating payloads"); + let series = aggregate_normalize_series(&payloads); + + common_series_assertions(&series); + + println!("{:?}", series.keys()); + + series +} + +pub(super) async fn validate() { + println!("==== getting series data from agent-only pipeline ==== "); + let agent_series = get_series_from_pipeline(fake_intake_agent_address()).await; + + println!("==== getting series data from agent-vector pipeline ===="); + let vector_series = get_series_from_pipeline(fake_intake_vector_address()).await; + + assert_eq!(agent_series, vector_series); +} diff --git a/tests/e2e/datadog/metrics/sketches.rs b/tests/e2e/datadog/metrics/sketches.rs new file mode 100644 index 0000000000000..4d1e9811bab31 --- /dev/null +++ b/tests/e2e/datadog/metrics/sketches.rs @@ -0,0 +1,104 @@ +use std::collections::BTreeMap; + +#[allow(warnings, clippy::pedantic, clippy::nursery)] +mod ddmetric_proto { + include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); +} + +use ddmetric_proto::{sketch_payload::Sketch, SketchPayload}; + +use super::*; + +const SKETCHES_ENDPOINT: &str = "/api/beta/sketches"; + +// unique identification of a Sketch +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct SketchContext { + metric_name: String, + tags: Vec, +} + +fn aggregate_normalize_sketches( + payloads: &mut Vec, +) -> BTreeMap { + let mut aggregate = BTreeMap::new(); + + for payload in payloads { + for sketch in &mut payload.sketches { + // filter out the metrics we don't care about + if !sketch.metric.starts_with("foo_metric") { + continue; + } + + let ctx = SketchContext { + metric_name: sketch.metric.clone(), + tags: sketch.tags.clone(), + }; + + if sketch.host.ends_with("-vector") { + sketch.host.truncate(sketch.host.len() - "-vector".len()); + } + + if !aggregate.contains_key(&ctx) { + aggregate.insert(ctx, sketch.clone()); + println!("{:?}", sketch); + } + } + } + + aggregate +} + +// runs assertions that each set of payloads should be true to regardless +// of the pipeline +fn common_sketch_assertions(sketches: &BTreeMap) { + // we should have received some metrics from the emitter + assert!(sketches.len() > 0); + println!("metric sketch received: {}", sketches.len()); + + // specifically we should have received each of these + let mut found = vec![(false, "distribution")]; + sketches.keys().for_each(|ctx| { + found.iter_mut().for_each(|found| { + if ctx + .metric_name + .starts_with(&format!("foo_metric.{}", found.1)) + { + println!("received {}", found.1); + found.0 = true; + } + }); + }); + + found + .iter() + .for_each(|(found, mtype)| assert!(found, "Didn't receive metric type {}", *mtype)); +} + +async fn get_sketches_from_pipeline(address: String) -> BTreeMap { + println!("getting sketch payloads"); + let payloads = + get_fakeintake_payloads::(&address, SKETCHES_ENDPOINT).await; + + println!("unpacking payloads"); + let mut payloads = unpack_proto_payloads(&payloads); + + println!("aggregating payloads"); + let sketches = aggregate_normalize_sketches(&mut payloads); + + common_sketch_assertions(&sketches); + + println!("{:?}", sketches.keys()); + + sketches +} + +pub(super) async fn validate() { + println!("==== getting sketch data from agent-only pipeline ==== "); + let agent_sketches = get_sketches_from_pipeline(fake_intake_agent_address()).await; + + println!("==== getting sketch data from agent-vector pipeline ===="); + let vector_sketches = get_sketches_from_pipeline(fake_intake_vector_address()).await; + + assert_eq!(agent_sketches, vector_sketches); +} From c66de4f990e937f7baff0d857833cbdcd723f3c5 Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 13 Oct 2023 17:01:56 -0600 Subject: [PATCH 56/99] cleanup --- scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile | 2 +- scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile index 718a61a07b953..e8769539c8092 100644 --- a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile +++ b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile @@ -3,6 +3,6 @@ FROM python:3.7-alpine COPY . /app WORKDIR /app -RUN pip install -r requirements.txt +RUN pip install -r requirements.txt CMD [ "python3", "./client.py"] diff --git a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py index 41585f2686d73..b4a9b7390be6b 100644 --- a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py +++ b/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py @@ -39,7 +39,7 @@ print("rate") statsd.increment('foo_metric.rate', tags=['a_tag:1']) - print("guage") + print("gauge") statsd.gauge('foo_metric.gauge', i, tags=["a_tag:2"]) print("set") From cde2a2d1065a4cd6cc666db6c70ceea8d4b5aed2 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 9 Jan 2024 15:15:25 -0700 Subject: [PATCH 57/99] add notes about the e2e prefix needed --- scripts/e2e/README.md | 4 ++++ scripts/e2e/e2e-datadog-logs/test.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 8ad7e5f90697c..45422e823c3cc 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -11,3 +11,7 @@ invocation of the end-to-end tests from the integration tests in `vdev`, to corr code separation and fundamental differences between the two classes of tests. See https://github.com/vectordotdev/vector/issues/18829 for more information. + +For now, any subdir here needs to be unique from the other integration tests outside this dir. +For example there is already a `datadog-logs` integration test, hence the e2e test is in a sub- +dir called `e2e-datadog-logs`. diff --git a/scripts/e2e/e2e-datadog-logs/test.yaml b/scripts/e2e/e2e-datadog-logs/test.yaml index 32fc49769a4aa..09785f7811f86 100644 --- a/scripts/e2e/e2e-datadog-logs/test.yaml +++ b/scripts/e2e/e2e-datadog-logs/test.yaml @@ -25,5 +25,9 @@ paths: - "src/internal_events/datadog_*" - "src/sinks/datadog/logs/**" - "src/sinks/util/**" +# NOTE: currently we need the prefix 'e2e' even though it looks redundant, +# because the vdev code does not otherwise have a way to distinguish between +# the other `datadog-logs` int test. +# but once GH issue 18829 is completed, this will become unecessary. - "scripts/e2e/e2e-datadog-logs/**" - "tests/data/e2e/datadog/logs/**" From bedb3d3d62cbe84e3336cf3ba7d06cb132f522ca Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 10 Jan 2024 11:34:11 -0700 Subject: [PATCH 58/99] re-use runner image --- scripts/e2e/e2e-datadog-logs/compose.yaml | 13 ++++--- scripts/integration/Dockerfile | 43 +++++++++++++++++------ vdev/src/commands/test.rs | 1 + vdev/src/testing/integration.rs | 17 +++++++-- vdev/src/testing/runner.rs | 21 +++++++---- 5 files changed, 72 insertions(+), 23 deletions(-) diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml index 7d8d5eba5e535..05db907e8f963 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/e2e-datadog-logs/compose.yaml @@ -20,16 +20,21 @@ services: # to the `fakeintake-vector` service. vector: depends_on: + - log_generator - fakeintake-vector build: context: ${PWD} - dockerfile: tests/data/e2e/datadog/Dockerfile - args: - - RUST_VERSION=${RUST_VERSION} - - FEATURES=e2e-tests-datadog + # dockerfile: tests/data/e2e/datadog/Dockerfile + # args: + # - RUST_VERSION=${RUST_VERSION} + # - FEATURES=e2e-tests-datadog + image: ${CONFIG_VECTOR_IMAGE} + environment: + - FEATURES=e2e-tests-datadog working_dir: /home/vector network_mode: host command: + - "/usr/bin/vector" - "-vvv" - "-c" - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index f2ff5ef201d4d..f8217ee6e5598 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -1,23 +1,46 @@ ARG RUST_VERSION -FROM docker.io/rust:${RUST_VERSION}-slim-bookworm +ARG FEATURES +ARG DEBIAN_RELEASE=slim-bookworm -RUN apt-get update && apt-get install -y --no-install-recommends \ +FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} + +RUN apt-get update && apt-get -y --no-install-recommends install \ build-essential \ cmake \ curl \ - g++ \ - libclang1 \ + git \ + clang \ + libclang-dev \ libsasl2-dev \ + libstdc++-11-dev \ libssl-dev \ - llvm \ - pkg-config \ - zlib1g-dev \ + libxxhash-dev \ unzip \ - git \ - && rm -rf /var/lib/apt/lists/* + zlib1g-dev \ + zlib1g -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.25 --locked +RUN git clone https://github.com/rui314/mold.git \ + && mkdir mold/build \ + && cd mold/build \ + && git checkout v2.0.0 \ + && ../install-build-deps.sh \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ + && cmake --build . -j $(nproc) \ + && cmake --install . COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs RUN bash /install-protoc.sh + +RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.25 --locked + +WORKDIR /vector +COPY . . +ARG FEATURES + +RUN --mount=type=cache,target=/vector/target \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + /usr/local/bin/mold -run cargo build --tests --lib --bin vector \ + --no-default-features --features $FEATURES && \ + cp target/debug/vector /usr/bin/vector diff --git a/vdev/src/commands/test.rs b/vdev/src/commands/test.rs index 4c9f39d60e01a..4d28b48408de2 100644 --- a/vdev/src/commands/test.rs +++ b/vdev/src/commands/test.rs @@ -51,6 +51,7 @@ impl Cli { runner.test( &parse_env(self.env.unwrap_or_default()), &BTreeMap::default(), + None, &args, ) } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 3b1f85959818a..b0544c2038082 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -37,9 +37,10 @@ impl IntegrationTest { let environment = environment.into(); let (test_dir, config) = IntegrationTestConfig::load(&integration)?; let envs_dir = EnvsDir::new(&integration); - let Some(env_config) = config.environments().get(&environment).map(Clone::clone) else { + let Some(mut env_config) = config.environments().get(&environment).map(Clone::clone) else { bail!("Could not find environment named {environment:?}"); }; + let network_name = format!("vector-integration-tests-{integration}"); let compose = Compose::new(test_dir, env_config.clone(), network_name.clone())?; @@ -52,6 +53,8 @@ impl IntegrationTest { compose.is_some().then_some(network_name), )?; + env_config.insert("VECTOR_IMAGE".to_string(), Some(runner.image_name())); + Ok(Self { integration, environment, @@ -70,6 +73,10 @@ impl IntegrationTest { self.config.check_required()?; if !active { + // For end-to-end tests, we want to run vector as a service, leveraging the + // image for the runner. So we must build that image before starting the + // compose so that it is available. + self.runner.build(Some(&self.config.features))?; self.start()?; } @@ -115,8 +122,12 @@ impl IntegrationTest { args.push(self.retries.to_string()); } - self.runner - .test(&env_vars, &self.config.runner.env, &args)?; + self.runner.test( + &env_vars, + &self.config.runner.env, + Some(&self.config.features), + &args, + )?; if !active { self.runner.remove()?; diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index e4fd18cdd015b..5b67671945525 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -80,8 +80,13 @@ pub fn get_agent_test_runner(container: bool) -> Result> { } pub trait TestRunner { - fn test(&self, outer_env: &Environment, inner_env: &Environment, args: &[String]) - -> Result<()>; + fn test( + &self, + outer_env: &Environment, + inner_env: &Environment, + features: Option<&[String]>, + args: &[String], + ) -> Result<()>; } pub trait ContainerTestRunner: TestRunner { @@ -129,7 +134,7 @@ pub trait ContainerTestRunner: TestRunner { Ok(RunnerState::Missing) } - fn ensure_running(&self) -> Result<()> { + fn ensure_running(&self, features: Option<&[String]>) -> Result<()> { match self.state()? { RunnerState::Running | RunnerState::Restarting => (), RunnerState::Created | RunnerState::Exited => self.start()?, @@ -140,7 +145,7 @@ pub trait ContainerTestRunner: TestRunner { self.start()?; } RunnerState::Missing => { - self.build()?; + self.build(features)?; self.ensure_volumes()?; self.create()?; self.start()?; @@ -168,7 +173,7 @@ pub trait ContainerTestRunner: TestRunner { Ok(()) } - fn build(&self) -> Result<()> { + fn build(&self, features: Option<&[String]>) -> Result<()> { let dockerfile: PathBuf = [app::path(), "scripts", "integration", "Dockerfile"] .iter() .collect(); @@ -187,6 +192,8 @@ pub trait ContainerTestRunner: TestRunner { "vector-test-runner=true", "--build-arg", &format!("RUST_VERSION={}", get_rust_version()), + "--build-arg", + &format!("FEATURES={}", features.unwrap_or(&[]).join(",")), ".", ]); @@ -264,9 +271,10 @@ where &self, outer_env: &Environment, inner_env: &Environment, + features: Option<&[String]>, args: &[String], ) -> Result<()> { - self.ensure_running()?; + self.ensure_running(features)?; let mut command = dockercmd(["exec"]); if *IS_A_TTY { @@ -398,6 +406,7 @@ impl TestRunner for LocalTestRunner { &self, outer_env: &Environment, inner_env: &Environment, + _features: Option<&[String]>, args: &[String], ) -> Result<()> { let mut command = Command::new(TEST_COMMAND[0]); From aab25000e60319935f0dea20a4e368e77c6c191d Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 10 Jan 2024 12:23:36 -0700 Subject: [PATCH 59/99] fixes --- scripts/e2e/e2e-datadog-logs/compose.yaml | 6 ++-- tests/data/e2e/datadog/Dockerfile | 41 ----------------------- tests/e2e/datadog/logs/mod.rs | 4 +++ tests/e2e/datadog/mod.rs | 4 +-- vdev/src/testing/integration.rs | 9 ++--- 5 files changed, 12 insertions(+), 52 deletions(-) delete mode 100644 tests/data/e2e/datadog/Dockerfile diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml index 05db907e8f963..ec4fe5f9ecff6 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/e2e-datadog-logs/compose.yaml @@ -24,10 +24,8 @@ services: - fakeintake-vector build: context: ${PWD} - # dockerfile: tests/data/e2e/datadog/Dockerfile - # args: - # - RUST_VERSION=${RUST_VERSION} - # - FEATURES=e2e-tests-datadog + # re-using the integration test runner image since it already has + # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} environment: - FEATURES=e2e-tests-datadog diff --git a/tests/data/e2e/datadog/Dockerfile b/tests/data/e2e/datadog/Dockerfile deleted file mode 100644 index a000766d915c3..0000000000000 --- a/tests/data/e2e/datadog/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -ARG RUST_VERSION -ARG FEATURES -ARG DEBIAN_RELEASE=bookworm - -# -# VECTOR BUILDER -# -FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} as builder -RUN apt-get update && apt-get -y --no-install-recommends install build-essential git clang cmake libclang-dev libsasl2-dev libstdc++-11-dev libssl-dev libxxhash-dev zlib1g-dev zlib1g -RUN git clone https://github.com/rui314/mold.git \ - && mkdir mold/build \ - && cd mold/build \ - && git checkout v2.0.0 \ - && ../install-build-deps.sh \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ - && cmake --build . -j $(nproc) \ - && cmake --install . - -WORKDIR /vector -COPY . . -ARG FEATURES -RUN scripts/environment/install-protoc.sh -RUN --mount=type=cache,target=/vector/target \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - /usr/local/bin/mold -run cargo build --bin vector \ - --no-default-features --features $FEATURES && \ - cp target/debug/vector . - -# -# TARGET -# -FROM docker.io/debian:${DEBIAN_RELEASE}-slim -RUN apt-get update && apt-get -y --no-install-recommends install zlib1g && rm -rf /var/lib/apt/lists/* -COPY --from=builder /vector/vector /usr/bin/vector -RUN mkdir -p /var/lib/vector - -# Smoke test -RUN ["vector", "--version"] - -ENTRYPOINT ["/usr/bin/vector"] diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 663e243c58682..2ca431a7997f0 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -58,6 +58,10 @@ fn common_assertions(payloads: &mut [Value]) { async fn validate() { trace_init(); + // a small sleep here is kind of hard to avoid. Regardless of dependencies flagged for the + // containers, we need the events to flow between them. + std::thread::sleep(std::time::Duration::from_secs(5)); + println!("getting log payloads from agent-only pipeline"); let mut agent_payloads = get_payloads_agent(LOGS_ENDPOINT).await; diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 3336d666904ff..e7507b2814151 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -51,9 +51,7 @@ async fn get_payloads_agent(endpoint: &str) -> Vec { // Not sure what this is but the logs endpoint receives an empty payload in the beginning if raw_payloads.len() > 0 && endpoint == "/api/v2/logs" { - if raw_payloads[0].data.as_array().unwrap().len() == 0 && raw_payloads[0].encoding == "" { - raw_payloads.remove(0); - } + raw_payloads.retain(|raw_payload| raw_payload.data.as_array().unwrap().len() != 0); } raw_payloads.into_iter().map(|raw| raw.data).collect() diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index b0544c2038082..abb1af7a1e5a6 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -73,10 +73,6 @@ impl IntegrationTest { self.config.check_required()?; if !active { - // For end-to-end tests, we want to run vector as a service, leveraging the - // image for the runner. So we must build that image before starting the - // compose so that it is available. - self.runner.build(Some(&self.config.features))?; self.start()?; } @@ -137,6 +133,11 @@ impl IntegrationTest { } pub fn start(&self) -> Result<()> { + // For end-to-end tests, we want to run vector as a service, leveraging the + // image for the runner. So we must build that image before starting the + // compose so that it is available. + self.runner.build(Some(&self.config.features))?; + self.config.check_required()?; if let Some(compose) = &self.compose { self.runner.ensure_network()?; From e9474f7c3bdb4892c77a6f84b1a1ccec86610428 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 10 Jan 2024 14:14:23 -0700 Subject: [PATCH 60/99] e2e tests now run in CI --- Cargo.toml | 14 +++++++++++++- scripts/integration/Dockerfile | 2 -- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index eb30709eae3ed..373d5b0898a27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -738,8 +738,15 @@ enterprise = [ # Identifies that the build is a nightly build nightly = [] -# Testing-related features +# Integration testing-related features all-integration-tests = [ + # TODO: This is a temporary solution. When this GH issue + # https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor + # to have a separate vdev e2e subcommand, and we can then use "all-e2e-tests" in + # that subcommand's processing. Until then the e2e tests must be included here + # in order for them to be exercised in CI. + "all-e2e-tests", + "amqp-integration-tests", "appsignal-integration-tests", "aws-integration-tests", @@ -845,6 +852,11 @@ disable-resolv-conf = [] shutdown-tests = ["api", "sinks-blackhole", "sinks-console", "sinks-prometheus", "sources", "transforms-lua", "transforms-remap", "unix"] cli-tests = ["sinks-blackhole", "sinks-socket", "sources-demo_logs", "sources-file"] +# End-to-End testing-related features +all-e2e-tests = [ + "e2e-tests-datadog" +] + e2e-tests-datadog = [ "sources-datadog_agent", "sinks-datadog_logs" diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index b5eef9580b0e6..f976155d47e8d 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -34,8 +34,6 @@ COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs RUN bash /install-protoc.sh -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.25 --locked - WORKDIR /vector COPY . . ARG FEATURES From 262821d567aba655cd7ac3ca9e9208356b43524c Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 10 Jan 2024 14:32:04 -0700 Subject: [PATCH 61/99] clippy --- tests/e2e/datadog/logs/mod.rs | 9 +++++---- tests/e2e/datadog/mod.rs | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 2ca431a7997f0..7cf1e52d9f8fd 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -1,4 +1,5 @@ use serde_json::Value; +use tracing::info; use vector::test_util::trace_init; @@ -45,11 +46,11 @@ fn assert_timestamp_hostname(payloads: &mut [Value]) -> usize { // runs assertions that each set of payloads should be true to regardless // of the pipeline fn common_assertions(payloads: &mut [Value]) { - assert!(payloads.len() > 0); + assert!(!payloads.is_empty()); let n_log_events = assert_timestamp_hostname(payloads); - println!("log events received: {n_log_events}"); + info!("log events received: {n_log_events}"); assert!(n_log_events == expected_log_events()); } @@ -62,12 +63,12 @@ async fn validate() { // containers, we need the events to flow between them. std::thread::sleep(std::time::Duration::from_secs(5)); - println!("getting log payloads from agent-only pipeline"); + info!("getting log payloads from agent-only pipeline"); let mut agent_payloads = get_payloads_agent(LOGS_ENDPOINT).await; common_assertions(&mut agent_payloads); - println!("getting log payloads from agent-vector pipeline"); + info!("getting log payloads from agent-vector pipeline"); let mut vector_payloads = get_payloads_vector(LOGS_ENDPOINT).await; common_assertions(&mut vector_payloads); diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index e7507b2814151..c76d3d234e4ca 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -50,8 +50,8 @@ async fn get_payloads_agent(endpoint: &str) -> Vec { .payloads; // Not sure what this is but the logs endpoint receives an empty payload in the beginning - if raw_payloads.len() > 0 && endpoint == "/api/v2/logs" { - raw_payloads.retain(|raw_payload| raw_payload.data.as_array().unwrap().len() != 0); + if !raw_payloads.is_empty() && endpoint == "/api/v2/logs" { + raw_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()) } raw_payloads.into_iter().map(|raw| raw.data).collect() From 71a3f4259ae753294127e308728a13d5c4df2a6e Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 10 Jan 2024 15:59:21 -0700 Subject: [PATCH 62/99] remove unused struct --- vdev/src/util.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vdev/src/util.rs b/vdev/src/util.rs index ec1adfca53d1b..e559b20f496dd 100644 --- a/vdev/src/util.rs +++ b/vdev/src/util.rs @@ -52,11 +52,6 @@ pub fn get_channel() -> String { std::env::var("CHANNEL").unwrap_or_else(|_| "custom".to_string()) } -#[derive(Deserialize)] -pub struct RustToolChain { - pub channel: String, -} - pub fn exists(path: impl AsRef + Debug) -> Result { match fs::metadata(path.as_ref()) { Ok(_) => Ok(true), From 2b149404910362549510c3fb3e31c4516f39d4df Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 12 Jan 2024 13:41:32 -0700 Subject: [PATCH 63/99] true up the int test comment changes --- .github/workflows/integration-comment.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 1eb84ec58f786..755c0998012bc 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -203,7 +203,9 @@ jobs: command: bash scripts/ci-integration-test.sh datadog-traces - name: e2e-datadog-logs - if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') || contains(github.event.comment.body, '/ci-run-all') }} + if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') + || contains(github.event.comment.body, '/ci-run-integration-all') + || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 with: timeout_minutes: 30 @@ -211,7 +213,9 @@ jobs: command: bash scripts/ci-integration-test.sh e2e-datadog-logs - name: datadog-e2e-metrics - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') || contains(github.event.comment.body, '/ci-run-all') }} + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') + || contains(github.event.comment.body, '/ci-run-integration-all') + || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 with: timeout_minutes: 30 From 54fe46d3d5bb5e17b2f5cdaa85bc38da4fe1edbb Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 12 Jan 2024 13:46:23 -0700 Subject: [PATCH 64/99] cleanup --- scripts/e2e/e2e-datadog-metrics/compose.yaml | 2 -- tests/data/e2e/datadog/metrics/agent_only.yaml | 7 ------- tests/data/e2e/datadog/metrics/agent_vector.yaml | 7 ------- tests/e2e/datadog/metrics/mod.rs | 2 -- 4 files changed, 18 deletions(-) diff --git a/scripts/e2e/e2e-datadog-metrics/compose.yaml b/scripts/e2e/e2e-datadog-metrics/compose.yaml index c9c9fbe585dfe..191f34cacceee 100644 --- a/scripts/e2e/e2e-datadog-metrics/compose.yaml +++ b/scripts/e2e/e2e-datadog-metrics/compose.yaml @@ -25,7 +25,6 @@ services: - fakeintake-agent environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - # - DD_API_KEY=DEADBEEF - DD_HOSTNAME=datadog-agent volumes: # The Agent config file @@ -38,7 +37,6 @@ services: - vector environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - # - DD_API_KEY=DEADBEEF - DD_HOSTNAME=datadog-agent-vector volumes: # The Agent config file diff --git a/tests/data/e2e/datadog/metrics/agent_only.yaml b/tests/data/e2e/datadog/metrics/agent_only.yaml index 1d7429e01b900..0ed0518c84b18 100644 --- a/tests/data/e2e/datadog/metrics/agent_only.yaml +++ b/tests/data/e2e/datadog/metrics/agent_only.yaml @@ -3,16 +3,9 @@ log_level: 'warn' # disable bunch of stuff we don't need inventories_configuration_enabled: false -# enable_metadata_collection: false enable_gohai: false cloud_provider_metadata: [] -# enable_payloads: - # series: false - # events: false - # service_checks: false - # sketches: false - apm_config: enabled: false diff --git a/tests/data/e2e/datadog/metrics/agent_vector.yaml b/tests/data/e2e/datadog/metrics/agent_vector.yaml index 28d43898e6f44..c1566946a96b6 100644 --- a/tests/data/e2e/datadog/metrics/agent_vector.yaml +++ b/tests/data/e2e/datadog/metrics/agent_vector.yaml @@ -3,16 +3,9 @@ log_level: 'warn' # disable bunch of stuff we don't need inventories_configuration_enabled: false -# enable_metadata_collection: false enable_gohai: false cloud_provider_metadata: [] -# enable_payloads: - # series: false - # events: false - # service_checks: false - # sketches: false - apm_config: enabled: false diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index aac43fc5010ba..c614a25299151 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -49,6 +49,4 @@ async fn validate() { series::validate().await; sketches::validate().await; - - // std::thread::sleep(std::time::Duration::from_secs(120)); } From 48ec676e889e400fc8ca7e1cd7a5791150727a86 Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 12 Jan 2024 15:59:58 -0700 Subject: [PATCH 65/99] rework the massaging and comparison model --- scripts/e2e/e2e-datadog-metrics/compose.yaml | 10 +- src/common/datadog.rs | 42 ++- tests/e2e/datadog/logs/mod.rs | 4 +- tests/e2e/datadog/metrics/series.rs | 302 ++++++++++++------- tests/e2e/datadog/metrics/sketches.rs | 46 ++- tests/e2e/datadog/mod.rs | 3 +- 6 files changed, 257 insertions(+), 150 deletions(-) diff --git a/scripts/e2e/e2e-datadog-metrics/compose.yaml b/scripts/e2e/e2e-datadog-metrics/compose.yaml index 191f34cacceee..f5a6ad99cab32 100644 --- a/scripts/e2e/e2e-datadog-metrics/compose.yaml +++ b/scripts/e2e/e2e-datadog-metrics/compose.yaml @@ -49,13 +49,15 @@ services: - fakeintake-vector build: context: ${PWD} - dockerfile: tests/data/e2e/datadog/Dockerfile - args: - - RUST_VERSION=${RUST_VERSION} - - FEATURES=e2e-tests-datadog + # re-using the integration test runner image since it already has + # compiled vector on it. + image: ${CONFIG_VECTOR_IMAGE} + environment: + - FEATURES=e2e-tests-datadog working_dir: /home/vector network_mode: host command: + - "/usr/bin/vector" - "-vvv" - "-c" - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 7f968a1433dfe..5077453861fbd 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -9,39 +9,55 @@ use vector_lib::{event::DatadogMetricOriginMetadata, sensitive_string::Sensitive pub(crate) const DD_US_SITE: &str = "datadoghq.com"; pub(crate) const DD_EU_SITE: &str = "datadoghq.eu"; +/// DatadogSeriesMetric #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub(crate) struct DatadogSeriesMetric { - pub(crate) metric: String, - pub(crate) r#type: DatadogMetricType, - pub(crate) interval: Option, - pub(crate) points: Vec>, - pub(crate) tags: Option>, +pub struct DatadogSeriesMetric { + /// metric + pub metric: String, + /// metric type + pub r#type: DatadogMetricType, + /// interval + pub interval: Option, + /// points + pub points: Vec>, + /// tags + pub tags: Option>, + /// host #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) host: Option, + pub host: Option, + /// source_type_name #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) source_type_name: Option, + pub source_type_name: Option, + /// device #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) device: Option, + pub device: Option, + /// metadata #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) metadata: Option, + pub metadata: Option, } +/// Datadog series metric metadata #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub(crate) struct DatadogSeriesMetricMetadata { +pub struct DatadogSeriesMetricMetadata { #[serde(skip_serializing_if = "Option::is_none")] pub(crate) origin: Option, } +/// Datadog Metric Type #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum DatadogMetricType { +pub enum DatadogMetricType { + /// Gauge Gauge, + /// Count Count, + /// Rate Rate, } +/// Datadog Point #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub(crate) struct DatadogPoint(pub(crate) i64, pub(crate) T); +pub struct DatadogPoint(pub i64, pub T); /// Gets the base API endpoint to use for any calls to Datadog. /// diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 1c1ccc07dee07..54d1eb0aba959 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -56,9 +56,9 @@ fn common_assertions(payloads: &mut [Value]) { } // reduces the payload down to just the log data -fn reduce_to_data(payloads: &mut Vec>) -> Vec { +fn reduce_to_data(payloads: &mut [FakeIntakePayload]) -> Vec { payloads - .into_iter() + .iter_mut() .map(|payload| payload.data.take()) .collect() } diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index 666f2692afea9..dba39a9d33878 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -9,10 +9,15 @@ use ddmetric_proto::{ metric_payload::{MetricSeries, MetricType}, MetricPayload, }; +use tracing::info; +use vector::common::datadog::DatadogSeriesMetric; + +use self::ddmetric_proto::metric_payload::{MetricPoint, Resource}; use super::*; -const SERIES_ENDPOINT: &str = "/api/v2/series"; +const SERIES_ENDPOINT_V1: &str = "/api/v1/series"; +const SERIES_ENDPOINT_V2: &str = "/api/v2/series"; // unique identification of a Series #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] @@ -22,21 +27,37 @@ struct SeriesContext { r#type: i32, } -// 1. filters out the metrics not generated by the Emitter -// 2. Aggregates the metric value across the different payloads. -// This is necessary because we can't guarantee the payloads sent from -// Agent/Vector will contain the exact same sets of data across a time period. -// 3. Normalizes the `host` value, and the `interval` -fn aggregate_normalize_series( - payloads: &Vec, -) -> BTreeMap { - let mut aggregate = BTreeMap::new(); - - for metric_payload in payloads { - for serie in &metric_payload.series { - // filter out the metrics we don't care about +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct TimeBucket(i64, i64); + +fn get_time_bucket(point: &MetricPoint, interval: i64, metric_type: MetricType) -> TimeBucket { + match metric_type { + MetricType::Unspecified => panic!("received an unspecified metric type"), + MetricType::Rate => TimeBucket(point.timestamp - interval, point.timestamp), + MetricType::Gauge | MetricType::Count => TimeBucket(point.timestamp, point.timestamp), + } +} + +type TimeSeriesData = BTreeMap>; + +/// This type represents the massaged intake data collected from the upstream. +/// The idea is to be able to store what was received in a way that allows us to +/// compare what is important to compare, and accounting for the bits that are not +/// guaranteed to line up. +/// +/// For instance, the services that are running, may start at different times, thus the +/// timestamps (TimeBucket) for data points received are not guaranteed to match up. +type SeriesIntake = BTreeMap; + +// massages the raw payloads into our intake structure +fn generate_series_intake(payloads: &[MetricPayload]) -> SeriesIntake { + let mut intake = BTreeMap::new(); + + payloads.iter().for_each(|payload| { + payload.series.iter().for_each(|serie| { + // filter out the metrics we don't care about (ones not generated by the client) if !serie.metric.starts_with("foo_metric") { - continue; + return; } let ctx = SeriesContext { @@ -45,91 +66,35 @@ fn aggregate_normalize_series( r#type: serie.r#type, }; - if !aggregate.contains_key(&ctx) { - aggregate.insert(ctx, serie.clone()); - continue; - } - - let existing = aggregate.get_mut(&ctx).unwrap(); - - match serie.r#type() { - MetricType::Unspecified => { - panic!("unspecified metric type") - } - MetricType::Count => { - existing.points.extend_from_slice(&serie.points); - } - MetricType::Rate => { - existing.points.extend_from_slice(&serie.points); - } - MetricType::Gauge => { - // last reported one wins - if let Some(point) = serie.points.last() { - let current = existing.points.last().unwrap(); - if point.timestamp > current.timestamp && point.value > 0.0 { - existing.points = vec![point.clone()]; - } - } - } - } - } - } - - // remove the timestamps and sum the points and normalize the other metadata - for (_ctx, series) in &mut aggregate { - // the Agent sets a non-zero default interval for all metric types. - // Vector on ingest from the Agent, only sets the interval if it is a Rate. - if series.r#type() != MetricType::Rate && series.interval != 0 { - println!( - "serie {:?} non-rate metric has interval set. Setting to zero.", - _ctx.metric_name - ); - series.interval = 0; - } - - let mut value = 0.0; - - series.points.retain(|point| point.value > 0.0); - - for point in &mut series.points { - point.timestamp = 0; - - if series.interval > 0 { - value += point.value * series.interval as f64; - } else { - value += point.value; + if !intake.contains_key(&ctx) { + intake.insert(ctx.clone(), BTreeMap::new()); } - point.value = 0.0; - } - if series.points.len() > 0 { - series.points[0].value = value; - } + let entry: &mut BTreeMap> = intake.get_mut(&ctx).unwrap(); - for resource in &mut series.resources { - if resource.r#type == "host" { - if resource.name.ends_with("-vector") { - resource - .name - .truncate(resource.name.len() - "-vector".len()); + serie.points.iter().for_each(|point| { + let tb = get_time_bucket(point, serie.interval, serie.r#type()); + if !entry.contains_key(&tb) { + entry.insert(tb.clone(), vec![]); } - } - } - - println!("{:?}", series); - } + entry.get_mut(&tb).unwrap().push(point.value); + }); + }); + }); - aggregate + intake } // runs assertions that each set of payloads should be true to regardless // of the pipeline -fn common_series_assertions(series: &BTreeMap) { +fn common_series_assertions(series: &SeriesIntake) { // we should have received some metrics from the emitter - assert!(series.len() > 0); - println!("metric series received: {}", series.len()); + assert!(!series.is_empty()); + info!("metric series received: {}", series.len()); // specifically we should have received each of these - let mut found = vec![ + let mut found = [ + // NOTE: no count expected due to the in-app type being Rate + // (https://docs.datadoghq.com/metrics/types/?tab=count#submission-types-and-datadog-in-app-types) (false, "rate"), (false, "gauge"), (false, "set"), @@ -141,7 +106,7 @@ fn common_series_assertions(series: &BTreeMap) { .metric_name .starts_with(&format!("foo_metric.{}", found.1)) { - println!("received {}", found.1); + info!("received {}", found.1); found.0 = true; } }); @@ -152,30 +117,159 @@ fn common_series_assertions(series: &BTreeMap) { .for_each(|(found, mtype)| assert!(found, "Didn't receive metric type {}", *mtype)); } -async fn get_series_from_pipeline(address: String) -> BTreeMap { - println!("getting series payloads"); +impl From<&DatadogSeriesMetric> for MetricSeries { + fn from(input: &DatadogSeriesMetric) -> Self { + let mut resources = vec![]; + if let Some(host) = &input.host { + resources.push(Resource { + r#type: "host".to_string(), + name: host.clone(), + }); + } + + let mut points = vec![]; + input.points.iter().for_each(|point| { + points.push(MetricPoint { + value: point.1, + timestamp: point.0, + }) + }); + + let interval = input.interval.unwrap_or(0) as i64; + + let r#type = match input.r#type { + vector::common::datadog::DatadogMetricType::Gauge => 3, + vector::common::datadog::DatadogMetricType::Count => 1, + vector::common::datadog::DatadogMetricType::Rate => 2, + }; + + MetricSeries { + resources, + metric: input.metric.clone(), + tags: input.tags.clone().unwrap_or_default(), + points, + r#type, + unit: "".to_string(), + source_type_name: input.clone().source_type_name.unwrap_or_default(), + interval, + metadata: None, + } + } +} + +fn convert_v1_payloads_v2(input: &[DatadogSeriesMetric]) -> Vec { + let mut output = vec![]; + + input.iter().for_each(|serie| { + output.push(MetricPayload { + series: vec![serie.into()], + }); + }); + + output +} + +fn unpack_v1_series(in_payloads: &[FakeIntakePayloadJson]) -> Vec { + let mut out_series = vec![]; + + in_payloads.iter().for_each(|payload| { + let series = payload.data.as_array().unwrap(); + + series.iter().for_each(|serie| { + let ser: DatadogSeriesMetric = serde_json::from_value(serie.clone()).unwrap(); + out_series.push(ser); + }); + }); + + out_series +} + +async fn get_v1_series_from_pipeline(address: String) -> SeriesIntake { + info!("getting v1 series payloads"); let payloads = - get_fakeintake_payloads::(&address, SERIES_ENDPOINT).await; + get_fakeintake_payloads::(&address, SERIES_ENDPOINT_V1).await; + + info!("unpacking payloads"); + let payloads = unpack_v1_series(&payloads.payloads); + info!("converting payloads"); + let payloads = convert_v1_payloads_v2(&payloads); - println!("unpacking payloads"); + info!("aggregating payloads"); + let intake = generate_series_intake(&payloads); + + common_series_assertions(&intake); + + info!("{:?}", intake); + + intake +} + +async fn get_v2_series_from_pipeline(address: String) -> SeriesIntake { + info!("getting v2 series payloads"); + let payloads = + get_fakeintake_payloads::(&address, SERIES_ENDPOINT_V2).await; + + info!("unpacking payloads"); let payloads = unpack_proto_payloads::(&payloads); - println!("aggregating payloads"); - let series = aggregate_normalize_series(&payloads); + info!("aggregating payloads"); + let intake = generate_series_intake(&payloads); - common_series_assertions(&series); + common_series_assertions(&intake); - println!("{:?}", series.keys()); + info!("{:?}", intake); - series + intake } pub(super) async fn validate() { - println!("==== getting series data from agent-only pipeline ==== "); - let agent_series = get_series_from_pipeline(fake_intake_agent_address()).await; + info!("==== getting series data from agent-only pipeline ==== "); + let agent_intake = get_v2_series_from_pipeline(fake_intake_agent_address()).await; + + info!("==== getting series data from agent-vector pipeline ===="); + let vector_intake = get_v1_series_from_pipeline(fake_intake_vector_address()).await; + + assert_eq!( + agent_intake.len(), + vector_intake.len(), + "different number of unique Series contexts received" + ); - println!("==== getting series data from agent-vector pipeline ===="); - let vector_series = get_series_from_pipeline(fake_intake_vector_address()).await; + agent_intake + .iter() + .zip(vector_intake.iter()) + .for_each(|(agent_ts, vector_ts)| { + assert_eq!(agent_ts.0, vector_ts.0, "Mismatch of series context"); + + let metric_type = agent_ts.0.r#type; + + // gauge: last one wins. + // we can't rely on comparing each value due to the fact that we can't guarantee consistent sampling + if metric_type == 3 { + let last_agent_point = agent_ts.1.iter().last(); + let last_vector_point = vector_ts.1.iter().last(); + + assert_eq!( + last_agent_point, last_vector_point, + "Mismatch of gauge data" + ); + } - assert_eq!(agent_series, vector_series); + // rate: summation. + if metric_type == 2 { + let mut agent_sum = 0.0; + agent_ts + .1 + .iter() + .for_each(|(_tb, points)| points.iter().for_each(|v| agent_sum += v)); + + let mut vector_sum = 0.0; + vector_ts + .1 + .iter() + .for_each(|(_tb, points)| points.iter().for_each(|v| vector_sum += v)); + + assert_eq!(agent_sum, vector_sum, "Mismatch of rate data"); + } + }); } diff --git a/tests/e2e/datadog/metrics/sketches.rs b/tests/e2e/datadog/metrics/sketches.rs index 4d1e9811bab31..3305e6cd89185 100644 --- a/tests/e2e/datadog/metrics/sketches.rs +++ b/tests/e2e/datadog/metrics/sketches.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{btree_map::Entry, BTreeMap}; #[allow(warnings, clippy::pedantic, clippy::nursery)] mod ddmetric_proto { @@ -6,11 +6,14 @@ mod ddmetric_proto { } use ddmetric_proto::{sketch_payload::Sketch, SketchPayload}; +use tracing::info; use super::*; const SKETCHES_ENDPOINT: &str = "/api/beta/sketches"; +// TODO this needs a re-work to align with the SeriesIntake model in series.rs + // unique identification of a Sketch #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] struct SketchContext { @@ -39,9 +42,9 @@ fn aggregate_normalize_sketches( sketch.host.truncate(sketch.host.len() - "-vector".len()); } - if !aggregate.contains_key(&ctx) { - aggregate.insert(ctx, sketch.clone()); - println!("{:?}", sketch); + if let Entry::Vacant(e) = aggregate.entry(ctx) { + e.insert(sketch.clone()); + info!("{:?}", sketch); } } } @@ -53,51 +56,42 @@ fn aggregate_normalize_sketches( // of the pipeline fn common_sketch_assertions(sketches: &BTreeMap) { // we should have received some metrics from the emitter - assert!(sketches.len() > 0); - println!("metric sketch received: {}", sketches.len()); + assert!(!sketches.is_empty()); + info!("metric sketch received: {}", sketches.len()); - // specifically we should have received each of these - let mut found = vec![(false, "distribution")]; + let mut found = false; sketches.keys().for_each(|ctx| { - found.iter_mut().for_each(|found| { - if ctx - .metric_name - .starts_with(&format!("foo_metric.{}", found.1)) - { - println!("received {}", found.1); - found.0 = true; - } - }); + if ctx.metric_name.starts_with("foo_metric.distribution") { + found = true; + } }); - found - .iter() - .for_each(|(found, mtype)| assert!(found, "Didn't receive metric type {}", *mtype)); + assert!(found, "Didn't receive metric type distribution"); } async fn get_sketches_from_pipeline(address: String) -> BTreeMap { - println!("getting sketch payloads"); + info!("getting sketch payloads"); let payloads = get_fakeintake_payloads::(&address, SKETCHES_ENDPOINT).await; - println!("unpacking payloads"); + info!("unpacking payloads"); let mut payloads = unpack_proto_payloads(&payloads); - println!("aggregating payloads"); + info!("aggregating payloads"); let sketches = aggregate_normalize_sketches(&mut payloads); common_sketch_assertions(&sketches); - println!("{:?}", sketches.keys()); + info!("{:?}", sketches.keys()); sketches } pub(super) async fn validate() { - println!("==== getting sketch data from agent-only pipeline ==== "); + info!("==== getting sketch data from agent-only pipeline ==== "); let agent_sketches = get_sketches_from_pipeline(fake_intake_agent_address()).await; - println!("==== getting sketch data from agent-vector pipeline ===="); + info!("==== getting sketch data from agent-vector pipeline ===="); let vector_sketches = get_sketches_from_pipeline(fake_intake_vector_address()).await; assert_eq!(agent_sketches, vector_sketches); diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 0dbe1a46226a9..2e64f40ab1e93 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -19,7 +19,8 @@ fn fake_intake_agent_address() -> String { struct FakeIntakePayload { // When string, base64 encoded data: D, - encoding: String, + #[serde(rename = "encoding")] + _encoding: String, #[serde(rename = "timestamp")] _timestamp: String, } From 61d25beba76c4c582b8aa889c7673e9dfa4dfa6e Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 12 Jan 2024 16:07:54 -0700 Subject: [PATCH 66/99] compare to all versions --- scripts/e2e/e2e-datadog-metrics/test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/e2e/e2e-datadog-metrics/test.yaml b/scripts/e2e/e2e-datadog-metrics/test.yaml index 0253384160a4c..d975e1000b6e7 100644 --- a/scripts/e2e/e2e-datadog-metrics/test.yaml +++ b/scripts/e2e/e2e-datadog-metrics/test.yaml @@ -12,10 +12,8 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' matrix: - version: [latest] - # TODO use below when not debugging # validate against the latest nightly and also stable v6 and v7 - # version: ['latest', '6', '7'] + version: ['latest', '6', '7'] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From 06277c25e6b2d7bf4980d88bde6bd4e6d178300f Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 16 Jan 2024 15:42:58 -0700 Subject: [PATCH 67/99] cleanup and address TODO for sketches --- scripts/e2e/e2e-datadog-logs/compose.yaml | 50 ++++++------ tests/e2e/datadog/logs/mod.rs | 4 - tests/e2e/datadog/metrics/mod.rs | 3 - tests/e2e/datadog/metrics/series.rs | 4 +- tests/e2e/datadog/metrics/sketches.rs | 93 +++++++++++++++-------- 5 files changed, 90 insertions(+), 64 deletions(-) diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml index ec4fe5f9ecff6..6a34814740d11 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/e2e-datadog-logs/compose.yaml @@ -4,6 +4,9 @@ services: # Generates random log data for consumption by the custom Agent check log_generator: image: docker.io/mingrammer/flog + depends_on: + - datadog-agent-vector + - datadog-agent command: - "-f" - "json" @@ -16,35 +19,11 @@ services: volumes: - log_path:/var/log/ - # Receives log data from the `datadog-agent-vector` service and sends - # to the `fakeintake-vector` service. - vector: - depends_on: - - log_generator - - fakeintake-vector - build: - context: ${PWD} - # re-using the integration test runner image since it already has - # compiled vector on it. - image: ${CONFIG_VECTOR_IMAGE} - environment: - - FEATURES=e2e-tests-datadog - working_dir: /home/vector - network_mode: host - command: - - "/usr/bin/vector" - - "-vvv" - - "-c" - - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" - volumes: - - ${PWD}:/home/vector - # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service datadog-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} depends_on: - - log_generator - fakeintake-agent environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} @@ -65,7 +44,6 @@ services: datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} depends_on: - - log_generator - vector environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} @@ -81,6 +59,28 @@ services: # The custom log to tail, created by the `log_generator` service - log_path:/var/log/ + # Receives log data from the `datadog-agent-vector` service and sends + # to the `fakeintake-vector` service. + vector: + depends_on: + - fakeintake-vector + build: + context: ${PWD} + # re-using the integration test runner image since it already has + # compiled vector on it. + image: ${CONFIG_VECTOR_IMAGE} + environment: + - FEATURES=e2e-tests-datadog + working_dir: /home/vector + network_mode: host + command: + - "/usr/bin/vector" + - "-vvv" + - "-c" + - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" + volumes: + - ${PWD}:/home/vector + # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-agent: diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 54d1eb0aba959..8fe0669c95ac2 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -67,10 +67,6 @@ fn reduce_to_data(payloads: &mut [FakeIntakePayload]) -> Vec { async fn validate() { trace_init(); - // a small sleep here is kind of hard to avoid. Regardless of dependencies flagged for the - // containers, we need the events to flow between them. - std::thread::sleep(std::time::Duration::from_secs(5)); - info!("getting log payloads from agent-only pipeline"); let mut agent_payloads = get_fakeintake_payloads::( &fake_intake_agent_address(), diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index c614a25299151..d614916047262 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -43,9 +43,6 @@ where async fn validate() { trace_init(); - // TODO need to see if can configure the agent flush interval - std::thread::sleep(std::time::Duration::from_secs(20)); - series::validate().await; sketches::validate().await; diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index dba39a9d33878..221879a14409c 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -194,7 +194,7 @@ async fn get_v1_series_from_pipeline(address: String) -> SeriesIntake { info!("converting payloads"); let payloads = convert_v1_payloads_v2(&payloads); - info!("aggregating payloads"); + info!("generating series intake"); let intake = generate_series_intake(&payloads); common_series_assertions(&intake); @@ -212,7 +212,7 @@ async fn get_v2_series_from_pipeline(address: String) -> SeriesIntake { info!("unpacking payloads"); let payloads = unpack_proto_payloads::(&payloads); - info!("aggregating payloads"); + info!("generating series intake"); let intake = generate_series_intake(&payloads); common_series_assertions(&intake); diff --git a/tests/e2e/datadog/metrics/sketches.rs b/tests/e2e/datadog/metrics/sketches.rs index 3305e6cd89185..47ea682cbde3c 100644 --- a/tests/e2e/datadog/metrics/sketches.rs +++ b/tests/e2e/datadog/metrics/sketches.rs @@ -1,19 +1,20 @@ -use std::collections::{btree_map::Entry, BTreeMap}; +use std::collections::BTreeMap; #[allow(warnings, clippy::pedantic, clippy::nursery)] mod ddmetric_proto { include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); } -use ddmetric_proto::{sketch_payload::Sketch, SketchPayload}; +use ddmetric_proto::{ + sketch_payload::sketch::{Distribution, Dogsketch}, + SketchPayload, +}; use tracing::info; use super::*; const SKETCHES_ENDPOINT: &str = "/api/beta/sketches"; -// TODO this needs a re-work to align with the SeriesIntake model in series.rs - // unique identification of a Sketch #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] struct SketchContext { @@ -21,40 +22,65 @@ struct SketchContext { tags: Vec, } -fn aggregate_normalize_sketches( - payloads: &mut Vec, -) -> BTreeMap { - let mut aggregate = BTreeMap::new(); - - for payload in payloads { - for sketch in &mut payload.sketches { - // filter out the metrics we don't care about +type TimeSketchData = BTreeMap>; + +/// This type represents the massaged intake data collected from the upstream. +/// The idea is to be able to store what was received in a way that allows us to +/// compare what is important to compare, and accounting for the bits that are not +/// guaranteed to line up. +/// +/// For instance, the services that are running, may start at different times, thus the +/// timestamps for data points received are not guaranteed to match up. +type SketchIntake = + BTreeMap, TimeSketchData)>; + +// massages the raw payloads into our intake structure +fn generate_sketch_intake(payloads: &mut [SketchPayload]) -> SketchIntake { + let mut intake = SketchIntake::new(); + + payloads.iter_mut().for_each(|payload| { + payload.sketches.iter_mut().for_each(|sketch| { + // filter out the metrics we don't care about (ones not generated by the client) if !sketch.metric.starts_with("foo_metric") { - continue; + return; } - let ctx = SketchContext { metric_name: sketch.metric.clone(), tags: sketch.tags.clone(), }; - if sketch.host.ends_with("-vector") { - sketch.host.truncate(sketch.host.len() - "-vector".len()); - } - - if let Entry::Vacant(e) = aggregate.entry(ctx) { - e.insert(sketch.clone()); - info!("{:?}", sketch); + if !intake.contains_key(&ctx) { + intake.insert(ctx.clone(), (TimeSketchData::new(), TimeSketchData::new())); } - } - } + let entry: &mut (TimeSketchData, TimeSketchData) = + intake.get_mut(&ctx).unwrap(); + + sketch.dogsketches.iter_mut().for_each(|ds| { + let ts = ds.ts; + if !entry.0.contains_key(&ts) { + entry.0.insert(ts, vec![]); + } + ds.ts = 0; + entry.0.get_mut(&ts).unwrap().push(ds.clone()); + }); + + sketch.distributions.iter_mut().for_each(|dt| { + let ts = dt.ts; + if !entry.1.contains_key(&ts) { + entry.1.insert(ts, vec![]); + } + dt.ts = 0; + entry.1.get_mut(&ts).unwrap().push(dt.clone()); + }); + }); + }); - aggregate + intake } // runs assertions that each set of payloads should be true to regardless // of the pipeline -fn common_sketch_assertions(sketches: &BTreeMap) { +fn common_sketch_assertions(sketches: &SketchIntake) { // we should have received some metrics from the emitter assert!(!sketches.is_empty()); info!("metric sketch received: {}", sketches.len()); @@ -69,7 +95,7 @@ fn common_sketch_assertions(sketches: &BTreeMap) { assert!(found, "Didn't receive metric type distribution"); } -async fn get_sketches_from_pipeline(address: String) -> BTreeMap { +async fn get_sketches_from_pipeline(address: String) -> SketchIntake { info!("getting sketch payloads"); let payloads = get_fakeintake_payloads::(&address, SKETCHES_ENDPOINT).await; @@ -77,12 +103,12 @@ async fn get_sketches_from_pipeline(address: String) -> BTreeMap Date: Tue, 16 Jan 2024 15:50:50 -0700 Subject: [PATCH 68/99] feedback ds: workflows --- .github/workflows/integration-comment.yml | 2 +- .github/workflows/integration.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 7464164883663..eb13703d8930f 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -206,7 +206,7 @@ jobs: if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 with: - timeout_minutes: 30 + timeout_minutes: 35 max_attempts: 3 command: bash scripts/ci-integration-test.sh e2e-datadog-logs diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f7708fe5c156e..3f3433e6d5d58 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -211,7 +211,7 @@ jobs: name: e2e-datadog-logs uses: nick-fields/retry@v2 with: - timeout_minutes: 60 + timeout_minutes: 35 max_attempts: 3 command: bash scripts/ci-integration-test.sh e2e-datadog-logs From 19d9dfd2baaa5eeadeecf9dad020f91c5745f47c Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 16 Jan 2024 15:56:42 -0700 Subject: [PATCH 69/99] feedback ds: simpler compose --- scripts/e2e/e2e-datadog-logs/compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml index ec4fe5f9ecff6..097b4f6b7b2d5 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/e2e-datadog-logs/compose.yaml @@ -54,9 +54,9 @@ services: - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml + - ${PWD}/tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml # The custom logs check - - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro + - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro # The custom log to tail, created by the `log_generator` service - log_path:/var/log/ @@ -75,9 +75,9 @@ services: - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + - ${PWD}/tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # The custom logs check - - ../../../tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro + - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro # The custom log to tail, created by the `log_generator` service - log_path:/var/log/ From c0b3989678b1013eae774e62695317366620bead Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 16 Jan 2024 16:09:38 -0700 Subject: [PATCH 70/99] feedback ds: agent versions clarity --- scripts/e2e/e2e-datadog-logs/compose.yaml | 4 ++-- scripts/e2e/e2e-datadog-logs/test.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/e2e-datadog-logs/compose.yaml index 097b4f6b7b2d5..5c1814097f08a 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/e2e-datadog-logs/compose.yaml @@ -42,7 +42,7 @@ services: # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service datadog-agent: - image: docker.io/datadog/agent:${CONFIG_VERSION} + image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} depends_on: - log_generator - fakeintake-agent @@ -63,7 +63,7 @@ services: # Tails a custom log created by `log_generator` and sends log data to # the `vector` service datadog-agent-vector: - image: docker.io/datadog/agent:${CONFIG_VERSION} + image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} depends_on: - log_generator - vector diff --git a/scripts/e2e/e2e-datadog-logs/test.yaml b/scripts/e2e/e2e-datadog-logs/test.yaml index 09785f7811f86..0610bb939ec2c 100644 --- a/scripts/e2e/e2e-datadog-logs/test.yaml +++ b/scripts/e2e/e2e-datadog-logs/test.yaml @@ -13,8 +13,8 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' matrix: - # validate against the latest nightly and also stable v6 and v7 - version: ['latest', '6', '7'] + # validate against the latest Agent nightly and also stable v6 and v7 + agent_version: ['latest', '6', '7'] # changes to these files/paths will invoke the integration test in CI From af7c2fbfdebb33ee4ea75b9269f01ff9f29871c4 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 17 Jan 2024 10:47:42 -0700 Subject: [PATCH 71/99] feedback ds: comments --- tests/e2e/datadog/logs/mod.rs | 3 +++ tests/e2e/datadog/mod.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 7cf1e52d9f8fd..5f9a1997bb3df 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -35,6 +35,9 @@ fn assert_timestamp_hostname(payloads: &mut [Value]) -> usize { .as_object_mut() .expect("log entries should be objects"); + // timestamp is available in the flog generated logs as a datetime but + // there does not appear to be a way to add a custom parser in the Agent + // to handle it. assert!(log.remove("timestamp").is_some()); assert!(log.remove("hostname").is_some()); }) diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index c76d3d234e4ca..12baef8fe5c9a 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -49,7 +49,7 @@ async fn get_payloads_agent(endpoint: &str) -> Vec { .await .payloads; - // Not sure what this is but the logs endpoint receives an empty payload in the beginning + // the logs endpoint receives an empty healthcheck payload in the beginning if !raw_payloads.is_empty() && endpoint == "/api/v2/logs" { raw_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()) } From aa19307571c4b146db00713f9bf796dd785c889c Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 17 Jan 2024 11:49:45 -0700 Subject: [PATCH 72/99] clippy --- tests/e2e/datadog/metrics/sketches.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/e2e/datadog/metrics/sketches.rs b/tests/e2e/datadog/metrics/sketches.rs index 47ea682cbde3c..f01d9c2cb1a31 100644 --- a/tests/e2e/datadog/metrics/sketches.rs +++ b/tests/e2e/datadog/metrics/sketches.rs @@ -57,18 +57,14 @@ fn generate_sketch_intake(payloads: &mut [SketchPayload]) -> SketchIntake { sketch.dogsketches.iter_mut().for_each(|ds| { let ts = ds.ts; - if !entry.0.contains_key(&ts) { - entry.0.insert(ts, vec![]); - } + entry.0.entry(ts).or_default(); ds.ts = 0; entry.0.get_mut(&ts).unwrap().push(ds.clone()); }); sketch.distributions.iter_mut().for_each(|dt| { let ts = dt.ts; - if !entry.1.contains_key(&ts) { - entry.1.insert(ts, vec![]); - } + entry.1.entry(ts).or_default(); dt.ts = 0; entry.1.get_mut(&ts).unwrap().push(dt.clone()); }); From fb7fc57f36930f2be76e15f7d76fe46d100ef680 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 17 Jan 2024 11:55:00 -0700 Subject: [PATCH 73/99] touchups --- .github/workflows/integration-comment.yml | 2 +- .github/workflows/integration.yml | 2 +- scripts/e2e/e2e-datadog-metrics/compose.yaml | 8 ++++---- scripts/e2e/e2e-datadog-metrics/test.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 3c5ed71252362..dce7c64b99367 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -218,7 +218,7 @@ jobs: || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 with: - timeout_minutes: 30 + timeout_minutes: 35 max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-e2e-metrics diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7d1d8c2eb6b3d..19ff019c6e9cc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -221,7 +221,7 @@ jobs: name: e2e-datadog-metrics uses: nick-fields/retry@v2 with: - timeout_minutes: 60 + timeout_minutes: 35 max_attempts: 3 command: bash scripts/ci-integration-test.sh e2e-datadog-metrics diff --git a/scripts/e2e/e2e-datadog-metrics/compose.yaml b/scripts/e2e/e2e-datadog-metrics/compose.yaml index f5a6ad99cab32..d0860b83b71e3 100644 --- a/scripts/e2e/e2e-datadog-metrics/compose.yaml +++ b/scripts/e2e/e2e-datadog-metrics/compose.yaml @@ -20,7 +20,7 @@ services: # Sends metric data received from the Emitter to the `fakeintake-agent` service datadog-agent: - image: docker.io/datadog/agent:${CONFIG_VERSION} + image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} depends_on: - fakeintake-agent environment: @@ -28,11 +28,11 @@ services: - DD_HOSTNAME=datadog-agent volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml + - ${PWD}/tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml # Sends metric data received from the Emitter to the `vector` service datadog-agent-vector: - image: docker.io/datadog/agent:${CONFIG_VERSION} + image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} depends_on: - vector environment: @@ -40,7 +40,7 @@ services: - DD_HOSTNAME=datadog-agent-vector volumes: # The Agent config file - - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + - ${PWD}/tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # Receives metric data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. diff --git a/scripts/e2e/e2e-datadog-metrics/test.yaml b/scripts/e2e/e2e-datadog-metrics/test.yaml index d975e1000b6e7..b9025d08ebf3d 100644 --- a/scripts/e2e/e2e-datadog-metrics/test.yaml +++ b/scripts/e2e/e2e-datadog-metrics/test.yaml @@ -12,8 +12,8 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' matrix: - # validate against the latest nightly and also stable v6 and v7 - version: ['latest', '6', '7'] + # validate against the Agent latest nightly and also stable v6 and v7 + agent_version: ['latest', '6', '7'] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From 4cf483edd1f5958bc7dcf33d9f61ce46948b3601 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 17 Jan 2024 13:41:04 -0700 Subject: [PATCH 74/99] feedback bg: assert_eq --- tests/e2e/datadog/logs/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 5f9a1997bb3df..630c3a1ff191a 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -55,7 +55,7 @@ fn common_assertions(payloads: &mut [Value]) { info!("log events received: {n_log_events}"); - assert!(n_log_events == expected_log_events()); + assert_eq!(n_log_events, expected_log_events()); } #[tokio::test] From d372567bdcbea1f5d1cff39b53f0a81ac7c54c1a Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 17 Jan 2024 16:43:33 -0700 Subject: [PATCH 75/99] increase timeout workflow --- .github/workflows/integration.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3f3433e6d5d58..c141f9d00107b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -90,7 +90,7 @@ jobs: || needs.changes.outputs.webhdfs == 'true' ) ) - timeout-minutes: 75 + timeout-minutes: 120 steps: - uses: actions/checkout@v3 with: @@ -206,6 +206,8 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: e2e-datadog-logs From 66006b76bd2586e8fc746028768320446ce05973 Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 18 Jan 2024 10:27:21 -0700 Subject: [PATCH 76/99] run in 8 core runner --- .github/workflows/integration-comment.yml | 53 ++++++++++++++++++----- .github/workflows/integration.yml | 52 +++++++++++++++++----- 2 files changed, 83 insertions(+), 22 deletions(-) diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index eb13703d8930f..b661c8fbd4883 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -202,14 +202,6 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces - - name: e2e-datadog-logs - if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') || contains(github.event.comment.body, '/ci-run-all') }} - uses: nick-fields/retry@v2 - with: - timeout_minutes: 35 - max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs - - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') || contains(github.event.comment.body, '/ci-run-integration-all') @@ -454,11 +446,39 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh webhdfs + e2e-tests: + needs: prep-pr + runs-on: [linux, ubuntu-20.04-8core] + if: contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') + || contains(github.event.comment.body, '/ci-run-integration-all') + || contains(github.event.comment.body, '/ci-run-all') + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - run: sudo npm -g install @datadog/datadog-ci + + - run: docker image prune -af ; docker container prune -f + - name: e2e-datadog-logs + if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') + || contains(github.event.comment.body, '/ci-run-integration-all') + || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh e2e-datadog-logs + update-pr-status: name: Signal result to PR runs-on: ubuntu-latest - needs: integration-tests + needs: + - integration-tests + - e2e-tests if: always() && (contains(github.event.comment.body, '/ci-run-integration') || contains(github.event.comment.body, '/ci-run-all')) + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - name: Generate authentication token id: generate_token @@ -466,6 +486,7 @@ jobs: with: app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} + - name: Validate issue comment if: github.event_name == 'issue_comment' uses: tspascoal/get-user-teams-membership@v3 @@ -475,12 +496,22 @@ jobs: GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: (PR comment) Get PR branch + if: github.event_name == 'issue_comment' && env.FAILED != 'true' uses: xt0rted/pull-request-comment-branch@v2 id: comment-branch - - name: (PR comment) Submit PR result as ${{ needs.integration-tests.result }} + - name: (PR comment) Submit PR result as success + if: github.event_name == 'issue_comment' && env.FAILED != 'true' uses: myrotvorets/set-commit-status-action@v2.0.0 with: sha: ${{ steps.comment-branch.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} - status: ${{ needs.integration-tests.result }} + status: 'success' + + - run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c141f9d00107b..20c9140a6bb97 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -206,17 +206,6 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-traces - - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f - - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && - (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') - name: e2e-datadog-logs - uses: nick-fields/retry@v2 - with: - timeout_minutes: 35 - max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs - - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap uses: nick-fields/retry@v2 @@ -413,6 +402,46 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh webhdfs + e2e-tests: + name: E2E Tests + runs-on: [linux, ubuntu-20.04-8core] + needs: changes + if: always() && ( + github.event_name == 'merge_group' || ( + needs.changes.outputs.all-int == 'true' + || needs.changes.outputs.e2e-datadog-logs == 'true' + ) + ) + timeout-minutes: 45 + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - run: sudo npm -g install @datadog/datadog-ci + + - run: docker image prune -af ; docker container prune -f + + - name: Determine if secrets are defined (PR author is team member). + if: github.event_name == 'pull_request' + env: + GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + run: | + if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then + echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV" + else + echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" + fi + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-datadog-logs + uses: nick-fields/retry@v2 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh e2e-datadog-logs + integration-test-suite: name: Integration Test Suite runs-on: ubuntu-latest @@ -420,6 +449,7 @@ jobs: needs: - changes - integration-tests + - e2e-tests env: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: From 34c9f7f832331a821e7551f40616b049e94030fe Mon Sep 17 00:00:00 2001 From: neuronull Date: Thu, 18 Jan 2024 10:28:10 -0700 Subject: [PATCH 77/99] check --- .github/workflows/integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 20c9140a6bb97..bca0f22e3158d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -64,7 +64,6 @@ jobs: || needs.changes.outputs.datadog-logs == 'true' || needs.changes.outputs.datadog-metrics == 'true' || needs.changes.outputs.datadog-traces == 'true' - || needs.changes.outputs.e2e-datadog-logs == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' || needs.changes.outputs.elasticsearch == 'true' From 010d8abbffeda23b249e724f2f1496cd5297a4d0 Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 22 Jan 2024 09:03:53 -0700 Subject: [PATCH 78/99] separate e2e workflow --- .github/workflows/changes.yml | 35 ++++++++++- .github/workflows/e2e.yml | 100 ++++++++++++++++++++++++++++++ .github/workflows/integration.yml | 41 ------------ 3 files changed, 132 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 12a252b79a8cc..f8c59afb84dca 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -19,6 +19,10 @@ on: required: false type: boolean default: false + e2e_tests: + required: false + type: boolean + default: false source: required: false type: boolean @@ -64,8 +68,6 @@ on: value: ${{ jobs.int_tests.outputs.datadog-metrics }} datadog-traces: value: ${{ jobs.int_tests.outputs.datadog-traces }} - e2e-datadog-logs: - value: ${{ jobs.int_tests.outputs.e2e-datadog-logs }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: @@ -112,6 +114,11 @@ on: value: ${{ jobs.int_tests.outputs.splunk }} webhdfs: value: ${{ jobs.int_tests.outputs.webhdfs }} + # e2e tests + all-e2e: + value: ${{ jobs.e2e_tests.outputs.all-e2e }} + e2e-datadog-logs: + value: ${{ jobs.e2e_tests.outputs.datadog-logs }} jobs: # Detects changes that are not specific to integration tests @@ -198,7 +205,6 @@ jobs: datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} datadog-traces: ${{ steps.filter.outputs.datadog-traces }} - e2e-datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} @@ -237,3 +243,26 @@ jobs: base: ${{ inputs.base_ref }} ref: ${{ inputs.head_ref }} filters: int_test_filters.yaml + + # Detects changes that are specific to e2e tests + e2e_tests: + runs-on: ubuntu-latest + if: ${{ inputs.e2e_tests }} + outputs: + all-e2e: ${{ steps.filter.outputs.all-int}} + datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} + steps: + - uses: actions/checkout@v3 + + # creates a yaml file that contains the filters for each test, + # extracted from the output of the `vdev int ci-paths` command, which + # sources the paths from the scripts/integration/.../test.yaml files + - name: Create filter rules for e2e tests + run: cargo vdev int ci-paths > int_test_filters.yaml + + - uses: dorny/paths-filter@v2 + id: filter + with: + base: ${{ inputs.base_ref }} + ref: ${{ inputs.head_ref }} + filters: int_test_filters.yaml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000000..5d0217ff6bde7 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,100 @@ +# End to End Suite +# + +name: E2E Test Suite + +on: + pull_request: + # Needs to pass by default in MQ + merge_group: + types: [checks_requested] + schedule: + # At midnight UTC Tue-Sat + - cron: '0 0 * * 2-6' + +concurrency: + # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue + group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.head_sha }} + cancel-in-progress: true + +env: + CONTAINER_TOOL: "docker" + DD_ENV: "ci" + DD_API_KEY: ${{ secrets.DD_API_KEY }} + TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} + RUST_BACKTRACE: full + TEST_LOG: vector=debug + VERBOSE: true + CI: true + PROFILE: debug + # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps + # can be removed when we switch back to the upstream openssl-sys crate + CARGO_NET_GIT_FETCH_WITH_CLI: true + +jobs: + + changes: + if: github.event_name == 'pull_request' + uses: ./.github/workflows/changes.yml + with: + base_ref: ${{ github.event.pull_request.base.ref }} + head_ref: ${{ github.event.pull_request.head.ref }} + source: false + e2e_tests: true + secrets: inherit + + e2e-tests: + name: E2E Tests + runs-on: [linux, ubuntu-20.04-8core] + needs: changes + if: always() && ( + github.event_name == 'schedule' || ( + needs.changes.outputs.all-e2e == 'true' + || needs.changes.outputs.e2e-datadog-logs == 'true' + ) + ) + timeout-minutes: 45 + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - run: sudo npm -g install @datadog/datadog-ci + + - run: docker image prune -af ; docker container prune -f + + - name: Determine if secrets are defined (PR author is team member). + if: github.event_name == 'pull_request' + env: + GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + run: | + if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then + echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV" + else + echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" + fi + + - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-datadog-logs + uses: nick-fields/retry@v2 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh e2e-datadog-logs + + e2e-test-suite: + name: E2E Test Suite + runs-on: ubuntu-latest + if: always() + needs: e2e-tests + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} + steps: + - run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bca0f22e3158d..99c582ac1ca18 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -401,46 +401,6 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh webhdfs - e2e-tests: - name: E2E Tests - runs-on: [linux, ubuntu-20.04-8core] - needs: changes - if: always() && ( - github.event_name == 'merge_group' || ( - needs.changes.outputs.all-int == 'true' - || needs.changes.outputs.e2e-datadog-logs == 'true' - ) - ) - timeout-minutes: 45 - steps: - - uses: actions/checkout@v3 - with: - submodules: "recursive" - - - run: sudo npm -g install @datadog/datadog-ci - - - run: docker image prune -af ; docker container prune -f - - - name: Determine if secrets are defined (PR author is team member). - if: github.event_name == 'pull_request' - env: - GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} - run: | - if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then - echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV" - else - echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" - fi - - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && - (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') - name: e2e-datadog-logs - uses: nick-fields/retry@v2 - with: - timeout_minutes: 35 - max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs - integration-test-suite: name: Integration Test Suite runs-on: ubuntu-latest @@ -448,7 +408,6 @@ jobs: needs: - changes - integration-tests - - e2e-tests env: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: From ffc9d5d27a70a36a017b3a90214bf1e2bf518791 Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 22 Jan 2024 09:13:20 -0700 Subject: [PATCH 79/99] comment --- .github/workflows/integration-comment.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index b661c8fbd4883..4f42fe915aff1 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -449,9 +449,6 @@ jobs: e2e-tests: needs: prep-pr runs-on: [linux, ubuntu-20.04-8core] - if: contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') - || contains(github.event.comment.body, '/ci-run-integration-all') - || contains(github.event.comment.body, '/ci-run-all') steps: - uses: actions/checkout@v3 with: From c33af87461b258d632065af3775b572bc1e50409 Mon Sep 17 00:00:00 2001 From: neuronull Date: Mon, 22 Jan 2024 09:24:25 -0700 Subject: [PATCH 80/99] fix workflows --- .github/workflows/changes.yml | 14 +++++--------- .github/workflows/e2e.yml | 10 ++++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 45b2991942f54..d72ee3063b1f7 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -72,10 +72,6 @@ on: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: value: ${{ jobs.int_tests.outputs.docker-logs }} - e2e-datadog-logs: - value: ${{ jobs.int_tests.outputs.e2e-datadog-logs }} - e2e-datadog-metrics: - value: ${{ jobs.int_tests.outputs.e2e-datadog-metrics }} elasticsearch: value: ${{ jobs.int_tests.outputs.elasticsearch }} eventstoredb: @@ -123,6 +119,8 @@ on: value: ${{ jobs.e2e_tests.outputs.all-e2e }} e2e-datadog-logs: value: ${{ jobs.e2e_tests.outputs.datadog-logs }} + e2e-datadog-metrics: + value: ${{ jobs.e2e_tests.outputs.datadog-metrics }} jobs: # Detects changes that are not specific to integration tests @@ -211,11 +209,6 @@ jobs: datadog-traces: ${{ steps.filter.outputs.datadog-traces }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} - - # TODO: when https://github.com/vectordotdev/vector/issues/18829 is undertaken, - # part of that will involve updating this job to also run `cargo vdev e2e ci-paths` - e2e-datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} - e2e-datadog-metrics: ${{ steps.filter.outputs.e2e-datadog-metrics }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} eventstoredb: ${{ steps.filter.outputs.eventstoredb }} fluent: ${{ steps.filter.outputs.fluent }} @@ -259,7 +252,10 @@ jobs: if: ${{ inputs.e2e_tests }} outputs: all-e2e: ${{ steps.filter.outputs.all-int}} + # TODO: when https://github.com/vectordotdev/vector/issues/18829 is undertaken, + # part of that will involve updating this job to also run `cargo vdev e2e ci-paths` datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} + datadog-metrics: ${{ steps.filter.outputs.e2e-datadog-metrics }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5d0217ff6bde7..40b58ffa80c96 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -83,6 +83,16 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh e2e-datadog-logs + - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-datadog-metrics + uses: nick-fields/retry@v2 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh e2e-datadog-metrics + + e2e-test-suite: name: E2E Test Suite runs-on: ubuntu-latest From b71cfcc903d0da78a1e650e1ea651066e35de995 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 23 Jan 2024 16:19:33 +0000 Subject: [PATCH 81/99] prune timing --- .github/workflows/integration.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 99c582ac1ca18..575e09da7f13f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -89,7 +89,7 @@ jobs: || needs.changes.outputs.webhdfs == 'true' ) ) - timeout-minutes: 120 + timeout-minutes: 90 steps: - uses: actions/checkout@v3 with: @@ -187,6 +187,8 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-logs + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-metrics @@ -213,8 +215,6 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh dnstap - - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f - - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }} name: docker-logs uses: nick-fields/retry@v2 @@ -263,6 +263,8 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh greptimedb + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.humio == 'true' }} name: humio uses: nick-fields/retry@v2 @@ -319,8 +321,6 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh mongodb - - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f - - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }} name: nats uses: nick-fields/retry@v2 @@ -337,6 +337,8 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh nginx + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.opentelemetry == 'true' }} name: opentelemetry uses: nick-fields/retry@v2 From ac56ba4ce2dbe3c1527df3ccff575ed74a247992 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 23 Jan 2024 14:09:32 -0700 Subject: [PATCH 82/99] allow build_all option to start subcommand --- scripts/ci-integration-test.sh | 2 +- vdev/src/commands/integration/start.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/ci-integration-test.sh b/scripts/ci-integration-test.sh index 9687f1c1eafe6..49780e7bba378 100755 --- a/scripts/ci-integration-test.sh +++ b/scripts/ci-integration-test.sh @@ -21,7 +21,7 @@ set -x INTEGRATION=$1 -cargo vdev -v int start "${INTEGRATION}" +cargo vdev -v int start -a "${INTEGRATION}" sleep 30 cargo vdev -v int test --retries 2 -a "${INTEGRATION}" RET=$? diff --git a/vdev/src/commands/integration/start.rs b/vdev/src/commands/integration/start.rs index da8947e74997a..ef487a8c487b7 100644 --- a/vdev/src/commands/integration/start.rs +++ b/vdev/src/commands/integration/start.rs @@ -10,6 +10,10 @@ pub struct Cli { /// The integration name integration: String, + /// Whether to compile the test runner with all integration test features + #[arg(short = 'a', long)] + build_all: bool, + /// The desired environment name to start. If omitted, the first environment name is used. environment: Option, } @@ -24,6 +28,6 @@ impl Cli { let env = envs.keys().next().expect("Integration has no environments"); env.clone() }; - IntegrationTest::new(self.integration, environment, false, 0)?.start() + IntegrationTest::new(self.integration, environment, self.build_all, 0)?.start() } } From 97aa5a89738fdcd80e5cf2ab5f580aafcb374180 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 23 Jan 2024 16:16:28 -0700 Subject: [PATCH 83/99] hack to bypass this temporary issue with CI --- Makefile | 2 +- vdev/src/testing/integration.rs | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 87f50cc350d72..934e7c5eb7755 100644 --- a/Makefile +++ b/Makefile @@ -361,7 +361,7 @@ test-integration: test-integration-databend test-integration-docker-logs test-in test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar -test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-datadog-e2e-logs +test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-e2e-datadog-logs test-integration: test-integration-datadog-traces test-integration-shutdown test-integration-%-cleanup: diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index abb1af7a1e5a6..b3b0f1d79ab09 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -133,10 +133,18 @@ impl IntegrationTest { } pub fn start(&self) -> Result<()> { - // For end-to-end tests, we want to run vector as a service, leveraging the - // image for the runner. So we must build that image before starting the - // compose so that it is available. - self.runner.build(Some(&self.config.features))?; + // TODO this is a hack until https://github.com/vectordotdev/vector/pull/18840 + // During that PR, will be able to conditionally call this based on the trait + // definition for the E2E tests. + // The reason we need this temporary hack is because adding the features as a build + // argument results in each integration's runner container being unique, and thus + // the job in CI runs out of space. + if self.integration.contains("e2e-") { + // For end-to-end tests, we want to run vector as a service, leveraging the + // image for the runner. So we must build that image before starting the + // compose so that it is available. + self.runner.build(Some(&self.config.features))?; + } self.config.check_required()?; if let Some(compose) = &self.compose { From 1e8bfb0a8b70cdad32214438b02af9f26e05aecf Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 24 Jan 2024 11:02:28 -0700 Subject: [PATCH 84/99] feedback bg --- tests/e2e/datadog/logs/mod.rs | 12 ++--- tests/e2e/datadog/metrics/mod.rs | 3 +- tests/e2e/datadog/metrics/series.rs | 64 +++++++++++++++------------ tests/e2e/datadog/metrics/sketches.rs | 6 +-- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 80c71b4d7f9b0..de9210388ed39 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -59,10 +59,10 @@ fn common_assertions(payloads: &mut [Value]) { } // reduces the payload down to just the log data -fn reduce_to_data(payloads: &mut [FakeIntakePayload]) -> Vec { +fn reduce_to_data(payloads: Vec>) -> Vec { payloads - .iter_mut() - .map(|payload| payload.data.take()) + .into_iter() + .map(|mut payload| payload.data.take()) .collect() } @@ -87,19 +87,19 @@ async fn validate() { agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()) } - let mut agent_payloads = reduce_to_data(&mut agent_payloads); + let mut agent_payloads = reduce_to_data(agent_payloads); common_assertions(&mut agent_payloads); info!("getting log payloads from agent-vector pipeline"); - let mut vector_payloads = get_fakeintake_payloads::( + let vector_payloads = get_fakeintake_payloads::( &fake_intake_vector_address(), LOGS_ENDPOINT, ) .await .payloads; - let mut vector_payloads = reduce_to_data(&mut vector_payloads); + let mut vector_payloads = reduce_to_data(vector_payloads); common_assertions(&mut vector_payloads); diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs index e11139cdab780..875d74f0fc3c3 100644 --- a/tests/e2e/datadog/metrics/mod.rs +++ b/tests/e2e/datadog/metrics/mod.rs @@ -1,6 +1,7 @@ use base64::{prelude::BASE64_STANDARD, Engine}; use bytes::Bytes; use flate2::read::ZlibDecoder; +use std::io::Read; use vector::test_util::trace_init; @@ -12,7 +13,7 @@ use super::*; fn decompress_payload(payload: Vec) -> std::io::Result> { let mut decompressor = ZlibDecoder::new(&payload[..]); let mut decompressed = Vec::new(); - let result = std::io::copy(&mut decompressor, &mut decompressed); + let result = decompressor.read_to_end(&mut decompressed); result.map(|_| decompressed) } diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index 221879a14409c..1b101b69283d6 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -74,7 +74,7 @@ fn generate_series_intake(payloads: &[MetricPayload]) -> SeriesIntake { serie.points.iter().for_each(|point| { let tb = get_time_bucket(point, serie.interval, serie.r#type()); if !entry.contains_key(&tb) { - entry.insert(tb.clone(), vec![]); + entry.insert(tb.clone(), Vec::new()); } entry.get_mut(&tb).unwrap().push(point.value); }); @@ -127,13 +127,14 @@ impl From<&DatadogSeriesMetric> for MetricSeries { }); } - let mut points = vec![]; - input.points.iter().for_each(|point| { - points.push(MetricPoint { + let points = input + .points + .iter() + .map(|point| MetricPoint { value: point.1, timestamp: point.0, }) - }); + .collect(); let interval = input.interval.unwrap_or(0) as i64; @@ -158,30 +159,37 @@ impl From<&DatadogSeriesMetric> for MetricSeries { } fn convert_v1_payloads_v2(input: &[DatadogSeriesMetric]) -> Vec { - let mut output = vec![]; - - input.iter().for_each(|serie| { - output.push(MetricPayload { + input + .iter() + .map(|serie| MetricPayload { series: vec![serie.into()], - }); - }); - - output + }) + .collect() } fn unpack_v1_series(in_payloads: &[FakeIntakePayloadJson]) -> Vec { - let mut out_series = vec![]; + // let mut out_series = vec![]; - in_payloads.iter().for_each(|payload| { - let series = payload.data.as_array().unwrap(); + // in_payloads.iter().for_each(|payload| { + // let series = payload.data.as_array().unwrap(); - series.iter().for_each(|serie| { - let ser: DatadogSeriesMetric = serde_json::from_value(serie.clone()).unwrap(); - out_series.push(ser); - }); - }); + // series.iter().for_each(|serie| { + // let ser: DatadogSeriesMetric = serde_json::from_value(serie.clone()).unwrap(); + // out_series.push(ser); + // }); + // }); - out_series + // out_series + + in_payloads + .iter() + .flat_map(|payload| { + let series = payload.data.as_array().unwrap(); + series + .iter() + .map(|serie| serde_json::from_value(serie.clone()).unwrap()) + }) + .collect() } async fn get_v1_series_from_pipeline(address: String) -> SeriesIntake { @@ -257,17 +265,17 @@ pub(super) async fn validate() { // rate: summation. if metric_type == 2 { - let mut agent_sum = 0.0; - agent_ts + let agent_sum: f64 = agent_ts .1 .iter() - .for_each(|(_tb, points)| points.iter().for_each(|v| agent_sum += v)); + .map(|(_tb, points)| points.iter().sum::()) + .sum(); - let mut vector_sum = 0.0; - vector_ts + let vector_sum: f64 = vector_ts .1 .iter() - .for_each(|(_tb, points)| points.iter().for_each(|v| vector_sum += v)); + .map(|(_tb, points)| points.iter().sum::()) + .sum(); assert_eq!(agent_sum, vector_sum, "Mismatch of rate data"); } diff --git a/tests/e2e/datadog/metrics/sketches.rs b/tests/e2e/datadog/metrics/sketches.rs index f01d9c2cb1a31..f506e4f6e1176 100644 --- a/tests/e2e/datadog/metrics/sketches.rs +++ b/tests/e2e/datadog/metrics/sketches.rs @@ -35,7 +35,7 @@ type SketchIntake = BTreeMap, TimeSketchData)>; // massages the raw payloads into our intake structure -fn generate_sketch_intake(payloads: &mut [SketchPayload]) -> SketchIntake { +fn generate_sketch_intake(mut payloads: Vec) -> SketchIntake { let mut intake = SketchIntake::new(); payloads.iter_mut().for_each(|payload| { @@ -97,10 +97,10 @@ async fn get_sketches_from_pipeline(address: String) -> SketchIntake { get_fakeintake_payloads::(&address, SKETCHES_ENDPOINT).await; info!("unpacking payloads"); - let mut payloads = unpack_proto_payloads(&payloads); + let payloads = unpack_proto_payloads(&payloads); info!("generating sketch intake"); - let sketches = generate_sketch_intake(&mut payloads); + let sketches = generate_sketch_intake(payloads); common_sketch_assertions(&sketches); From af5c4a9b363eb8752beca233e09265360af09402 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 24 Jan 2024 13:25:31 -0700 Subject: [PATCH 85/99] feedback bg --- tests/e2e/datadog/metrics/series.rs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index 1b101b69283d6..fd4e5cf8bf1ff 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -168,19 +168,6 @@ fn convert_v1_payloads_v2(input: &[DatadogSeriesMetric]) -> Vec { } fn unpack_v1_series(in_payloads: &[FakeIntakePayloadJson]) -> Vec { - // let mut out_series = vec![]; - - // in_payloads.iter().for_each(|payload| { - // let series = payload.data.as_array().unwrap(); - - // series.iter().for_each(|serie| { - // let ser: DatadogSeriesMetric = serde_json::from_value(serie.clone()).unwrap(); - // out_series.push(ser); - // }); - // }); - - // out_series - in_payloads .iter() .flat_map(|payload| { @@ -254,8 +241,16 @@ pub(super) async fn validate() { // gauge: last one wins. // we can't rely on comparing each value due to the fact that we can't guarantee consistent sampling if metric_type == 3 { - let last_agent_point = agent_ts.1.iter().last(); - let last_vector_point = vector_ts.1.iter().last(); + let last_agent_point = agent_ts + .1 + .last_key_value() + .expect("should have received points") + .1; + let last_vector_point = vector_ts + .1 + .last_key_value() + .expect("should have received points") + .1; assert_eq!( last_agent_point, last_vector_point, From c9d33d34c790dc9eadb7d8193c2af42e91ede171 Mon Sep 17 00:00:00 2001 From: neuronull Date: Fri, 26 Jan 2024 16:53:04 -0700 Subject: [PATCH 86/99] feedback ds: metric type check --- tests/e2e/datadog/metrics/series.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index fd4e5cf8bf1ff..a41315cff2abb 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -238,6 +238,13 @@ pub(super) async fn validate() { let metric_type = agent_ts.0.r#type; + // Dogstatsd emits counters but the output type from the Agent is Rate. + // https://docs.datadoghq.com/metrics/types/?tab=rate#submission-types-and-datadog-in-app-types + assert!( + metric_type == 2 || metric_type == 3, + "Metric type should always be rate or gauge." + ); + // gauge: last one wins. // we can't rely on comparing each value due to the fact that we can't guarantee consistent sampling if metric_type == 3 { From 031cdcc6d99d96e8113df54c9f5fc1e286c02d39 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 11:12:29 -0700 Subject: [PATCH 87/99] add descriptive comment --- tests/e2e/datadog/metrics/series.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index a41315cff2abb..8edf959bf82ec 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -230,6 +230,24 @@ pub(super) async fn validate() { "different number of unique Series contexts received" ); + // The assertions we make below can be summarized as follows: + // - For each metric type, we have a different set of assertions which are relevant to + // the expectations for that metric type's behavior. For example, gauges are a last + // write wins. While rates are taken as a summation over the time interval. + // + // - The intake from each case (Agent only, and Agent->Vector) is stored in a data structure + // that allows us to compare the consistency in the overall shape of the data collected + // during the entire test duration, regardless of how requests are batched. This is because + // the data is stored based on a timebucket, and not per request. And the timebuckets holding + // data points are collected in BTreeMap, which means the sort order will be consistent, + // regardless of whether the actual timestamps between the Agent only vs Agent+Vector cases + // are identical (which they may not be, since there is no guarantee the compose services + // started at the same time, nor that the Agent will instances will process in the same time). + // + // Together, this means that data sets passing these validations confirm that the Vector version + // used in the test case is not introducing inconsistencies in the data flowing between the + // Agent and the Datadog backend. + agent_intake .iter() .zip(vector_intake.iter()) From 773bb2414ccb2bf8bbe9ee17359c18df168f5654 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 11:15:27 -0700 Subject: [PATCH 88/99] spelling --- tests/e2e/datadog/metrics/series.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index 8edf959bf82ec..0094f014e372d 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -238,7 +238,7 @@ pub(super) async fn validate() { // - The intake from each case (Agent only, and Agent->Vector) is stored in a data structure // that allows us to compare the consistency in the overall shape of the data collected // during the entire test duration, regardless of how requests are batched. This is because - // the data is stored based on a timebucket, and not per request. And the timebuckets holding + // the data is stored based on a time bucket, and not per request. And the time buckets holding // data points are collected in BTreeMap, which means the sort order will be consistent, // regardless of whether the actual timestamps between the Agent only vs Agent+Vector cases // are identical (which they may not be, since there is no guarantee the compose services From 6da0b03f65370fc69d1eec29829a3006906cdbb0 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 15:02:42 -0700 Subject: [PATCH 89/99] chore(vdev): refactor e2e tests into own subcommand (#19666) --- .github/actions/spelling/allow.txt | 1 + .github/workflows/changes.yml | 12 +- Cargo.toml | 7 - scripts/e2e/README.md | 18 +-- scripts/e2e/datadog-logs/README.md | 19 +++ .../compose.yaml | 0 .../test.yaml | 0 scripts/e2e/datadog-metrics/README.md | 19 +++ .../compose.yaml | 0 .../dogstatsd_client/Dockerfile | 0 .../dogstatsd_client/client.py | 0 .../dogstatsd_client/requirements.txt | 0 .../test.yaml | 0 vdev/src/commands/compose_tests/ci_paths.rs | 20 +++ vdev/src/commands/compose_tests/mod.rs | 5 + vdev/src/commands/compose_tests/show.rs | 93 ++++++++++++ vdev/src/commands/compose_tests/start.rs | 16 ++ vdev/src/commands/compose_tests/stop.rs | 12 ++ vdev/src/commands/compose_tests/test.rs | 39 +++++ vdev/src/commands/e2e/ci_paths.rs | 16 ++ vdev/src/commands/e2e/mod.rs | 13 ++ vdev/src/commands/e2e/show.rs | 18 +++ vdev/src/commands/e2e/start.rs | 21 +++ vdev/src/commands/e2e/stop.rs | 22 +++ vdev/src/commands/e2e/test.rs | 45 ++++++ vdev/src/commands/integration/ci_paths.rs | 17 +-- vdev/src/commands/integration/show.rs | 84 +---------- vdev/src/commands/integration/start.rs | 15 +- vdev/src/commands/integration/stop.rs | 12 +- vdev/src/commands/integration/test.rs | 38 ++--- vdev/src/commands/mod.rs | 3 + vdev/src/testing/config.rs | 37 ++--- vdev/src/testing/integration.rs | 139 +++++++++++------- 33 files changed, 496 insertions(+), 245 deletions(-) create mode 100644 scripts/e2e/datadog-logs/README.md rename scripts/e2e/{e2e-datadog-logs => datadog-logs}/compose.yaml (100%) rename scripts/e2e/{e2e-datadog-logs => datadog-logs}/test.yaml (100%) create mode 100644 scripts/e2e/datadog-metrics/README.md rename scripts/e2e/{e2e-datadog-metrics => datadog-metrics}/compose.yaml (100%) rename scripts/e2e/{e2e-datadog-metrics => datadog-metrics}/dogstatsd_client/Dockerfile (100%) rename scripts/e2e/{e2e-datadog-metrics => datadog-metrics}/dogstatsd_client/client.py (100%) rename scripts/e2e/{e2e-datadog-metrics => datadog-metrics}/dogstatsd_client/requirements.txt (100%) rename scripts/e2e/{e2e-datadog-metrics => datadog-metrics}/test.yaml (100%) create mode 100644 vdev/src/commands/compose_tests/ci_paths.rs create mode 100644 vdev/src/commands/compose_tests/mod.rs create mode 100644 vdev/src/commands/compose_tests/show.rs create mode 100644 vdev/src/commands/compose_tests/start.rs create mode 100644 vdev/src/commands/compose_tests/stop.rs create mode 100644 vdev/src/commands/compose_tests/test.rs create mode 100644 vdev/src/commands/e2e/ci_paths.rs create mode 100644 vdev/src/commands/e2e/mod.rs create mode 100644 vdev/src/commands/e2e/show.rs create mode 100644 vdev/src/commands/e2e/start.rs create mode 100644 vdev/src/commands/e2e/stop.rs create mode 100644 vdev/src/commands/e2e/test.rs diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 74703aa20ce4f..b3cb42844ef5a 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -52,6 +52,7 @@ dkr Dockerfiles DOOV Douban +E2ETest Enot Evercoss Explay diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index d72ee3063b1f7..83af6244ecc19 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -45,7 +45,7 @@ on: k8s: value: ${{ jobs.source.outputs.k8s }} all-int: - value: ${{ jobs.int_tests.outputs.all-int }} + value: ${{ jobs.int_tests.outputs.all-tests }} amqp: value: ${{ jobs.int_tests.outputs.amqp }} appsignal: @@ -116,7 +116,7 @@ on: value: ${{ jobs.int_tests.outputs.webhdfs }} # e2e tests all-e2e: - value: ${{ jobs.e2e_tests.outputs.all-e2e }} + value: ${{ jobs.e2e_tests.outputs.all-tests }} e2e-datadog-logs: value: ${{ jobs.e2e_tests.outputs.datadog-logs }} e2e-datadog-metrics: @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest if: ${{ inputs.int_tests }} outputs: - all-int: ${{ steps.filter.outputs.all-int}} + all-tests: ${{ steps.filter.outputs.all-tests}} amqp: ${{ steps.filter.outputs.amqp }} appsignal: ${{ steps.filter.outputs.appsignal}} aws: ${{ steps.filter.outputs.aws }} @@ -251,9 +251,7 @@ jobs: runs-on: ubuntu-latest if: ${{ inputs.e2e_tests }} outputs: - all-e2e: ${{ steps.filter.outputs.all-int}} - # TODO: when https://github.com/vectordotdev/vector/issues/18829 is undertaken, - # part of that will involve updating this job to also run `cargo vdev e2e ci-paths` + all-tests: ${{ steps.filter.outputs.all-tests}} datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.e2e-datadog-metrics }} steps: @@ -263,7 +261,7 @@ jobs: # extracted from the output of the `vdev int ci-paths` command, which # sources the paths from the scripts/integration/.../test.yaml files - name: Create filter rules for e2e tests - run: cargo vdev int ci-paths > int_test_filters.yaml + run: cargo vdev e2e ci-paths > int_test_filters.yaml - uses: dorny/paths-filter@v2 id: filter diff --git a/Cargo.toml b/Cargo.toml index 3907d50481841..3c2f1579f4597 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -740,13 +740,6 @@ nightly = [] # Integration testing-related features all-integration-tests = [ - # TODO: This is a temporary solution. When this GH issue - # https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor - # to have a separate vdev e2e subcommand, and we can then use "all-e2e-tests" in - # that subcommand's processing. Until then the e2e tests must be included here - # in order for them to be exercised in CI. - "all-e2e-tests", - "amqp-integration-tests", "appsignal-integration-tests", "aws-integration-tests", diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 45422e823c3cc..62e7f8e9f64c7 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -1,17 +1,9 @@ This directory contains a set of end-to-end test frameworks for vector which are executed by the `vdev` tool. -Currently these e2e tests are executed with the same `vdev` subcommand as the integration tests, -`cargo vdev integration`. +These end-to-end (e2e) tests are executed with the `vdev e2e` subcommand, which behaves +identically to the `vdev integration` subcommand. See the README in the `scripts/integration` +subdirectory for more information. -See the README in the `scripts/integration` subdirectory for more information. - -A pending future enhancement is to create a `vdev` subcommand `e2e`, that will separate the -invocation of the end-to-end tests from the integration tests in `vdev`, to correspond to the -code separation and fundamental differences between the two classes of tests. - -See https://github.com/vectordotdev/vector/issues/18829 for more information. - -For now, any subdir here needs to be unique from the other integration tests outside this dir. -For example there is already a `datadog-logs` integration test, hence the e2e test is in a sub- -dir called `e2e-datadog-logs`. +The e2e tests are more of a black box test, in which we spin up a full vector instance as one +of the compose services that runs alongside the others. diff --git a/scripts/e2e/datadog-logs/README.md b/scripts/e2e/datadog-logs/README.md new file mode 100644 index 0000000000000..7b3eaac8d9407 --- /dev/null +++ b/scripts/e2e/datadog-logs/README.md @@ -0,0 +1,19 @@ +This e2e test covers the `datadog_agent` source, and the +`datadog_logs` sink. + +Fake logs are generated in the emitter service and written +to a file. + +Two Agent containers are spun up to read the log file, one +for the Agent only case and one for the Agent -> Vector case. + +In the Agent only case, the Agent sends the logs to `fakeintake` +(another service) directly. This is the baseline. + +In the Agent-Vector case, the Agent send the logs to the vector +service, and the `datadog_logs` sink sends to a separate +`fakeintake` service. This is the compare case. + +The two sets of data should be shaped the same in terms of when +the events were received, and the content of the events, but the +timestamps themselves are not guaranteed to align. diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml similarity index 100% rename from scripts/e2e/e2e-datadog-logs/compose.yaml rename to scripts/e2e/datadog-logs/compose.yaml diff --git a/scripts/e2e/e2e-datadog-logs/test.yaml b/scripts/e2e/datadog-logs/test.yaml similarity index 100% rename from scripts/e2e/e2e-datadog-logs/test.yaml rename to scripts/e2e/datadog-logs/test.yaml diff --git a/scripts/e2e/datadog-metrics/README.md b/scripts/e2e/datadog-metrics/README.md new file mode 100644 index 0000000000000..07f0361f9ac6a --- /dev/null +++ b/scripts/e2e/datadog-metrics/README.md @@ -0,0 +1,19 @@ +This e2e test covers the `datadog_agent` source, and the +`datadog_metrics` sink. + +An emitter compose service runs a python DogStatsD program, +to generate various metric types for the test cases. + +Two Agent containers are spun up to receive the metrics, one +for the Agent only case and one for the Agent -> Vector case. + +In the Agent only case, the Agent sends the metrics to `fakeintake` +(another service) directly. This is the baseline. + +In the Agent-Vector case, the Agent send the metrics to the vector +service, and the `datadog_metrics` sink sends to a separate +`fakeintake` service. This is the compare case. + +The two sets of data should be shaped the same in terms of when +the events were received, and the content of the events, but the +timestamps themselves are not guaranteed to align. diff --git a/scripts/e2e/e2e-datadog-metrics/compose.yaml b/scripts/e2e/datadog-metrics/compose.yaml similarity index 100% rename from scripts/e2e/e2e-datadog-metrics/compose.yaml rename to scripts/e2e/datadog-metrics/compose.yaml diff --git a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile b/scripts/e2e/datadog-metrics/dogstatsd_client/Dockerfile similarity index 100% rename from scripts/e2e/e2e-datadog-metrics/dogstatsd_client/Dockerfile rename to scripts/e2e/datadog-metrics/dogstatsd_client/Dockerfile diff --git a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py b/scripts/e2e/datadog-metrics/dogstatsd_client/client.py similarity index 100% rename from scripts/e2e/e2e-datadog-metrics/dogstatsd_client/client.py rename to scripts/e2e/datadog-metrics/dogstatsd_client/client.py diff --git a/scripts/e2e/e2e-datadog-metrics/dogstatsd_client/requirements.txt b/scripts/e2e/datadog-metrics/dogstatsd_client/requirements.txt similarity index 100% rename from scripts/e2e/e2e-datadog-metrics/dogstatsd_client/requirements.txt rename to scripts/e2e/datadog-metrics/dogstatsd_client/requirements.txt diff --git a/scripts/e2e/e2e-datadog-metrics/test.yaml b/scripts/e2e/datadog-metrics/test.yaml similarity index 100% rename from scripts/e2e/e2e-datadog-metrics/test.yaml rename to scripts/e2e/datadog-metrics/test.yaml diff --git a/vdev/src/commands/compose_tests/ci_paths.rs b/vdev/src/commands/compose_tests/ci_paths.rs new file mode 100644 index 0000000000000..e18b9ce0bc2d5 --- /dev/null +++ b/vdev/src/commands/compose_tests/ci_paths.rs @@ -0,0 +1,20 @@ +use anyhow::Result; + +use crate::testing::config::ComposeTestConfig; + +pub(crate) fn exec(path: &str) -> Result<()> { + // placeholder for changes that should run all integration tests + println!("all-tests: []"); + + // paths for each integration are defined in their respective config files. + for (test_name, config) in ComposeTestConfig::collect_all(path)? { + if let Some(paths) = config.paths { + println!("{test_name}:"); + for path in paths { + println!("- {path:?}"); + } + } + } + + Ok(()) +} diff --git a/vdev/src/commands/compose_tests/mod.rs b/vdev/src/commands/compose_tests/mod.rs new file mode 100644 index 0000000000000..27a033f199a12 --- /dev/null +++ b/vdev/src/commands/compose_tests/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod ci_paths; +pub(crate) mod show; +pub(crate) mod start; +pub(crate) mod stop; +pub(crate) mod test; diff --git a/vdev/src/commands/compose_tests/show.rs b/vdev/src/commands/compose_tests/show.rs new file mode 100644 index 0000000000000..672026ef442ec --- /dev/null +++ b/vdev/src/commands/compose_tests/show.rs @@ -0,0 +1,93 @@ +use anyhow::Result; + +use crate::testing::{config::ComposeTestConfig, config::Environment, state}; + +pub fn exec(integration: &Option, path: &str) -> Result<()> { + match integration { + None => { + let entries = ComposeTestConfig::collect_all(path)?; + let width = entries + .keys() + .fold(16, |width, entry| width.max(entry.len())); + println!("{:width$} Environment Name(s)", "Integration Name"); + println!("{:width$} -------------------", "----------------"); + for (integration, config) in entries { + let envs_dir = state::EnvsDir::new(&integration); + let active_env = envs_dir.active()?; + let environments = config + .environments() + .keys() + .map(|environment| format(&active_env, environment)) + .collect::>() + .join(" "); + println!("{integration:width$} {environments}"); + } + } + Some(integration) => { + let (_test_dir, config) = ComposeTestConfig::load(path, integration)?; + let envs_dir = state::EnvsDir::new(integration); + let active_env = envs_dir.active()?; + + if let Some(args) = &config.args { + println!("Test args: {}", args.join(" ")); + } else { + println!("Test args: N/A"); + } + + if config.features.is_empty() { + println!("Features: N/A"); + } else { + println!("Features: {}", config.features.join(",")); + } + + println!( + "Test filter: {}", + config.test_filter.as_deref().unwrap_or("N/A") + ); + + println!("Environment:"); + print_env(" ", &config.env); + println!("Runner:"); + println!(" Environment:"); + print_env(" ", &config.runner.env); + println!(" Volumes:"); + if config.runner.volumes.is_empty() { + println!(" N/A"); + } else { + for (target, mount) in &config.runner.volumes { + println!(" {target} => {mount}"); + } + } + println!( + " Needs docker socket: {}", + config.runner.needs_docker_socket + ); + + println!("Environments:"); + for environment in config.environments().keys() { + println!(" {}", format(&active_env, environment)); + } + } + } + Ok(()) +} + +fn format(active_env: &Option, environment: &str) -> String { + match active_env { + Some(active) if active == environment => format!("{environment} (active)"), + _ => environment.into(), + } +} + +fn print_env(prefix: &str, environment: &Environment) { + if environment.is_empty() { + println!("{prefix}N/A"); + } else { + for (key, value) in environment { + match value { + Some(value) => println!("{prefix}{key}={value:?}"), + None => println!("{prefix}{key} (passthrough)"), + } + } + } +} diff --git a/vdev/src/commands/compose_tests/start.rs b/vdev/src/commands/compose_tests/start.rs new file mode 100644 index 0000000000000..70dd45ab61927 --- /dev/null +++ b/vdev/src/commands/compose_tests/start.rs @@ -0,0 +1,16 @@ +use anyhow::Result; + +use crate::testing::{config::ComposeTestConfig, integration::ComposeTestT}; + +pub(crate) fn exec(integration: &str, environment: &Option) -> Result<()> { + let environment = if let Some(environment) = environment { + environment.clone() + } else { + let (_test_dir, config) = ComposeTestConfig::load(T::DIRECTORY, integration)?; + let envs = config.environments(); + let env = envs.keys().next().expect("Integration has no environments"); + env.clone() + }; + + T::start(&T::generate(integration, environment, false, 0)?) +} diff --git a/vdev/src/commands/compose_tests/stop.rs b/vdev/src/commands/compose_tests/stop.rs new file mode 100644 index 0000000000000..7a780faf4d507 --- /dev/null +++ b/vdev/src/commands/compose_tests/stop.rs @@ -0,0 +1,12 @@ +use anyhow::Result; + +use crate::testing::{integration::ComposeTestT, state::EnvsDir}; + +pub(crate) fn exec(test_name: &str, all_features: bool) -> Result<()> { + if let Some(active) = EnvsDir::new(test_name).active()? { + T::stop(&T::generate(test_name, active, all_features, 0)?) + } else { + println!("No environment for {test_name} is active."); + Ok(()) + } +} diff --git a/vdev/src/commands/compose_tests/test.rs b/vdev/src/commands/compose_tests/test.rs new file mode 100644 index 0000000000000..25c6175b5e6a5 --- /dev/null +++ b/vdev/src/commands/compose_tests/test.rs @@ -0,0 +1,39 @@ +use anyhow::{bail, Result}; + +use crate::testing::{config::ComposeTestConfig, integration::ComposeTestT, state::EnvsDir}; + +pub fn exec( + integration: &str, + environment: &Option, + build_all: bool, + retries: u8, + args: &[String], +) -> Result<()> { + let (_test_dir, config) = ComposeTestConfig::load(T::DIRECTORY, integration)?; + let envs = config.environments(); + + let active = EnvsDir::new(integration).active()?; + + match (&environment, &active) { + (Some(environment), Some(active)) if environment != active => { + bail!("Requested environment {environment:?} does not match active one {active:?}") + } + (Some(environment), _) => T::test( + &T::generate(integration, environment, build_all, retries)?, + args.to_owned(), + ), + (None, Some(active)) => T::test( + &T::generate(integration, active, build_all, retries)?, + args.to_owned(), + ), + (None, None) => { + for env_name in envs.keys() { + T::test( + &T::generate(integration, env_name, build_all, retries)?, + args.to_owned(), + )?; + } + Ok(()) + } + } +} diff --git a/vdev/src/commands/e2e/ci_paths.rs b/vdev/src/commands/e2e/ci_paths.rs new file mode 100644 index 0000000000000..d3679a3b7bb5a --- /dev/null +++ b/vdev/src/commands/e2e/ci_paths.rs @@ -0,0 +1,16 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::config::E2E_TESTS_DIR; + +/// Output paths in the repository that are associated with an integration. +/// If any changes are made to these paths, that integration should be tested. +#[derive(Args, Debug)] +#[command()] +pub struct Cli {} + +impl Cli { + pub fn exec(&self) -> Result<()> { + crate::commands::compose_tests::ci_paths::exec(E2E_TESTS_DIR) + } +} diff --git a/vdev/src/commands/e2e/mod.rs b/vdev/src/commands/e2e/mod.rs new file mode 100644 index 0000000000000..be9a6b77af2d8 --- /dev/null +++ b/vdev/src/commands/e2e/mod.rs @@ -0,0 +1,13 @@ +crate::cli_subcommands! { + r#"Manage end-to-end test environments... + +These test setups are organized into a set of integrations, located in subdirectories +`scripts/e2e`. For each integration, there is a matrix of environments, described in the +`matrix` setting in the `test.yaml` file contained therein."# + + mod show, + mod start, + mod stop, + mod test, + mod ci_paths, +} diff --git a/vdev/src/commands/e2e/show.rs b/vdev/src/commands/e2e/show.rs new file mode 100644 index 0000000000000..cc1645ba18019 --- /dev/null +++ b/vdev/src/commands/e2e/show.rs @@ -0,0 +1,18 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::config::E2E_TESTS_DIR; + +/// Show information about e2e-tests +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The desired e2e test name + test: Option, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::show::exec(&self.test, E2E_TESTS_DIR) + } +} diff --git a/vdev/src/commands/e2e/start.rs b/vdev/src/commands/e2e/start.rs new file mode 100644 index 0000000000000..d3d7eec503381 --- /dev/null +++ b/vdev/src/commands/e2e/start.rs @@ -0,0 +1,21 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::E2ETest; + +/// Start an environment +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The e2e test name + test: String, + + /// The desired environment name to start. If omitted, the first environment name is used. + environment: Option, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::start::exec::(&self.test, &self.environment) + } +} diff --git a/vdev/src/commands/e2e/stop.rs b/vdev/src/commands/e2e/stop.rs new file mode 100644 index 0000000000000..2a11fd5b4c976 --- /dev/null +++ b/vdev/src/commands/e2e/stop.rs @@ -0,0 +1,22 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::E2ETest; + +/// Stop an e2e-test environment +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The e2e test name to stop + test: String, + + /// If true, remove the runner container compiled with all integration test features + #[arg(short = 'a', long)] + all_features: bool, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::stop::exec::(&self.test, self.all_features) + } +} diff --git a/vdev/src/commands/e2e/test.rs b/vdev/src/commands/e2e/test.rs new file mode 100644 index 0000000000000..45d70a3fabad3 --- /dev/null +++ b/vdev/src/commands/e2e/test.rs @@ -0,0 +1,45 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::E2ETest; + +/// Execute end-to-end tests +/// +/// If an environment is named, it is used to run the test. If the environment was not previously started, +/// it is started before the test is run and stopped afterwards. +/// +/// If no environment is named, but one has been started already, that environment is used for the test. +/// +/// Otherwise, all environments are started, the test run, and then stopped, one by one. +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The desired e2e test + e2e_test: String, + + /// The desired environment (optional) + environment: Option, + + /// Whether to compile the test runner with all integration test features + #[arg(short = 'a', long)] + build_all: bool, + + /// Number of retries to allow on each integration test case. + #[arg(short = 'r', long)] + retries: Option, + + /// Extra test command arguments + args: Vec, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::test::exec::( + &self.e2e_test, + &self.environment, + self.build_all, + self.retries.unwrap_or_default(), + &self.args, + ) + } +} diff --git a/vdev/src/commands/integration/ci_paths.rs b/vdev/src/commands/integration/ci_paths.rs index c37b682a2ec7b..a91034c396f63 100644 --- a/vdev/src/commands/integration/ci_paths.rs +++ b/vdev/src/commands/integration/ci_paths.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::config::IntegrationTestConfig; +use crate::testing::config::INTEGRATION_TESTS_DIR; /// Output paths in the repository that are associated with an integration. /// If any changes are made to these paths, that integration should be tested. @@ -11,19 +11,6 @@ pub struct Cli {} impl Cli { pub fn exec(&self) -> Result<()> { - // placeholder for changes that should run all integration tests - println!("all-int: []"); - - // paths for each integration are defined in their respective config files. - for (integration, config) in IntegrationTestConfig::collect_all()? { - if let Some(paths) = config.paths { - println!("{integration}:"); - for path in paths { - println!("- \"{path}\""); - } - } - } - - Ok(()) + crate::commands::compose_tests::ci_paths::exec(INTEGRATION_TESTS_DIR) } } diff --git a/vdev/src/commands/integration/show.rs b/vdev/src/commands/integration/show.rs index c6394657534ec..410219e536312 100644 --- a/vdev/src/commands/integration/show.rs +++ b/vdev/src/commands/integration/show.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::{config::Environment, config::IntegrationTestConfig, state}; +use crate::testing::config::INTEGRATION_TESTS_DIR; /// Show information about integrations #[derive(Args, Debug)] @@ -13,86 +13,6 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - match self.integration { - None => { - let entries = IntegrationTestConfig::collect_all()?; - let width = entries - .keys() - .fold(16, |width, entry| width.max(entry.len())); - println!("{:width$} Environment Name(s)", "Integration Name"); - println!("{:width$} -------------------", "----------------"); - for (integration, config) in entries { - let envs_dir = state::EnvsDir::new(&integration); - let active_env = envs_dir.active()?; - let environments = config - .environments() - .keys() - .map(|environment| format(&active_env, environment)) - .collect::>() - .join(" "); - println!("{integration:width$} {environments}"); - } - } - Some(integration) => { - let (_test_dir, config) = IntegrationTestConfig::load(&integration)?; - let envs_dir = state::EnvsDir::new(&integration); - let active_env = envs_dir.active()?; - - if let Some(args) = &config.args { - println!("Test args: {}", args.join(" ")); - } else { - println!("Test args: N/A"); - } - - if config.features.is_empty() { - println!("Features: N/A"); - } else { - println!("Features: {}", config.features.join(",")); - } - - println!("Test filter: {}", config.test_filter.as_deref().unwrap_or("N/A")); - - println!("Environment:"); - print_env(" ", &config.env); - println!("Runner:"); - println!(" Environment:"); - print_env(" ", &config.runner.env); - println!(" Volumes:"); - if config.runner.volumes.is_empty() { - println!(" N/A"); - } else { - for (target, mount) in &config.runner.volumes { - println!(" {target} => {mount}"); - } - } - println!(" Needs docker socket: {}", config.runner.needs_docker_socket); - - println!("Environments:"); - for environment in config.environments().keys() { - println!(" {}", format(&active_env, environment)); - } - } - } - Ok(()) - } -} - -fn format(active_env: &Option, environment: &str) -> String { - match active_env { - Some(active) if active == environment => format!("{environment} (active)"), - _ => environment.into(), - } -} - -fn print_env(prefix: &str, environment: &Environment) { - if environment.is_empty() { - println!("{prefix}N/A"); - } else { - for (key, value) in environment { - match value { - Some(value) => println!("{prefix}{key}={value:?}"), - None => println!("{prefix}{key} (passthrough)"), - } - } + crate::commands::compose_tests::show::exec(&self.integration, INTEGRATION_TESTS_DIR) } } diff --git a/vdev/src/commands/integration/start.rs b/vdev/src/commands/integration/start.rs index da8947e74997a..8396ae7530f5d 100644 --- a/vdev/src/commands/integration/start.rs +++ b/vdev/src/commands/integration/start.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::{config::IntegrationTestConfig, integration::IntegrationTest}; +use crate::testing::integration::IntegrationTest; /// Start an environment #[derive(Args, Debug)] @@ -16,14 +16,9 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - let environment = if let Some(environment) = self.environment { - environment - } else { - let (_test_dir, config) = IntegrationTestConfig::load(&self.integration)?; - let envs = config.environments(); - let env = envs.keys().next().expect("Integration has no environments"); - env.clone() - }; - IntegrationTest::new(self.integration, environment, false, 0)?.start() + crate::commands::compose_tests::start::exec::( + &self.integration, + &self.environment, + ) } } diff --git a/vdev/src/commands/integration/stop.rs b/vdev/src/commands/integration/stop.rs index 22955d5fa7daf..0bacde212596d 100644 --- a/vdev/src/commands/integration/stop.rs +++ b/vdev/src/commands/integration/stop.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::{integration::IntegrationTest, state::EnvsDir}; +use crate::testing::integration::IntegrationTest; /// Stop an integration test environment #[derive(Args, Debug)] @@ -17,11 +17,9 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - if let Some(active) = EnvsDir::new(&self.integration).active()? { - IntegrationTest::new(self.integration, active, self.all_features, 0)?.stop() - } else { - println!("No environment for {:?} is active.", self.integration); - Ok(()) - } + crate::commands::compose_tests::stop::exec::( + &self.integration, + self.all_features, + ) } } diff --git a/vdev/src/commands/integration/test.rs b/vdev/src/commands/integration/test.rs index ca57d895bd0c8..e41015527cdca 100644 --- a/vdev/src/commands/integration/test.rs +++ b/vdev/src/commands/integration/test.rs @@ -1,7 +1,7 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use clap::Args; -use crate::testing::{config::IntegrationTestConfig, integration::IntegrationTest, state::EnvsDir}; +use crate::testing::integration::IntegrationTest; /// Execute integration tests /// @@ -34,32 +34,12 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - let (_test_dir, config) = IntegrationTestConfig::load(&self.integration)?; - let envs = config.environments(); - - let active = EnvsDir::new(&self.integration).active()?; - - let retries = self.retries.unwrap_or_default(); - - match (self.environment, active) { - (Some(environment), Some(active)) if environment != active => { - bail!("Requested environment {environment:?} does not match active one {active:?}") - } - (Some(environment), _) => { - IntegrationTest::new(self.integration, environment, self.build_all, retries)? - .test(self.args) - } - (None, Some(active)) => { - IntegrationTest::new(self.integration, active, self.build_all, retries)? - .test(self.args) - } - (None, None) => { - for env_name in envs.keys() { - IntegrationTest::new(&self.integration, env_name, self.build_all, retries)? - .test(self.args.clone())?; - } - Ok(()) - } - } + crate::commands::compose_tests::test::exec::( + &self.integration, + &self.environment, + self.build_all, + self.retries.unwrap_or_default(), + &self.args, + ) } } diff --git a/vdev/src/commands/mod.rs b/vdev/src/commands/mod.rs index fc9c9f8c539e6..9c5f0fb2e87d9 100644 --- a/vdev/src/commands/mod.rs +++ b/vdev/src/commands/mod.rs @@ -1,6 +1,8 @@ use clap::Parser; use clap_verbosity_flag::{InfoLevel, Verbosity}; +mod compose_tests; + /// This macro simplifies the generation of CLI subcommand invocation structures by combining the /// creation of the command enum and implementation of the dispatch function into one simple list. #[macro_export] @@ -75,6 +77,7 @@ cli_commands! { mod complete, mod config, mod crate_versions, + mod e2e, mod exec, mod features, mod fmt, diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 7e1ca36cd6452..314e7a0441ef1 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -12,6 +12,9 @@ use crate::{app, util}; const FILE_NAME: &str = "test.yaml"; +pub const INTEGRATION_TESTS_DIR: &str = "integration"; +pub const E2E_TESTS_DIR: &str = "e2e"; + pub type Environment = BTreeMap>; #[derive(Deserialize, Debug)] @@ -96,7 +99,7 @@ impl ComposeConfig { #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub struct IntegrationTestConfig { +pub struct ComposeTestConfig { /// The list of arguments to add to the command line for the test runner pub args: Option>, /// The set of environment variables to set in both the services and the runner. Variables with @@ -135,11 +138,11 @@ pub struct IntegrationRunnerConfig { pub needs_docker_socket: bool, } -impl IntegrationTestConfig { +impl ComposeTestConfig { fn parse_file(config_file: &Path) -> Result { let contents = fs::read_to_string(config_file) .with_context(|| format!("failed to read {}", config_file.display()))?; - let config: IntegrationTestConfig = serde_yaml::from_str(&contents).with_context(|| { + let config: Self = serde_yaml::from_str(&contents).with_context(|| { format!( "failed to parse integration test configuration file {}", config_file.display() @@ -166,22 +169,13 @@ impl IntegrationTestConfig { .collect() } - pub fn load(integration: &str) -> Result<(PathBuf, Self)> { - let mut test_dir: PathBuf = [app::path(), "scripts", "integration", integration] + pub fn load(root_dir: &str, integration: &str) -> Result<(PathBuf, Self)> { + let test_dir: PathBuf = [app::path(), "scripts", root_dir, integration] .iter() .collect(); + if !test_dir.is_dir() { - // try the e2e dir now - - // TODO: This is a temporary solution, looking in both dirs. When this GH issue - // https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor - // to have a separate e2e subcommand that both int and e2e will leverage. - test_dir = [app::path(), "scripts", "e2e", integration] - .iter() - .collect(); - if !test_dir.is_dir() { - bail!("unknown integration: {}", integration); - } + bail!("unknown integration: {}", integration); } let config = Self::parse_file(&test_dir.join(FILE_NAME))?; @@ -203,17 +197,12 @@ impl IntegrationTestConfig { Ok(()) } - pub fn collect_all() -> Result> { + pub fn collect_all(root_dir: &str) -> Result> { let mut configs = BTreeMap::new(); - // TODO: This is a temporary solution, looking in both dirs. When this GH issue - // https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor - // to have a separate e2e subcommand that both int and e2e will leverage. - let int_tests_dir: PathBuf = [app::path(), "scripts", "integration"].iter().collect(); - let e2e_tests_dir: PathBuf = [app::path(), "scripts", "e2e"].iter().collect(); + let tests_dir: PathBuf = [app::path(), "scripts", root_dir].iter().collect(); - Self::collect_all_dir(&int_tests_dir, &mut configs)?; - Self::collect_all_dir(&e2e_tests_dir, &mut configs)?; + Self::collect_all_dir(&tests_dir, &mut configs)?; Ok(configs) } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index abb1af7a1e5a6..63161862f8d52 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -3,8 +3,9 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command use anyhow::{bail, Context, Result}; use tempfile::{Builder, NamedTempFile}; -use super::config::ComposeConfig; -use super::config::{Environment, IntegrationTestConfig}; +use super::config::{ + ComposeConfig, ComposeTestConfig, Environment, E2E_TESTS_DIR, INTEGRATION_TESTS_DIR, +}; use super::runner::{ ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, }; @@ -14,10 +15,13 @@ use crate::testing::config::get_rust_version; const NETWORK_ENV_VAR: &str = "VECTOR_NETWORK"; -pub struct IntegrationTest { - integration: String, +const INTEGRATION_FEATURE_FLAG: &str = "all-integration-tests"; +const E2E_FEATURE_FLAG: &str = "all-e2e-tests"; + +pub(crate) struct ComposeTest { + test_name: String, environment: String, - config: IntegrationTestConfig, + config: ComposeTestConfig, envs_dir: EnvsDir, runner: IntegrationTestRunner, compose: Option, @@ -26,26 +30,30 @@ pub struct IntegrationTest { retries: u8, } -impl IntegrationTest { - pub fn new( - integration: impl Into, +pub(crate) trait ComposeTestT { + const DIRECTORY: &'static str; + + const FEATURE_FLAG: &'static str; + + fn generate( + test_name: impl Into, environment: impl Into, build_all: bool, retries: u8, - ) -> Result { - let integration = integration.into(); + ) -> Result { + let test_name = test_name.into(); let environment = environment.into(); - let (test_dir, config) = IntegrationTestConfig::load(&integration)?; - let envs_dir = EnvsDir::new(&integration); + let (test_dir, config) = ComposeTestConfig::load(Self::DIRECTORY, &test_name)?; + let envs_dir = EnvsDir::new(&test_name); let Some(mut env_config) = config.environments().get(&environment).map(Clone::clone) else { bail!("Could not find environment named {environment:?}"); }; - let network_name = format!("vector-integration-tests-{integration}"); + let network_name = format!("vector-integration-tests-{test_name}"); let compose = Compose::new(test_dir, env_config.clone(), network_name.clone())?; // None if compiling with all integration test feature flag. - let runner_name = (!build_all).then(|| integration.clone()); + let runner_name = (!build_all).then(|| test_name.clone()); let runner = IntegrationTestRunner::new( runner_name, @@ -55,8 +63,8 @@ impl IntegrationTest { env_config.insert("VECTOR_IMAGE".to_string(), Some(runner.image_name())); - Ok(Self { - integration, + Ok(ComposeTest { + test_name, environment, config, envs_dir, @@ -68,33 +76,35 @@ impl IntegrationTest { }) } - pub fn test(self, extra_args: Vec) -> Result<()> { - let active = self.envs_dir.check_active(&self.environment)?; - self.config.check_required()?; + fn test(compose_test: &ComposeTest, extra_args: Vec) -> Result<()> { + let active = compose_test + .envs_dir + .check_active(&compose_test.environment)?; + compose_test.config.check_required()?; if !active { - self.start()?; + Self::start(compose_test)?; } - let mut env_vars = self.config.env.clone(); + let mut env_vars = compose_test.config.env.clone(); // Make sure the test runner has the same config environment vars as the services do. - for (key, value) in config_env(&self.env_config) { + for (key, value) in config_env(&compose_test.env_config) { env_vars.insert(key, Some(value)); } env_vars.insert("TEST_LOG".to_string(), Some("info".into())); - let mut args = self.config.args.clone().unwrap_or_default(); + let mut args = compose_test.config.args.clone().unwrap_or_default(); args.push("--features".to_string()); - args.push(if self.build_all { - "all-integration-tests".to_string() + args.push(if compose_test.build_all { + Self::FEATURE_FLAG.to_string() } else { - self.config.features.join(",") + compose_test.config.features.join(",") }); // If the test field is not present then use the --lib flag - match self.config.test { + match compose_test.config.test { Some(ref test_arg) => { args.push("--test".to_string()); args.push(test_arg.to_string()); @@ -103,7 +113,7 @@ impl IntegrationTest { } // Ensure the test_filter args are passed as well - if let Some(ref filter) = self.config.test_filter { + if let Some(ref filter) = compose_test.config.test_filter { args.push(filter.to_string()); } args.extend(extra_args); @@ -113,63 +123,90 @@ impl IntegrationTest { args.push("--no-capture".to_string()); } - if self.retries > 0 { + if compose_test.retries > 0 { args.push("--retries".to_string()); - args.push(self.retries.to_string()); + args.push(compose_test.retries.to_string()); } - self.runner.test( + compose_test.runner.test( &env_vars, - &self.config.runner.env, - Some(&self.config.features), + &compose_test.config.runner.env, + Some(&compose_test.config.features), &args, )?; if !active { - self.runner.remove()?; - self.stop()?; + compose_test.runner.remove()?; + Self::stop(compose_test)?; } Ok(()) } - pub fn start(&self) -> Result<()> { + fn start(compose_test: &ComposeTest) -> Result<()> { // For end-to-end tests, we want to run vector as a service, leveraging the // image for the runner. So we must build that image before starting the // compose so that it is available. - self.runner.build(Some(&self.config.features))?; - - self.config.check_required()?; - if let Some(compose) = &self.compose { - self.runner.ensure_network()?; - - if self.envs_dir.check_active(&self.environment)? { + compose_test + .runner + .build(Some(&compose_test.config.features))?; + + compose_test.config.check_required()?; + if let Some(compose) = &compose_test.compose { + compose_test.runner.ensure_network()?; + + if compose_test + .envs_dir + .check_active(&compose_test.environment)? + { bail!("environment is already up"); } - compose.start(&self.env_config)?; + compose.start(&compose_test.env_config)?; - self.envs_dir.save(&self.environment, &self.env_config) + compose_test + .envs_dir + .save(&compose_test.environment, &compose_test.env_config) } else { Ok(()) } } - pub fn stop(&self) -> Result<()> { - if let Some(compose) = &self.compose { + fn stop(compose_test: &ComposeTest) -> Result<()> { + if let Some(compose) = &compose_test.compose { // TODO: Is this check really needed? - if self.envs_dir.load()?.is_none() { - bail!("No environment for {} is up.", self.integration); + if compose_test.envs_dir.load()?.is_none() { + bail!("No environment for {} is up.", compose_test.test_name); } - self.runner.remove()?; + compose_test.runner.remove()?; compose.stop()?; - self.envs_dir.remove()?; + compose_test.envs_dir.remove()?; } Ok(()) } } +/// Integration tests are located in the `scripts/integration` dir, +/// and are the full feature flag is `all-integration-tests`. +pub(crate) struct IntegrationTest; + +impl ComposeTestT for IntegrationTest { + const DIRECTORY: &'static str = INTEGRATION_TESTS_DIR; + + const FEATURE_FLAG: &'static str = INTEGRATION_FEATURE_FLAG; +} + +/// E2E tests are located in the `scripts/e2e` dir, +/// and are the full feature flag is `all-e2e-tests`. +pub(crate) struct E2ETest; + +impl ComposeTestT for E2ETest { + const DIRECTORY: &'static str = E2E_TESTS_DIR; + + const FEATURE_FLAG: &'static str = E2E_FEATURE_FLAG; +} + struct Compose { original_path: PathBuf, test_dir: PathBuf, From 34f9e7ad7ff1f1d4d5d5b70d55fc5b289b4e4c36 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:04:07 -0700 Subject: [PATCH 90/99] merge conflict --- vdev/src/commands/compose_tests/start.rs | 8 ++++++-- vdev/src/commands/e2e/start.rs | 10 +++++++++- vdev/src/commands/integration/start.rs | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/vdev/src/commands/compose_tests/start.rs b/vdev/src/commands/compose_tests/start.rs index 70dd45ab61927..3ccb7cdb57125 100644 --- a/vdev/src/commands/compose_tests/start.rs +++ b/vdev/src/commands/compose_tests/start.rs @@ -2,7 +2,11 @@ use anyhow::Result; use crate::testing::{config::ComposeTestConfig, integration::ComposeTestT}; -pub(crate) fn exec(integration: &str, environment: &Option) -> Result<()> { +pub(crate) fn exec( + integration: &str, + environment: &Option, + build_all: bool, +) -> Result<()> { let environment = if let Some(environment) = environment { environment.clone() } else { @@ -12,5 +16,5 @@ pub(crate) fn exec(integration: &str, environment: &Option, } impl Cli { pub fn exec(self) -> Result<()> { - crate::commands::compose_tests::start::exec::(&self.test, &self.environment) + crate::commands::compose_tests::start::exec::( + &self.test, + &self.environment, + self.build_all, + ) } } diff --git a/vdev/src/commands/integration/start.rs b/vdev/src/commands/integration/start.rs index 46d4355e8c177..51a247da5fc3f 100644 --- a/vdev/src/commands/integration/start.rs +++ b/vdev/src/commands/integration/start.rs @@ -23,7 +23,7 @@ impl Cli { crate::commands::compose_tests::start::exec::( &self.integration, &self.environment, - &self.build_all, + self.build_all, ) } } From 12318fbba08230400cd7a8017e08ec4b3db7d9a3 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:23:34 -0700 Subject: [PATCH 91/99] update workflows --- .github/workflows/e2e.yml | 4 +- .github/workflows/integration-comment.yml | 74 +++++++++++------------ .github/workflows/integration.yml | 70 ++++++++++----------- scripts/ci-integration-test.sh | 9 +-- 4 files changed, 79 insertions(+), 78 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 40b58ffa80c96..5dd648c33def0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -81,7 +81,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs + command: bash scripts/ci-integration-test.sh e2e datadog-logs - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -90,7 +90,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-metrics + command: bash scripts/ci-integration-test.sh e2e datadog-metrics e2e-test-suite: diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 481ee1db273ad..383309a2fe83b 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -100,7 +100,7 @@ jobs: # First one requires more time, as we need to build the image from scratch timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh amqp + command: bash scripts/ci-integration-test.sh int amqp - name: appsignal if: ${{ contains(github.event.comment.body, '/ci-run-integration-appsignal') @@ -110,7 +110,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh appsignal + command: bash scripts/ci-integration-test.sh int appsignal - name: aws if: ${{ contains(github.event.comment.body, '/ci-run-integration-aws') @@ -120,7 +120,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh aws + command: bash scripts/ci-integration-test.sh int aws - name: axiom if: ${{ contains(github.event.comment.body, '/ci-run-integration-axiom') @@ -130,7 +130,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh axiom + command: bash scripts/ci-integration-test.sh int axiom - name: azure if: ${{ contains(github.event.comment.body, '/ci-run-integration-azure') @@ -140,7 +140,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh azure + command: bash scripts/ci-integration-test.sh int azure - name: clickhouse if: ${{ contains(github.event.comment.body, '/ci-run-integration-clickhouse') @@ -150,7 +150,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh clickhouse + command: bash scripts/ci-integration-test.sh int clickhouse - name: databend if: ${{ contains(github.event.comment.body, '/ci-run-integration-databend') @@ -160,7 +160,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh databend + command: bash scripts/ci-integration-test.sh int databend - name: datadog-agent if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-agent') @@ -170,7 +170,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-agent + command: bash scripts/ci-integration-test.sh int datadog-agent - name: datadog-logs if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-logs') @@ -180,7 +180,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-logs + command: bash scripts/ci-integration-test.sh int datadog-logs - name: datadog-metrics if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-metrics') @@ -190,7 +190,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-metrics + command: bash scripts/ci-integration-test.sh int datadog-metrics - name: datadog-traces if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-traces') @@ -200,7 +200,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-traces + command: bash scripts/ci-integration-test.sh int datadog-traces - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') @@ -210,7 +210,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh dnstap + command: bash scripts/ci-integration-test.sh int dnstap - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -222,7 +222,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh docker-logs + command: bash scripts/ci-integration-test.sh int docker-logs - name: elasticsearch if: ${{ contains(github.event.comment.body, '/ci-run-integration-elasticsearch') @@ -232,7 +232,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh elasticsearch + command: bash scripts/ci-integration-test.sh int elasticsearch - name: eventstoredb if: ${{ contains(github.event.comment.body, '/ci-run-integration-eventstoredb') @@ -242,7 +242,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh eventstoredb + command: bash scripts/ci-integration-test.sh int eventstoredb - name: fluent if: ${{ contains(github.event.comment.body, '/ci-run-integration-fluent') @@ -252,7 +252,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh fluent + command: bash scripts/ci-integration-test.sh int fluent - name: gcp if: ${{ contains(github.event.comment.body, '/ci-run-integration-gcp') @@ -262,7 +262,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh gcp + command: bash scripts/ci-integration-test.sh int gcp - name: greptimedb if: ${{ contains(github.event.comment.body, '/ci-run-integration-greptimedb') @@ -272,7 +272,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh greptimedb + command: bash scripts/ci-integration-test.sh int greptimedb - name: humio if: ${{ contains(github.event.comment.body, '/ci-run-integration-humio') @@ -282,7 +282,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh humio + command: bash scripts/ci-integration-test.sh int humio - name: http-client if: ${{ contains(github.event.comment.body, '/ci-run-integration-http-client') @@ -292,7 +292,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh http-client + command: bash scripts/ci-integration-test.sh int http-client - name: influxdb if: ${{ contains(github.event.comment.body, '/ci-run-integration-influxdb') @@ -302,7 +302,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh influxdb + command: bash scripts/ci-integration-test.sh int influxdb - name: kafka if: ${{ contains(github.event.comment.body, '/ci-run-integration-kafka') @@ -312,7 +312,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh kafka + command: bash scripts/ci-integration-test.sh int kafka - name: logstash if: ${{ contains(github.event.comment.body, '/ci-run-integration-logstash') @@ -322,7 +322,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh logstash + command: bash scripts/ci-integration-test.sh int logstash - name: loki if: ${{ contains(github.event.comment.body, '/ci-run-integration-loki') @@ -332,7 +332,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh loki + command: bash scripts/ci-integration-test.sh int loki - name: mongodb if: ${{ contains(github.event.comment.body, '/ci-run-integration-mongodb') @@ -342,7 +342,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh mongodb + command: bash scripts/ci-integration-test.sh int mongodb - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -354,7 +354,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nats + command: bash scripts/ci-integration-test.sh int nats - name: nginx if: ${{ contains(github.event.comment.body, '/ci-run-integration-nginx') @@ -364,7 +364,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nginx + command: bash scripts/ci-integration-test.sh int nginx - name: opentelemetry if: ${{ contains(github.event.comment.body, '/ci-run-integration-opentelemetry') @@ -374,7 +374,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh opentelemetry + command: bash scripts/ci-integration-test.sh int opentelemetry - name: postgres if: ${{ contains(github.event.comment.body, '/ci-run-integration-postgres') @@ -384,7 +384,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh postgres + command: bash scripts/ci-integration-test.sh int postgres - name: prometheus if: ${{ contains(github.event.comment.body, '/ci-run-integration-prometheus') @@ -394,7 +394,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh prometheus + command: bash scripts/ci-integration-test.sh int prometheus - name: pulsar if: ${{ contains(github.event.comment.body, '/ci-run-integration-pulsar') @@ -404,7 +404,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh pulsar + command: bash scripts/ci-integration-test.sh int pulsar - name: redis if: ${{ contains(github.event.comment.body, '/ci-run-integration-redis') @@ -414,7 +414,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh redis + command: bash scripts/ci-integration-test.sh int redis - name: shutdown if: ${{ contains(github.event.comment.body, '/ci-run-integration-shutdown') @@ -424,7 +424,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh shutdown + command: bash scripts/ci-integration-test.sh int shutdown - name: splunk if: ${{ contains(github.event.comment.body, '/ci-run-integration-splunk') @@ -434,7 +434,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh splunk + command: bash scripts/ci-integration-test.sh int splunk - name: webhdfs if: ${{ contains(github.event.comment.body, '/ci-run-integration-webhdfs') @@ -444,7 +444,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh webhdfs + command: bash scripts/ci-integration-test.sh int webhdfs e2e-tests: needs: prep-pr @@ -465,7 +465,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs + command: bash scripts/ci-integration-test.sh e2e datadog-logs - name: datadog-e2e-metrics if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') @@ -475,7 +475,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-e2e-metrics + command: bash scripts/ci-integration-test.sh e2e datadog-metrics update-pr-status: name: Signal result to PR diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 575e09da7f13f..20305d07032e1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -116,7 +116,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh amqp + command: bash scripts/ci-integration-test.sh int amqp - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -125,7 +125,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh appsignal + command: bash scripts/ci-integration-test.sh int appsignal - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.aws == 'true' }} name: aws @@ -133,7 +133,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh aws + command: bash scripts/ci-integration-test.sh int aws - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -142,7 +142,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh axiom + command: bash scripts/ci-integration-test.sh int axiom - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.azure == 'true' }} name: azure @@ -150,7 +150,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh azure + command: bash scripts/ci-integration-test.sh int azure - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.clickhouse == 'true' }} name: clickhouse @@ -158,7 +158,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh clickhouse + command: bash scripts/ci-integration-test.sh int clickhouse - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -167,7 +167,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh databend + command: bash scripts/ci-integration-test.sh int databend - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -176,7 +176,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-agent + command: bash scripts/ci-integration-test.sh int datadog-agent - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -185,7 +185,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-logs + command: bash scripts/ci-integration-test.sh int datadog-logs - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -196,7 +196,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-metrics + command: bash scripts/ci-integration-test.sh int datadog-metrics - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -205,7 +205,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-traces + command: bash scripts/ci-integration-test.sh int datadog-traces - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap @@ -213,7 +213,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh dnstap + command: bash scripts/ci-integration-test.sh int dnstap - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }} name: docker-logs @@ -221,7 +221,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh docker-logs + command: bash scripts/ci-integration-test.sh int docker-logs - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.elasticsearch == 'true' }} name: elasticsearch @@ -229,7 +229,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh elasticsearch + command: bash scripts/ci-integration-test.sh int elasticsearch - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.eventstoredb == 'true' }} name: eventstoredb @@ -237,7 +237,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh eventstoredb + command: bash scripts/ci-integration-test.sh int eventstoredb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.fluent == 'true' }} name: fluent @@ -245,7 +245,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh fluent + command: bash scripts/ci-integration-test.sh int fluent - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.gcp == 'true' }} name: gcp @@ -253,7 +253,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh gcp + command: bash scripts/ci-integration-test.sh int gcp - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.greptimedb == 'true' }} name: greptimedb @@ -261,7 +261,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh greptimedb + command: bash scripts/ci-integration-test.sh int greptimedb - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -271,7 +271,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh humio + command: bash scripts/ci-integration-test.sh int humio - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.http-client == 'true' }} name: http-client @@ -279,7 +279,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh http-client + command: bash scripts/ci-integration-test.sh int http-client - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.influxdb == 'true' }} name: influxdb @@ -287,7 +287,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh influxdb + command: bash scripts/ci-integration-test.sh int influxdb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.kafka == 'true' }} name: kafka @@ -295,7 +295,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh kafka + command: bash scripts/ci-integration-test.sh int kafka - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.logstash == 'true' }} name: logstash @@ -303,7 +303,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh logstash + command: bash scripts/ci-integration-test.sh int logstash - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.loki == 'true' }} name: loki @@ -311,7 +311,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh loki + command: bash scripts/ci-integration-test.sh int loki - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mongodb == 'true' }} name: mongodb @@ -319,7 +319,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh mongodb + command: bash scripts/ci-integration-test.sh int mongodb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }} name: nats @@ -327,7 +327,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nats + command: bash scripts/ci-integration-test.sh int nats - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nginx == 'true' }} name: nginx @@ -335,7 +335,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nginx + command: bash scripts/ci-integration-test.sh int nginx - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -345,7 +345,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh opentelemetry + command: bash scripts/ci-integration-test.sh int opentelemetry - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }} name: postgres @@ -353,7 +353,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh postgres + command: bash scripts/ci-integration-test.sh int postgres - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.prometheus == 'true' }} name: prometheus @@ -361,7 +361,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh prometheus + command: bash scripts/ci-integration-test.sh int prometheus - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.pulsar == 'true' }} name: pulsar @@ -369,7 +369,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh pulsar + command: bash scripts/ci-integration-test.sh int pulsar - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.redis == 'true' }} name: redis @@ -377,7 +377,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh redis + command: bash scripts/ci-integration-test.sh int redis - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' }} name: shutdown @@ -385,7 +385,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh shutdown + command: bash scripts/ci-integration-test.sh int shutdown - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.splunk == 'true' }} name: splunk @@ -393,7 +393,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh splunk + command: bash scripts/ci-integration-test.sh int splunk - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.webhdfs == 'true' }} name: webhdfs @@ -401,7 +401,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh webhdfs + command: bash scripts/ci-integration-test.sh int webhdfs integration-test-suite: name: Integration Test Suite diff --git a/scripts/ci-integration-test.sh b/scripts/ci-integration-test.sh index 49780e7bba378..13cc85026356d 100755 --- a/scripts/ci-integration-test.sh +++ b/scripts/ci-integration-test.sh @@ -19,12 +19,13 @@ fi set -x -INTEGRATION=$1 +TEST_TYPE=$1 # either "int" or "e2e" +TEST_NAME=$2 -cargo vdev -v int start -a "${INTEGRATION}" +cargo vdev -v "${TEST_TYPE}" start -a "${TEST_NAME}" sleep 30 -cargo vdev -v int test --retries 2 -a "${INTEGRATION}" +cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" RET=$? -cargo vdev -v int stop -a "${INTEGRATION}" +cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" ./scripts/upload-test-results.sh exit $RET From 68d0fb7816692dff86856caf5f24e360f616e620 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:25:25 -0700 Subject: [PATCH 92/99] rename --- .github/workflows/e2e.yml | 4 +- .github/workflows/integration-comment.yml | 74 +++++++++---------- .github/workflows/integration.yml | 70 +++++++++--------- ...integration-test.sh => ci-int-e2e-test.sh} | 0 4 files changed, 74 insertions(+), 74 deletions(-) rename scripts/{ci-integration-test.sh => ci-int-e2e-test.sh} (100%) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5dd648c33def0..14ebd8d19dacc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -81,7 +81,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e datadog-logs + command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -90,7 +90,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e datadog-metrics + command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics e2e-test-suite: diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 383309a2fe83b..c84246d116a8b 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -100,7 +100,7 @@ jobs: # First one requires more time, as we need to build the image from scratch timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int amqp + command: bash scripts/ci-int-e2e-test.sh int amqp - name: appsignal if: ${{ contains(github.event.comment.body, '/ci-run-integration-appsignal') @@ -110,7 +110,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int appsignal + command: bash scripts/ci-int-e2e-test.sh int appsignal - name: aws if: ${{ contains(github.event.comment.body, '/ci-run-integration-aws') @@ -120,7 +120,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int aws + command: bash scripts/ci-int-e2e-test.sh int aws - name: axiom if: ${{ contains(github.event.comment.body, '/ci-run-integration-axiom') @@ -130,7 +130,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int axiom + command: bash scripts/ci-int-e2e-test.sh int axiom - name: azure if: ${{ contains(github.event.comment.body, '/ci-run-integration-azure') @@ -140,7 +140,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int azure + command: bash scripts/ci-int-e2e-test.sh int azure - name: clickhouse if: ${{ contains(github.event.comment.body, '/ci-run-integration-clickhouse') @@ -150,7 +150,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int clickhouse + command: bash scripts/ci-int-e2e-test.sh int clickhouse - name: databend if: ${{ contains(github.event.comment.body, '/ci-run-integration-databend') @@ -160,7 +160,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int databend + command: bash scripts/ci-int-e2e-test.sh int databend - name: datadog-agent if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-agent') @@ -170,7 +170,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-agent + command: bash scripts/ci-int-e2e-test.sh int datadog-agent - name: datadog-logs if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-logs') @@ -180,7 +180,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-logs + command: bash scripts/ci-int-e2e-test.sh int datadog-logs - name: datadog-metrics if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-metrics') @@ -190,7 +190,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-metrics + command: bash scripts/ci-int-e2e-test.sh int datadog-metrics - name: datadog-traces if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-traces') @@ -200,7 +200,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-traces + command: bash scripts/ci-int-e2e-test.sh int datadog-traces - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') @@ -210,7 +210,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int dnstap + command: bash scripts/ci-int-e2e-test.sh int dnstap - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -222,7 +222,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int docker-logs + command: bash scripts/ci-int-e2e-test.sh int docker-logs - name: elasticsearch if: ${{ contains(github.event.comment.body, '/ci-run-integration-elasticsearch') @@ -232,7 +232,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int elasticsearch + command: bash scripts/ci-int-e2e-test.sh int elasticsearch - name: eventstoredb if: ${{ contains(github.event.comment.body, '/ci-run-integration-eventstoredb') @@ -242,7 +242,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int eventstoredb + command: bash scripts/ci-int-e2e-test.sh int eventstoredb - name: fluent if: ${{ contains(github.event.comment.body, '/ci-run-integration-fluent') @@ -252,7 +252,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int fluent + command: bash scripts/ci-int-e2e-test.sh int fluent - name: gcp if: ${{ contains(github.event.comment.body, '/ci-run-integration-gcp') @@ -262,7 +262,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int gcp + command: bash scripts/ci-int-e2e-test.sh int gcp - name: greptimedb if: ${{ contains(github.event.comment.body, '/ci-run-integration-greptimedb') @@ -272,7 +272,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int greptimedb + command: bash scripts/ci-int-e2e-test.sh int greptimedb - name: humio if: ${{ contains(github.event.comment.body, '/ci-run-integration-humio') @@ -282,7 +282,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int humio + command: bash scripts/ci-int-e2e-test.sh int humio - name: http-client if: ${{ contains(github.event.comment.body, '/ci-run-integration-http-client') @@ -292,7 +292,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int http-client + command: bash scripts/ci-int-e2e-test.sh int http-client - name: influxdb if: ${{ contains(github.event.comment.body, '/ci-run-integration-influxdb') @@ -302,7 +302,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int influxdb + command: bash scripts/ci-int-e2e-test.sh int influxdb - name: kafka if: ${{ contains(github.event.comment.body, '/ci-run-integration-kafka') @@ -312,7 +312,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int kafka + command: bash scripts/ci-int-e2e-test.sh int kafka - name: logstash if: ${{ contains(github.event.comment.body, '/ci-run-integration-logstash') @@ -322,7 +322,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int logstash + command: bash scripts/ci-int-e2e-test.sh int logstash - name: loki if: ${{ contains(github.event.comment.body, '/ci-run-integration-loki') @@ -332,7 +332,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int loki + command: bash scripts/ci-int-e2e-test.sh int loki - name: mongodb if: ${{ contains(github.event.comment.body, '/ci-run-integration-mongodb') @@ -342,7 +342,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int mongodb + command: bash scripts/ci-int-e2e-test.sh int mongodb - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -354,7 +354,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int nats + command: bash scripts/ci-int-e2e-test.sh int nats - name: nginx if: ${{ contains(github.event.comment.body, '/ci-run-integration-nginx') @@ -364,7 +364,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int nginx + command: bash scripts/ci-int-e2e-test.sh int nginx - name: opentelemetry if: ${{ contains(github.event.comment.body, '/ci-run-integration-opentelemetry') @@ -374,7 +374,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int opentelemetry + command: bash scripts/ci-int-e2e-test.sh int opentelemetry - name: postgres if: ${{ contains(github.event.comment.body, '/ci-run-integration-postgres') @@ -384,7 +384,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int postgres + command: bash scripts/ci-int-e2e-test.sh int postgres - name: prometheus if: ${{ contains(github.event.comment.body, '/ci-run-integration-prometheus') @@ -394,7 +394,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int prometheus + command: bash scripts/ci-int-e2e-test.sh int prometheus - name: pulsar if: ${{ contains(github.event.comment.body, '/ci-run-integration-pulsar') @@ -404,7 +404,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int pulsar + command: bash scripts/ci-int-e2e-test.sh int pulsar - name: redis if: ${{ contains(github.event.comment.body, '/ci-run-integration-redis') @@ -414,7 +414,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int redis + command: bash scripts/ci-int-e2e-test.sh int redis - name: shutdown if: ${{ contains(github.event.comment.body, '/ci-run-integration-shutdown') @@ -424,7 +424,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int shutdown + command: bash scripts/ci-int-e2e-test.sh int shutdown - name: splunk if: ${{ contains(github.event.comment.body, '/ci-run-integration-splunk') @@ -434,7 +434,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int splunk + command: bash scripts/ci-int-e2e-test.sh int splunk - name: webhdfs if: ${{ contains(github.event.comment.body, '/ci-run-integration-webhdfs') @@ -444,7 +444,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int webhdfs + command: bash scripts/ci-int-e2e-test.sh int webhdfs e2e-tests: needs: prep-pr @@ -465,7 +465,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e datadog-logs + command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs - name: datadog-e2e-metrics if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') @@ -475,7 +475,7 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e datadog-metrics + command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics update-pr-status: name: Signal result to PR diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 20305d07032e1..afc862265f8bc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -116,7 +116,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int amqp + command: bash scripts/ci-int-e2e-test.sh int amqp - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -125,7 +125,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int appsignal + command: bash scripts/ci-int-e2e-test.sh int appsignal - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.aws == 'true' }} name: aws @@ -133,7 +133,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int aws + command: bash scripts/ci-int-e2e-test.sh int aws - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -142,7 +142,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int axiom + command: bash scripts/ci-int-e2e-test.sh int axiom - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.azure == 'true' }} name: azure @@ -150,7 +150,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int azure + command: bash scripts/ci-int-e2e-test.sh int azure - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.clickhouse == 'true' }} name: clickhouse @@ -158,7 +158,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int clickhouse + command: bash scripts/ci-int-e2e-test.sh int clickhouse - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -167,7 +167,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int databend + command: bash scripts/ci-int-e2e-test.sh int databend - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -176,7 +176,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-agent + command: bash scripts/ci-int-e2e-test.sh int datadog-agent - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -185,7 +185,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-logs + command: bash scripts/ci-int-e2e-test.sh int datadog-logs - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -196,7 +196,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-metrics + command: bash scripts/ci-int-e2e-test.sh int datadog-metrics - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -205,7 +205,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int datadog-traces + command: bash scripts/ci-int-e2e-test.sh int datadog-traces - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap @@ -213,7 +213,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int dnstap + command: bash scripts/ci-int-e2e-test.sh int dnstap - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }} name: docker-logs @@ -221,7 +221,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int docker-logs + command: bash scripts/ci-int-e2e-test.sh int docker-logs - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.elasticsearch == 'true' }} name: elasticsearch @@ -229,7 +229,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int elasticsearch + command: bash scripts/ci-int-e2e-test.sh int elasticsearch - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.eventstoredb == 'true' }} name: eventstoredb @@ -237,7 +237,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int eventstoredb + command: bash scripts/ci-int-e2e-test.sh int eventstoredb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.fluent == 'true' }} name: fluent @@ -245,7 +245,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int fluent + command: bash scripts/ci-int-e2e-test.sh int fluent - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.gcp == 'true' }} name: gcp @@ -253,7 +253,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int gcp + command: bash scripts/ci-int-e2e-test.sh int gcp - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.greptimedb == 'true' }} name: greptimedb @@ -261,7 +261,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int greptimedb + command: bash scripts/ci-int-e2e-test.sh int greptimedb - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -271,7 +271,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int humio + command: bash scripts/ci-int-e2e-test.sh int humio - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.http-client == 'true' }} name: http-client @@ -279,7 +279,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int http-client + command: bash scripts/ci-int-e2e-test.sh int http-client - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.influxdb == 'true' }} name: influxdb @@ -287,7 +287,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int influxdb + command: bash scripts/ci-int-e2e-test.sh int influxdb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.kafka == 'true' }} name: kafka @@ -295,7 +295,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int kafka + command: bash scripts/ci-int-e2e-test.sh int kafka - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.logstash == 'true' }} name: logstash @@ -303,7 +303,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int logstash + command: bash scripts/ci-int-e2e-test.sh int logstash - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.loki == 'true' }} name: loki @@ -311,7 +311,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int loki + command: bash scripts/ci-int-e2e-test.sh int loki - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mongodb == 'true' }} name: mongodb @@ -319,7 +319,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int mongodb + command: bash scripts/ci-int-e2e-test.sh int mongodb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }} name: nats @@ -327,7 +327,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int nats + command: bash scripts/ci-int-e2e-test.sh int nats - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nginx == 'true' }} name: nginx @@ -335,7 +335,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int nginx + command: bash scripts/ci-int-e2e-test.sh int nginx - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -345,7 +345,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int opentelemetry + command: bash scripts/ci-int-e2e-test.sh int opentelemetry - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }} name: postgres @@ -353,7 +353,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int postgres + command: bash scripts/ci-int-e2e-test.sh int postgres - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.prometheus == 'true' }} name: prometheus @@ -361,7 +361,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int prometheus + command: bash scripts/ci-int-e2e-test.sh int prometheus - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.pulsar == 'true' }} name: pulsar @@ -369,7 +369,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int pulsar + command: bash scripts/ci-int-e2e-test.sh int pulsar - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.redis == 'true' }} name: redis @@ -377,7 +377,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int redis + command: bash scripts/ci-int-e2e-test.sh int redis - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' }} name: shutdown @@ -385,7 +385,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int shutdown + command: bash scripts/ci-int-e2e-test.sh int shutdown - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.splunk == 'true' }} name: splunk @@ -393,7 +393,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int splunk + command: bash scripts/ci-int-e2e-test.sh int splunk - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.webhdfs == 'true' }} name: webhdfs @@ -401,7 +401,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh int webhdfs + command: bash scripts/ci-int-e2e-test.sh int webhdfs integration-test-suite: name: Integration Test Suite diff --git a/scripts/ci-integration-test.sh b/scripts/ci-int-e2e-test.sh similarity index 100% rename from scripts/ci-integration-test.sh rename to scripts/ci-int-e2e-test.sh From 5820a0b8afafa6fdc7151f2735c8dab62372d842 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:36:33 -0700 Subject: [PATCH 93/99] fix e2e logic --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 14ebd8d19dacc..a46d4d09e496c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -50,7 +50,8 @@ jobs: if: always() && ( github.event_name == 'schedule' || ( needs.changes.outputs.all-e2e == 'true' - || needs.changes.outputs.e2e-datadog-logs == 'true' + || needs.changes.outputs.datadog-logs == 'true' + || needs.changes.outputs.datadog-metrics == 'true' ) ) timeout-minutes: 45 From 32e133cbd7d1b4db15db582f0ea2889390c164d2 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:40:07 -0700 Subject: [PATCH 94/99] fix e2e --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a46d4d09e496c..210b7315f5569 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -50,8 +50,8 @@ jobs: if: always() && ( github.event_name == 'schedule' || ( needs.changes.outputs.all-e2e == 'true' - || needs.changes.outputs.datadog-logs == 'true' - || needs.changes.outputs.datadog-metrics == 'true' + || needs.changes.outputs.e2e-datadog-logs == 'true' + || needs.changes.outputs.e2e-datadog-metrics == 'true' ) ) timeout-minutes: 45 From f11099ab11f257a278a8a9436b3a85aa4e3a84a2 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:44:02 -0700 Subject: [PATCH 95/99] fix e2e --- .github/workflows/changes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 83af6244ecc19..4443d1c04ba51 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -252,8 +252,8 @@ jobs: if: ${{ inputs.e2e_tests }} outputs: all-tests: ${{ steps.filter.outputs.all-tests}} - datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} - datadog-metrics: ${{ steps.filter.outputs.e2e-datadog-metrics }} + datadog-logs: ${{ steps.filter.outputs.datadog-logs }} + datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} steps: - uses: actions/checkout@v3 From 5933c06cd6a6eea26e51ec2dfc4eb7e920e1b298 Mon Sep 17 00:00:00 2001 From: neuronull Date: Tue, 30 Jan 2024 16:49:03 -0700 Subject: [PATCH 96/99] script usage --- scripts/ci-int-e2e-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci-int-e2e-test.sh b/scripts/ci-int-e2e-test.sh index 13cc85026356d..3b569db530f5f 100755 --- a/scripts/ci-int-e2e-test.sh +++ b/scripts/ci-int-e2e-test.sh @@ -11,9 +11,9 @@ if [[ -z "${CI:-}" ]]; then exit 1 fi -if [ $# -ne 1 ] +if [ $# -ne 2 ] then - echo "usage: $0 INTEGRATION" + echo "usage: $0 [int|e2e] TEST_NAME" exit 1 fi From 731b2786407b3f51821af51f14498f05c9e98426 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 31 Jan 2024 09:17:23 -0700 Subject: [PATCH 97/99] touchups to workflows --- .github/workflows/e2e.yml | 4 ++++ .github/workflows/integration-comment.yml | 4 ++-- .github/workflows/integration.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 210b7315f5569..7de89479412df 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,5 +1,9 @@ # End to End Suite # +# Runs on: +# - PRs if there are code changes to the source files that are noted in `vdev e2e ci_paths` +# - MQ (always pass) +# - Scheduled: at midnight UTC Tues-Sat name: E2E Test Suite diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index c84246d116a8b..3d88d173d676f 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -458,7 +458,7 @@ jobs: - run: docker image prune -af ; docker container prune -f - name: e2e-datadog-logs - if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') + if: ${{ contains(github.event.comment.body, '/ci-run-e2e-datadog-logs') || contains(github.event.comment.body, '/ci-run-integration-all') || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 @@ -468,7 +468,7 @@ jobs: command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs - name: datadog-e2e-metrics - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') + if: ${{ contains(github.event.comment.body, '/ci-run-e2e-datadog-metrics') || contains(github.event.comment.body, '/ci-run-integration-all') || contains(github.event.comment.body, '/ci-run-all') }} uses: nick-fields/retry@v2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index afc862265f8bc..7ddc0322f2802 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -89,7 +89,7 @@ jobs: || needs.changes.outputs.webhdfs == 'true' ) ) - timeout-minutes: 90 + timeout-minutes: 75 steps: - uses: actions/checkout@v3 with: From 87ffb7c27189a49431893d3d43c0666d32c682f3 Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 31 Jan 2024 11:57:01 -0700 Subject: [PATCH 98/99] separate dockerfile for the e2e tests --- scripts/integration/Dockerfile | 41 ++++++++------------------------- vdev/src/commands/test.rs | 1 + vdev/src/testing/integration.rs | 3 ++- vdev/src/testing/runner.rs | 13 +++++++---- 4 files changed, 20 insertions(+), 38 deletions(-) diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index f976155d47e8d..6205c96eb56fe 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -1,46 +1,23 @@ ARG RUST_VERSION -ARG FEATURES -ARG DEBIAN_RELEASE=slim-bookworm +FROM docker.io/rust:${RUST_VERSION}-slim-bookworm -FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} - -RUN apt-get update && apt-get -y --no-install-recommends install \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cmake \ curl \ - git \ - clang \ - libclang-dev \ + g++ \ + libclang1 \ libsasl2-dev \ - libstdc++-11-dev \ libssl-dev \ - libxxhash-dev \ - unzip \ + llvm \ + pkg-config \ zlib1g-dev \ - zlib1g - -RUN git clone https://github.com/rui314/mold.git \ - && mkdir mold/build \ - && cd mold/build \ - && git checkout v2.0.0 \ - && ../install-build-deps.sh \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ - && cmake --build . -j $(nproc) \ - && cmake --install . + unzip \ + git \ + && rm -rf /var/lib/apt/lists/* RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.64 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs RUN bash /install-protoc.sh - -WORKDIR /vector -COPY . . -ARG FEATURES - -RUN --mount=type=cache,target=/vector/target \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - /usr/local/bin/mold -run cargo build --tests --lib --bin vector \ - --no-default-features --features $FEATURES && \ - cp target/debug/vector /usr/bin/vector diff --git a/vdev/src/commands/test.rs b/vdev/src/commands/test.rs index 4d28b48408de2..c54c45a3429c1 100644 --- a/vdev/src/commands/test.rs +++ b/vdev/src/commands/test.rs @@ -53,6 +53,7 @@ impl Cli { &BTreeMap::default(), None, &args, + "", ) } } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 2ade5213977e0..be5c714b7739a 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -133,6 +133,7 @@ pub(crate) trait ComposeTestT { &compose_test.config.runner.env, Some(&compose_test.config.features), &args, + Self::DIRECTORY, )?; if !active { @@ -149,7 +150,7 @@ pub(crate) trait ComposeTestT { if Self::DIRECTORY == E2E_TESTS_DIR { compose_test .runner - .build(Some(&compose_test.config.features))?; + .build(Some(&compose_test.config.features), Self::DIRECTORY)?; } compose_test.config.check_required()?; diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index 5b67671945525..01f167e0a4599 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -86,6 +86,7 @@ pub trait TestRunner { inner_env: &Environment, features: Option<&[String]>, args: &[String], + directory: &str, ) -> Result<()>; } @@ -134,7 +135,7 @@ pub trait ContainerTestRunner: TestRunner { Ok(RunnerState::Missing) } - fn ensure_running(&self, features: Option<&[String]>) -> Result<()> { + fn ensure_running(&self, features: Option<&[String]>, directory: &str) -> Result<()> { match self.state()? { RunnerState::Running | RunnerState::Restarting => (), RunnerState::Created | RunnerState::Exited => self.start()?, @@ -145,7 +146,7 @@ pub trait ContainerTestRunner: TestRunner { self.start()?; } RunnerState::Missing => { - self.build(features)?; + self.build(features, directory)?; self.ensure_volumes()?; self.create()?; self.start()?; @@ -173,8 +174,8 @@ pub trait ContainerTestRunner: TestRunner { Ok(()) } - fn build(&self, features: Option<&[String]>) -> Result<()> { - let dockerfile: PathBuf = [app::path(), "scripts", "integration", "Dockerfile"] + fn build(&self, features: Option<&[String]>, directory: &str) -> Result<()> { + let dockerfile: PathBuf = [app::path(), "scripts", directory, "Dockerfile"] .iter() .collect(); let mut command = dockercmd(["build"]); @@ -273,8 +274,9 @@ where inner_env: &Environment, features: Option<&[String]>, args: &[String], + directory: &str, ) -> Result<()> { - self.ensure_running(features)?; + self.ensure_running(features, directory)?; let mut command = dockercmd(["exec"]); if *IS_A_TTY { @@ -408,6 +410,7 @@ impl TestRunner for LocalTestRunner { inner_env: &Environment, _features: Option<&[String]>, args: &[String], + _directory: &str, ) -> Result<()> { let mut command = Command::new(TEST_COMMAND[0]); command.args(&TEST_COMMAND[1..]); From 81f22296a68726baacd0861938fa3676570110fb Mon Sep 17 00:00:00 2001 From: neuronull Date: Wed, 31 Jan 2024 12:08:56 -0700 Subject: [PATCH 99/99] add the Dockerfile --- scripts/e2e/Dockerfile | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 scripts/e2e/Dockerfile diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile new file mode 100644 index 0000000000000..f976155d47e8d --- /dev/null +++ b/scripts/e2e/Dockerfile @@ -0,0 +1,46 @@ +ARG RUST_VERSION +ARG FEATURES +ARG DEBIAN_RELEASE=slim-bookworm + +FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} + +RUN apt-get update && apt-get -y --no-install-recommends install \ + build-essential \ + cmake \ + curl \ + git \ + clang \ + libclang-dev \ + libsasl2-dev \ + libstdc++-11-dev \ + libssl-dev \ + libxxhash-dev \ + unzip \ + zlib1g-dev \ + zlib1g + +RUN git clone https://github.com/rui314/mold.git \ + && mkdir mold/build \ + && cd mold/build \ + && git checkout v2.0.0 \ + && ../install-build-deps.sh \ + && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ + && cmake --build . -j $(nproc) \ + && cmake --install . + +RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.64 --locked + +COPY scripts/environment/install-protoc.sh / +COPY tests/data/ca/certs /certs +RUN bash /install-protoc.sh + +WORKDIR /vector +COPY . . +ARG FEATURES + +RUN --mount=type=cache,target=/vector/target \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + /usr/local/bin/mold -run cargo build --tests --lib --bin vector \ + --no-default-features --features $FEATURES && \ + cp target/debug/vector /usr/bin/vector