chore(tests): Add end-to-end tests with the Datadog Agent

.github/actions/spelling/allow.txt

@@ -277,6 +277,7 @@ downsides
 downwardapi
 emoji
 esbuild
+fakeintake
 fargate
 fibonacci
 fileapi

.github/actions/spelling/expect.txt

@@ -449,6 +449,7 @@ gnux
 gny
 Godbolt
 gogoproto
+gohai
 goldberg
 goldmark
 GPB

.github/workflows/changes.yml

@@ -64,6 +64,8 @@ on:
         value: ${{ jobs.int_tests.outputs.datadog-metrics }}
       datadog-traces:
         value: ${{ jobs.int_tests.outputs.datadog-traces }}
+      e2e-datadog-logs:
+        value: ${{ jobs.int_tests.outputs.e2e-datadog-logs }}
       dnstap:
         value: ${{ jobs.int_tests.outputs.dnstap }}
       docker-logs:
@@ -196,6 +198,7 @@ jobs:
       datadog-logs: ${{ steps.filter.outputs.datadog-logs }}
       datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }}
       datadog-traces: ${{ steps.filter.outputs.datadog-traces }}
+      e2e-datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }}
       dnstap: ${{ steps.filter.outputs.dnstap }}
       docker-logs: ${{ steps.filter.outputs.docker-logs }}
       elasticsearch: ${{ steps.filter.outputs.elasticsearch }}

.github/workflows/integration-comment.yml

@@ -202,6 +202,14 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-integration-test.sh datadog-traces
 
+      - name: e2e-datadog-logs
+        if: ${{ contains(github.event.comment.body, '/ci-run-integration-e2e-datadog-logs') || contains(github.event.comment.body, '/ci-run-all') }}
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 35
+          max_attempts: 3
+          command: bash scripts/ci-integration-test.sh e2e-datadog-logs
+
       - name: dnstap
         if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap')
           || contains(github.event.comment.body, '/ci-run-integration-all')

.github/workflows/integration.yml

@@ -64,6 +64,7 @@ jobs:
             || needs.changes.outputs.datadog-logs == 'true'
             || needs.changes.outputs.datadog-metrics == 'true'
             || needs.changes.outputs.datadog-traces == 'true'
+            || needs.changes.outputs.e2e-datadog-logs == 'true'
             || needs.changes.outputs.dnstap == 'true'
             || needs.changes.outputs.docker-logs == 'true'
             || needs.changes.outputs.elasticsearch == 'true'
@@ -205,6 +206,15 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-integration-test.sh  datadog-traces
 
+      - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') &&
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
+        name: e2e-datadog-logs
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 35
+          max_attempts: 3
+          command: bash scripts/ci-integration-test.sh  e2e-datadog-logs
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }}
         name: dnstap
         uses: nick-fields/retry@v2

Cargo.toml

@@ -22,6 +22,10 @@ bench = false
 name = "integration"
 path = "tests/integration/lib.rs"
 
+[[test]]
+name = "e2e"
+path = "tests/e2e/mod.rs"
+
 [[bin]]
 name = "graphql-schema"
 path = "src/api/schema/gen.rs"
@@ -734,8 +738,15 @@ enterprise = [
 # Identifies that the build is a nightly build
 nightly = []
 
-# Testing-related features
+# Integration testing-related features
 all-integration-tests = [
+  # TODO: This is a temporary solution. When this GH issue
+  # https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor
+  # to have a separate vdev e2e subcommand, and we can then use "all-e2e-tests" in
+  # that subcommand's processing. Until then the e2e tests must be included here
+  # in order for them to be exercised in CI.
+  "all-e2e-tests",
+
   "amqp-integration-tests",
   "appsignal-integration-tests",
   "aws-integration-tests",
@@ -840,6 +851,17 @@ webhdfs-integration-tests = ["sinks-webhdfs"]
 disable-resolv-conf = []
 shutdown-tests = ["api", "sinks-blackhole", "sinks-console", "sinks-prometheus", "sources", "transforms-lua", "transforms-remap", "unix"]
 cli-tests = ["sinks-blackhole", "sinks-socket", "sources-demo_logs", "sources-file"]
+
+# End-to-End testing-related features
+all-e2e-tests = [
+  "e2e-tests-datadog"
+]
+
+e2e-tests-datadog = [
+  "sources-datadog_agent",
+  "sinks-datadog_logs"
+]
+
 vector-api-tests = [
   "sources-demo_logs",
   "transforms-log_to_metric",

Makefile

@@ -361,7 +361,7 @@ test-integration: test-integration-databend test-integration-docker-logs test-in
 test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb
 test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats
 test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar
-test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs
+test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-datadog-e2e-logs
 test-integration: test-integration-datadog-traces test-integration-shutdown
 
 test-integration-%-cleanup:

scripts/e2e/README.md

@@ -0,0 +1,17 @@
+This directory contains a set of end-to-end test frameworks for vector which are executed by the
+`vdev` tool.
+
+Currently these e2e tests are executed with the same `vdev` subcommand as the integration tests,
+`cargo vdev integration`.
+
+See the README in the `scripts/integration` subdirectory for more information.
+
+A pending future enhancement is to create a `vdev` subcommand `e2e`, that will separate the
+invocation of the end-to-end tests from the integration tests in `vdev`, to correspond to the
+code separation and fundamental differences between the two classes of tests.
+
+See https://github.com/vectordotdev/vector/issues/18829 for more information.
+
+For now, any subdir here needs to be unique from the other integration tests outside this dir.
+For example there is already a `datadog-logs` integration test, hence the e2e test is in a sub-
+dir called `e2e-datadog-logs`.

scripts/e2e/e2e-datadog-logs/compose.yaml

@@ -0,0 +1,100 @@
+version: '3'
+
+services:
+  # Generates random log data for consumption by the custom Agent check
+  log_generator:
+    image: docker.io/mingrammer/flog
+    command:
+      - "-f"
+      - "json"
+      - "-n"
+      - "1000"
+      - "-t"
+      - "log"
+      - "-o"
+      - "/var/log/a_custom.log"
+    volumes:
+      - log_path:/var/log/
+
+  # Receives log data from the `datadog-agent-vector` service and sends
+  # to the `fakeintake-vector` service.
+  vector:
+    depends_on:
+      - log_generator
+      - fakeintake-vector
+    build:
+      context: ${PWD}
+    # re-using the integration test runner image since it already has
+    # compiled vector on it.
+    image: ${CONFIG_VECTOR_IMAGE}
+    environment:
+      - FEATURES=e2e-tests-datadog
+    working_dir: /home/vector
+    network_mode: host
+    command:
+      - "/usr/bin/vector"
+      - "-vvv"
+      - "-c"
+      - "/home/vector/tests/data/e2e/datadog/logs/vector.toml"
+    volumes:
+      - ${PWD}:/home/vector
+
+  # Tails a custom log created by `log_generator` and sends log data to
+  # the `fakeintake-agent` service
+  datadog-agent:
+    image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION}
+    depends_on:
+      - log_generator
+      - fakeintake-agent
+    environment:
+    - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+    - DD_HOSTNAME=datadog-agent
+    - DD_ENABLE_PAYLOADS_EVENTS=false
+    - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false
+    - DD_CONTAINER_EXCLUDE="name:.*"
+    volumes:
+    # The Agent config file
+    - ${PWD}/tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml
+    # The custom logs check
+    - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro
+    # The custom log to tail, created by the `log_generator` service
+    - log_path:/var/log/
+
+  # Tails a custom log created by `log_generator` and sends log data to
+  # the `vector` service
+  datadog-agent-vector:
+    image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION}
+    depends_on:
+      - log_generator
+      - vector
+    environment:
+    - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required}
+    - DD_HOSTNAME=datadog-agent-vector
+    - DD_ENABLE_PAYLOADS_EVENTS=false
+    - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false
+    - DD_CONTAINER_EXCLUDE="name:.*"
+    volumes:
+    # The Agent config file
+    - ${PWD}/tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml
+    # The custom logs check
+    - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro
+    # The custom log to tail, created by the `log_generator` service
+    - log_path:/var/log/
+
+  # Receives log data from the `datadog-agent` service. Is queried by the test runner
+  # which does the validation of consistency with the other fakeintake service.
+  fakeintake-agent:
+    image: docker.io/datadog/fakeintake:latest
+
+  # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner
+  # which does the validation of consistency with the other fakeintake service.
+  fakeintake-vector:
+    image: docker.io/datadog/fakeintake:latest
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+
+volumes:
+  log_path: {}
+  target: {}

scripts/e2e/e2e-datadog-logs/test.yaml

@@ -0,0 +1,33 @@
+features:
+- e2e-tests-datadog
+
+test: "e2e"
+
+test_filter: "datadog::logs::"
+
+runner:
+  env:
+    EXPECTED_LOG_EVENTS: '1000'
+    VECTOR_RECEIVE_PORT: '8081'
+    FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80'
+    FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80'
+
+matrix:
+  # validate against the latest Agent nightly and also stable v6 and v7
+  agent_version: ['latest', '6', '7']
+
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/common/datadog.rs"
+- "src/sources/datadog_agent/**"
+- "src/internal_events/datadog_*"
+- "src/sinks/datadog/logs/**"
+- "src/sinks/util/**"
+# NOTE: currently we need the prefix 'e2e' even though it looks redundant,
+# because the vdev code does not otherwise have a way to distinguish between
+# the other `datadog-logs` int test.
+# but once GH issue 18829 is completed, this will become unecessary.
+- "scripts/e2e/e2e-datadog-logs/**"
+- "tests/data/e2e/datadog/logs/**"

scripts/integration/Dockerfile

@@ -1,23 +1,46 @@
 ARG RUST_VERSION
-FROM docker.io/rust:${RUST_VERSION}-slim-bookworm
+ARG FEATURES
+ARG DEBIAN_RELEASE=slim-bookworm
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE}
+
+RUN apt-get update && apt-get -y --no-install-recommends install \
     build-essential \
     cmake \
     curl \
-    g++ \
-    libclang1 \
+    git \
+    clang \
+    libclang-dev \
     libsasl2-dev \
+    libstdc++-11-dev \
     libssl-dev \
-    llvm \
-    pkg-config \
-    zlib1g-dev \
+    libxxhash-dev \
     unzip \
-    git \
-  && rm -rf /var/lib/apt/lists/*
+    zlib1g-dev \
+    zlib1g
+
+RUN git clone https://github.com/rui314/mold.git \
+    && mkdir mold/build \
+    && cd mold/build \
+    && git checkout v2.0.0 \
+    && ../install-build-deps.sh \
+    && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \
+    && cmake --build . -j $(nproc) \
+    && cmake --install .
 
 RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.64 --locked
 
 COPY scripts/environment/install-protoc.sh /
 COPY tests/data/ca/certs /certs
 RUN bash /install-protoc.sh
+
+WORKDIR /vector
+COPY . .
+ARG FEATURES
+
+RUN --mount=type=cache,target=/vector/target \
+    --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    /usr/local/bin/mold -run cargo build --tests --lib --bin vector \
+    --no-default-features --features $FEATURES && \
+    cp target/debug/vector /usr/bin/vector

tests/data/e2e/datadog/logs/agent_only.yaml

@@ -0,0 +1,28 @@
+api_key: DEADBEEF
+log_level: 'debug'
+
+# disable bunch of stuff we don't need
+inventories_configuration_enabled: false
+enable_metadata_collection: false
+enable_gohai: false
+
+apm_config:
+  enabled: false
+
+process_config:
+  container_collection:
+    enabled: false
+  process_discovery:
+    enabled: false
+  disable_realtime_checks: true
+
+use_dogstatsd: false
+
+# configure logs
+logs_enabled: true
+
+logs_config:
+  logs_dd_url: fakeintake-agent:80
+  logs_no_ssl: true
+  force_use_http: true
+  batch_wait: 1

tests/data/e2e/datadog/logs/agent_vector.yaml

@@ -0,0 +1,33 @@
+api_key: DEADBEEF
+log_level: 'debug'
+
+# disable bunch of stuff we don't need
+inventories_configuration_enabled: false
+enable_metadata_collection: false
+enable_gohai: false
+
+apm_config:
+  enabled: false
+
+process_config:
+  container_collection:
+    enabled: false
+  process_discovery:
+    enabled: false
+  disable_realtime_checks: true
+
+use_dogstatsd: false
+
+# configure logs
+logs_enabled: true
+
+logs_config:
+  logs_no_ssl: true
+  force_use_http: true
+  batch_wait: 1
+
+# send to vector
+vector:
+  logs:
+    enabled: true
+    url: "http://vector:8181"