chore(tests): Add end-to-end tests with the Datadog Agent (vectordotd…

…ev#18538) * start * check spelling allow * reorg * try consume a log * trying on logs * saving progress on vdev changes * more vdev drafts * Revert "more vdev drafts" This reverts commit 758a06a. * Revert "saving progress on vdev changes" This reverts commit 7a69290. * extract Vector instance to its own container * cleanup the compose file * Generate logs on fly * cleanup * cleanup * cleanup * remove sleep * add to CI * spell checker * fix issue with DD changes detection * more workflow fix * increase timeout as experiment * update vector config for log endpoint field * touch up * start * CI * feedback bg * add scaffolding for the differentiation of series API versions * fix(datadog_metrics sink): fix the integration tests which weren't actually validating anything * fix workflows * clippy * fix filter for traces * add first pass * add testing coverage * cargo.lock * reduce duplicated code * cleanup * clippy * feedback ds: remove check for sort by name * feedback ds: extend unit tests for v2 * feedback ds: extend the int test coverage * Revert "feedback ds: remove check for sort by name" This reverts commit c95a326. * add explicit sort check * add env var for v1 support * check events * add note in deprecations * feedback ds * ds feedback logs PR * config pain; start aggregation * feedback ds: move to scripts/e2e * add v6 and v7 agent versions to test against * remove dead code allow * move to e2e dir * improve comparison logic * code reduction * add sketches, some reorg * cleanup * add notes about the e2e prefix needed * re-use runner image * fixes * e2e tests now run in CI * clippy * remove unused struct * true up the int test comment changes * cleanup * rework the massaging and comparison model * compare to all versions * cleanup and address TODO for sketches * feedback ds: workflows * feedback ds: simpler compose * feedback ds: agent versions clarity * feedback ds: comments * clippy * touchups * feedback bg: assert_eq * increase timeout workflow * run in 8 core runner * check * separate e2e workflow * comment * fix workflows * prune timing * allow build_all option to start subcommand * hack to bypass this temporary issue with CI * feedback bg * feedback bg * feedback ds: metric type check * add descriptive comment * spelling * chore(vdev): refactor e2e tests into own subcommand (vectordotdev#19666) * merge conflict * update workflows * rename * fix e2e logic * fix e2e * fix e2e * script usage * touchups to workflows * separate dockerfile for the e2e tests * add the Dockerfile
discord · Sep 23, 2024 · 58dcafd · 58dcafd
1 parent a5d5c7f
commit 58dcafd
Show file tree

Hide file tree

Showing 56 changed files with 2,071 additions and 306 deletions.
diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt
@@ -52,6 +52,7 @@ dkr
 Dockerfiles
 DOOV
 Douban
+E2ETest
 Enot
 Evercoss
 Explay
@@ -278,6 +279,7 @@ downsides
 downwardapi
 emoji
 esbuild
+fakeintake
 fargate
 fibonacci
 fileapi

diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -449,6 +449,7 @@ gnux
 gny
 Godbolt
 gogoproto
+gohai
 goldberg
 goldmark
 GPB

diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml
@@ -19,6 +19,10 @@ on:
         required: false
         type: boolean
         default: false
+      e2e_tests:
+        required: false
+        type: boolean
+        default: false
       source:
         required: false
         type: boolean
@@ -41,7 +45,7 @@ on:
       k8s:
         value: ${{ jobs.source.outputs.k8s }}
       all-int:
-        value: ${{ jobs.int_tests.outputs.all-int }}
+        value: ${{ jobs.int_tests.outputs.all-tests }}
       amqp:
         value: ${{ jobs.int_tests.outputs.amqp }}
       appsignal:
@@ -110,6 +114,13 @@ on:
         value: ${{ jobs.int_tests.outputs.splunk }}
       webhdfs:
         value: ${{ jobs.int_tests.outputs.webhdfs }}
+      # e2e tests
+      all-e2e:
+        value: ${{ jobs.e2e_tests.outputs.all-tests }}
+      e2e-datadog-logs:
+        value: ${{ jobs.e2e_tests.outputs.datadog-logs }}
+      e2e-datadog-metrics:
+        value: ${{ jobs.e2e_tests.outputs.datadog-metrics }}
 
 jobs:
   # Detects changes that are not specific to integration tests
@@ -184,7 +195,7 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ inputs.int_tests }}
     outputs:
-      all-int: ${{ steps.filter.outputs.all-int}}
+      all-tests: ${{ steps.filter.outputs.all-tests}}
       amqp: ${{ steps.filter.outputs.amqp }}
       appsignal: ${{ steps.filter.outputs.appsignal}}
       aws: ${{ steps.filter.outputs.aws }}
@@ -234,3 +245,27 @@ jobs:
           base: ${{ inputs.base_ref }}
           ref: ${{ inputs.head_ref }}
           filters: int_test_filters.yaml
+
+  # Detects changes that are specific to e2e tests
+  e2e_tests:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.e2e_tests }}
+    outputs:
+      all-tests: ${{ steps.filter.outputs.all-tests}}
+      datadog-logs: ${{ steps.filter.outputs.datadog-logs }}
+      datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }}
+    steps:
+      - uses: actions/checkout@v3
+
+      # creates a yaml file that contains the filters for each test,
+      # extracted from the output of the `vdev int ci-paths` command, which
+      # sources the paths from the scripts/integration/.../test.yaml files
+      - name: Create filter rules for e2e tests
+        run: cargo vdev e2e ci-paths > int_test_filters.yaml
+
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: ${{ inputs.base_ref }}
+          ref: ${{ inputs.head_ref }}
+          filters: int_test_filters.yaml
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -0,0 +1,115 @@
+# End to End Suite
+#
+# Runs on:
+#   - PRs if there are code changes to the source files that are noted in `vdev e2e ci_paths`
+#   - MQ (always pass)
+#   - Scheduled: at midnight UTC Tues-Sat
+
+name: E2E Test Suite
+
+on:
+  pull_request:
+  # Needs to pass by default in MQ
+  merge_group:
+    types: [checks_requested]
+  schedule:
+    # At midnight UTC Tue-Sat
+    - cron: '0 0 * * 2-6'
+
+concurrency:
+  # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue
+  group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.head_sha }}
+  cancel-in-progress: true
+
+env:
+  CONTAINER_TOOL: "docker"
+  DD_ENV: "ci"
+  DD_API_KEY: ${{ secrets.DD_API_KEY }}
+  TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }}
+  RUST_BACKTRACE: full
+  TEST_LOG: vector=debug
+  VERBOSE: true
+  CI: true
+  PROFILE: debug
+  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
+  # can be removed when we switch back to the upstream openssl-sys crate
+  CARGO_NET_GIT_FETCH_WITH_CLI: true
+
+jobs:
+
+  changes:
+    if: github.event_name == 'pull_request'
+    uses: ./.github/workflows/changes.yml
+    with:
+      base_ref: ${{ github.event.pull_request.base.ref }}
+      head_ref: ${{ github.event.pull_request.head.ref  }}
+      source: false
+      e2e_tests: true
+    secrets: inherit
+
+  e2e-tests:
+    name: E2E Tests
+    runs-on: [linux, ubuntu-20.04-8core]
+    needs: changes
+    if: always() && (
+      github.event_name == 'schedule' || (
+          needs.changes.outputs.all-e2e == 'true'
+            || needs.changes.outputs.e2e-datadog-logs == 'true'
+            || needs.changes.outputs.e2e-datadog-metrics == 'true'
+        )
+      )
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - run: sudo npm -g install @datadog/datadog-ci
+
+      - run: docker image prune -af ; docker container prune -f
+
+      - name: Determine if secrets are defined (PR author is team member).
+        if: github.event_name == 'pull_request'
+        env:
+          GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
+        run: |
+          if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then
+            echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV"
+          else
+            echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV"
+          fi
+
+      - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') &&
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
+        name: e2e-datadog-logs
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 35
+          max_attempts: 3
+          command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs
+
+      - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') &&
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
+        name: e2e-datadog-metrics
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 35
+          max_attempts: 3
+          command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics
+
+
+  e2e-test-suite:
+    name: E2E Test Suite
+    runs-on: ubuntu-latest
+    if: always()
+    needs: e2e-tests
+    env:
+      FAILED: ${{ contains(needs.*.result, 'failure') }}
+    steps:
+      - run: |
+          echo "failed=${{ env.FAILED }}"
+          if [[ "$FAILED" == "true" ]] ; then
+            exit 1
+          else
+            exit 0
+          fi
-Original file line number
+Diff line change
@@ Expand Up / @@ -449,6 +449,7 @@ gnux @@
     gny
     Godbolt
     gogoproto
+    gohai
     goldberg
     goldmark
     GPB
@@ Expand Down @@