diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 74703aa20ce4f..b3cb42844ef5a 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -52,6 +52,7 @@ dkr Dockerfiles DOOV Douban +E2ETest Enot Evercoss Explay diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index f8c59afb84dca..4443d1c04ba51 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -45,7 +45,7 @@ on: k8s: value: ${{ jobs.source.outputs.k8s }} all-int: - value: ${{ jobs.int_tests.outputs.all-int }} + value: ${{ jobs.int_tests.outputs.all-tests }} amqp: value: ${{ jobs.int_tests.outputs.amqp }} appsignal: @@ -116,9 +116,11 @@ on: value: ${{ jobs.int_tests.outputs.webhdfs }} # e2e tests all-e2e: - value: ${{ jobs.e2e_tests.outputs.all-e2e }} + value: ${{ jobs.e2e_tests.outputs.all-tests }} e2e-datadog-logs: value: ${{ jobs.e2e_tests.outputs.datadog-logs }} + e2e-datadog-metrics: + value: ${{ jobs.e2e_tests.outputs.datadog-metrics }} jobs: # Detects changes that are not specific to integration tests @@ -193,7 +195,7 @@ jobs: runs-on: ubuntu-latest if: ${{ inputs.int_tests }} outputs: - all-int: ${{ steps.filter.outputs.all-int}} + all-tests: ${{ steps.filter.outputs.all-tests}} amqp: ${{ steps.filter.outputs.amqp }} appsignal: ${{ steps.filter.outputs.appsignal}} aws: ${{ steps.filter.outputs.aws }} @@ -249,8 +251,9 @@ jobs: runs-on: ubuntu-latest if: ${{ inputs.e2e_tests }} outputs: - all-e2e: ${{ steps.filter.outputs.all-int}} - datadog-logs: ${{ steps.filter.outputs.e2e-datadog-logs }} + all-tests: ${{ steps.filter.outputs.all-tests}} + datadog-logs: ${{ steps.filter.outputs.datadog-logs }} + datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} steps: - uses: actions/checkout@v3 @@ -258,7 +261,7 @@ jobs: # extracted from the output of the `vdev int ci-paths` command, which # sources the paths from the scripts/integration/.../test.yaml files - name: Create filter rules for e2e tests - run: cargo vdev int ci-paths > int_test_filters.yaml + run: cargo vdev e2e ci-paths > int_test_filters.yaml - uses: dorny/paths-filter@v2 id: filter diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5d0217ff6bde7..210b7315f5569 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,6 +51,7 @@ jobs: github.event_name == 'schedule' || ( needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true' + || needs.changes.outputs.e2e-datadog-metrics == 'true' ) ) timeout-minutes: 45 @@ -81,7 +82,17 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs + command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + + - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-datadog-metrics + uses: nick-fields/retry@v2 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + e2e-test-suite: name: E2E Test Suite diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index 4f42fe915aff1..c84246d116a8b 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -100,7 +100,7 @@ jobs: # First one requires more time, as we need to build the image from scratch timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh amqp + command: bash scripts/ci-int-e2e-test.sh int amqp - name: appsignal if: ${{ contains(github.event.comment.body, '/ci-run-integration-appsignal') @@ -110,7 +110,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh appsignal + command: bash scripts/ci-int-e2e-test.sh int appsignal - name: aws if: ${{ contains(github.event.comment.body, '/ci-run-integration-aws') @@ -120,7 +120,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh aws + command: bash scripts/ci-int-e2e-test.sh int aws - name: axiom if: ${{ contains(github.event.comment.body, '/ci-run-integration-axiom') @@ -130,7 +130,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh axiom + command: bash scripts/ci-int-e2e-test.sh int axiom - name: azure if: ${{ contains(github.event.comment.body, '/ci-run-integration-azure') @@ -140,7 +140,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh azure + command: bash scripts/ci-int-e2e-test.sh int azure - name: clickhouse if: ${{ contains(github.event.comment.body, '/ci-run-integration-clickhouse') @@ -150,7 +150,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh clickhouse + command: bash scripts/ci-int-e2e-test.sh int clickhouse - name: databend if: ${{ contains(github.event.comment.body, '/ci-run-integration-databend') @@ -160,7 +160,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh databend + command: bash scripts/ci-int-e2e-test.sh int databend - name: datadog-agent if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-agent') @@ -170,7 +170,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-agent + command: bash scripts/ci-int-e2e-test.sh int datadog-agent - name: datadog-logs if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-logs') @@ -180,7 +180,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-logs + command: bash scripts/ci-int-e2e-test.sh int datadog-logs - name: datadog-metrics if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-metrics') @@ -190,7 +190,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-metrics + command: bash scripts/ci-int-e2e-test.sh int datadog-metrics - name: datadog-traces if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-traces') @@ -200,7 +200,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-traces + command: bash scripts/ci-int-e2e-test.sh int datadog-traces - name: dnstap if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') @@ -210,7 +210,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh dnstap + command: bash scripts/ci-int-e2e-test.sh int dnstap - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -222,7 +222,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh docker-logs + command: bash scripts/ci-int-e2e-test.sh int docker-logs - name: elasticsearch if: ${{ contains(github.event.comment.body, '/ci-run-integration-elasticsearch') @@ -232,7 +232,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh elasticsearch + command: bash scripts/ci-int-e2e-test.sh int elasticsearch - name: eventstoredb if: ${{ contains(github.event.comment.body, '/ci-run-integration-eventstoredb') @@ -242,7 +242,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh eventstoredb + command: bash scripts/ci-int-e2e-test.sh int eventstoredb - name: fluent if: ${{ contains(github.event.comment.body, '/ci-run-integration-fluent') @@ -252,7 +252,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh fluent + command: bash scripts/ci-int-e2e-test.sh int fluent - name: gcp if: ${{ contains(github.event.comment.body, '/ci-run-integration-gcp') @@ -262,7 +262,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh gcp + command: bash scripts/ci-int-e2e-test.sh int gcp - name: greptimedb if: ${{ contains(github.event.comment.body, '/ci-run-integration-greptimedb') @@ -272,7 +272,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh greptimedb + command: bash scripts/ci-int-e2e-test.sh int greptimedb - name: humio if: ${{ contains(github.event.comment.body, '/ci-run-integration-humio') @@ -282,7 +282,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh humio + command: bash scripts/ci-int-e2e-test.sh int humio - name: http-client if: ${{ contains(github.event.comment.body, '/ci-run-integration-http-client') @@ -292,7 +292,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh http-client + command: bash scripts/ci-int-e2e-test.sh int http-client - name: influxdb if: ${{ contains(github.event.comment.body, '/ci-run-integration-influxdb') @@ -302,7 +302,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh influxdb + command: bash scripts/ci-int-e2e-test.sh int influxdb - name: kafka if: ${{ contains(github.event.comment.body, '/ci-run-integration-kafka') @@ -312,7 +312,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh kafka + command: bash scripts/ci-int-e2e-test.sh int kafka - name: logstash if: ${{ contains(github.event.comment.body, '/ci-run-integration-logstash') @@ -322,7 +322,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh logstash + command: bash scripts/ci-int-e2e-test.sh int logstash - name: loki if: ${{ contains(github.event.comment.body, '/ci-run-integration-loki') @@ -332,7 +332,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh loki + command: bash scripts/ci-int-e2e-test.sh int loki - name: mongodb if: ${{ contains(github.event.comment.body, '/ci-run-integration-mongodb') @@ -342,7 +342,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh mongodb + command: bash scripts/ci-int-e2e-test.sh int mongodb - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -354,7 +354,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nats + command: bash scripts/ci-int-e2e-test.sh int nats - name: nginx if: ${{ contains(github.event.comment.body, '/ci-run-integration-nginx') @@ -364,7 +364,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nginx + command: bash scripts/ci-int-e2e-test.sh int nginx - name: opentelemetry if: ${{ contains(github.event.comment.body, '/ci-run-integration-opentelemetry') @@ -374,7 +374,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh opentelemetry + command: bash scripts/ci-int-e2e-test.sh int opentelemetry - name: postgres if: ${{ contains(github.event.comment.body, '/ci-run-integration-postgres') @@ -384,7 +384,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh postgres + command: bash scripts/ci-int-e2e-test.sh int postgres - name: prometheus if: ${{ contains(github.event.comment.body, '/ci-run-integration-prometheus') @@ -394,7 +394,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh prometheus + command: bash scripts/ci-int-e2e-test.sh int prometheus - name: pulsar if: ${{ contains(github.event.comment.body, '/ci-run-integration-pulsar') @@ -404,7 +404,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh pulsar + command: bash scripts/ci-int-e2e-test.sh int pulsar - name: redis if: ${{ contains(github.event.comment.body, '/ci-run-integration-redis') @@ -414,7 +414,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh redis + command: bash scripts/ci-int-e2e-test.sh int redis - name: shutdown if: ${{ contains(github.event.comment.body, '/ci-run-integration-shutdown') @@ -424,7 +424,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh shutdown + command: bash scripts/ci-int-e2e-test.sh int shutdown - name: splunk if: ${{ contains(github.event.comment.body, '/ci-run-integration-splunk') @@ -434,7 +434,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh splunk + command: bash scripts/ci-int-e2e-test.sh int splunk - name: webhdfs if: ${{ contains(github.event.comment.body, '/ci-run-integration-webhdfs') @@ -444,7 +444,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh webhdfs + command: bash scripts/ci-int-e2e-test.sh int webhdfs e2e-tests: needs: prep-pr @@ -465,7 +465,17 @@ jobs: with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-integration-test.sh e2e-datadog-logs + command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + + - name: datadog-e2e-metrics + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-e2e-metrics') + || contains(github.event.comment.body, '/ci-run-integration-all') + || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 35 + max_attempts: 3 + command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics update-pr-status: name: Signal result to PR diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 575e09da7f13f..afc862265f8bc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -116,7 +116,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh amqp + command: bash scripts/ci-int-e2e-test.sh int amqp - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -125,7 +125,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh appsignal + command: bash scripts/ci-int-e2e-test.sh int appsignal - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.aws == 'true' }} name: aws @@ -133,7 +133,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh aws + command: bash scripts/ci-int-e2e-test.sh int aws - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -142,7 +142,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh axiom + command: bash scripts/ci-int-e2e-test.sh int axiom - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.azure == 'true' }} name: azure @@ -150,7 +150,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh azure + command: bash scripts/ci-int-e2e-test.sh int azure - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.clickhouse == 'true' }} name: clickhouse @@ -158,7 +158,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh clickhouse + command: bash scripts/ci-int-e2e-test.sh int clickhouse - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -167,7 +167,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh databend + command: bash scripts/ci-int-e2e-test.sh int databend - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -176,7 +176,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-agent + command: bash scripts/ci-int-e2e-test.sh int datadog-agent - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -185,7 +185,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-logs + command: bash scripts/ci-int-e2e-test.sh int datadog-logs - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -196,7 +196,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-metrics + command: bash scripts/ci-int-e2e-test.sh int datadog-metrics - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') @@ -205,7 +205,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh datadog-traces + command: bash scripts/ci-int-e2e-test.sh int datadog-traces - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} name: dnstap @@ -213,7 +213,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh dnstap + command: bash scripts/ci-int-e2e-test.sh int dnstap - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }} name: docker-logs @@ -221,7 +221,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh docker-logs + command: bash scripts/ci-int-e2e-test.sh int docker-logs - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.elasticsearch == 'true' }} name: elasticsearch @@ -229,7 +229,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh elasticsearch + command: bash scripts/ci-int-e2e-test.sh int elasticsearch - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.eventstoredb == 'true' }} name: eventstoredb @@ -237,7 +237,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh eventstoredb + command: bash scripts/ci-int-e2e-test.sh int eventstoredb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.fluent == 'true' }} name: fluent @@ -245,7 +245,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh fluent + command: bash scripts/ci-int-e2e-test.sh int fluent - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.gcp == 'true' }} name: gcp @@ -253,7 +253,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh gcp + command: bash scripts/ci-int-e2e-test.sh int gcp - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.greptimedb == 'true' }} name: greptimedb @@ -261,7 +261,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh greptimedb + command: bash scripts/ci-int-e2e-test.sh int greptimedb - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -271,7 +271,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh humio + command: bash scripts/ci-int-e2e-test.sh int humio - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.http-client == 'true' }} name: http-client @@ -279,7 +279,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh http-client + command: bash scripts/ci-int-e2e-test.sh int http-client - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.influxdb == 'true' }} name: influxdb @@ -287,7 +287,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh influxdb + command: bash scripts/ci-int-e2e-test.sh int influxdb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.kafka == 'true' }} name: kafka @@ -295,7 +295,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh kafka + command: bash scripts/ci-int-e2e-test.sh int kafka - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.logstash == 'true' }} name: logstash @@ -303,7 +303,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh logstash + command: bash scripts/ci-int-e2e-test.sh int logstash - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.loki == 'true' }} name: loki @@ -311,7 +311,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh loki + command: bash scripts/ci-int-e2e-test.sh int loki - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mongodb == 'true' }} name: mongodb @@ -319,7 +319,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh mongodb + command: bash scripts/ci-int-e2e-test.sh int mongodb - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }} name: nats @@ -327,7 +327,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nats + command: bash scripts/ci-int-e2e-test.sh int nats - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nginx == 'true' }} name: nginx @@ -335,7 +335,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh nginx + command: bash scripts/ci-int-e2e-test.sh int nginx - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f @@ -345,7 +345,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh opentelemetry + command: bash scripts/ci-int-e2e-test.sh int opentelemetry - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }} name: postgres @@ -353,7 +353,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh postgres + command: bash scripts/ci-int-e2e-test.sh int postgres - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.prometheus == 'true' }} name: prometheus @@ -361,7 +361,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh prometheus + command: bash scripts/ci-int-e2e-test.sh int prometheus - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.pulsar == 'true' }} name: pulsar @@ -369,7 +369,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh pulsar + command: bash scripts/ci-int-e2e-test.sh int pulsar - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.redis == 'true' }} name: redis @@ -377,7 +377,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh redis + command: bash scripts/ci-int-e2e-test.sh int redis - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' }} name: shutdown @@ -385,7 +385,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh shutdown + command: bash scripts/ci-int-e2e-test.sh int shutdown - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.splunk == 'true' }} name: splunk @@ -393,7 +393,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh splunk + command: bash scripts/ci-int-e2e-test.sh int splunk - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.webhdfs == 'true' }} name: webhdfs @@ -401,7 +401,7 @@ jobs: with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-integration-test.sh webhdfs + command: bash scripts/ci-int-e2e-test.sh int webhdfs integration-test-suite: name: Integration Test Suite diff --git a/Cargo.toml b/Cargo.toml index f304ede634691..3c2f1579f4597 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -740,13 +740,6 @@ nightly = [] # Integration testing-related features all-integration-tests = [ - # TODO: This is a temporary solution. When this GH issue - # https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor - # to have a separate vdev e2e subcommand, and we can then use "all-e2e-tests" in - # that subcommand's processing. Until then the e2e tests must be included here - # in order for them to be exercised in CI. - "all-e2e-tests", - "amqp-integration-tests", "appsignal-integration-tests", "aws-integration-tests", @@ -859,7 +852,8 @@ all-e2e-tests = [ e2e-tests-datadog = [ "sources-datadog_agent", - "sinks-datadog_logs" + "sinks-datadog_logs", + "sinks-datadog_metrics" ] vector-api-tests = [ diff --git a/scripts/ci-integration-test.sh b/scripts/ci-int-e2e-test.sh similarity index 60% rename from scripts/ci-integration-test.sh rename to scripts/ci-int-e2e-test.sh index 49780e7bba378..3b569db530f5f 100755 --- a/scripts/ci-integration-test.sh +++ b/scripts/ci-int-e2e-test.sh @@ -11,20 +11,21 @@ if [[ -z "${CI:-}" ]]; then exit 1 fi -if [ $# -ne 1 ] +if [ $# -ne 2 ] then - echo "usage: $0 INTEGRATION" + echo "usage: $0 [int|e2e] TEST_NAME" exit 1 fi set -x -INTEGRATION=$1 +TEST_TYPE=$1 # either "int" or "e2e" +TEST_NAME=$2 -cargo vdev -v int start -a "${INTEGRATION}" +cargo vdev -v "${TEST_TYPE}" start -a "${TEST_NAME}" sleep 30 -cargo vdev -v int test --retries 2 -a "${INTEGRATION}" +cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" RET=$? -cargo vdev -v int stop -a "${INTEGRATION}" +cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" ./scripts/upload-test-results.sh exit $RET diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 45422e823c3cc..62e7f8e9f64c7 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -1,17 +1,9 @@ This directory contains a set of end-to-end test frameworks for vector which are executed by the `vdev` tool. -Currently these e2e tests are executed with the same `vdev` subcommand as the integration tests, -`cargo vdev integration`. +These end-to-end (e2e) tests are executed with the `vdev e2e` subcommand, which behaves +identically to the `vdev integration` subcommand. See the README in the `scripts/integration` +subdirectory for more information. -See the README in the `scripts/integration` subdirectory for more information. - -A pending future enhancement is to create a `vdev` subcommand `e2e`, that will separate the -invocation of the end-to-end tests from the integration tests in `vdev`, to correspond to the -code separation and fundamental differences between the two classes of tests. - -See https://github.com/vectordotdev/vector/issues/18829 for more information. - -For now, any subdir here needs to be unique from the other integration tests outside this dir. -For example there is already a `datadog-logs` integration test, hence the e2e test is in a sub- -dir called `e2e-datadog-logs`. +The e2e tests are more of a black box test, in which we spin up a full vector instance as one +of the compose services that runs alongside the others. diff --git a/scripts/e2e/datadog-logs/README.md b/scripts/e2e/datadog-logs/README.md new file mode 100644 index 0000000000000..7b3eaac8d9407 --- /dev/null +++ b/scripts/e2e/datadog-logs/README.md @@ -0,0 +1,19 @@ +This e2e test covers the `datadog_agent` source, and the +`datadog_logs` sink. + +Fake logs are generated in the emitter service and written +to a file. + +Two Agent containers are spun up to read the log file, one +for the Agent only case and one for the Agent -> Vector case. + +In the Agent only case, the Agent sends the logs to `fakeintake` +(another service) directly. This is the baseline. + +In the Agent-Vector case, the Agent send the logs to the vector +service, and the `datadog_logs` sink sends to a separate +`fakeintake` service. This is the compare case. + +The two sets of data should be shaped the same in terms of when +the events were received, and the content of the events, but the +timestamps themselves are not guaranteed to align. diff --git a/scripts/e2e/e2e-datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml similarity index 97% rename from scripts/e2e/e2e-datadog-logs/compose.yaml rename to scripts/e2e/datadog-logs/compose.yaml index 5c1814097f08a..4ca9633af2e4e 100644 --- a/scripts/e2e/e2e-datadog-logs/compose.yaml +++ b/scripts/e2e/datadog-logs/compose.yaml @@ -4,6 +4,9 @@ services: # Generates random log data for consumption by the custom Agent check log_generator: image: docker.io/mingrammer/flog + depends_on: + - datadog-agent-vector + - datadog-agent command: - "-f" - "json" @@ -16,35 +19,11 @@ services: volumes: - log_path:/var/log/ - # Receives log data from the `datadog-agent-vector` service and sends - # to the `fakeintake-vector` service. - vector: - depends_on: - - log_generator - - fakeintake-vector - build: - context: ${PWD} - # re-using the integration test runner image since it already has - # compiled vector on it. - image: ${CONFIG_VECTOR_IMAGE} - environment: - - FEATURES=e2e-tests-datadog - working_dir: /home/vector - network_mode: host - command: - - "/usr/bin/vector" - - "-vvv" - - "-c" - - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" - volumes: - - ${PWD}:/home/vector - # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service datadog-agent: image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} depends_on: - - log_generator - fakeintake-agent environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} @@ -65,7 +44,6 @@ services: datadog-agent-vector: image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} depends_on: - - log_generator - vector environment: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} @@ -81,6 +59,28 @@ services: # The custom log to tail, created by the `log_generator` service - log_path:/var/log/ + # Receives log data from the `datadog-agent-vector` service and sends + # to the `fakeintake-vector` service. + vector: + depends_on: + - fakeintake-vector + build: + context: ${PWD} + # re-using the integration test runner image since it already has + # compiled vector on it. + image: ${CONFIG_VECTOR_IMAGE} + environment: + - FEATURES=e2e-tests-datadog + working_dir: /home/vector + network_mode: host + command: + - "/usr/bin/vector" + - "-vvv" + - "-c" + - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" + volumes: + - ${PWD}:/home/vector + # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-agent: diff --git a/scripts/e2e/e2e-datadog-logs/test.yaml b/scripts/e2e/datadog-logs/test.yaml similarity index 100% rename from scripts/e2e/e2e-datadog-logs/test.yaml rename to scripts/e2e/datadog-logs/test.yaml diff --git a/scripts/e2e/datadog-metrics/README.md b/scripts/e2e/datadog-metrics/README.md new file mode 100644 index 0000000000000..07f0361f9ac6a --- /dev/null +++ b/scripts/e2e/datadog-metrics/README.md @@ -0,0 +1,19 @@ +This e2e test covers the `datadog_agent` source, and the +`datadog_metrics` sink. + +An emitter compose service runs a python DogStatsD program, +to generate various metric types for the test cases. + +Two Agent containers are spun up to receive the metrics, one +for the Agent only case and one for the Agent -> Vector case. + +In the Agent only case, the Agent sends the metrics to `fakeintake` +(another service) directly. This is the baseline. + +In the Agent-Vector case, the Agent send the metrics to the vector +service, and the `datadog_metrics` sink sends to a separate +`fakeintake` service. This is the compare case. + +The two sets of data should be shaped the same in terms of when +the events were received, and the content of the events, but the +timestamps themselves are not guaranteed to align. diff --git a/scripts/e2e/datadog-metrics/compose.yaml b/scripts/e2e/datadog-metrics/compose.yaml new file mode 100644 index 0000000000000..d0860b83b71e3 --- /dev/null +++ b/scripts/e2e/datadog-metrics/compose.yaml @@ -0,0 +1,82 @@ +version: '3' + +services: + + # Emits metrics to the Agent only path + dogstatsd-client-agent: + build: ./dogstatsd_client + environment: + - STATSD_HOST=datadog-agent + depends_on: + - datadog-agent + + # Emits metrics to the Agent-Vector path + dogstatsd-client-vector: + build: ./dogstatsd_client + environment: + - STATSD_HOST=datadog-agent-vector + depends_on: + - datadog-agent-vector + + # Sends metric data received from the Emitter to the `fakeintake-agent` service + datadog-agent: + image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} + depends_on: + - fakeintake-agent + environment: + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent + volumes: + # The Agent config file + - ${PWD}/tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml + + # Sends metric data received from the Emitter to the `vector` service + datadog-agent-vector: + image: docker.io/datadog/agent:${CONFIG_AGENT_VERSION} + depends_on: + - vector + environment: + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector + volumes: + # The Agent config file + - ${PWD}/tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + + # Receives metric data from the `datadog-agent-vector` service and sends + # to the `fakeintake-vector` service. + vector: + depends_on: + - fakeintake-vector + build: + context: ${PWD} + # re-using the integration test runner image since it already has + # compiled vector on it. + image: ${CONFIG_VECTOR_IMAGE} + environment: + - FEATURES=e2e-tests-datadog + working_dir: /home/vector + network_mode: host + command: + - "/usr/bin/vector" + - "-vvv" + - "-c" + - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" + volumes: + - ${PWD}:/home/vector + + # Receives metric data from the `datadog-agent` service. Is queried by the test runner + # which does the validation of consistency with the other fakeintake service. + fakeintake-agent: + image: docker.io/datadog/fakeintake:latest + + # Receives metric data from the `datadog-agent-vector` service. Is queried by the test runner + # which does the validation of consistency with the other fakeintake service. + fakeintake-vector: + image: docker.io/datadog/fakeintake:latest + +networks: + default: + name: ${VECTOR_NETWORK} + +volumes: + target: {} diff --git a/scripts/e2e/datadog-metrics/dogstatsd_client/Dockerfile b/scripts/e2e/datadog-metrics/dogstatsd_client/Dockerfile new file mode 100644 index 0000000000000..e8769539c8092 --- /dev/null +++ b/scripts/e2e/datadog-metrics/dogstatsd_client/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.7-alpine + +COPY . /app +WORKDIR /app + +RUN pip install -r requirements.txt + +CMD [ "python3", "./client.py"] diff --git a/scripts/e2e/datadog-metrics/dogstatsd_client/client.py b/scripts/e2e/datadog-metrics/dogstatsd_client/client.py new file mode 100644 index 0000000000000..b4a9b7390be6b --- /dev/null +++ b/scripts/e2e/datadog-metrics/dogstatsd_client/client.py @@ -0,0 +1,56 @@ +from datadog import initialize, statsd +import time +import os +import random + +STATSD_HOST = os.getenv('STATSD_HOST') + +print(f"initializing for {STATSD_HOST}") + +options = { + 'statsd_host':STATSD_HOST, + 'statsd_port':8125 +} + +initialize(**options) + +# Give the Agent time to actually spin up. +# The container may return "ready" but the +# Agent process is still booting. +time.sleep(10) + +hist_data = [ + 9, 5, 0, 2, 16, 17, 8, 16, 10, 13, + 15, 3, 9, 13, 11, 17, 5, 18, 14, 9, + 4, 16, 9, 17, 4, 11, 7, 14, 8, 12, + 10, 9, 11, 3, 18, 12, 17, 12, 3, 19, + 9, 11, 19, 9, 15, 2, 7, 10, 4, 14 +] + +dist_data = [ + 18, 5, 19, 0, 13, 12, 5, 12, 10, 4, + 1, 5, 7, 1, 14, 16, 20, 0, 8, 2, 4, + 20, 8, 4, 20, 6, 20, 3, 10, 11, 12, + 15, 2, 12, 5, 19, 19, 5, 9, 6, 18, + 19, 11, 6, 17, 5, 0, 1, 17, 17 +] + +for i in range(50): + print("rate") + statsd.increment('foo_metric.rate', tags=['a_tag:1']) + + print("gauge") + statsd.gauge('foo_metric.gauge', i, tags=["a_tag:2"]) + + print("set") + statsd.set('foo_metric.set', i, tags=["a_tag:3"]) + + print("histogram") + statsd.histogram('foo_metric.histogram', hist_data[i], tags=["a_tag:4"]) + + print("distribution") + statsd.distribution('foo_metric.distribution', dist_data[i], tags=["a_tag:5"]) + + statsd.flush() + time.sleep(0.01) + diff --git a/scripts/e2e/datadog-metrics/dogstatsd_client/requirements.txt b/scripts/e2e/datadog-metrics/dogstatsd_client/requirements.txt new file mode 100644 index 0000000000000..43023b823d991 --- /dev/null +++ b/scripts/e2e/datadog-metrics/dogstatsd_client/requirements.txt @@ -0,0 +1 @@ +datadog diff --git a/scripts/e2e/datadog-metrics/test.yaml b/scripts/e2e/datadog-metrics/test.yaml new file mode 100644 index 0000000000000..b9025d08ebf3d --- /dev/null +++ b/scripts/e2e/datadog-metrics/test.yaml @@ -0,0 +1,27 @@ +features: +- e2e-tests-datadog + +test: "e2e" + +test_filter: 'datadog::metrics::' + +runner: + env: + VECTOR_RECEIVE_PORT: '8081' + FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' + FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' + +matrix: + # validate against the Agent latest nightly and also stable v6 and v7 + agent_version: ['latest', '6', '7'] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: +- "src/common/datadog.rs" +- "src/sources/datadog_agent/**" +- "src/internal_events/datadog_*" +- "src/sinks/datadog/metrics/**" +- "src/sinks/util/**" +- "scripts/integration/datadog-e2e/metrics/**" +- "tests/data/e2e/datadog/metrics/**" diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 7f968a1433dfe..5077453861fbd 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -9,39 +9,55 @@ use vector_lib::{event::DatadogMetricOriginMetadata, sensitive_string::Sensitive pub(crate) const DD_US_SITE: &str = "datadoghq.com"; pub(crate) const DD_EU_SITE: &str = "datadoghq.eu"; +/// DatadogSeriesMetric #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub(crate) struct DatadogSeriesMetric { - pub(crate) metric: String, - pub(crate) r#type: DatadogMetricType, - pub(crate) interval: Option, - pub(crate) points: Vec>, - pub(crate) tags: Option>, +pub struct DatadogSeriesMetric { + /// metric + pub metric: String, + /// metric type + pub r#type: DatadogMetricType, + /// interval + pub interval: Option, + /// points + pub points: Vec>, + /// tags + pub tags: Option>, + /// host #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) host: Option, + pub host: Option, + /// source_type_name #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) source_type_name: Option, + pub source_type_name: Option, + /// device #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) device: Option, + pub device: Option, + /// metadata #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) metadata: Option, + pub metadata: Option, } +/// Datadog series metric metadata #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub(crate) struct DatadogSeriesMetricMetadata { +pub struct DatadogSeriesMetricMetadata { #[serde(skip_serializing_if = "Option::is_none")] pub(crate) origin: Option, } +/// Datadog Metric Type #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum DatadogMetricType { +pub enum DatadogMetricType { + /// Gauge Gauge, + /// Count Count, + /// Rate Rate, } +/// Datadog Point #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -pub(crate) struct DatadogPoint(pub(crate) i64, pub(crate) T); +pub struct DatadogPoint(pub i64, pub T); /// Gets the base API endpoint to use for any calls to Datadog. /// diff --git a/tests/data/e2e/datadog/logs/agent_only.yaml b/tests/data/e2e/datadog/logs/agent_only.yaml index aeebb113c1075..dcdfcab9b40dd 100644 --- a/tests/data/e2e/datadog/logs/agent_only.yaml +++ b/tests/data/e2e/datadog/logs/agent_only.yaml @@ -5,6 +5,7 @@ log_level: 'debug' inventories_configuration_enabled: false enable_metadata_collection: false enable_gohai: false +cloud_provider_metadata: [] apm_config: enabled: false diff --git a/tests/data/e2e/datadog/logs/agent_vector.yaml b/tests/data/e2e/datadog/logs/agent_vector.yaml index 1686986746971..411da03025e0c 100644 --- a/tests/data/e2e/datadog/logs/agent_vector.yaml +++ b/tests/data/e2e/datadog/logs/agent_vector.yaml @@ -5,6 +5,7 @@ log_level: 'debug' inventories_configuration_enabled: false enable_metadata_collection: false enable_gohai: false +cloud_provider_metadata: [] apm_config: enabled: false diff --git a/tests/data/e2e/datadog/metrics/agent_only.yaml b/tests/data/e2e/datadog/metrics/agent_only.yaml new file mode 100644 index 0000000000000..0ed0518c84b18 --- /dev/null +++ b/tests/data/e2e/datadog/metrics/agent_only.yaml @@ -0,0 +1,26 @@ +api_key: DEADBEEF +log_level: 'warn' + +# disable bunch of stuff we don't need +inventories_configuration_enabled: false +enable_gohai: false +cloud_provider_metadata: [] + +apm_config: + enabled: false + +process_config: + container_collection: + enabled: false + process_discovery: + enabled: false + disable_realtime_checks: true + +logs_enabled: false + +# configure dogstatsd +use_dogstatsd: true +dogstatsd_non_local_traffic: true + +# configure metrics +dd_url: http://fakeintake-agent:80 diff --git a/tests/data/e2e/datadog/metrics/agent_vector.yaml b/tests/data/e2e/datadog/metrics/agent_vector.yaml new file mode 100644 index 0000000000000..c1566946a96b6 --- /dev/null +++ b/tests/data/e2e/datadog/metrics/agent_vector.yaml @@ -0,0 +1,29 @@ +api_key: DEADBEEF +log_level: 'warn' + +# disable bunch of stuff we don't need +inventories_configuration_enabled: false +enable_gohai: false +cloud_provider_metadata: [] + +apm_config: + enabled: false + +process_config: + container_collection: + enabled: false + process_discovery: + enabled: false + disable_realtime_checks: true + +logs_enabled: false + +# configure dogstatsd +use_dogstatsd: true +dogstatsd_non_local_traffic: true + +# send to vector +vector: + metrics: + enabled: true + url: "http://vector:8181" diff --git a/tests/data/e2e/datadog/metrics/vector.toml b/tests/data/e2e/datadog/metrics/vector.toml new file mode 100644 index 0000000000000..c99e2f60ab9ba --- /dev/null +++ b/tests/data/e2e/datadog/metrics/vector.toml @@ -0,0 +1,16 @@ +data_dir = "/tmp/" + +[sources.agent] +type = "datadog_agent" +address = "0.0.0.0:8181" +multiple_outputs = true +disable_logs = true +disable_traces = true +store_api_key = false + +[sinks.dd] +inputs = [ "agent.metrics" ] +type = "datadog_metrics" +default_api_key = "unused" +endpoint = "http://fakeintake-vector:80" +batch.timeout_secs = 1 diff --git a/tests/e2e/datadog/logs/mod.rs b/tests/e2e/datadog/logs/mod.rs index 630c3a1ff191a..de9210388ed39 100644 --- a/tests/e2e/datadog/logs/mod.rs +++ b/tests/e2e/datadog/logs/mod.rs @@ -58,21 +58,48 @@ fn common_assertions(payloads: &mut [Value]) { assert_eq!(n_log_events, expected_log_events()); } +// reduces the payload down to just the log data +fn reduce_to_data(payloads: Vec>) -> Vec { + payloads + .into_iter() + .map(|mut payload| payload.data.take()) + .collect() +} + #[tokio::test] async fn validate() { trace_init(); - // a small sleep here is kind of hard to avoid. Regardless of dependencies flagged for the - // containers, we need the events to flow between them. - std::thread::sleep(std::time::Duration::from_secs(5)); + // Even with configuring docker service dependencies, we need a small buffer of time + // to ensure events flow through to fakeintake before asking for them + std::thread::sleep(std::time::Duration::from_secs(2)); info!("getting log payloads from agent-only pipeline"); - let mut agent_payloads = get_payloads_agent(LOGS_ENDPOINT).await; + let mut agent_payloads = get_fakeintake_payloads::( + &fake_intake_agent_address(), + LOGS_ENDPOINT, + ) + .await + .payloads; + + // the logs endpoint receives an empty healthcheck payload in the beginning + if !agent_payloads.is_empty() { + agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()) + } + + let mut agent_payloads = reduce_to_data(agent_payloads); common_assertions(&mut agent_payloads); info!("getting log payloads from agent-vector pipeline"); - let mut vector_payloads = get_payloads_vector(LOGS_ENDPOINT).await; + let vector_payloads = get_fakeintake_payloads::( + &fake_intake_vector_address(), + LOGS_ENDPOINT, + ) + .await + .payloads; + + let mut vector_payloads = reduce_to_data(vector_payloads); common_assertions(&mut vector_payloads); diff --git a/tests/e2e/datadog/metrics/mod.rs b/tests/e2e/datadog/metrics/mod.rs new file mode 100644 index 0000000000000..875d74f0fc3c3 --- /dev/null +++ b/tests/e2e/datadog/metrics/mod.rs @@ -0,0 +1,54 @@ +use base64::{prelude::BASE64_STANDARD, Engine}; +use bytes::Bytes; +use flate2::read::ZlibDecoder; +use std::io::Read; + +use vector::test_util::trace_init; + +mod series; +mod sketches; + +use super::*; + +fn decompress_payload(payload: Vec) -> std::io::Result> { + let mut decompressor = ZlibDecoder::new(&payload[..]); + let mut decompressed = Vec::new(); + let result = decompressor.read_to_end(&mut decompressed); + result.map(|_| decompressed) +} + +fn unpack_proto_payloads(in_payloads: &FakeIntakeResponseRaw) -> Vec +where + T: prost::Message + std::default::Default, +{ + let mut out_payloads = vec![]; + + in_payloads.payloads.iter().for_each(|payload| { + // decode base64 + let payload = BASE64_STANDARD + .decode(&payload.data) + .expect("Invalid base64 data"); + + // decompress + let bytes = Bytes::from(decompress_payload(payload).unwrap()); + + let payload = T::decode(bytes).unwrap(); + + out_payloads.push(payload); + }); + + out_payloads +} + +#[tokio::test] +async fn validate() { + trace_init(); + + // Even with configuring docker service dependencies, we need a small buffer of time + // to ensure events flow through to fakeintake before asking for them + std::thread::sleep(std::time::Duration::from_secs(2)); + + series::validate().await; + + sketches::validate().await; +} diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs new file mode 100644 index 0000000000000..0094f014e372d --- /dev/null +++ b/tests/e2e/datadog/metrics/series.rs @@ -0,0 +1,303 @@ +use std::collections::BTreeMap; + +#[allow(warnings, clippy::pedantic, clippy::nursery)] +mod ddmetric_proto { + include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); +} + +use ddmetric_proto::{ + metric_payload::{MetricSeries, MetricType}, + MetricPayload, +}; +use tracing::info; +use vector::common::datadog::DatadogSeriesMetric; + +use self::ddmetric_proto::metric_payload::{MetricPoint, Resource}; + +use super::*; + +const SERIES_ENDPOINT_V1: &str = "/api/v1/series"; +const SERIES_ENDPOINT_V2: &str = "/api/v2/series"; + +// unique identification of a Series +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct SeriesContext { + metric_name: String, + tags: Vec, + r#type: i32, +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct TimeBucket(i64, i64); + +fn get_time_bucket(point: &MetricPoint, interval: i64, metric_type: MetricType) -> TimeBucket { + match metric_type { + MetricType::Unspecified => panic!("received an unspecified metric type"), + MetricType::Rate => TimeBucket(point.timestamp - interval, point.timestamp), + MetricType::Gauge | MetricType::Count => TimeBucket(point.timestamp, point.timestamp), + } +} + +type TimeSeriesData = BTreeMap>; + +/// This type represents the massaged intake data collected from the upstream. +/// The idea is to be able to store what was received in a way that allows us to +/// compare what is important to compare, and accounting for the bits that are not +/// guaranteed to line up. +/// +/// For instance, the services that are running, may start at different times, thus the +/// timestamps (TimeBucket) for data points received are not guaranteed to match up. +type SeriesIntake = BTreeMap; + +// massages the raw payloads into our intake structure +fn generate_series_intake(payloads: &[MetricPayload]) -> SeriesIntake { + let mut intake = BTreeMap::new(); + + payloads.iter().for_each(|payload| { + payload.series.iter().for_each(|serie| { + // filter out the metrics we don't care about (ones not generated by the client) + if !serie.metric.starts_with("foo_metric") { + return; + } + + let ctx = SeriesContext { + metric_name: serie.metric.clone(), + tags: serie.tags.clone(), + r#type: serie.r#type, + }; + + if !intake.contains_key(&ctx) { + intake.insert(ctx.clone(), BTreeMap::new()); + } + let entry: &mut BTreeMap> = intake.get_mut(&ctx).unwrap(); + + serie.points.iter().for_each(|point| { + let tb = get_time_bucket(point, serie.interval, serie.r#type()); + if !entry.contains_key(&tb) { + entry.insert(tb.clone(), Vec::new()); + } + entry.get_mut(&tb).unwrap().push(point.value); + }); + }); + }); + + intake +} + +// runs assertions that each set of payloads should be true to regardless +// of the pipeline +fn common_series_assertions(series: &SeriesIntake) { + // we should have received some metrics from the emitter + assert!(!series.is_empty()); + info!("metric series received: {}", series.len()); + + // specifically we should have received each of these + let mut found = [ + // NOTE: no count expected due to the in-app type being Rate + // (https://docs.datadoghq.com/metrics/types/?tab=count#submission-types-and-datadog-in-app-types) + (false, "rate"), + (false, "gauge"), + (false, "set"), + (false, "histogram"), + ]; + series.keys().for_each(|ctx| { + found.iter_mut().for_each(|found| { + if ctx + .metric_name + .starts_with(&format!("foo_metric.{}", found.1)) + { + info!("received {}", found.1); + found.0 = true; + } + }); + }); + + found + .iter() + .for_each(|(found, mtype)| assert!(found, "Didn't receive metric type {}", *mtype)); +} + +impl From<&DatadogSeriesMetric> for MetricSeries { + fn from(input: &DatadogSeriesMetric) -> Self { + let mut resources = vec![]; + if let Some(host) = &input.host { + resources.push(Resource { + r#type: "host".to_string(), + name: host.clone(), + }); + } + + let points = input + .points + .iter() + .map(|point| MetricPoint { + value: point.1, + timestamp: point.0, + }) + .collect(); + + let interval = input.interval.unwrap_or(0) as i64; + + let r#type = match input.r#type { + vector::common::datadog::DatadogMetricType::Gauge => 3, + vector::common::datadog::DatadogMetricType::Count => 1, + vector::common::datadog::DatadogMetricType::Rate => 2, + }; + + MetricSeries { + resources, + metric: input.metric.clone(), + tags: input.tags.clone().unwrap_or_default(), + points, + r#type, + unit: "".to_string(), + source_type_name: input.clone().source_type_name.unwrap_or_default(), + interval, + metadata: None, + } + } +} + +fn convert_v1_payloads_v2(input: &[DatadogSeriesMetric]) -> Vec { + input + .iter() + .map(|serie| MetricPayload { + series: vec![serie.into()], + }) + .collect() +} + +fn unpack_v1_series(in_payloads: &[FakeIntakePayloadJson]) -> Vec { + in_payloads + .iter() + .flat_map(|payload| { + let series = payload.data.as_array().unwrap(); + series + .iter() + .map(|serie| serde_json::from_value(serie.clone()).unwrap()) + }) + .collect() +} + +async fn get_v1_series_from_pipeline(address: String) -> SeriesIntake { + info!("getting v1 series payloads"); + let payloads = + get_fakeintake_payloads::(&address, SERIES_ENDPOINT_V1).await; + + info!("unpacking payloads"); + let payloads = unpack_v1_series(&payloads.payloads); + info!("converting payloads"); + let payloads = convert_v1_payloads_v2(&payloads); + + info!("generating series intake"); + let intake = generate_series_intake(&payloads); + + common_series_assertions(&intake); + + info!("{:?}", intake); + + intake +} + +async fn get_v2_series_from_pipeline(address: String) -> SeriesIntake { + info!("getting v2 series payloads"); + let payloads = + get_fakeintake_payloads::(&address, SERIES_ENDPOINT_V2).await; + + info!("unpacking payloads"); + let payloads = unpack_proto_payloads::(&payloads); + + info!("generating series intake"); + let intake = generate_series_intake(&payloads); + + common_series_assertions(&intake); + + info!("{:?}", intake); + + intake +} + +pub(super) async fn validate() { + info!("==== getting series data from agent-only pipeline ==== "); + let agent_intake = get_v2_series_from_pipeline(fake_intake_agent_address()).await; + + info!("==== getting series data from agent-vector pipeline ===="); + let vector_intake = get_v1_series_from_pipeline(fake_intake_vector_address()).await; + + assert_eq!( + agent_intake.len(), + vector_intake.len(), + "different number of unique Series contexts received" + ); + + // The assertions we make below can be summarized as follows: + // - For each metric type, we have a different set of assertions which are relevant to + // the expectations for that metric type's behavior. For example, gauges are a last + // write wins. While rates are taken as a summation over the time interval. + // + // - The intake from each case (Agent only, and Agent->Vector) is stored in a data structure + // that allows us to compare the consistency in the overall shape of the data collected + // during the entire test duration, regardless of how requests are batched. This is because + // the data is stored based on a time bucket, and not per request. And the time buckets holding + // data points are collected in BTreeMap, which means the sort order will be consistent, + // regardless of whether the actual timestamps between the Agent only vs Agent+Vector cases + // are identical (which they may not be, since there is no guarantee the compose services + // started at the same time, nor that the Agent will instances will process in the same time). + // + // Together, this means that data sets passing these validations confirm that the Vector version + // used in the test case is not introducing inconsistencies in the data flowing between the + // Agent and the Datadog backend. + + agent_intake + .iter() + .zip(vector_intake.iter()) + .for_each(|(agent_ts, vector_ts)| { + assert_eq!(agent_ts.0, vector_ts.0, "Mismatch of series context"); + + let metric_type = agent_ts.0.r#type; + + // Dogstatsd emits counters but the output type from the Agent is Rate. + // https://docs.datadoghq.com/metrics/types/?tab=rate#submission-types-and-datadog-in-app-types + assert!( + metric_type == 2 || metric_type == 3, + "Metric type should always be rate or gauge." + ); + + // gauge: last one wins. + // we can't rely on comparing each value due to the fact that we can't guarantee consistent sampling + if metric_type == 3 { + let last_agent_point = agent_ts + .1 + .last_key_value() + .expect("should have received points") + .1; + let last_vector_point = vector_ts + .1 + .last_key_value() + .expect("should have received points") + .1; + + assert_eq!( + last_agent_point, last_vector_point, + "Mismatch of gauge data" + ); + } + + // rate: summation. + if metric_type == 2 { + let agent_sum: f64 = agent_ts + .1 + .iter() + .map(|(_tb, points)| points.iter().sum::()) + .sum(); + + let vector_sum: f64 = vector_ts + .1 + .iter() + .map(|(_tb, points)| points.iter().sum::()) + .sum(); + + assert_eq!(agent_sum, vector_sum, "Mismatch of rate data"); + } + }); +} diff --git a/tests/e2e/datadog/metrics/sketches.rs b/tests/e2e/datadog/metrics/sketches.rs new file mode 100644 index 0000000000000..f506e4f6e1176 --- /dev/null +++ b/tests/e2e/datadog/metrics/sketches.rs @@ -0,0 +1,127 @@ +use std::collections::BTreeMap; + +#[allow(warnings, clippy::pedantic, clippy::nursery)] +mod ddmetric_proto { + include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); +} + +use ddmetric_proto::{ + sketch_payload::sketch::{Distribution, Dogsketch}, + SketchPayload, +}; +use tracing::info; + +use super::*; + +const SKETCHES_ENDPOINT: &str = "/api/beta/sketches"; + +// unique identification of a Sketch +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +struct SketchContext { + metric_name: String, + tags: Vec, +} + +type TimeSketchData = BTreeMap>; + +/// This type represents the massaged intake data collected from the upstream. +/// The idea is to be able to store what was received in a way that allows us to +/// compare what is important to compare, and accounting for the bits that are not +/// guaranteed to line up. +/// +/// For instance, the services that are running, may start at different times, thus the +/// timestamps for data points received are not guaranteed to match up. +type SketchIntake = + BTreeMap, TimeSketchData)>; + +// massages the raw payloads into our intake structure +fn generate_sketch_intake(mut payloads: Vec) -> SketchIntake { + let mut intake = SketchIntake::new(); + + payloads.iter_mut().for_each(|payload| { + payload.sketches.iter_mut().for_each(|sketch| { + // filter out the metrics we don't care about (ones not generated by the client) + if !sketch.metric.starts_with("foo_metric") { + return; + } + let ctx = SketchContext { + metric_name: sketch.metric.clone(), + tags: sketch.tags.clone(), + }; + + if !intake.contains_key(&ctx) { + intake.insert(ctx.clone(), (TimeSketchData::new(), TimeSketchData::new())); + } + let entry: &mut (TimeSketchData, TimeSketchData) = + intake.get_mut(&ctx).unwrap(); + + sketch.dogsketches.iter_mut().for_each(|ds| { + let ts = ds.ts; + entry.0.entry(ts).or_default(); + ds.ts = 0; + entry.0.get_mut(&ts).unwrap().push(ds.clone()); + }); + + sketch.distributions.iter_mut().for_each(|dt| { + let ts = dt.ts; + entry.1.entry(ts).or_default(); + dt.ts = 0; + entry.1.get_mut(&ts).unwrap().push(dt.clone()); + }); + }); + }); + + intake +} + +// runs assertions that each set of payloads should be true to regardless +// of the pipeline +fn common_sketch_assertions(sketches: &SketchIntake) { + // we should have received some metrics from the emitter + assert!(!sketches.is_empty()); + info!("metric sketch received: {}", sketches.len()); + + let mut found = false; + sketches.keys().for_each(|ctx| { + if ctx.metric_name.starts_with("foo_metric.distribution") { + found = true; + } + }); + + assert!(found, "Didn't receive metric type distribution"); +} + +async fn get_sketches_from_pipeline(address: String) -> SketchIntake { + info!("getting sketch payloads"); + let payloads = + get_fakeintake_payloads::(&address, SKETCHES_ENDPOINT).await; + + info!("unpacking payloads"); + let payloads = unpack_proto_payloads(&payloads); + + info!("generating sketch intake"); + let sketches = generate_sketch_intake(payloads); + + common_sketch_assertions(&sketches); + + info!("{:?}", sketches); + + sketches +} + +pub(super) async fn validate() { + info!("==== getting sketch data from agent-only pipeline ==== "); + let agent_sketches = get_sketches_from_pipeline(fake_intake_agent_address()).await; + + info!("==== getting sketch data from agent-vector pipeline ===="); + let vector_sketches = get_sketches_from_pipeline(fake_intake_vector_address()).await; + + agent_sketches + .iter() + .zip(vector_sketches.iter()) + .for_each(|(agent_s, vector_s)| { + assert_eq!(agent_s.0, vector_s.0, "Mismatch of sketch context"); + + assert_eq!(agent_s.1, vector_s.1, "Mismatch of sketch data"); + }); +} diff --git a/tests/e2e/datadog/mod.rs b/tests/e2e/datadog/mod.rs index 12baef8fe5c9a..2e64f40ab1e93 100644 --- a/tests/e2e/datadog/mod.rs +++ b/tests/e2e/datadog/mod.rs @@ -1,67 +1,73 @@ pub mod logs; +pub mod metrics; use reqwest::{Client, Method}; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use serde_json::Value; -fn fake_intake_vector_endpoint() -> String { +fn fake_intake_vector_address() -> String { std::env::var("FAKE_INTAKE_VECTOR_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()) } -fn fake_intake_agent_endpoint() -> String { +fn fake_intake_agent_address() -> String { std::env::var("FAKE_INTAKE_AGENT_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8083".to_string()) } -// Fakeintake response #[derive(Deserialize, Debug)] -struct FakeIntakeResponse { - payloads: Vec, +struct FakeIntakePayload { + // When string, base64 encoded + data: D, + #[serde(rename = "encoding")] + _encoding: String, + #[serde(rename = "timestamp")] + _timestamp: String, +} + +type FakeIntakePayloadJson = FakeIntakePayload; + +type FakeIntakePayloadRaw = FakeIntakePayload; + +trait FakeIntakeResponseT { + fn build_url(base: &str, endpoint: &str) -> String; } -#[allow(dead_code)] #[derive(Deserialize, Debug)] -struct FakeIntakePayload { - data: Value, - encoding: String, - timestamp: String, +struct FakeIntakeResponse

{ + payloads: Vec

, } -async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> FakeIntakeResponse { - let url = format!( - "{}/fakeintake/payloads?endpoint={}&format=json", - base, endpoint, - ); +type FakeIntakeResponseJson = FakeIntakeResponse; - Client::new() - .request(Method::GET, &url) - .send() - .await - .unwrap_or_else(|_| panic!("Sending GET request to {} failed", &url)) - .json::() - .await - .expect("Parsing fakeintake payloads failed") +impl FakeIntakeResponseT for FakeIntakeResponseJson { + fn build_url(base: &str, endpoint: &str) -> String { + format!( + "{}/fakeintake/payloads?endpoint={}&format=json", + base, endpoint, + ) + } } -async fn get_payloads_agent(endpoint: &str) -> Vec { - let mut raw_payloads = get_fakeintake_payloads(&fake_intake_agent_endpoint(), endpoint) - .await - .payloads; +type FakeIntakeResponseRaw = FakeIntakeResponse; - // the logs endpoint receives an empty healthcheck payload in the beginning - if !raw_payloads.is_empty() && endpoint == "/api/v2/logs" { - raw_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty()) +impl FakeIntakeResponseT for FakeIntakeResponseRaw { + fn build_url(base: &str, endpoint: &str) -> String { + format!("{}/fakeintake/payloads?endpoint={}", base, endpoint,) } - - raw_payloads.into_iter().map(|raw| raw.data).collect() } -async fn get_payloads_vector(endpoint: &str) -> Vec { - get_fakeintake_payloads(&fake_intake_vector_endpoint(), endpoint) +async fn get_fakeintake_payloads(base: &str, endpoint: &str) -> R +where + R: FakeIntakeResponseT + DeserializeOwned, +{ + let url = &R::build_url(base, endpoint); + Client::new() + .request(Method::GET, url) + .send() .await - .payloads - .into_iter() - .map(|raw| raw.data) - .collect() + .unwrap_or_else(|_| panic!("Sending GET request to {} failed", url)) + .json::() + .await + .expect("Parsing fakeintake payloads failed") } diff --git a/vdev/src/commands/compose_tests/ci_paths.rs b/vdev/src/commands/compose_tests/ci_paths.rs new file mode 100644 index 0000000000000..e18b9ce0bc2d5 --- /dev/null +++ b/vdev/src/commands/compose_tests/ci_paths.rs @@ -0,0 +1,20 @@ +use anyhow::Result; + +use crate::testing::config::ComposeTestConfig; + +pub(crate) fn exec(path: &str) -> Result<()> { + // placeholder for changes that should run all integration tests + println!("all-tests: []"); + + // paths for each integration are defined in their respective config files. + for (test_name, config) in ComposeTestConfig::collect_all(path)? { + if let Some(paths) = config.paths { + println!("{test_name}:"); + for path in paths { + println!("- {path:?}"); + } + } + } + + Ok(()) +} diff --git a/vdev/src/commands/compose_tests/mod.rs b/vdev/src/commands/compose_tests/mod.rs new file mode 100644 index 0000000000000..27a033f199a12 --- /dev/null +++ b/vdev/src/commands/compose_tests/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod ci_paths; +pub(crate) mod show; +pub(crate) mod start; +pub(crate) mod stop; +pub(crate) mod test; diff --git a/vdev/src/commands/compose_tests/show.rs b/vdev/src/commands/compose_tests/show.rs new file mode 100644 index 0000000000000..672026ef442ec --- /dev/null +++ b/vdev/src/commands/compose_tests/show.rs @@ -0,0 +1,93 @@ +use anyhow::Result; + +use crate::testing::{config::ComposeTestConfig, config::Environment, state}; + +pub fn exec(integration: &Option, path: &str) -> Result<()> { + match integration { + None => { + let entries = ComposeTestConfig::collect_all(path)?; + let width = entries + .keys() + .fold(16, |width, entry| width.max(entry.len())); + println!("{:width$} Environment Name(s)", "Integration Name"); + println!("{:width$} -------------------", "----------------"); + for (integration, config) in entries { + let envs_dir = state::EnvsDir::new(&integration); + let active_env = envs_dir.active()?; + let environments = config + .environments() + .keys() + .map(|environment| format(&active_env, environment)) + .collect::>() + .join(" "); + println!("{integration:width$} {environments}"); + } + } + Some(integration) => { + let (_test_dir, config) = ComposeTestConfig::load(path, integration)?; + let envs_dir = state::EnvsDir::new(integration); + let active_env = envs_dir.active()?; + + if let Some(args) = &config.args { + println!("Test args: {}", args.join(" ")); + } else { + println!("Test args: N/A"); + } + + if config.features.is_empty() { + println!("Features: N/A"); + } else { + println!("Features: {}", config.features.join(",")); + } + + println!( + "Test filter: {}", + config.test_filter.as_deref().unwrap_or("N/A") + ); + + println!("Environment:"); + print_env(" ", &config.env); + println!("Runner:"); + println!(" Environment:"); + print_env(" ", &config.runner.env); + println!(" Volumes:"); + if config.runner.volumes.is_empty() { + println!(" N/A"); + } else { + for (target, mount) in &config.runner.volumes { + println!(" {target} => {mount}"); + } + } + println!( + " Needs docker socket: {}", + config.runner.needs_docker_socket + ); + + println!("Environments:"); + for environment in config.environments().keys() { + println!(" {}", format(&active_env, environment)); + } + } + } + Ok(()) +} + +fn format(active_env: &Option, environment: &str) -> String { + match active_env { + Some(active) if active == environment => format!("{environment} (active)"), + _ => environment.into(), + } +} + +fn print_env(prefix: &str, environment: &Environment) { + if environment.is_empty() { + println!("{prefix}N/A"); + } else { + for (key, value) in environment { + match value { + Some(value) => println!("{prefix}{key}={value:?}"), + None => println!("{prefix}{key} (passthrough)"), + } + } + } +} diff --git a/vdev/src/commands/compose_tests/start.rs b/vdev/src/commands/compose_tests/start.rs new file mode 100644 index 0000000000000..3ccb7cdb57125 --- /dev/null +++ b/vdev/src/commands/compose_tests/start.rs @@ -0,0 +1,20 @@ +use anyhow::Result; + +use crate::testing::{config::ComposeTestConfig, integration::ComposeTestT}; + +pub(crate) fn exec( + integration: &str, + environment: &Option, + build_all: bool, +) -> Result<()> { + let environment = if let Some(environment) = environment { + environment.clone() + } else { + let (_test_dir, config) = ComposeTestConfig::load(T::DIRECTORY, integration)?; + let envs = config.environments(); + let env = envs.keys().next().expect("Integration has no environments"); + env.clone() + }; + + T::start(&T::generate(integration, environment, build_all, 0)?) +} diff --git a/vdev/src/commands/compose_tests/stop.rs b/vdev/src/commands/compose_tests/stop.rs new file mode 100644 index 0000000000000..7a780faf4d507 --- /dev/null +++ b/vdev/src/commands/compose_tests/stop.rs @@ -0,0 +1,12 @@ +use anyhow::Result; + +use crate::testing::{integration::ComposeTestT, state::EnvsDir}; + +pub(crate) fn exec(test_name: &str, all_features: bool) -> Result<()> { + if let Some(active) = EnvsDir::new(test_name).active()? { + T::stop(&T::generate(test_name, active, all_features, 0)?) + } else { + println!("No environment for {test_name} is active."); + Ok(()) + } +} diff --git a/vdev/src/commands/compose_tests/test.rs b/vdev/src/commands/compose_tests/test.rs new file mode 100644 index 0000000000000..25c6175b5e6a5 --- /dev/null +++ b/vdev/src/commands/compose_tests/test.rs @@ -0,0 +1,39 @@ +use anyhow::{bail, Result}; + +use crate::testing::{config::ComposeTestConfig, integration::ComposeTestT, state::EnvsDir}; + +pub fn exec( + integration: &str, + environment: &Option, + build_all: bool, + retries: u8, + args: &[String], +) -> Result<()> { + let (_test_dir, config) = ComposeTestConfig::load(T::DIRECTORY, integration)?; + let envs = config.environments(); + + let active = EnvsDir::new(integration).active()?; + + match (&environment, &active) { + (Some(environment), Some(active)) if environment != active => { + bail!("Requested environment {environment:?} does not match active one {active:?}") + } + (Some(environment), _) => T::test( + &T::generate(integration, environment, build_all, retries)?, + args.to_owned(), + ), + (None, Some(active)) => T::test( + &T::generate(integration, active, build_all, retries)?, + args.to_owned(), + ), + (None, None) => { + for env_name in envs.keys() { + T::test( + &T::generate(integration, env_name, build_all, retries)?, + args.to_owned(), + )?; + } + Ok(()) + } + } +} diff --git a/vdev/src/commands/e2e/ci_paths.rs b/vdev/src/commands/e2e/ci_paths.rs new file mode 100644 index 0000000000000..d3679a3b7bb5a --- /dev/null +++ b/vdev/src/commands/e2e/ci_paths.rs @@ -0,0 +1,16 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::config::E2E_TESTS_DIR; + +/// Output paths in the repository that are associated with an integration. +/// If any changes are made to these paths, that integration should be tested. +#[derive(Args, Debug)] +#[command()] +pub struct Cli {} + +impl Cli { + pub fn exec(&self) -> Result<()> { + crate::commands::compose_tests::ci_paths::exec(E2E_TESTS_DIR) + } +} diff --git a/vdev/src/commands/e2e/mod.rs b/vdev/src/commands/e2e/mod.rs new file mode 100644 index 0000000000000..be9a6b77af2d8 --- /dev/null +++ b/vdev/src/commands/e2e/mod.rs @@ -0,0 +1,13 @@ +crate::cli_subcommands! { + r#"Manage end-to-end test environments... + +These test setups are organized into a set of integrations, located in subdirectories +`scripts/e2e`. For each integration, there is a matrix of environments, described in the +`matrix` setting in the `test.yaml` file contained therein."# + + mod show, + mod start, + mod stop, + mod test, + mod ci_paths, +} diff --git a/vdev/src/commands/e2e/show.rs b/vdev/src/commands/e2e/show.rs new file mode 100644 index 0000000000000..cc1645ba18019 --- /dev/null +++ b/vdev/src/commands/e2e/show.rs @@ -0,0 +1,18 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::config::E2E_TESTS_DIR; + +/// Show information about e2e-tests +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The desired e2e test name + test: Option, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::show::exec(&self.test, E2E_TESTS_DIR) + } +} diff --git a/vdev/src/commands/e2e/start.rs b/vdev/src/commands/e2e/start.rs new file mode 100644 index 0000000000000..63e11ee584740 --- /dev/null +++ b/vdev/src/commands/e2e/start.rs @@ -0,0 +1,29 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::E2ETest; + +/// Start an environment +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The e2e test name + test: String, + + /// Whether to compile the test runner with all integration test features + #[arg(short = 'a', long)] + build_all: bool, + + /// The desired environment name to start. If omitted, the first environment name is used. + environment: Option, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::start::exec::( + &self.test, + &self.environment, + self.build_all, + ) + } +} diff --git a/vdev/src/commands/e2e/stop.rs b/vdev/src/commands/e2e/stop.rs new file mode 100644 index 0000000000000..2a11fd5b4c976 --- /dev/null +++ b/vdev/src/commands/e2e/stop.rs @@ -0,0 +1,22 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::E2ETest; + +/// Stop an e2e-test environment +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The e2e test name to stop + test: String, + + /// If true, remove the runner container compiled with all integration test features + #[arg(short = 'a', long)] + all_features: bool, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::stop::exec::(&self.test, self.all_features) + } +} diff --git a/vdev/src/commands/e2e/test.rs b/vdev/src/commands/e2e/test.rs new file mode 100644 index 0000000000000..45d70a3fabad3 --- /dev/null +++ b/vdev/src/commands/e2e/test.rs @@ -0,0 +1,45 @@ +use anyhow::Result; +use clap::Args; + +use crate::testing::integration::E2ETest; + +/// Execute end-to-end tests +/// +/// If an environment is named, it is used to run the test. If the environment was not previously started, +/// it is started before the test is run and stopped afterwards. +/// +/// If no environment is named, but one has been started already, that environment is used for the test. +/// +/// Otherwise, all environments are started, the test run, and then stopped, one by one. +#[derive(Args, Debug)] +#[command()] +pub struct Cli { + /// The desired e2e test + e2e_test: String, + + /// The desired environment (optional) + environment: Option, + + /// Whether to compile the test runner with all integration test features + #[arg(short = 'a', long)] + build_all: bool, + + /// Number of retries to allow on each integration test case. + #[arg(short = 'r', long)] + retries: Option, + + /// Extra test command arguments + args: Vec, +} + +impl Cli { + pub fn exec(self) -> Result<()> { + crate::commands::compose_tests::test::exec::( + &self.e2e_test, + &self.environment, + self.build_all, + self.retries.unwrap_or_default(), + &self.args, + ) + } +} diff --git a/vdev/src/commands/integration/ci_paths.rs b/vdev/src/commands/integration/ci_paths.rs index c37b682a2ec7b..a91034c396f63 100644 --- a/vdev/src/commands/integration/ci_paths.rs +++ b/vdev/src/commands/integration/ci_paths.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::config::IntegrationTestConfig; +use crate::testing::config::INTEGRATION_TESTS_DIR; /// Output paths in the repository that are associated with an integration. /// If any changes are made to these paths, that integration should be tested. @@ -11,19 +11,6 @@ pub struct Cli {} impl Cli { pub fn exec(&self) -> Result<()> { - // placeholder for changes that should run all integration tests - println!("all-int: []"); - - // paths for each integration are defined in their respective config files. - for (integration, config) in IntegrationTestConfig::collect_all()? { - if let Some(paths) = config.paths { - println!("{integration}:"); - for path in paths { - println!("- \"{path}\""); - } - } - } - - Ok(()) + crate::commands::compose_tests::ci_paths::exec(INTEGRATION_TESTS_DIR) } } diff --git a/vdev/src/commands/integration/show.rs b/vdev/src/commands/integration/show.rs index c6394657534ec..410219e536312 100644 --- a/vdev/src/commands/integration/show.rs +++ b/vdev/src/commands/integration/show.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::{config::Environment, config::IntegrationTestConfig, state}; +use crate::testing::config::INTEGRATION_TESTS_DIR; /// Show information about integrations #[derive(Args, Debug)] @@ -13,86 +13,6 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - match self.integration { - None => { - let entries = IntegrationTestConfig::collect_all()?; - let width = entries - .keys() - .fold(16, |width, entry| width.max(entry.len())); - println!("{:width$} Environment Name(s)", "Integration Name"); - println!("{:width$} -------------------", "----------------"); - for (integration, config) in entries { - let envs_dir = state::EnvsDir::new(&integration); - let active_env = envs_dir.active()?; - let environments = config - .environments() - .keys() - .map(|environment| format(&active_env, environment)) - .collect::>() - .join(" "); - println!("{integration:width$} {environments}"); - } - } - Some(integration) => { - let (_test_dir, config) = IntegrationTestConfig::load(&integration)?; - let envs_dir = state::EnvsDir::new(&integration); - let active_env = envs_dir.active()?; - - if let Some(args) = &config.args { - println!("Test args: {}", args.join(" ")); - } else { - println!("Test args: N/A"); - } - - if config.features.is_empty() { - println!("Features: N/A"); - } else { - println!("Features: {}", config.features.join(",")); - } - - println!("Test filter: {}", config.test_filter.as_deref().unwrap_or("N/A")); - - println!("Environment:"); - print_env(" ", &config.env); - println!("Runner:"); - println!(" Environment:"); - print_env(" ", &config.runner.env); - println!(" Volumes:"); - if config.runner.volumes.is_empty() { - println!(" N/A"); - } else { - for (target, mount) in &config.runner.volumes { - println!(" {target} => {mount}"); - } - } - println!(" Needs docker socket: {}", config.runner.needs_docker_socket); - - println!("Environments:"); - for environment in config.environments().keys() { - println!(" {}", format(&active_env, environment)); - } - } - } - Ok(()) - } -} - -fn format(active_env: &Option, environment: &str) -> String { - match active_env { - Some(active) if active == environment => format!("{environment} (active)"), - _ => environment.into(), - } -} - -fn print_env(prefix: &str, environment: &Environment) { - if environment.is_empty() { - println!("{prefix}N/A"); - } else { - for (key, value) in environment { - match value { - Some(value) => println!("{prefix}{key}={value:?}"), - None => println!("{prefix}{key} (passthrough)"), - } - } + crate::commands::compose_tests::show::exec(&self.integration, INTEGRATION_TESTS_DIR) } } diff --git a/vdev/src/commands/integration/start.rs b/vdev/src/commands/integration/start.rs index ef487a8c487b7..51a247da5fc3f 100644 --- a/vdev/src/commands/integration/start.rs +++ b/vdev/src/commands/integration/start.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::{config::IntegrationTestConfig, integration::IntegrationTest}; +use crate::testing::integration::IntegrationTest; /// Start an environment #[derive(Args, Debug)] @@ -20,14 +20,10 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - let environment = if let Some(environment) = self.environment { - environment - } else { - let (_test_dir, config) = IntegrationTestConfig::load(&self.integration)?; - let envs = config.environments(); - let env = envs.keys().next().expect("Integration has no environments"); - env.clone() - }; - IntegrationTest::new(self.integration, environment, self.build_all, 0)?.start() + crate::commands::compose_tests::start::exec::( + &self.integration, + &self.environment, + self.build_all, + ) } } diff --git a/vdev/src/commands/integration/stop.rs b/vdev/src/commands/integration/stop.rs index 22955d5fa7daf..0bacde212596d 100644 --- a/vdev/src/commands/integration/stop.rs +++ b/vdev/src/commands/integration/stop.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Args; -use crate::testing::{integration::IntegrationTest, state::EnvsDir}; +use crate::testing::integration::IntegrationTest; /// Stop an integration test environment #[derive(Args, Debug)] @@ -17,11 +17,9 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - if let Some(active) = EnvsDir::new(&self.integration).active()? { - IntegrationTest::new(self.integration, active, self.all_features, 0)?.stop() - } else { - println!("No environment for {:?} is active.", self.integration); - Ok(()) - } + crate::commands::compose_tests::stop::exec::( + &self.integration, + self.all_features, + ) } } diff --git a/vdev/src/commands/integration/test.rs b/vdev/src/commands/integration/test.rs index ca57d895bd0c8..e41015527cdca 100644 --- a/vdev/src/commands/integration/test.rs +++ b/vdev/src/commands/integration/test.rs @@ -1,7 +1,7 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use clap::Args; -use crate::testing::{config::IntegrationTestConfig, integration::IntegrationTest, state::EnvsDir}; +use crate::testing::integration::IntegrationTest; /// Execute integration tests /// @@ -34,32 +34,12 @@ pub struct Cli { impl Cli { pub fn exec(self) -> Result<()> { - let (_test_dir, config) = IntegrationTestConfig::load(&self.integration)?; - let envs = config.environments(); - - let active = EnvsDir::new(&self.integration).active()?; - - let retries = self.retries.unwrap_or_default(); - - match (self.environment, active) { - (Some(environment), Some(active)) if environment != active => { - bail!("Requested environment {environment:?} does not match active one {active:?}") - } - (Some(environment), _) => { - IntegrationTest::new(self.integration, environment, self.build_all, retries)? - .test(self.args) - } - (None, Some(active)) => { - IntegrationTest::new(self.integration, active, self.build_all, retries)? - .test(self.args) - } - (None, None) => { - for env_name in envs.keys() { - IntegrationTest::new(&self.integration, env_name, self.build_all, retries)? - .test(self.args.clone())?; - } - Ok(()) - } - } + crate::commands::compose_tests::test::exec::( + &self.integration, + &self.environment, + self.build_all, + self.retries.unwrap_or_default(), + &self.args, + ) } } diff --git a/vdev/src/commands/mod.rs b/vdev/src/commands/mod.rs index fc9c9f8c539e6..9c5f0fb2e87d9 100644 --- a/vdev/src/commands/mod.rs +++ b/vdev/src/commands/mod.rs @@ -1,6 +1,8 @@ use clap::Parser; use clap_verbosity_flag::{InfoLevel, Verbosity}; +mod compose_tests; + /// This macro simplifies the generation of CLI subcommand invocation structures by combining the /// creation of the command enum and implementation of the dispatch function into one simple list. #[macro_export] @@ -75,6 +77,7 @@ cli_commands! { mod complete, mod config, mod crate_versions, + mod e2e, mod exec, mod features, mod fmt, diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 7e1ca36cd6452..314e7a0441ef1 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -12,6 +12,9 @@ use crate::{app, util}; const FILE_NAME: &str = "test.yaml"; +pub const INTEGRATION_TESTS_DIR: &str = "integration"; +pub const E2E_TESTS_DIR: &str = "e2e"; + pub type Environment = BTreeMap>; #[derive(Deserialize, Debug)] @@ -96,7 +99,7 @@ impl ComposeConfig { #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub struct IntegrationTestConfig { +pub struct ComposeTestConfig { /// The list of arguments to add to the command line for the test runner pub args: Option>, /// The set of environment variables to set in both the services and the runner. Variables with @@ -135,11 +138,11 @@ pub struct IntegrationRunnerConfig { pub needs_docker_socket: bool, } -impl IntegrationTestConfig { +impl ComposeTestConfig { fn parse_file(config_file: &Path) -> Result { let contents = fs::read_to_string(config_file) .with_context(|| format!("failed to read {}", config_file.display()))?; - let config: IntegrationTestConfig = serde_yaml::from_str(&contents).with_context(|| { + let config: Self = serde_yaml::from_str(&contents).with_context(|| { format!( "failed to parse integration test configuration file {}", config_file.display() @@ -166,22 +169,13 @@ impl IntegrationTestConfig { .collect() } - pub fn load(integration: &str) -> Result<(PathBuf, Self)> { - let mut test_dir: PathBuf = [app::path(), "scripts", "integration", integration] + pub fn load(root_dir: &str, integration: &str) -> Result<(PathBuf, Self)> { + let test_dir: PathBuf = [app::path(), "scripts", root_dir, integration] .iter() .collect(); + if !test_dir.is_dir() { - // try the e2e dir now - - // TODO: This is a temporary solution, looking in both dirs. When this GH issue - // https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor - // to have a separate e2e subcommand that both int and e2e will leverage. - test_dir = [app::path(), "scripts", "e2e", integration] - .iter() - .collect(); - if !test_dir.is_dir() { - bail!("unknown integration: {}", integration); - } + bail!("unknown integration: {}", integration); } let config = Self::parse_file(&test_dir.join(FILE_NAME))?; @@ -203,17 +197,12 @@ impl IntegrationTestConfig { Ok(()) } - pub fn collect_all() -> Result> { + pub fn collect_all(root_dir: &str) -> Result> { let mut configs = BTreeMap::new(); - // TODO: This is a temporary solution, looking in both dirs. When this GH issue - // https://github.com/vectordotdev/vector/issues/18829 , is worked, we will refactor - // to have a separate e2e subcommand that both int and e2e will leverage. - let int_tests_dir: PathBuf = [app::path(), "scripts", "integration"].iter().collect(); - let e2e_tests_dir: PathBuf = [app::path(), "scripts", "e2e"].iter().collect(); + let tests_dir: PathBuf = [app::path(), "scripts", root_dir].iter().collect(); - Self::collect_all_dir(&int_tests_dir, &mut configs)?; - Self::collect_all_dir(&e2e_tests_dir, &mut configs)?; + Self::collect_all_dir(&tests_dir, &mut configs)?; Ok(configs) } diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index b3b0f1d79ab09..2ade5213977e0 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -3,8 +3,9 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command use anyhow::{bail, Context, Result}; use tempfile::{Builder, NamedTempFile}; -use super::config::ComposeConfig; -use super::config::{Environment, IntegrationTestConfig}; +use super::config::{ + ComposeConfig, ComposeTestConfig, Environment, E2E_TESTS_DIR, INTEGRATION_TESTS_DIR, +}; use super::runner::{ ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _, CONTAINER_TOOL, DOCKER_SOCKET, }; @@ -14,10 +15,13 @@ use crate::testing::config::get_rust_version; const NETWORK_ENV_VAR: &str = "VECTOR_NETWORK"; -pub struct IntegrationTest { - integration: String, +const INTEGRATION_FEATURE_FLAG: &str = "all-integration-tests"; +const E2E_FEATURE_FLAG: &str = "all-e2e-tests"; + +pub(crate) struct ComposeTest { + test_name: String, environment: String, - config: IntegrationTestConfig, + config: ComposeTestConfig, envs_dir: EnvsDir, runner: IntegrationTestRunner, compose: Option, @@ -26,26 +30,30 @@ pub struct IntegrationTest { retries: u8, } -impl IntegrationTest { - pub fn new( - integration: impl Into, +pub(crate) trait ComposeTestT { + const DIRECTORY: &'static str; + + const FEATURE_FLAG: &'static str; + + fn generate( + test_name: impl Into, environment: impl Into, build_all: bool, retries: u8, - ) -> Result { - let integration = integration.into(); + ) -> Result { + let test_name = test_name.into(); let environment = environment.into(); - let (test_dir, config) = IntegrationTestConfig::load(&integration)?; - let envs_dir = EnvsDir::new(&integration); + let (test_dir, config) = ComposeTestConfig::load(Self::DIRECTORY, &test_name)?; + let envs_dir = EnvsDir::new(&test_name); let Some(mut env_config) = config.environments().get(&environment).map(Clone::clone) else { bail!("Could not find environment named {environment:?}"); }; - let network_name = format!("vector-integration-tests-{integration}"); + let network_name = format!("vector-integration-tests-{test_name}"); let compose = Compose::new(test_dir, env_config.clone(), network_name.clone())?; // None if compiling with all integration test feature flag. - let runner_name = (!build_all).then(|| integration.clone()); + let runner_name = (!build_all).then(|| test_name.clone()); let runner = IntegrationTestRunner::new( runner_name, @@ -55,8 +63,8 @@ impl IntegrationTest { env_config.insert("VECTOR_IMAGE".to_string(), Some(runner.image_name())); - Ok(Self { - integration, + Ok(ComposeTest { + test_name, environment, config, envs_dir, @@ -68,33 +76,35 @@ impl IntegrationTest { }) } - pub fn test(self, extra_args: Vec) -> Result<()> { - let active = self.envs_dir.check_active(&self.environment)?; - self.config.check_required()?; + fn test(compose_test: &ComposeTest, extra_args: Vec) -> Result<()> { + let active = compose_test + .envs_dir + .check_active(&compose_test.environment)?; + compose_test.config.check_required()?; if !active { - self.start()?; + Self::start(compose_test)?; } - let mut env_vars = self.config.env.clone(); + let mut env_vars = compose_test.config.env.clone(); // Make sure the test runner has the same config environment vars as the services do. - for (key, value) in config_env(&self.env_config) { + for (key, value) in config_env(&compose_test.env_config) { env_vars.insert(key, Some(value)); } env_vars.insert("TEST_LOG".to_string(), Some("info".into())); - let mut args = self.config.args.clone().unwrap_or_default(); + let mut args = compose_test.config.args.clone().unwrap_or_default(); args.push("--features".to_string()); - args.push(if self.build_all { - "all-integration-tests".to_string() + args.push(if compose_test.build_all { + Self::FEATURE_FLAG.to_string() } else { - self.config.features.join(",") + compose_test.config.features.join(",") }); // If the test field is not present then use the --lib flag - match self.config.test { + match compose_test.config.test { Some(ref test_arg) => { args.push("--test".to_string()); args.push(test_arg.to_string()); @@ -103,7 +113,7 @@ impl IntegrationTest { } // Ensure the test_filter args are passed as well - if let Some(ref filter) = self.config.test_filter { + if let Some(ref filter) = compose_test.config.test_filter { args.push(filter.to_string()); } args.extend(extra_args); @@ -113,71 +123,92 @@ impl IntegrationTest { args.push("--no-capture".to_string()); } - if self.retries > 0 { + if compose_test.retries > 0 { args.push("--retries".to_string()); - args.push(self.retries.to_string()); + args.push(compose_test.retries.to_string()); } - self.runner.test( + compose_test.runner.test( &env_vars, - &self.config.runner.env, - Some(&self.config.features), + &compose_test.config.runner.env, + Some(&compose_test.config.features), &args, )?; if !active { - self.runner.remove()?; - self.stop()?; + compose_test.runner.remove()?; + Self::stop(compose_test)?; } Ok(()) } - pub fn start(&self) -> Result<()> { - // TODO this is a hack until https://github.com/vectordotdev/vector/pull/18840 - // During that PR, will be able to conditionally call this based on the trait - // definition for the E2E tests. - // The reason we need this temporary hack is because adding the features as a build - // argument results in each integration's runner container being unique, and thus - // the job in CI runs out of space. - if self.integration.contains("e2e-") { - // For end-to-end tests, we want to run vector as a service, leveraging the - // image for the runner. So we must build that image before starting the - // compose so that it is available. - self.runner.build(Some(&self.config.features))?; + fn start(compose_test: &ComposeTest) -> Result<()> { + // For end-to-end tests, we want to run vector as a service, leveraging the + // image for the runner. So we must build that image before starting the + // compose so that it is available. + if Self::DIRECTORY == E2E_TESTS_DIR { + compose_test + .runner + .build(Some(&compose_test.config.features))?; } - self.config.check_required()?; - if let Some(compose) = &self.compose { - self.runner.ensure_network()?; + compose_test.config.check_required()?; + if let Some(compose) = &compose_test.compose { + compose_test.runner.ensure_network()?; - if self.envs_dir.check_active(&self.environment)? { + if compose_test + .envs_dir + .check_active(&compose_test.environment)? + { bail!("environment is already up"); } - compose.start(&self.env_config)?; + compose.start(&compose_test.env_config)?; - self.envs_dir.save(&self.environment, &self.env_config) + compose_test + .envs_dir + .save(&compose_test.environment, &compose_test.env_config) } else { Ok(()) } } - pub fn stop(&self) -> Result<()> { - if let Some(compose) = &self.compose { + fn stop(compose_test: &ComposeTest) -> Result<()> { + if let Some(compose) = &compose_test.compose { // TODO: Is this check really needed? - if self.envs_dir.load()?.is_none() { - bail!("No environment for {} is up.", self.integration); + if compose_test.envs_dir.load()?.is_none() { + bail!("No environment for {} is up.", compose_test.test_name); } - self.runner.remove()?; + compose_test.runner.remove()?; compose.stop()?; - self.envs_dir.remove()?; + compose_test.envs_dir.remove()?; } Ok(()) } } +/// Integration tests are located in the `scripts/integration` dir, +/// and are the full feature flag is `all-integration-tests`. +pub(crate) struct IntegrationTest; + +impl ComposeTestT for IntegrationTest { + const DIRECTORY: &'static str = INTEGRATION_TESTS_DIR; + + const FEATURE_FLAG: &'static str = INTEGRATION_FEATURE_FLAG; +} + +/// E2E tests are located in the `scripts/e2e` dir, +/// and are the full feature flag is `all-e2e-tests`. +pub(crate) struct E2ETest; + +impl ComposeTestT for E2ETest { + const DIRECTORY: &'static str = E2E_TESTS_DIR; + + const FEATURE_FLAG: &'static str = E2E_FEATURE_FLAG; +} + struct Compose { original_path: PathBuf, test_dir: PathBuf,