integration.yml

# Integration Test Suite
#
# This workflow runs the integration tests. If the workflow is triggered in the merge queue, all integration tests
# are run. If the workflow is triggered in a PR commit, then the files changed in the PR are evaluated to determine
# if any integration tests will run.

name: Integration Test Suite

on:
  pull_request:
  merge_group:
    types: [checks_requested]

concurrency:
  # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue
  group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.head_sha }}
  cancel-in-progress: true

env:
  AWS_ACCESS_KEY_ID: "dummy"
  AWS_SECRET_ACCESS_KEY: "dummy"
  CONTAINER_TOOL: "docker"
  DD_ENV: "ci"
  DD_API_KEY: ${{ secrets.DD_API_KEY }}
  TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }}
  TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }}
  AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }}
  RUST_BACKTRACE: full
  TEST_LOG: vector=debug
  VERBOSE: true
  CI: true
  PROFILE: debug
  # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
  # can be removed when we switch back to the upstream openssl-sys crate
  CARGO_NET_GIT_FETCH_WITH_CLI: true

jobs:
  changes:
    if: github.event_name == 'pull_request'
    uses: ./.github/workflows/changes.yml
    with:
      base_ref: ${{ github.event.pull_request.base.ref }}
      head_ref: ${{ github.event.pull_request.head.ref }}
      source: false
      int_tests: true
    secrets: inherit

  check-all:
    runs-on: ubuntu-latest
    needs: changes
    outputs:
      should_run: ${{ steps.check.outputs.should_run }}
    steps:
      - name: Download JSON artifact from changes.yml
        uses: actions/download-artifact@v4
        with:
          name: int_tests_changes

      - name: Check if all values are false
        id: check
        run: |
          # Check if all values are 'false'
          json=$(cat int_tests_changes.json)
          all_false=$(echo "$json" | jq -r 'to_entries | all(.value == false)')

          if [[ "$all_false" == "true" ]]; then
            echo "No changes detected. Skipping integration tests."
            echo "should_run=false" >> $GITHUB_OUTPUT
          else
            echo "Detected changes. Proceeding with integration tests."
            echo "should_run=true" >> $GITHUB_OUTPUT
          fi

  setup:
    runs-on: ubuntu-latest
    needs: check-all
    if: ${{ needs.check-all.outputs.should_run == 'true' }}
    steps:
      - uses: actions/checkout@v4

      - run: sudo npm -g install @datadog/datadog-ci
      - run: sudo -E bash scripts/ci-free-disk-space.sh
      - run: docker image prune -af ; docker container prune -f

      - name: Determine if secrets are defined (PR author is team member)
        if: github.event_name == 'pull_request'
        env:
          GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
        run: |
          if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then
            echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV"
          else
            echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV"
          fi

  integration-tests:
    runs-on: ubuntu-20.04
    needs:
      - changes
      - setup
    strategy:
      matrix:
        # If you modify this list, please also update the `int_tests` job in changes.yml.
        service: [
          "amqp", "appsignal", "aws", "axiom", "azure", "clickhouse", "databend", "datadog-agent",
          "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch",
          "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash",
          "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar",
          "redis", "splunk", "webhdfs"
        ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"

      - run: docker image prune -af ; docker container prune -f

      - name: Download JSON artifact from changes.yml
        uses: actions/download-artifact@v4
        with:
          name: int_tests_changes

      - name: Run Integration Tests for ${{ matrix.service }}
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: |
            # Parse the JSON and check if the specific integration test should run.
            should_run=$(jq '."${{ matrix.service }}"' int_tests_changes.json)

            # Check if any of the three conditions is true
            if [[ "${{ github.event_name }}" == "merge_group" || \
                  "${{ needs.changes.outputs.all-int }}" == "true" || \
                  "$should_run" == "true" ]]; then
              echo "Running test for ${{ matrix.service }}"
              bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }}
            else
              echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met."
            fi


  integration-test-suite:
    name: Integration Test Suite
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    if: always()
    needs:
      - changes
      - check-all
      - setup
      - integration-tests
    env:
      FAILED: ${{ contains(needs.*.result, 'failure') }}
    steps:
      - run: |
          echo "failed=${{ env.FAILED }}"
          if [[ "$FAILED" == "true" ]] ; then
            exit 1
          else
            exit 0
          fi