integration-test.yml

name: "Integration Test"

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - '.github/workflows/integration-test.yml'
      - 'cmd/**'
      - 'docker/**'
      - 'internal/**'
      - 'packaging/bundle/**'
      - 'instrumentation/packaging/**'
      - 'pkg/**'
      - 'tests/**'
      - 'Makefile'
      - 'Makefile.Common'
      - 'go.mod'
      - 'go.sum'
      - 'packaging/jmx-metric-gatherer-release.txt'
      - '!**.md'
      - '!packaging/technical-addon/**'

concurrency:
  group: integration-test-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  GO_VERSION: "1.23.6"
jobs:
  agent-bundle-linux:
    runs-on: ${{ fromJSON('["ubuntu-24.04", "otel-arm64"]')[matrix.ARCH == 'arm64'] }}
    strategy:
      matrix:
        ARCH: [ "amd64", "arm64" ]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        id: bundle-cache
        with:
          path: .cache/buildx/agent-bundle-${{ matrix.ARCH }}
          key: agent-bundle-buildx-${{ matrix.ARCH }}-${{ hashFiles('packaging/bundle/**') }}
          restore-keys: |
            agent-bundle-buildx-${{ matrix.ARCH }}-
      - run: make -C packaging/bundle agent-bundle-linux ARCH=${{ matrix.ARCH }}
        env:
          BUNDLE_CACHE_HIT: "${{ steps.bundle-cache.outputs.cache-hit }}"
      - uses: actions/upload-artifact@v4
        with:
          name: agent-bundle-linux-${{ matrix.ARCH }}
          path: ./dist/agent-bundle_linux_${{ matrix.ARCH }}.tar.gz

  otelcol:
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        ARCH: [ "amd64", "arm64", "ppc64le" ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'
      - run: make binaries-linux_${{ matrix.ARCH }}
      - uses: actions/upload-artifact@v4
        with:
          name: otelcol-${{ matrix.ARCH }}
          path: |
            ./bin/*

  docker-otelcol:
    name: docker-otelcol
    runs-on: ubuntu-24.04
    needs: [ "agent-bundle-linux", "otelcol" ]
    services:
      # Start a local registry for pushing the multiarch manifest and images
      registry:
        image: registry:2
        ports:
          - 5000:5000
    steps:
      # Multiarch images require more disk space
      - uses: jlumbroso/free-disk-space@v1.3.1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'
      - uses: actions/download-artifact@v4
        with:
          pattern: agent-bundle-linux-*
          merge-multiple: true
          path: ./dist
      - uses: actions/download-artifact@v4
        with:
          pattern: otelcol-*
          merge-multiple: true
          path: ./bin
      - uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64,ppc64le
          image: tonistiigi/binfmt:qemu-v7.0.0
      - uses: docker/setup-buildx-action@v3
        id: multiarch-otelcol-builder
        with:
          driver: docker-container   # Create a builder with the docker-container driver required for multiarch builds
          driver-opts: network=host  # Required for the builder to push to the local registry service
      - run: make docker-otelcol SKIP_COMPILE=true SKIP_BUNDLE=true ARCH=amd64,arm64,ppc64le IMAGE_NAME=localhost:5000/otelcol IMAGE_TAG=latest PUSH=true
        env:
          MULTIARCH_OTELCOL_BUILDER: ${{ steps.multiarch-otelcol-builder.outputs.name }}  # Use the builder created by the docker/setup-buildx-action step
      - name: Save image archive for each platform to be loaded by downstream jobs
        run: |
          for arch in "amd64" "arm64" "ppc64le"; do
            mkdir -p docker-otelcol/${arch}
            docker pull --platform=linux/${arch} localhost:5000/otelcol:latest
            docker tag localhost:5000/otelcol:latest otelcol:latest
            docker save -o ./docker-otelcol/${arch}/image.tar otelcol:latest
            docker rmi -f localhost:5000/otelcol:latest otelcol:latest
          done
      - uses: actions/upload-artifact@v4
        with:
          name: docker-otelcol-amd64
          path: ./docker-otelcol/amd64
      - uses: actions/upload-artifact@v4
        with:
          name: docker-otelcol-arm64
          path: ./docker-otelcol/arm64
      - uses: actions/upload-artifact@v4
        with:
          name: docker-otelcol-ppc64le
          path: ./docker-otelcol/ppc64le

  integration-vet:
    name: integration-vet
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        ARCH: [ "amd64", "arm64" ]
      fail-fast: false
    needs: [ "docker-otelcol", "otelcol" ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'
      - uses: actions/download-artifact@v4
        with:
          name: otelcol-${{ matrix.ARCH }}
          path: ./bin
      - uses: actions/download-artifact@v4
        with:
          name: docker-otelcol-${{ matrix.ARCH }}
          path: ./docker-otelcol/${{ matrix.ARCH }}
      - uses: docker/setup-qemu-action@v3
        if: ${{ matrix.ARCH != 'amd64' }}
        with:
          platforms: ${{ matrix.ARCH }}
          image: tonistiigi/binfmt:qemu-v7.0.0
      - run: docker load -i ./docker-otelcol/${{ matrix.ARCH }}/image.tar
      - run: ln -sf otelcol_linux_${{ matrix.ARCH }} ./bin/otelcol
      - run: chmod a+x ./bin/*
      - run: make integration-vet
        env:
          SPLUNK_OTEL_COLLECTOR_IMAGE: 'otelcol:latest'

  integration-test-docker:
    runs-on: ubuntu-24.04
    needs: [ "docker-otelcol", "otelcol" ]
    strategy:
      matrix:
        ARCH: [ "amd64", "arm64" ]
        PROFILE: [ "integration", "smartagent" ]
      fail-fast: false
    env:
      TEST_OUTPUT: ${{ github.job }}-${{ matrix.PROFILE }}-${{ matrix.ARCH }}.out
    steps:
      # Multiarch images require more disk space
      - uses: jlumbroso/free-disk-space@v1.3.1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Build ${{ matrix.PROFILE }} service images
        run: |
          images=$(yq '.services[] | select(.profiles[] | . == "${{ matrix.PROFILE }}") | .image' docker/docker-compose.yml | grep "quay.io/splunko11ytest/" | sort -u)
          for image in $images; do
            service=$(basename "$image" | cut -d ':' -f1)
            if [[ -f docker/${service}/Dockerfile ]]; then
              docker build --cache-from=quay.io/splunko11ytest/${service}:latest -t quay.io/splunko11ytest/${service}:latest docker/${service}
            fi
          done
          docker system prune -f
          docker builder prune -f
          docker images
      - run: docker compose -f docker/docker-compose.yml --profile ${{ matrix.PROFILE }} up -d --quiet-pull
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'
      - uses: actions/download-artifact@v4
        with:
          name: otelcol-${{ matrix.ARCH }}
          path: ./bin
      - uses: actions/download-artifact@v4
        with:
          name: docker-otelcol-${{ matrix.ARCH }}
          path: ./docker-otelcol/${{ matrix.ARCH }}
      - uses: docker/setup-qemu-action@v3
        if: ${{ matrix.ARCH != 'amd64' }}
        with:
          platforms: ${{ matrix.ARCH }}
          image: tonistiigi/binfmt:qemu-v7.0.0
      - run: docker load -i ./docker-otelcol/${{ matrix.ARCH }}/image.tar
      - run: ln -sf otelcol_linux_${{ matrix.ARCH }} ./bin/otelcol
      - run: chmod a+x ./bin/*
      - uses: shogo82148/actions-setup-redis@v1.38.0
        if: matrix.PROFILE == 'integration'
        with:
          auto-start: false
          redis-port: "6379"
      - run: redis-cli ping
        if: matrix.PROFILE == 'integration'
      - run: redis-cli set tempkey tempvalue
        if: matrix.PROFILE == 'integration'
      - name: Run Integration Test
        run: |
          set -o pipefail
          target="integration-test"
          if [[ "${{ matrix.PROFILE }}" = "smartagent" ]]; then
            target="smartagent-integration-test"
          fi
          make $target 2>&1 | tee $TEST_OUTPUT
          exit_status=${PIPESTATUS[0]}
          echo "Exit status: $exit_status"
          exit $exit_status
        env:
          SPLUNK_OTEL_COLLECTOR_IMAGE: 'otelcol:latest'
      # The Integration Test output is extremely large so we upload it as an artifact
      - name: Upload Integration Test Output as Artifact
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ${{ env.TEST_OUTPUT }}
          path: ${{ env.TEST_OUTPUT }}
          retention-days: 5

  integration-test-binary:
    runs-on: ${{ matrix.RUNNER }}
    needs: [ "agent-bundle-linux", "otelcol" ]
    strategy:
      matrix:
        RUNNER: [ "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04" ]
        PROFILE: [ "integration", "smartagent" ]
      fail-fast: false
    env:
      TEST_OUTPUT: ${{ github.job }}-${{ matrix.PROFILE }}-${{ matrix.RUNNER }}.out
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Build ${{ matrix.PROFILE }} service images
        run: |
          images=$(yq '.services[] | select(.profiles[] | . == "${{ matrix.PROFILE }}") | .image' docker/docker-compose.yml | grep "quay.io/splunko11ytest/" | sort -u)
          for image in $images; do
            service=$(basename "$image" | cut -d ':' -f1)
            if [[ -f docker/${service}/Dockerfile ]]; then
              docker build --cache-from=quay.io/splunko11ytest/${service}:latest -t quay.io/splunko11ytest/${service}:latest docker/${service}
            fi
          done
          docker system prune -f
          docker builder prune -f
          docker images
      - run: docker compose -f docker/docker-compose.yml --profile ${{ matrix.PROFILE }} up -d --quiet-pull
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'
      - uses: actions/download-artifact@v4
        with:
          pattern: agent-bundle-linux-amd64
          merge-multiple: true
          path: ./dist
      - run: sudo mkdir -p /usr/lib/splunk-otel-collector
      - run: sudo tar -xzf dist/agent-bundle_linux_amd64.tar.gz -C /usr/lib/splunk-otel-collector
      - run: sudo chown -R $USER /usr/lib/splunk-otel-collector
      - run: /usr/lib/splunk-otel-collector/agent-bundle/bin/patch-interpreter /usr/lib/splunk-otel-collector/agent-bundle
      - uses: actions/download-artifact@v4
        with:
          name: otelcol-amd64
          path: ./bin
      - run: ln -sf otelcol_linux_amd64 ./bin/otelcol
      - run: chmod a+x ./bin/*
      - uses: shogo82148/actions-setup-redis@v1.38.0
        if: matrix.PROFILE == 'integration'
        with:
          auto-start: false
          redis-port: "6379"
      - run: redis-cli ping
        if: matrix.PROFILE == 'integration'
      - run: redis-cli set tempkey tempvalue
        if: matrix.PROFILE == 'integration'
      - name: Run Integration Test
        run: |
          set -o pipefail
          target="integration-test"
          if [[ "${{ matrix.PROFILE }}" = "smartagent" ]]; then
            target="smartagent-integration-test"
          fi
          make $target 2>&1 | tee $TEST_OUTPUT
          exit_status=${PIPESTATUS[0]}
          echo "Exit status: $exit_status"
          exit $exit_status
        env:
          SPLUNK_OTEL_COLLECTOR_IMAGE: ""
      # The Integration Test output is extremely large so we upload it as an artifact
      - name: Upload Integration Test Output as Artifact
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ${{ env.TEST_OUTPUT }}
          path: ${{ env.TEST_OUTPUT }}
          retention-days: 5

  integration-test-discovery-matrix:
    runs-on: ubuntu-24.04
    outputs:
      matrix: ${{ steps.get-matrix.outputs.matrix }}
    steps:
      - name: Get matrix
        id: get-matrix
        run: |
          includes=""
          for service in "apache" "jmx/cassandra" "kafkametrics" "mongodb" "nginx" "envoy" "oracledb" "mysql" "redis"; do
            for arch in "amd64" "arm64"; do
              if [ "$service" = "mongodb" ]; then
                # tests for mongo "6.0" and "7.0" are flaky, skipping for now
                for mongodb_version in "4.0" "4.4" "5.0"; do
                  includes="${includes},{\"SERVICE\": \"${service}\", \"ARCH\": \"${arch}\", \"MONGODB_VERSION\": \"${mongodb_version}\"}"
                done
              elif [[ "$service" != "jmx/cassandra" || "arm64" != "$arch" ]]; then
                includes="${includes},{\"SERVICE\": \"${service}\", \"ARCH\": \"${arch}\"}"
              fi
            done
          done
          matrix="{\"include\": [${includes#,}]}"
          echo "$matrix" | jq
          echo "matrix=${matrix}" >> $GITHUB_OUTPUT

  integration-test-discovery:
      name: integration-test-discovery
      runs-on: ubuntu-24.04
      needs: [ "docker-otelcol", "otelcol", "integration-test-discovery-matrix" ]
      strategy:
        matrix: ${{ fromJSON(needs.integration-test-discovery-matrix.outputs.matrix) }}
        fail-fast: false
      steps:
        - uses: actions/checkout@v4
          with:
            fetch-depth: 0
        - run: echo "MONGODB_VERSION=${{ matrix.MONGODB_VERSION }}" > docker/.env
          if: matrix.SERVICE == 'mongodb'
        - run: docker compose -f docker/docker-compose.yml --profile integration-test-${{ matrix.SERVICE }}-discovery up -d --wait --build --quiet-pull
        - uses: actions/setup-go@v5
          with:
            go-version: ${{ env.GO_VERSION }}
            cache-dependency-path: '**/go.sum'
        - uses: actions/download-artifact@v4
          with:
            name: otelcol-${{ matrix.ARCH }}
            path: ./bin
        - uses: actions/download-artifact@v4
          with:
            name: docker-otelcol-${{ matrix.ARCH }}
            path: ./docker-otelcol/${{ matrix.ARCH }}
        - uses: docker/setup-qemu-action@v3
          if: ${{ matrix.ARCH != 'amd64' }}
          with:
            platforms: ${{ matrix.ARCH }}
            image: tonistiigi/binfmt:qemu-v7.0.0
        - run: docker load -i ./docker-otelcol/${{ matrix.ARCH }}/image.tar
        - run: ln -sf otelcol_linux_${{ matrix.ARCH }} ./bin/otelcol
        - run: chmod a+x ./bin/*
        - name: Run ${{ matrix.SERVICE }} Discovery Integration Test
          run: make integration-test-${{ matrix.SERVICE }}-discovery
          env:
            SPLUNK_OTEL_COLLECTOR_IMAGE: 'otelcol:latest'

  integration-test-discovery-k8s-matrix:
    runs-on: ubuntu-24.04
    outputs:
      matrix: ${{ steps.get-matrix-k8s.outputs.matrix }}
    steps:
      - name: Get matrix for Kubernetes
        id: get-matrix-k8s
        run: |
          includes=""
          for service in "envoy"; do
            for arch in "amd64"; do
                includes="${includes},{\"SERVICE\": \"${service}\", \"ARCH\": \"${arch}\"}"
            done
          done
          matrix="{\"include\": [${includes#,}]}"
          echo "$matrix" | jq
          echo "matrix=${matrix}" >> $GITHUB_OUTPUT

  integration-test-discovery-k8s:
    name: integration-test-discovery-k8s
    runs-on: ubuntu-24.04
    needs: [ "docker-otelcol", "otelcol", "integration-test-discovery-k8s-matrix" ]
    strategy:
      matrix: ${{ fromJSON(needs.integration-test-discovery-k8s-matrix.outputs.matrix) }}
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Create kind cluster
        uses: helm/kind-action@v1.12.0
        with:
          node_image: kindest/node:v1.30.0
          kubectl_version: v1.30.0
          cluster_name: kind
      - name: Deploy service under test
        run: |
          kubectl apply -f k8s/${{ matrix.SERVICE }}/*.yaml
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'
      - uses: actions/download-artifact@v4
        with:
          name: docker-otelcol-${{ matrix.ARCH }}
          path: ./docker-otelcol/${{ matrix.ARCH }}
      - run: docker load -i ./docker-otelcol/${{ matrix.ARCH }}/image.tar
      - name: Load Docker image in kind
        run: |
          kind load docker-image otelcol:latest --name kind
      - name: Run ${{ matrix.SERVICE }} Discovery Kubernetes Integration Test
        run: KUBECONFIG=$HOME/.kube/config SKIP_TEARDOWN=true make integration-test-${{ matrix.SERVICE }}-discovery-k8s
      - name: Print logs
        if: failure()
        run: |
          kubectl get pods
          kubectl logs $(kubectl get pod -l app=otelcol -o jsonpath="{.items[0].metadata.name}")