build-and-test.yml

name: build-and-test
on:
  push:
    branches: [ main ]
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+*'
  pull_request:
env:
  TEST_RESULTS: testbed/tests/results/junit/results.xml

jobs:
  windows-test:
    runs-on: windows-latest
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
        shell: bash
      - name: Cache Go
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            \Users\runneradmin\go\pkg\mod
            %LocalAppData%\go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Run Unit tests
        run: go test ./...
      - name: GitHub Issue Generator
        if: ${{ failure() && github.ref == 'ref/head/main' }}
        run: |
          cd internal/tools && go install go.opentelemetry.io/collector/cmd/issuegenerator
          issuegenerator $TEST_RESULTS
  setup-environment:
    runs-on: ubuntu-latest
    outputs:
      loadtest_matrix: ${{ steps.splitloadtest.outputs.loadtest_matrix }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - name: Install dependencies
        if: steps.module-cache.outputs.hit != 'true'
        run: |
          make gomoddownload
          make gotestinstall
      - name: Install Tools
        if: steps.tool-cache.outputs.cache-hit != 'true'
        run: make install-tools
      - name: Split Loadtest Jobs
        id: splitloadtest
        run: ./.github/workflows/scripts/setup_e2e_tests.sh
      - name: Split Stability Jobs
        id: splitstabilitytest
        run: ./.github/workflows/scripts/setup_stability_tests.sh
  lint:
    runs-on: ubuntu-latest
    needs: [setup-environment]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - name: Lint
        run: make -j4 golint
      - name: Checks 
        run: make -j4 checklicense misspell checkdoc impi
      - name: Codegem
        run: |
          make generate
          git diff --exit-code ':!*go.sum' || (echo 'Generated code is out of date, please run "make generate" and commit the changes in this PR.' && exit 1)
  unittest:
    runs-on: ubuntu-latest
    needs: [setup-environment]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - name: Run Unit Tests
        run: make gotest
  integration-tests:
    runs-on: ubuntu-latest
    needs: [setup-environment]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - name: Run Integration Tests
        run: make integration-tests-with-cover

  stability-tests:
    # Stability tests disabled in the makefile. We disable them here to avoid
    # consuming workers just to run a noop make target.
    # uncomment the following line to enable stability tests.
    # if: ${{ github.ref == 'ref/head/main' }}
    if: ${{ false }}
    runs-on: ubuntu-latest
    needs: [setup-environment]
    strategy:
      matrix: ${{ fromJson(needs.setup-environment.outputs.stabilitytest_matrix) }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Run Stability Tests
        run: |
          echo "Running ${matrix.test}..."
          TEST_ARGS="-test.run=${matrix.test}" make stability-tests
      - name: Upload Unit Test Results
        if: always()
        uses: actions/upload-artifact@v2
        with:
          name: unittests-results
          path: test-results/junit/results.xml

  build-examples:
    runs-on: ubuntu-latest
    needs: [setup-environment]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Build Examples
        run: make build-examples

  cross-compile:
    runs-on: ubuntu-latest
    needs: [setup-environment]
    strategy:
      matrix:
        binary:
          - darwin_amd64
          - linux_amd64
          - linux_arm64
          - windows_amd64
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - name: Build Collector ${{ matrix.binary }}
        run: make otelcontribcol-${{ matrix.binary }}
      - name: Upload Collector Binaries
        uses: actions/upload-artifact@v2
        with:
          name: collector-binaries
          path: ./bin/*
  loadtest:
    runs-on: ubuntu-latest
    needs: [setup-environment]
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.setup-environment.outputs.loadtest_matrix) }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - run: sudo chmod 0777 -R /opt
      - name: Fluentbit Cache
        id: fluentbit-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-fluentbit
        with:
          path: /opt/td-agent-bit
          key: fluentbit-cache-1.5.3
      - run: sudo ln -s /opt/td-agent-bit/bin/td-agent-bit /usr/local/bin/fluent-bit
      - name: Install fluentbit
        if: steps.fluentbit-cache.outputs.cache-hit != 'true'
        run: |
          wget https://packages.fluentbit.io/ubuntu/bionic/pool/main/t/td-agent-bit/td-agent-bit_1.5.3_amd64.deb
          sudo dpkg -i ./td-agent-bit*.deb
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Cache Go
        id: module-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-go-modules
        with:
          path: |
            /home/runner/go/pkg/mod
            /home/runner/.cache/go-build
          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
      - run: mkdir -p results && touch results/TESTRESULTS.md
      - run: make otelcontribcol
      - name: Loadtest
        run: make -C testbed run-tests 
        env:
          TEST_ARGS: "-test.run=${{ matrix.test }}"
      - name: Create Test Result Archive # some test results have invalid characters
        if: ${{ failure() || success() }}
        continue-on-error: true
        run: tar -cvf test_results.tar testbed/tests/results
      - name: Upload Test Results
        if: ${{ failure() || success() }}
        continue-on-error: true
        uses: actions/upload-artifact@v2
        with:
          name: test-results
          path: test_results.tar
      - name: GitHub Issue Generator
        if: ${{ failure() && github.ref == 'ref/head/main' }}
        run: issuegenerator $TEST_RESULTS
  build-package:
    runs-on: ubuntu-latest
    needs: [cross-compile]
    strategy:
      fail-fast: false
      matrix:
        package_type: ["deb", "rpm"]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Install Ruby
        uses: actions/setup-ruby@v1
        with:
          ruby-version: '2.6'
      - name: Install fpm
        run: gem install --no-document fpm -v 1.11.0
      - name: Download Collector Binaries
        uses: actions/download-artifact@v2
        with:
          name: collector-binaries
          path: bin/
      - run: chmod +x bin/*
      - name: Set Release Tag
        id: github_tag
        run: ./.github/workflows/scripts/set_release_tag.sh
      - name: Build ${{ matrix.package_type }} amd64 package
        run: ./internal/buildscripts/packaging/fpm/${{ matrix.package_type }}/build.sh "${{ steps.github_tag.outputs.tag }}" "amd64" "./dist/"
      - name: Build ${{ matrix.package_type }} arm64 package
        run: ./internal/buildscripts/packaging/fpm/${{ matrix.package_type }}/build.sh "${{ steps.github_tag.outputs.tag }}" "arm64" "./dist/"
      - name: Test ${{ matrix.package_type }} package
        run: |
          if [[ "${{ matrix.package_type }}" = "deb" ]]; then
              ./internal/buildscripts/packaging/fpm/test.sh dist/otel-contrib-collector*amd64.deb examples/tracing/otel-collector-config.yml
          else
              ./internal/buildscripts/packaging/fpm/test.sh dist/otel-contrib-collector*x86_64.rpm examples/tracing/otel-collector-config.yml
          fi
      - name: Upload Packages
        uses: actions/upload-artifact@v2
        with:
          name: collector-packages
          path: ./dist/*
  windows-msi:
    runs-on: windows-latest
    needs: [cross-compile]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Download Binaries
        uses: actions/download-artifact@v2
        with:
          name: collector-binaries
          path: ./bin/
      - name: Cache Wix
        id: wix-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-wix
        with:
          path: |
            "C:\Program Files (x86)\WiX Toolset v3.11"
          key: wix-3.11
      - name: Install Wix Toolset
        if: steps.wix-cache.outputs.cache-hit != 'true'
        run: .\internal\buildscripts\packaging\msi\make.ps1 Install-Tools
      - run: mkdir -p dist
      - name: Build MSI
        run: |
          $Version = if ($env:GITHUB_REF -match '^refs/tags/(\d+\.\d+\.\d+)') { $Matches[1] } else { "0.0.1" }
          .\internal\buildscripts\packaging\msi\make.ps1 New-MSI -Version $Version
      - name: Validate MSI
        run: .\internal\buildscripts\packaging\msi\make.ps1 Confirm-MSI
      - name: Upload MSI
        uses: actions/upload-artifact@v2
        with:
          name: collector-packages
          path: ./dist/*.msi

  publish-check:
    runs-on: ubuntu-latest
    needs: [build-package, windows-msi]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Download Binaries
        uses: actions/download-artifact@v2
        with:
          name: collector-binaries
      - name: Download Packages
        uses: actions/download-artifact@v2
        with:
          name: collector-packages
          path: ./dist/
      - name: Verify Distribution Files Exist
        id: check
        run: ./.github/workflows/scripts/verify-dist-files-exist.sh
  publish-dev:
    runs-on: ubuntu-latest
    needs: [lint, unittest, integration-tests, build-package, windows-msi]
    # this is disabled because we still use circleci for releases. enable it by uncommenting the 
    # following line once we move from CircleCI for releasing.
    # if: (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'open-telemetry/opentelemetry-collector-contrib'
    if: ${{ false }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
          mkdir bin/ dist/


      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - name: Download Binaries
        uses: actions/download-artifact@v2
        with:
          name: collector-binaries
          path: ./bin/
      - name: Download Packages
        uses: actions/download-artifact@v2
        with:
          name: collector-packages
          path: ./dist/
      - name: Add Permissions to Tool Binaries
        run: chmod -R +x ./dist
      - name: Verify Distribution Files Exist
        id: check
        run: ./.github/workflows/scripts/verify-dist-files-exist.sh
      - name: Build Docker Image
        if: steps.check.outputs.passed == 'true'
        run: |
            make docker-otelcol
            docker tag otelcontribcol:latest otel/opentelemetry-collector-contrib-dev:$GITHUB_SHA
            docker tag otelcontribcol:latest otel/opentelemetry-collector-contrib-dev:latest
      - name: Push Docker Image
        if: steps.check.outputs.passed == 'true'
        run: |
            docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
            docker push otel/opentelemetry-collector-contrib-dev:$GITHUB_SHA
            docker push otel/opentelemetry-collector-contrib-dev:latest
        env:
          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}}
  publish-stable:
    runs-on: ubuntu-latest
    needs: [lint, unittest, integration-tests, build-package, windows-msi]
    # this is disabled because we still use circleci for releases. enable it by uncommenting the 
    # following line once we move from CircleCI for releasing.
    # if: startsWith(github.ref, 'refs/tags/') && github.repository == 'open-telemetry/opentelemetry-collector-contrib'
    if: ${{ false }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Go
        uses: actions/setup-go@v2.1.3
        with:
          go-version: 1.16
      - name: Setup Go Environment
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
          mkdir bin/ dist/

      - name: Cache Tools
        id: tool-cache
        uses: actions/cache@v2
        env:
          cache-name: cache-tool-binaries
        with:
          path: /home/runner/go/bin
          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
      - name: Download Binaries
        uses: actions/download-artifact@v2
        with:
          name: collector-binaries
          path: ./bin/
      - name: Download Packages
        uses: actions/download-artifact@v2
        with:
          name: collector-packages
          path: ./dist/
      - name: Add Permissions to Tool Binaries
        run: chmod -R +x ./dist
      - name: Verify Distribution Files Exist
        id: check
        run: ./.github/workflows/scripts/verify-dist-files-exist.sh
      - name: Set Release Tag
        id: github_tag
        run: ./.github/workflows/scripts/set_release_tag.sh
      - name: Build Docker Image
        if: steps.check.outputs.passed == 'true'
        run: |
            make docker-otelcol
            docker tag otelcontribcol:latest otel/opentelemetry-collector-contrib:$RELEASE_TAG
            docker tag otelcontribcol:latest otel/opentelemetry-collector-contrib:latest
        env:
          RELEASE_TAG: ${{ steps.github_tag.outputs.tag }}
      - name: Push Docker Image
        if: steps.check.outputs.passed == 'true'
        run: |
            docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
            docker push otel/opentelemetry-collector-contrib:$RELEASE_TAG
            docker push otel/opentelemetry-collector-contrib:latest
        env:
          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}}
          RELEASE_TAG: ${{ steps.github_tag.outputs.tag }}
      - name: Create Github Release
        if: steps.check.outputs.passed == 'true'
        run: |
          cp bin/* dist/
          cd dist && shasum -a 256 * > checksums.txt
          ghr -t $GITHUB_TOKEN -u "GitHub Action" -r opentelemetry-collector-contrib --replace $RELEASE_TAG dist/
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RELEASE_TAG: ${{ steps.github_tag.outputs.tag }}