tests.yml

name: "Tests"

on:
  push:
    branches:
      - master
      - 'release/**'
  pull_request:
    types:
      - opened
      - synchronize
      - reopened
      - ready_for_review
    branches:
      - master
      - 'release/**'

env:
  CACHE_NAME_PREFIX: v1

jobs:
  run_pytest:
    name: Run pytest
    needs: calculate_matrix
    runs-on: ubuntu-latest
    timeout-minutes: 30
    
    strategy:
      fail-fast: false
      matrix: ${{fromJSON(needs.calculate_matrix.outputs.matrix)}}
    env:
      LOG_DIR: pytest_logs
      collect_analytics: false
      TEST_WITH_CPU: ${{ matrix.ml_backend == 'segment_anything_model' }}
    steps:
      - uses: hmarr/debug-action@v2.1.0

      - name: Free Disk Space
        if: matrix.ml_backend == 'mmdetection-3'
        uses: jlumbroso/free-disk-space@main
        with:
          # this might remove tools that are actually needed,
          # if set to "true" but frees about 6 GB
          tool-cache: true
          android: false
          dotnet: false
          haskell: false
          large-packages: false
          docker-images: false
          swap-storage: false

      - name: Extract branch name on direct push to a branch
        if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/')
        run: |
          echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV

      - name: Extract branch name on 'pull_request'
        if: github.event_name == 'pull_request'
        run: |
          echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_ENV

      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: "${{ env.GITHUB_SHA }}"

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.7'

      - uses: actions/cache@v3
        name: Configure pip cache
        id: pip-cache
        with:
          path: ~/.cache/pip
          key: ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-test.txt') }}
          restore-keys: |
            ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-pip-

      - name: Install dependencies
        run: |
          sudo apt-get clean
          sudo apt-get update
          sudo apt-get install virtualenv libsasl2-dev python3-dev libldap2-dev libssl-dev
          pip install -U pip==20.2
          pip install -r requirements-test.txt
          pip install -r requirements.txt
          pip install -e .

      - name: Check pytest version
        run: pytest --version

      - name: Build the stack
        env:
          DOCKER_BUILDKIT: 1
        run: docker-compose -f label_studio_ml/examples/${{ matrix.ml_backend }}/docker-compose.yml up -d

      - name: Wait for stack
        timeout-minutes: 10
        run: |
          while [ "$(curl -s -o /dev/null -L -w ''%{http_code}'' "http://localhost:9090/health")" != "200" ]; do
            echo "=> Waiting for service to become available" && sleep 2s
          done

      - name: Run general functional tests
        run: |
          cd label_studio_ml/
          pytest --ignore-glob='**/logs/*' --ignore-glob='**/data/*' --cov=. --cov-report=xml

      - name: Pull the logs
        if: always()
        env:
          DOCKER_BUILDKIT: 1
        run: docker-compose -f label_studio_ml/examples/${{ matrix.ml_backend }}/docker-compose.yml logs

      - name: "Upload cloud coverage to Codecov"
        uses: codecov/codecov-action@v3.1.4
        with:
          name: codecov
          files: ./label_studio_ml/coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true

  calculate_matrix:
    name: "Calculate test matrix"
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: hmarr/debug-action@v2.1.0

      - name: Extract branch name on direct push to a branch
        if: github.event_name == 'push'
        run: |
          echo "BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV
          echo "BASE_BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV
          echo "BASE_REF_NAME=${{ github.sha }}^" >> $GITHUB_ENV

      - name: Extract branch name on 'pull_request'
        if: github.event_name == 'pull_request'
        run: |
          echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
          echo "BASE_BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV
          echo "BASE_REF_NAME=${{ github.event.pull_request.base.sha }}" >> $GITHUB_ENV

      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: "${{ env.GITHUB_SHA }}"
          fetch-depth: 0

      - name: Build matrix
        id: set-matrix
        shell: bash
        env:
          BASE_REF_NAME: ${{ env.BASE_REF_NAME }}
        run: |
          set -xuo pipefail
          set +e
          CHANGED=$(git diff --name-only HEAD.."${BASE_REF_NAME}" -- label_studio_ml/ | xargs -L1 dirname | uniq | cut -d'/' -f2-)
          CHANGED_EXAMPLES=$(echo "$CHANGED" | grep examples )
          set -e
          ALL_EXAMPLES=$(ls -1d label_studio_ml/examples/*/ | cut -d'/' -f3- | cut -d'/' -f1)
          if [[ $GITHUB_REF == 'refs/heads/master' || $GITHUB_REF =~ ^refs/heads/release/.* ]]; then
            echo "All in matrix because we are on master or a release branch"
            echo "matrix={\"ml_backend\":[$(echo "$ALL_EXAMPLES" | xargs | sed -e 's/ /\",\"/g' -e 's/^/\"/' -e 's/$/\"/')]}" >> $GITHUB_OUTPUT
          elif [[ $(echo "$CHANGED" | grep -c -v ^$) == $(echo "$CHANGED_EXAMPLES" | grep -c -v ^$) ]]; then
            echo "Finding examples to test"
            if [[ $(echo "$CHANGED" | grep -c -v ^$) == 0 ]]; then
              echo "matrix={\"ml_backend\":[]}" >> $GITHUB_OUTPUT
            else
              echo "matrix={\"ml_backend\":[$(echo "$CHANGED_EXAMPLES" | cut -d'/' -f2- | xargs | sed -e 's/ /\",\"/g' -e 's/^/\"/' -e 's/$/\"/')]}" >> $GITHUB_OUTPUT
            fi
          else
            echo "All in matrix"
            echo "matrix={\"ml_backend\":[$(echo "$ALL_EXAMPLES" | xargs | sed -e 's/ /\",\"/g' -e 's/^/\"/' -e 's/$/\"/')]}" >> $GITHUB_OUTPUT
          fi