ci.yaml

name: "CI"

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]
  schedule:
    - cron: 0 14 * * *

jobs:
  lint:
    name: Linter
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout commit
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Setup Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Cache PyPI
        uses: actions/cache@v4
        with:
          key: ${{ runner.os }}-py-3.9-${{ hashFiles('requirements/*.txt') }}
          path: ${{ env.pythonLocation }}

      - name: Cache pre-commit hooks
        uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: ${{ runner.os }}-pre-commit-python-3.9-${{ hashFiles('.pre-commit-config.yaml') }}

      - name: Install dependencies
        run: |
          python -m pip install -U pip
          make setup

      - name: Run linters
        run: |
          make lint

  pretest:
    name: Prepare for tests
    needs: [lint]
    runs-on: ubuntu-latest
    env:
      PYTHONIOENCODING: utf-8
      NEURO_STAGING_URL: ${{ secrets.NEURO_STAGING_URL }}
      APOLO_TOKEN: ${{ secrets.NEURO_TOKEN }}
      APOLO_CLUSTER: default
      APOLO_PROJECT: e2e-tests
      APOLO_EXTRAS_PRESET: cpu-small
      # Note: ${{ github.sha }} not working, see https://github.com/actions/checkout/issues/299
      SHA: ${{ github.event.pull_request.head.sha || github.sha }}
    steps:
      - name: Checkout commit
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Setup Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Cache Python and its deps
        uses: actions/cache@v4
        with:
          key: ${{ runner.os }}-py-3.9-${{ hashFiles('requirements/base.txt') }}
          path: ${{ env.pythonLocation }}

      - name: Install and configure apolo
        run: |
          python -m pip install -r requirements/base.txt
          apolo config login-with-token ${{ env.APOLO_TOKEN }} ${{ env.NEURO_STAGING_URL }}
          apolo config switch-cluster ${{ env.APOLO_CLUSTER }}
          apolo config switch-project ${{ env.APOLO_PROJECT }}
          apolo --color=no config show
      - name: Login to ghcr.io
        uses: docker/login-action@v3.3.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build test image
        run: |
          export REPO_URL=https://github.com/neuro-inc/neuro-extras.git
          export PACKAGE="git+$REPO_URL@$SHA"
          docker build -t ghcr.io/neuro-inc/apolo-extras:$SHA \
            --build-arg APOLO_EXTRAS_PACKAGE=$PACKAGE .
      - name: Push test image
        run: |
          docker push ghcr.io/neuro-inc/apolo-extras:$SHA

  test:
    name: Run tests
    needs: [pretest]
    strategy:
      max-parallel: 2 # to avoid overloading the platform
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        os: [ubuntu, macos, windows]
        exclude:
          # not to overload the platform, we will remove this eventually
          # see https://github.com/neuro-inc/neuro-extras/pull/249
          - python-version: 3.10
            os: macos
          - python-version: 3.10
            os: windows
          - python-version: 3.11
            os: macos
          - python-version: 3.11
            os: windows
    runs-on: ${{ matrix.os }}-latest
    timeout-minutes: 90
    env:
      PYTHONIOENCODING: utf-8

    steps:
      - name: Checkout commit
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Authorize GCP
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.E2E_COOKIECUTTER_GCP_SA_KEY }}

      - name: Setup gcloud
        uses: google-github-actions/setup-gcloud@v2

      - name: Configure AWS access
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.E2E_COOKIECUTTER_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.E2E_COOKIECUTTER_AWS_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Install python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Cache Python and its deps
        uses: actions/cache@v4
        with:
          key: ${{ runner.os }}-py-${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }}
          path: ${{ env.pythonLocation }}

      - name: Cache pre-commit hooks
        uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: ${{ runner.os }}-pre-commit-python-${{ matrix.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}

      - name: Install libraries on Linux
        if: (matrix.os == 'ubuntu') || (matrix.os == 'macos')
        shell: bash
        run: |
          curl https://rclone.org/install.sh | sudo bash

      - name: Install python dependencies
        run: |
          make setup

      - name: Configure environment
        env:
          NEURO_STAGING_URL: ${{ secrets.NEURO_STAGING_URL }}
          APOLO_TOKEN: ${{ secrets.NEURO_TOKEN }}
          APOLO_CLUSTER: default
          APOLO_PROJECT: e2e-tests
          APOLO_EXTRAS_PRESET: cpu-small
          AZURE_SAS_TOKEN: ${{ secrets.AZURE_SAS_TOKEN }}
        run: |
          apolo config login-with-token ${{ env.APOLO_TOKEN }} ${{ env.NEURO_STAGING_URL }}
          apolo config switch-cluster ${{ env.APOLO_CLUSTER }}
          apolo config switch-project ${{ env.APOLO_PROJECT }}
          apolo --color=no config show
      - name: Read PR labels
        uses: joerick/pr-labels-action@v1.0.9
        id: pr_labels

      - name: Run all tests
        if: |
          github.ref_type == 'branch' && github.ref_name == 'master'
        env:
          COLOR: "yes"
          DOCKER_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKER_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
          APOLO_CLUSTER: default
          #APOLO_CLUSTER_SECONDARY: cato-poc  # TODO: uncomment when cato-poc cluster is fixed
          AZURE_SAS_TOKEN: ${{ secrets.AZURE_SAS_TOKEN }}
          # Note: ${{ github.sha }} not working, see https://github.com/actions/checkout/issues/299
          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          # Note: see tests/e2e/conftest.py:get_tested_archive_types()
          PYTEST_DATA_COPY_ARCHIVE_TYPES: ".tar.gz"
          PYTEST_PARALLEL: 6
        shell: bash
        run: |
          export APOLO_EXTRAS_IMAGE=ghcr.io/neuro-inc/apolo-extras:$SHA
          make test
      - name: Run smoke tests
        if: |
          !(github.ref_type == 'branch' && github.ref_name == 'master')
        env:
          COLOR: "yes"
          DOCKER_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKER_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
          APOLO_CLUSTER: default
          APOLO_EXTRAS_PRESET: cpu-small
          #APOLO_CLUSTER_SECONDARY: cato-poc  # TODO: uncomment when cato-poc cluster is fixed
          AZURE_SAS_TOKEN: ${{ secrets.AZURE_SAS_TOKEN }}
          # Note: ${{ github.sha }} not working, see https://github.com/actions/checkout/issues/299
          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          # Note: see tests/e2e/conftest.py:get_tested_archive_types()
          PYTEST_DATA_COPY_ARCHIVE_TYPES: ".tar.gz"
          PYTEST_PARALLEL: 6
        shell: bash
        run: |
          export APOLO_EXTRAS_IMAGE=ghcr.io/neuro-inc/apolo-extras:$SHA
          make test_smoke
      - name: Run data tests
        if: |
          !(github.ref_type == 'branch' && github.ref_name == 'master') &&
          contains(steps.pr_labels.outputs.labels, ' e2e-data ')
        env:
          COLOR: "yes"
          DOCKER_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKER_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
          APOLO_CLUSTER: default
          #APOLO_CLUSTER_SECONDARY: cato-poc  # TODO: uncomment when cato-poc cluster is fixed
          AZURE_SAS_TOKEN: ${{ secrets.AZURE_SAS_TOKEN }}
          # Note: ${{ github.sha }} not working, see https://github.com/actions/checkout/issues/299
          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          # Note: see tests/e2e/conftest.py:get_tested_archive_types()
          PYTEST_DATA_COPY_ARCHIVE_TYPES: ".tar.gz"
          PYTEST_PARALLEL: 6
        shell: bash
        run: |
          export APOLO_EXTRAS_IMAGE=ghcr.io/neuro-inc/apolo-extras:$SHA
          make test_data

      - name: Run image tests
        if: |
          !(github.ref_type == 'branch' && github.ref_name == 'master') &&
          contains(steps.pr_labels.outputs.labels, ' e2e-image ')
        env:
          COLOR: "yes"
          DOCKER_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKER_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
          APOLO_CLUSTER: default
          APOLO_EXTRAS_PRESET: cpu-small
          #APOLO_CLUSTER_SECONDARY: cato-poc  # TODO: uncomment when cato-poc cluster is fixed
          AZURE_SAS_TOKEN: ${{ secrets.AZURE_SAS_TOKEN }}
          # Note: ${{ github.sha }} not working, see https://github.com/actions/checkout/issues/299
          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          # Note: see tests/e2e/conftest.py:get_tested_archive_types()
          PYTEST_DATA_COPY_ARCHIVE_TYPES: ".tar.gz"
          PYTEST_PARALLEL: auto
        shell: bash
        run: |
          export APOLO_EXTRAS_IMAGE=ghcr.io/neuro-inc/apolo-extras:$SHA
          make test_image