integration-test-slow.yml

---
name: Integration Tests (Slow CI)
on:
  workflow_call:
    inputs:
      os:
        description: OS
        type: string
        required: true
      python-version:
        description: Python version
        type: string
        required: true
      test_environment:
        description: The test environment
        type: string
        required: true
      enable_tmate:
        description: Enable tmate session for debugging
        type: string
        required: false
        default: never
      tmate_timeout:
        description: Timeout for tmate session (minutes)
        type: number
        required: false
        default: 30
  workflow_dispatch:
    inputs:
      os:
        description: OS
        type: choice
        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
        required: false
        default: ubuntu-latest
      python-version:
        description: Python version
        type: choice
        options: ['3.8', '3.9', '3.10', '3.11']
        required: false
        default: '3.8'
      test_environment:
        description: The test environment
        type: choice
        options:
          # Default ZenML deployments
          - default
          - default-docker-orchestrator
          - default-airflow-orchestrator
          # Local ZenML server deployments
          - local-server
          - local-server-docker-orchestrator
          - local-server-airflow-orchestrator
          # Local ZenML docker-compose server deployments
          - docker-server-mysql
          - docker-server-mariadb
          - docker-server-docker-orchestrator-mysql
          - docker-server-docker-orchestrator-mariadb
          - docker-server-airflow-orchestrator-mysql
          - docker-server-airflow-orchestrator-mariadb
        required: false
        default: default
      enable_tmate:
        description: Enable tmate session for debugging
        type: choice
        options: [no, on-failure, always, before-tests]
        required: false
        default: 'no'
      tmate_timeout:
        description: Timeout for tmate session (minutes)
        type: number
        required: false
        default: 30
jobs:
  integration-tests-slow:
    name: integration-tests-slow
    runs-on: ${{ inputs.os }}
    strategy:
      fail-fast: false
    env:
      ZENML_DEBUG: 1
      ZENML_ANALYTICS_OPT_IN: false
      PYTHONIOENCODING: utf-8
      UV_HTTP_TIMEOUT: 600
      # on MAC OS, we need to set this environment variable
      # to fix problems with the fork() calls (see this thread
      # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_US_EAST_1_ENV_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_US_EAST_1_ENV_SECRET_ACCESS_KEY }}
      AWS_US_EAST_1_SERVER_URL: ${{ secrets.AWS_US_EAST_1_SERVER_URL }}
      AWS_US_EAST_1_SERVER_USERNAME: ${{ secrets.AWS_US_EAST_1_SERVER_USERNAME }}
      AWS_US_EAST_1_SERVER_PASSWORD: ${{ secrets.AWS_US_EAST_1_SERVER_PASSWORD }}
      GCP_US_EAST4_SERVER_URL: ${{ secrets.GCP_US_EAST4_SERVER_URL }}
      GCP_US_EAST4_SERVER_USERNAME: ${{ secrets.GCP_US_EAST4_SERVER_USERNAME }}
      GCP_US_EAST4_SERVER_PASSWORD: ${{ secrets.GCP_US_EAST4_SERVER_PASSWORD }}
    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') }}
    defaults:
      run:
        shell: bash
    steps:
      - name: Maximize space for Docker
        uses: easimon/maximize-build-space@v10
        with:
          root-reserve-mb: 20000
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          build-mount-path: /var/lib/docker/
        if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
          'airflow') || contains(inputs.test_environment, 'kubernetes'))
      - name: Reload Docker
        run: sudo systemctl restart docker
        if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
          'airflow') || contains(inputs.test_environment, 'kubernetes'))
      - uses: actions/checkout@v4.1.1
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          role-to-assume: ${{ secrets.AWS_US_EAST_1_ENV_ROLE_ARN }}
          aws-region: us-east-1
        if: contains(inputs.test_environment, 'aws')
      - name: Configure GCP credentials
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_US_EAST4_ENV_CREDENTIALS }}
        if: contains(inputs.test_environment, 'gcp')
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          install_components: gke-gcloud-auth-plugin
        if: contains(inputs.test_environment, 'gcp')
      - name: Setup environment
        uses: ./.github/actions/setup_environment
        with:
          cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
          python-version: ${{ inputs.python-version }}
          os: ${{ inputs.os }}
          runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
          runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
          discord_webhook: ${{ secrets.DISCORD_WEBHOOK }}
      - name: Install docker-compose for non-default environments
        if: inputs.test_environment != 'default'
        run: |
          pip install uv
          uv pip install --system "pyyaml==5.3.1" docker-compose
      - name: Install Linux System Dependencies
        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
        run: sudo apt install graphviz
      - name: Install MacOS System Dependencies
        if: runner.os=='macOS'
        run: brew install graphviz libomp
      - name: Install Windows System Dependencies
        if: runner.os=='Windows'
        run: choco install graphviz
      - name: Unbreak Python in GHA for 3.8-3.10
        if: runner.os=='macOS' && inputs.python-version != '3.11'
        # github actions overwrites brew's python. Force it to reassert itself, by
        # running in a separate step.
        # Workaround GitHub Actions Python issues
        # see https://github.com/Homebrew/homebrew-core/issues/165793#issuecomment-1989441193
        run: |
          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
          sudo rm -rf /Library/Frameworks/Python.framework/
          brew install --force python3 && brew unlink python3 && brew unlink python3 && brew link --overwrite python3
      - name: Unbreak Python in GHA for 3.11
        if: runner.os=='macOS' && inputs.python-version == '3.11'
        run: |
          # Unlink and re-link to prevent errors when github mac runner images
          # https://github.com/actions/setup-python/issues/577
          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
      - name: Install Docker and Colima on MacOS
        if: runner.os=='macOS'
        run: |
          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
          brew update
          brew install docker colima
          brew reinstall --force qemu

          # We need to mount the /private/tmp/zenml-test/ folder because
          # this folder is also mounted in the Docker containers that are
          # started by local ZenML orchestrators.
          colima start --mount /private/tmp/zenml-test/:w

          # This is required for the Docker Python SDK to work
          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
      - name: Install kubectl on Linux
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
      - name: Install kubectl on MacOS
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"
          sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
        if: runner.os=='macOS'
      - name: Install K3D
        run: |
          curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
        if: runner.os!='Windows' && contains(inputs.test_environment, 'kubeflow')
      - name: Login to Amazon ECR
        id: login-ecr
        run: |
          aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.us-east-1.amazonaws.com
        if: contains(inputs.test_environment, 'aws')
      - name: Login to Amazon EKS
        id: login-eks
        run: |
          aws eks --region us-east-1 update-kubeconfig --name zenml-ci-cluster --alias zenml-ci-aws-us-east-1
        if: contains(inputs.test_environment, 'aws')
      - name: Login to Google ECR
        run: |
          gcloud auth configure-docker --project zenml-ci
        if: contains(inputs.test_environment, 'gcp')
      - name: Login to Google GKE
        uses: google-github-actions/get-gke-credentials@v2
        with:
          cluster_name: zenml-ci-cluster
          location: us-east4
          project_id: zenml-ci
        if: contains(inputs.test_environment, 'gcp')
      - name: Setup tmate session before tests
        if: ${{ inputs.enable_tmate == 'before-tests' }}
        uses: mxschmitt/action-tmate@v3.17
        timeout-minutes: ${{ inputs.tmate_timeout }}
      - name: Integration Tests (Ubuntu) - Slow CI
          # Ubuntu Slow CI integration tests run on single machines
        if: runner.os != 'macOS' && runner.os != 'Windows'
        run: |
          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }}
      - name: Integration Tests (Non-Ubuntu) - Slow CI
        if: runner.os == 'macOS' || runner.os == 'Windows'
        run: |
          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }}
      - name: Setup tmate session after tests
        if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
        uses: mxschmitt/action-tmate@v3.17
        timeout-minutes: ${{ inputs.tmate_timeout }}
      - name: Verify Python Env unaffected
        run: |-
          zenml integration list
          uv pip list
          uv pip check || true