basic-tests.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Basic tests
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runs-on-as-json-public:
        description: "The array of labels (in json form) determining public runners."
        required: true
        type: string
      run-ui-tests:
        description: "Whether to run UI tests (true/false)"
        required: true
        type: string
      run-www-tests:
        description: "Whether to run WWW tests (true/false)"
        required: true
        type: string
      needs-api-codegen:
        description: "Whether to run API codegen (true/false)"
        required: true
        type: string
      basic-checks-only:
        description: "Whether to run only basic checks (true/false)"
        required: true
        type: string
      skip-pre-commits:
        description: "Whether to skip pre-commits (true/false)"
        required: true
        type: string
      default-python-version:
        description: "Which version of python should be used by default"
        required: true
        type: string
      canary-run:
        description: "Whether to run canary tests (true/false)"
        required: true
        type: string
      latest-versions-only:
        description: "Whether to run only latest version checks (true/false)"
        required: true
        type: string
      use-uv:
        description: "Whether to use uv in the image"
        required: true
        type: string
jobs:
  run-breeze-tests:
    timeout-minutes: 10
    name: Breeze unit tests
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - uses: actions/checkout@v4
        with:
          # Need to fetch all history for selective checks tests
          fetch-depth: 0
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        with:
          use-uv: ${{ inputs.use-uv }}
      - run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
        working-directory: ./dev/breeze/
  tests-ui:
    timeout-minutes: 10
    name: React UI tests
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    if: inputs.run-ui-tests == 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: Setup pnpm
        uses: pnpm/action-setup@v4.0.0
        with:
          version: 9
          run_install: false
      - name: "Setup node"
        uses: actions/setup-node@v4
        with:
          node-version: 21
          cache: 'pnpm'
          cache-dependency-path: 'airflow/ui/pnpm-lock.yaml'
      - name: "Restore eslint cache (ui)"
        uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          path: airflow/ui/node_modules/
          key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
        id: restore-eslint-cache
      - run: cd airflow/ui && pnpm install --frozen-lockfile
      - run: cd airflow/ui && pnpm test
        env:
          FORCE_COLOR: 2
      - name: "Save eslint cache (ui)"
        uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          path: airflow/ui/node_modules/
          key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
          if-no-files-found: 'error'
          retention-days: '2'
        if: steps.restore-eslint-cache.outputs.stash-hit != 'true'

  tests-www:
    timeout-minutes: 10
    name: React WWW tests
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    if: inputs.run-www-tests == 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Setup node"
        uses: actions/setup-node@v4
        with:
          node-version: 21
      - name: "Restore eslint cache (www)"
        uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          path: airflow/www/node_modules/
          key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }}
        id: restore-eslint-cache
      - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
      - run: yarn --cwd airflow/www/ run test
        env:
          FORCE_COLOR: 2
      - name: "Save eslint cache (www)"
        uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          path: airflow/www/node_modules/
          key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }}
          if-no-files-found: 'error'
          retention-days: '2'
        if: steps.restore-eslint-cache.outputs.stash-hit != 'true'

  install-pre-commit:
    timeout-minutes: 5
    name: "Install pre-commit for cache"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
    if: inputs.basic-checks-only == 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        with:
          use-uv: ${{ inputs.use-uv }}
        id: breeze
      - name: "Install pre-commit"
        uses: ./.github/actions/install-pre-commit
        id: pre-commit
        with:
          python-version: ${{steps.breeze.outputs.host-python-version}}

  # Those checks are run if no image needs to be built for checks. This is for simple changes that
  # Do not touch any of the python code or any of the important files that might require building
  # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
  static-checks-basic-checks-only:
    timeout-minutes: 30
    name: "Static checks: basic checks only"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    needs: install-pre-commit
    if: inputs.basic-checks-only == 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        with:
          use-uv: ${{ inputs.use-uv }}
        id: breeze
      - name: "Install pre-commit"
        uses: ./.github/actions/install-pre-commit
        id: pre-commit
        with:
          python-version: ${{steps.breeze.outputs.host-python-version}}
      - name: Fetch incoming commit ${{ github.sha }} with its parent
        uses: actions/checkout@v4
        with:
          ref: ${{ github.sha }}
          fetch-depth: 2
          persist-credentials: false
      - name: "Static checks: basic checks only"
        run: >
          breeze static-checks --show-diff-on-failure --color always --initialize-environment
          --skip-image-upgrade-check --commit-ref "${{ github.sha }}"
        env:
          VERBOSE: "false"
          SKIP_BREEZE_PRE_COMMITS: "true"
          SKIP: ${{ inputs.skip-pre-commits }}
          COLUMNS: "250"

  test-git-clone-on-windows:
    timeout-minutes: 5
    name: "Test git clone on Windows"
    runs-on: ["windows-latest"]
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          fetch-depth: 2
          persist-credentials: false

  upgrade-check:
    timeout-minutes: 45
    name: "Upgrade checks"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    needs: install-pre-commit
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
    if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        with:
          use-uv: ${{ inputs.use-uv }}
        id: breeze
      - name: "Install pre-commit"
        uses: ./.github/actions/install-pre-commit
        id: pre-commit
        with:
          python-version: ${{steps.breeze.outputs.host-python-version}}
      - name: "Autoupdate all pre-commits"
        run: pre-commit autoupdate
      - name: "Run automated upgrade for black"
        run: >
          pre-commit run
          --all-files --show-diff-on-failure --color always --verbose
          --hook-stage manual
          update-black-version
        if: always()
      - name: "Run automated upgrade for build dependencies"
        run: >
          pre-commit run
          --all-files --show-diff-on-failure --color always --verbose
          --hook-stage manual
          update-build-dependencies
        if: always()
        env:
          SKIP_TROVE_CLASSIFIERS_ONLY: "true"
      - name: "Run automated upgrade for chart dependencies"
        run: >
          pre-commit run
          --all-files --show-diff-on-failure --color always --verbose
          --hook-stage manual
          update-chart-dependencies
        if: always()
      # For UV we are not failing the upgrade installers check if it is updated because
      # it is upgraded very frequently, so we want to manually upgrade it rather than
      # get notified about it - until it stabilizes in 1.* version
      - name: "Run automated upgrade for uv (open to see if new version is updated)"
        run: >
          pre-commit run
          --all-files --show-diff-on-failure --color always --verbose
          --hook-stage manual update-installers-and-pre-commit || true
        if: always()
        env:
          UPGRADE_UV: "true"
          UPGRADE_PIP: "false"
          UPGRADE_PRE_COMMIT: "true"
      - name: "Run automated upgrade for pip"
        run: >
          pre-commit run
          --all-files --show-diff-on-failure --color always --verbose
          --hook-stage manual update-installers
        if: always()
        env:
          UPGRADE_UV: "false"
          UPGRADE_PIP: "true"

  test-airflow-release-commands:
    timeout-minutes: 80
    name: "Test Airflow release commands"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      VERBOSE: "true"
    if: inputs.canary-run == 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        with:
          use-uv: ${{ inputs.use-uv }}
      - name: "Cleanup dist files"
        run: rm -fv ./dist/*
      - name: Setup git for tagging
        run: |
          git config --global user.email "name@example.com"
          git config --global user.name "Your Name"
      - name: Install twine
        run: pip install twine
      - name: "Check Airflow create minor branch command"
        run: |
          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
          release-management create-minor-branch --version-branch 2-8 --answer yes
      - name: "Check Airflow RC process command"
        run: |
          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
          release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0 --answer yes
      - name: "Check Airflow release process command"
        run: |
          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
          release-management start-release --release-candidate 2.8.3rc1 --previous-release 2.8.0 --answer yes
      - name: "Fetch all git tags"
        run: git fetch --tags >/dev/null 2>&1 || true
      - name: "Test airflow core issue generation automatically"
        run: |
          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
          release-management generate-issue-content-core --limit-pr-count 25 --latest --verbose