ci-image-checks.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: CI Image Checks
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runs-on-as-json-default:
        description: "The array of labels (in json form) determining default runner used for the build."
        required: true
        type: string
      runs-on-as-json-docs-build:
        description: "The array of labels (in json form) determining the labels used for docs build."
        required: true
        type: string
      needs-mypy:
        description: "Whether to run mypy checks (true/false)"
        required: true
        type: string
      mypy-checks:
        description: "List of folders to run mypy checks on"
        required: false
        type: string
      python-versions-list-as-string:
        description: "The list of python versions as string separated by spaces"
        required: true
        type: string
      branch:
        description: "Branch used to run the CI jobs in (main/v2_*_test)."
        required: true
        type: string
      canary-run:
        description: "Whether this is a canary run (true/false)"
        required: true
        type: string
      default-python-version:
        description: "Which version of python should be used by default"
        required: true
        type: string
      docs-list-as-string:
        description: "Stringified list of docs to build (space separated)"
        required: true
        type: string
      upgrade-to-newer-dependencies:
        description: "Whether to upgrade to newer dependencies (true/false)"
        required: true
        type: string
      chicken-egg-providers:
        description: "List of providers that should be prepared from sources"
        required: false
        type: string
      basic-checks-only:
        description: "Whether to run only basic checks (true/false)"
        required: true
        type: string
      latest-versions-only:
        description: "Whether to run only latest versions (true/false)"
        required: true
        type: string
      ci-image-build:
        description: "Whether to build CI images (true/false)"
        required: true
        type: string
      skip-pre-commits:
        description: "Whether to skip pre-commits (true/false)"
        required: true
        type: string
      include-success-outputs:
        description: "Whether to include success outputs"
        required: true
        type: string
      debug-resources:
        description: "Whether to debug resources (true/false)"
        required: true
        type: string
      docs-build:
        description: "Whether to build docs (true/false)"
        required: true
        type: string
      needs-api-codegen:
        description: "Whether to run API codegen (true/false)"
        required: true
        type: string
      default-postgres-version:
        description: "The default version of the postgres to use"
        required: true
        type: string
      run-coverage:
        description: "Whether to run coverage or not (true/false)"
        required: true
        type: string
      use-uv:
        description: "Whether to use uv to build the image (true/false)"
        required: true
        type: string

jobs:
  install-pre-commit:
    timeout-minutes: 5
    name: "Install pre-commit for cache (only canary runs)"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
    if: inputs.basic-checks-only == 'false'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: inputs.canary-run == 'true'
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
        if: inputs.canary-run == 'true'
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        with:
          use-uv: ${{ inputs.use-uv }}
        id: breeze
        if: inputs.canary-run == 'true'
      - name: "Install pre-commit"
        uses: ./.github/actions/install-pre-commit
        id: pre-commit
        with:
          python-version: ${{steps.breeze.outputs.host-python-version}}
        if: inputs.canary-run == 'true'
      - name: "Prepare .tar file from pre-commit cache"
        run: |
          tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv
        shell: bash
        if: inputs.canary-run == 'true'
      - name: "Save pre-commit cache"
        uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          # yamllint disable rule:line-length
          key: cache-pre-commit-v4-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
          path: /tmp/cache-pre-commit.tar.gz
          if-no-files-found: 'error'
          retention-days: '2'
        if: inputs.canary-run == 'true'

  static-checks:
    timeout-minutes: 45
    name: "Static checks"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
    needs: install-pre-commit
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}"
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    if: inputs.basic-checks-only == 'false' && inputs.latest-versions-only != 'true'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: "linux/amd64"
          python: ${{ inputs.default-python-version }}
          use-uv: ${{ inputs.use-uv }}
        id: breeze
      - name: "Install pre-commit"
        uses: ./.github/actions/install-pre-commit
        id: pre-commit
        with:
          python-version: ${{steps.breeze.outputs.host-python-version}}
      - name: "Static checks"
        run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment
        env:
          VERBOSE: "false"
          SKIP: ${{ inputs.skip-pre-commits }}
          COLUMNS: "250"
          SKIP_GROUP_OUTPUT: "true"
          DEFAULT_BRANCH: ${{ inputs.branch }}
          RUFF_FORMAT: "github"

  mypy:
    timeout-minutes: 45
    name: "MyPy checks"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
    needs: install-pre-commit
    if: inputs.needs-mypy == 'true'
    strategy:
      fail-fast: false
      matrix:
        mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: "linux/amd64"
          python: ${{ inputs.default-python-version }}
          use-uv: ${{ inputs.use-uv }}
        id: breeze
      - name: "Install pre-commit"
        uses: ./.github/actions/install-pre-commit
        id: pre-commit
        with:
          python-version: ${{steps.breeze.outputs.host-python-version}}
      - name: "MyPy checks for ${{ matrix.mypy-check }}"
        run: pre-commit run --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files
        env:
          VERBOSE: "false"
          COLUMNS: "250"
          SKIP_GROUP_OUTPUT: "true"
          DEFAULT_BRANCH: ${{ inputs.branch }}
          RUFF_FORMAT: "github"
          INCLUDE_MYPY_VOLUME: "false"
          MYPY_CHECK: ${{ matrix.mypy-check }}

  build-docs:
    timeout-minutes: 150
    name: "Build documentation"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
    if: inputs.docs-build == 'true'
    strategy:
      fail-fast: false
      matrix:
        flag: ["--docs-only", "--spellcheck-only"]
    env:
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INCLUDE_NOT_READY_PROVIDERS: "true"
      INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: "linux/amd64"
          python: ${{ inputs.default-python-version }}
          use-uv: ${{ inputs.use-uv }}
      - name: "Restore docs inventory cache"
        uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          path: ./docs/_inventory_cache/
          # TODO(potiuk): do better with determining the key
          key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
        id: restore-docs-inventory-cache
      - name: "Building docs with ${{ matrix.flag }} flag"
        env:
          DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
        run: >
          breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }}
      - name: "Save docs inventory cache"
        uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
        with:
          path: ./docs/_inventory_cache/
          key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
          if-no-files-found: 'error'
          retention-days: '2'
        if: steps.restore-docs-inventory-cache != 'true'
      - name: "Upload build docs"
        uses: actions/upload-artifact@v4
        with:
          name: airflow-docs
          path: './docs/_build'
          retention-days: '7'
          if-no-files-found: 'error'
        if: matrix.flag == '--docs-only'

  publish-docs:
    timeout-minutes: 150
    name: "Publish documentation"
    needs: build-docs
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-docs-build) }}
    env:
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      INCLUDE_NOT_READY_PROVIDERS: "true"
      INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      VERBOSE: "true"
    if: inputs.canary-run == 'true' && inputs.branch == 'main'
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Download docs prepared as artifacts"
        uses: actions/download-artifact@v4
        with:
          name: airflow-docs
          path: './docs/_build'
      - name: Check disk space available
        run: df -h
      - name: Create /mnt/airflow-site directory
        run: sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}" /mnt/airflow-site
      - name: "Clone airflow-site"
        run: >
          git clone https://github.com/apache/airflow-site.git /mnt/airflow-site/airflow-site &&
          echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/airflow-site" >> "$GITHUB_ENV"
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: "linux/amd64"
          python: ${{ inputs.default-python-version }}
          use-uv: ${{ inputs.use-uv }}
      - name: "Publish docs"
        env:
          DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
        run: >
          breeze release-management publish-docs --override-versioned --run-in-parallel
          ${DOCS_LIST_AS_STRING}
      - name: Check disk space available
        run: df -h
      - name: "Generate back references for providers"
        run: breeze release-management add-back-references all-providers
      - name: "Generate back references for apache-airflow"
        run: breeze release-management add-back-references apache-airflow
      - name: "Generate back references for docker-stack"
        run: breeze release-management add-back-references docker-stack
      - name: "Generate back references for helm-chart"
        run: breeze release-management add-back-references helm-chart
      - name: Install AWS CLI v2
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
          unzip -q /tmp/awscliv2.zip -d /tmp
          rm /tmp/awscliv2.zip
          sudo /tmp/aws/install --update
          rm -rf /tmp/aws/
        if: inputs.branch == 'main'
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a  # v4.0.1
        if: inputs.branch == 'main'
        with:
          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-central-1
      - name: "Upload documentation to AWS S3"
        if: inputs.branch == 'main'
        run: aws s3 sync --delete ./docs/_build s3://apache-airflow-docs

  test-python-api-client:
    timeout-minutes: 60
    name: "Test Python API client"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
    if: inputs.needs-api-codegen == 'true'
    env:
      BACKEND: "postgres"
      BACKEND_VERSION: "${{ inputs.default-postgres-version }}"
      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
      ENABLE_COVERAGE: "${{ inputs.run-coverage }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      JOB_ID: "python-api-client-tests"
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          fetch-depth: 2
          persist-credentials: false
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v4
        with:
          repository: "apache/airflow-client-python"
          fetch-depth: 1
          persist-credentials: false
          path: ./airflow-client-python
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: "linux/amd64"
          python: ${{ inputs.default-python-version }}
          use-uv: ${{ inputs.use-uv }}
      - name: "Generate airflow python client"
        run: >
          breeze release-management prepare-python-client --package-format both
          --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python
      - name: "Show diff"
        run: git diff --color HEAD
        working-directory: ./airflow-client-python
      - name: "Python API client tests"
        run: breeze testing python-api-client-tests