Merge branch 'main' into backfill_dry_run

apache · Jan 11, 2025 · 711b10a · 711b10a
2 parents 4b877c9 + 1e8977a
commit 711b10a
Show file tree

Hide file tree

Showing 66 changed files with 896 additions and 581 deletions.
diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml
@@ -46,17 +46,22 @@ runs:
       with:
         use-uv: ${{ inputs.use-uv }}
       id: breeze
+    - name: Check free space
+      run: df -H
+      shell: bash
+    - name: Make /mnt/ directory writeable
+      run: sudo chown -R ${USER} /mnt
+      shell: bash
     - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
       uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
         key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
-        path: "/tmp/"
+        path: "/mnt/"
     - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
       env:
         PLATFORM: ${{ inputs.platform }}
         PYTHON: ${{ inputs.python }}
         IMAGE_TYPE: ${{ inputs.image-type }}
       run: >
-        breeze ${IMAGE_TYPE}-image load
-        --platform ${PLATFORM} --python ${PYTHON}
+        breeze ${IMAGE_TYPE}-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt"
       shell: bash
diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml
@@ -35,16 +35,22 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Check free space
+      run: df -H
+      shell: bash
+    - name: Make /mnt/ directory writeable
+      run: sudo chown -R ${USER} /mnt
+      shell: bash
     - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
       uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
         key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
-        path: "/tmp/"
+        path: "/mnt/"
       if: contains(inputs.python-versions-list-as-string, inputs.python)
     - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"
       env:
         PLATFORM: ${{ inputs.platform }}
         PYTHON: ${{ inputs.python }}
-      run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}"
+      run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt/"
       shell: bash
       if: contains(inputs.python-versions-list-as-string, inputs.python)
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
@@ -84,6 +84,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv to build the image (true/false)"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
   # tests to complete - it is run very early in the build process for "main" merges in order to refresh

diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml
@@ -60,6 +60,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   prod-image-extra-checks-main:
     name: PROD image extra checks (main)

diff --git a/.github/workflows/automatic-backport.yml b/.github/workflows/automatic-backport.yml
@@ -21,7 +21,8 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - main
-
+permissions:
+  contents: read
 jobs:
   get-pr-info:
     name: "Get PR information"

diff --git a/.github/workflows/backport-cli.yml b/.github/workflows/backport-cli.yml
@@ -41,6 +41,9 @@ on:  # yamllint disable-line rule:truthy
         type: string
 
 permissions:
+  # Those permissions are only active for workflow dispatch (only committers can trigger it) and workflow call
+  # Which is triggered automatically by "automatic-backport" push workflow (only when merging by committer)
+  # Branch protection  prevents from pushing to the "code" branches
   contents: write
   pull-requests: write
 jobs:

diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -60,6 +60,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv in the image"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   run-breeze-tests:
     timeout-minutes: 10

diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
@@ -96,6 +96,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Disable airflow repo cache read from main."
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   build-ci-images:
     strategy:
@@ -173,16 +175,22 @@ jobs:
           PUSH: ${{ inputs.push-image }}
           VERBOSE: "true"
           PLATFORM: ${{ inputs.platform }}
+      - name: Check free space
+        run: df -H
+        shell: bash
+      - name: Make /mnt/ directory writeable
+        run: sudo chown -R ${USER} /mnt
+        shell: bash
       - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         env:
           PLATFORM: ${{ inputs.platform }}
-        run: breeze ci-image save --platform "${PLATFORM}"
+        run: breeze ci-image save --platform "${PLATFORM}" --image-file-dir "/mnt"
         if: inputs.upload-image-artifact == 'true'
       - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
         with:
           key: ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}
-          path: "/tmp/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
+          path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
           if-no-files-found: 'error'
           retention-days: '2'
         if: inputs.upload-image-artifact == 'true'

diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml
@@ -108,7 +108,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv to build the image (true/false)"
         required: true
         type: string
-
+permissions:
+  contents: read
 jobs:
   install-pre-commit:
     timeout-minutes: 5

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -33,46 +33,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  selective-checks:
-    name: Selective checks
-    runs-on: ["ubuntu-22.04"]
-    env:
-      GITHUB_CONTEXT: ${{ toJson(github) }}
-    outputs:
-      needs-python-scans: ${{ steps.selective-checks.outputs.needs-python-scans }}
-      needs-javascript-scans: ${{ steps.selective-checks.outputs.needs-javascript-scans }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-          persist-credentials: false
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-        with:
-          use-uv: "true"
-      - name: "Get information about the Workflow"
-        id: source-run-info
-        run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
-        env:
-          SKIP_BREEZE_SELF_UPGRADE_CHECK: "true"
-      - name: Selective checks
-        id: selective-checks
-        env:
-          PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
-          COMMIT_REF: "${{ github.sha }}"
-          VERBOSE: "false"
-        run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
-
   analyze:
     name: Analyze
     runs-on: ["ubuntu-22.04"]
-    needs: [selective-checks]
     strategy:
       fail-fast: false
       matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
         language: ['python', 'javascript', 'actions']
     permissions:
       actions: read
@@ -84,37 +50,14 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-        if: |
-          matrix.language == 'actions' ||
-          matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
-          matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
 
-      # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
-        if: |
-          matrix.language == 'actions' ||
-          matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
-          matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
 
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
         uses: github/codeql-action/autobuild@v3
-        if: |
-          matrix.language == 'actions' ||
-          matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
-          matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v3
-        if: |
-          matrix.language == 'actions' ||
-          matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' ||
-          matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true'
diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
@@ -76,6 +76,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources or not (true/false)"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   update-constraints:
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}

diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
@@ -40,6 +40,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uvloop (true/false)"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   tests-helm:
     timeout-minutes: 80

diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml
@@ -64,6 +64,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   tests-core-integration:
     timeout-minutes: 130

diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -48,6 +48,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   tests-kubernetes:
     timeout-minutes: 60

diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml
@@ -21,7 +21,8 @@ name: CI
 on:  # yamllint disable-line rule:truthy
   pull_request:
     types: [labeled, unlabeled, opened, reopened, synchronize]
-
+permissions:
+  contents: read
 jobs:
   check-news-fragment:
     name: Check News Fragment

diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
@@ -116,8 +116,9 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether this is a prod-image build (true/false)"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
-
   build-prod-packages:
     name: "Build Airflow and provider packages"
     timeout-minutes: 10
@@ -282,17 +283,23 @@ jobs:
         if: inputs.build-provider-packages != 'true'
       - name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         run: breeze prod-image verify
+      - name: Check free space
+        run: df -H
+        shell: bash
+      - name: Make /mnt/ directory writeable
+        run: sudo chown -R ${USER} /mnt
+        shell: bash
       - name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         env:
           PLATFORM: ${{ inputs.platform }}
         run: >
-          breeze prod-image save --platform "${PLATFORM}"
+          breeze prod-image save --platform "${PLATFORM}" --image-file-dir "/mnt"
         if: inputs.upload-image-artifact == 'true'
       - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
         with:
           key: prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}
-          path: "/tmp/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
+          path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
           if-no-files-found: 'error'
           retention-days: '2'
         if: inputs.upload-image-artifact == 'true'
diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml
@@ -64,6 +64,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Disable airflow repo cache read from main."
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   myssql-client-image:
     uses: ./.github/workflows/prod-image-build.yml

diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml
@@ -88,6 +88,9 @@ jobs:
     # instead of an array of strings.
     # yamllint disable-line rule:line-length
     runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }}
+    permissions:
+      contents: read
+      packages: write
     strategy:
       fail-fast: false
       matrix:
@@ -161,6 +164,9 @@ jobs:
     # instead of an array of strings.
     # yamllint disable-line rule:line-length
     runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }}
+    permissions:
+      contents: read
+      packages: write
     strategy:
       fail-fast: false
       matrix:

diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml
@@ -116,6 +116,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   tests:
     timeout-minutes: 120

diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml
@@ -80,7 +80,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv or not (true/false)"
         required: true
         type: string
-
+permissions:
+  contents: read
 jobs:
   tests-min-sqlalchemy:
     name: "Min SQLAlchemy test"

diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml
@@ -44,7 +44,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether this is a canary run (true/false)"
         required: true
         type: string
-
+permissions:
+  contents: read
 jobs:
   task-sdk-tests:
     timeout-minutes: 80

diff --git a/.github/workflows/test-provider-packages.yml b/.github/workflows/test-provider-packages.yml
@@ -62,6 +62,8 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv"
         required: true
         type: string
+permissions:
+  contents: read
 jobs:
   prepare-install-verify-provider-packages:
     timeout-minutes: 80