Distribute CI/PROD regular builds among multiple workers (apache#37503)

Seems that Public GitHub runners started to run out of space for our parallell builds. Buid parallelisation has been done in order to make a good use of small number of runners available, However since we have many runners available from Apache Software Foundation, we can easily just have one image per worker now without risking unnecessary queuing because there is no worker available. This way each build is separate and requires less disk space to complete the build.
astronomer · Feb 17, 2024 · b3e971a · b3e971a
1 parent ea9efd7
commit b3e971a
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 94 deletions.
diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml
@@ -18,6 +18,10 @@
 ---
 name: 'Build CI images'
 description: 'Build CI images'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    required: true
 runs:
   using: "composite"
   steps:
@@ -29,18 +33,20 @@ runs:
         pip install rich>=12.4.4 pyyaml
         python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
-    - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}"
+    - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ inputs.python-version }}"
       shell: bash
-      run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure
-    - name: "Source constraints"
+      run: >
+        breeze ci-image build --push --tag-as-latest --upgrade-on-failure
+        --python "${{ inputs.python-version }}"
+    - name: "Source constraints: ${{ inputs.python-version }}"
       shell: bash
       run: >
-        breeze release-management generate-constraints --run-in-parallel
+        breeze release-management generate-constraints --python "${{ inputs.python-version }}"
         --airflow-constraints-mode constraints-source-providers
     - name: "Upload constraint artifacts"
       uses: actions/upload-artifact@v4
       with:
-        name: source-constraints
-        path: ./files/constraints-*/constraints-*.txt
+        name: source-constraints-${{ inputs.python-version }}
+        path: ./files/constraints-*/constraints-source-providers-*.txt
         retention-days: 7
         if-no-files-found: error
diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml
@@ -19,6 +19,9 @@
 name: 'Build PROD images'
 description: 'Build PROD images'
 inputs:
+  python-version:
+    description: 'Python versions to use'
+    required: true
   build-provider-packages:
     description: 'Whether to build provider packages from sources'
     required: true
@@ -64,7 +67,7 @@ runs:
     - name: "Download constraints from the CI build"
       uses: actions/download-artifact@v4
       with:
-        name: source-constraints
+        name: source-constraints-${{ inputs.python-version }}
         path: ./docker-context-files
       if: ${{ inputs.build-provider-packages == 'true' }}
     - name: "Download constraints from the Generate & Verify build"
@@ -73,21 +76,21 @@ runs:
         name: constraints
         path: ./docker-context-files
       if: ${{ inputs.build-provider-packages != 'true' }}
-    - name: "Build & Push PROD images with source providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+    - name: "Build & Push PROD images w/ source providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}"
       shell: bash
       run: >
-        breeze prod-image build --tag-as-latest --run-in-parallel --push
+        breeze prod-image build --tag-as-latest --push
         --install-packages-from-context --airflow-constraints-mode constraints-source-providers
-        --use-constraints-for-context-packages
+        --use-constraints-for-context-packages --python "${{ inputs.python-version }}"
       env:
         COMMIT_SHA: ${{ github.sha }}
       if: ${{ inputs.build-provider-packages == 'true' }}
-    - name: "Build & Push PROD images with PyPi providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+    - name: "Build & Push PROD images with PyPi providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}"
       shell: bash
       run: >
-        breeze prod-image build --tag-as-latest --run-in-parallel --push
+        breeze prod-image build --tag-as-latest --push
         --install-packages-from-context --airflow-constraints-mode constraints
-        --use-constraints-for-context-packages
+        --use-constraints-for-context-packages --python "${{ inputs.python-version }}"
       env:
         COMMIT_SHA: ${{ github.sha }}
       if: ${{ inputs.build-provider-packages != 'true' }}
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -60,7 +60,7 @@ jobs:
     env:
       TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
     outputs:
-      python-versions: "${{ steps.selective-checks.python-versions }}"
+      python-versions: ${{ steps.selective-checks.outputs.python-versions }}
       upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
       all-python-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
@@ -160,13 +160,16 @@ jobs:
           GITHUB_CONTEXT: ${{ toJson(github) }}
 
   build-ci-images:
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     permissions:
       contents: read
       packages: write
     timeout-minutes: 80
-    name: >
-      Build CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build CI image ${{ matrix.python-version }}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info]
     if: |
       needs.build-info.outputs.ci-image-build == 'true' &&
@@ -222,9 +225,10 @@ jobs:
       #  BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
       #  ISOLATED FROM THE RUNNER.
       ####################################################################################################
-      - name: >
-          Build CI Images ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-ci-images
+        with:
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -233,14 +237,16 @@ jobs:
           BUILD_TIMEOUT_MINUTES: 70
 
   build-prod-images:
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     permissions:
       contents: read
       packages: write
     timeout-minutes: 80
-    name: >
-      Build PROD images
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build PROD image ${{ matrix.python-version }}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, build-ci-images]
     if: |
       needs.build-info.outputs.prod-image-build == 'true' &&
@@ -301,13 +307,12 @@ jobs:
         uses: ./.github/actions/breeze
         with:
           python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
-      - name: >
-          Build PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}