feat(docker): store toolchains in separate repository

- we are constantly battling with not enough space in GitHub cache
- so instead of cache we will use separate GitHub repository to store
  the coreboot toolchains
- this also has the benefit that our self-hosted runners will not have
  to upload so much data all the time

Signed-off-by: AtomicFS <[email protected]>

Author: AtomicFS

.github/workflows/docker-build-and-test.yml

@@ -97,6 +97,7 @@ jobs:
             ca-certificates \
             curl \
             git \
+            git-lfs \
             jq \
             sudo \
             tzdata \
@@ -129,37 +130,40 @@ jobs:
           COREBOOT_HASH="$( git rev-parse --short HEAD )"
           echo "${COREBOOT_HASH}"
           echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
-      - name: Artefact and cache key
+
+      - name: Check if toolchain is stored in firmware-action-toolchains repo
+        continue-on-error: true
+        run: |
+          # Check if the toolchain exists without downloading it
+          wget --spider "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar"
+          touch .firmware-action-toolchains-exist
+      - name: Check if toolchain exist
+        id: toolchains-exist
+        run: |
+          if [ -f ".firmware-action-toolchains-exist" ]; then
+            echo "toolchain is stored in firmware-action-toolchains repository, skipping rest of the job"
+            echo "EXIST=true" >> "${GITHUB_OUTPUT}"
+          else
+            echo "toolchain is NOT stored in firmware-action-toolchains repository, will build it"
+            echo "EXIST=false" >> "${GITHUB_OUTPUT}"
+          fi
+
+      - name: Cache key
         id: cache-key
         run: |
           CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
           echo "${CACHE_KEY}"
           echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"
-
       - name: Restore cached toolchains
         id: cache-toolchains
         uses: actions/cache/restore@v4
         with:
           path: coreboot/util/crossgcc/xgcc
           key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc
-      - name: Restore cached utils
-        id: cache-utils
-        uses: actions/cache/restore@v4
-        with:
-          path: /usr/local/bin
-          key: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils
-
-      - name: Debug list crossgcc
-        if: steps.cache-toolchains.outputs.cache-hit == 'true'
-        run: |
-          ls -a1lh coreboot/util/crossgcc/xgcc
-      - name: Debug list utils
-        if: steps.cache-utils.outputs.cache-hit == 'true'
-        run: |
-          ls -a1lh /usr/local/bin
 
       - name: Install dependencies if needed
-        if: steps.cache-toolchains.outputs.cache-hit != 'true'
+        # != 'true' because on miss the cache-hit is empty
+        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
         run: |
           apt-get install -y --no-install-recommends \
             acpica-tools \
@@ -185,69 +189,57 @@ jobs:
             uuid-dev \
             zlib1g-dev
       - name: Install dependencies if needed (amd64)
-        if: matrix.arch == 'amd64' && steps.cache-toolchains.outputs.cache-hit != 'true'
+        if: matrix.arch == 'amd64' && steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
         run: |
           apt-get install -y --no-install-recommends \
             iucode-tool
 
       - name: Build coreboot toolchains
-        if: steps.cache-toolchains.outputs.cache-hit != 'true'
+        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
         run: |
           cd coreboot
           make crossgcc CPUS="$(nproc)"
-      - name: Build coreboot utils
-        if: steps.cache-utils.outputs.cache-hit != 'true'
-        run: |
-          cd coreboot
-          make -C util/ifdtool install
-          make -C util/cbfstool install
       - name: Compress toolchain binaries
         # This step should shrink the size of single toolchain from 1.5 GB down to around 700 MB
         # I think it is save to compress all binaries except libraries, hence the '-wholename'
-        if: steps.cache-toolchains.outputs.cache-hit != 'true'
+        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
         run: |
+          PWD=$( pwd )
           cd coreboot/util/crossgcc/xgcc
           # shellcheck disable=SC2016
           find . -type f -wholename '*/bin/*' -exec bash -c 'upx-ucl -9 "$1"' shell {} \; || true
 
+          cd "${PWD}"
+          # Tar toolchain to prevent permission loss
+          # Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+          mv "coreboot/util/crossgcc/xgcc" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
+          tar -cf "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
+          sha256sum "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar" > "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256"
+
       # Store toolchains and utils in cache
       - name: Cache toolchains
         uses: actions/cache/save@v4
-        if: steps.cache-toolchains.outputs.cache-hit != 'true'
+        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
         with:
           path: coreboot/util/crossgcc/xgcc
           key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc
-      - name: Cache utils
-        uses: actions/cache/save@v4
-        if: steps.cache-utils.outputs.cache-hit != 'true'
-        with:
-          path: /usr/local/bin
-          key: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils
 
-      # Upload toolchains and utils as artifacts
-      - name: Tar toolchain to prevent permission loss
-        # Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+      - name: Commit the toolchain into firmware-action-toolchains repository
+        if: steps.toolchains-exist.outputs.EXIST == 'false'
         run: |
-          if [ ! -f "coreboot/util/crossgcc/xgcc-tar/${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar" ]; then
-            mv coreboot/util/crossgcc/xgcc coreboot/util/crossgcc/${{ matrix.arch }}-xgcc
-            tar -cf ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar coreboot/util/crossgcc/${{ matrix.arch }}-xgcc
-          fi
-      - name: Upload toolchain
-        uses: actions/[email protected]
-        with:
-          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc
-          path: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar
-          retention-days: 30
-          include-hidden-files: true
-          compression-level: 9
-      - name: Upload utils
-        uses: actions/[email protected]
-        with:
-          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils
-          path: /usr/local/bin
-          retention-days: 30
-          include-hidden-files: true
-          compression-level: 9
+          # Clone repo without downloading LFS items
+          GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/9elements/firmware-action-toolchains.git
+          cd firmware-action-toolchains
+          git branch "feat/${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
+          mkdir -p "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
+          mv "../${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
+          mv "../${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
+          git lfs install
+          git add .
+          git commit -m "feat: add toolchain for coreboot ${{ steps.version.outputs.COREBOOT_VERSION }}"
+          git push --set-upstream "feat/${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
+
+
 
   #=========================
   # Build Docker containers
@@ -319,56 +311,33 @@ jobs:
           COREBOOT_HASH="$( git rev-parse --short HEAD )"
           echo "${COREBOOT_HASH}"
           echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
-      - name: Artefact and cache key
-        id: cache-key
-        if: startsWith(matrix.dockerfile, 'coreboot')
-        run: |
-          CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}"
-          echo "CACHE_KEY=${CACHE_KEY}"
-          echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"
 
       #=================================
       # Download artifacts for coreboot
       #=================================
 
-      - name: Download coreboot toolchains from current run (if possible)
-        id: artifacts-toolchains-current
+      - name: Download coreboot toolchains from firmware-action-toolchains repository
+        id: firmware-action-toolchains
         if: startsWith(matrix.dockerfile, 'coreboot')
-        uses: dawidd6/action-download-artifact@v6
-        with:
-          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-.*
-          name_is_regexp: true
-          if_no_artifact_found: warn
-          run_id: ${{ github.event.workflow_run.id }}
-        # It is possible that current run did not produce any artifacts (to save bandwidth on self-hosted runners)
-        # In which case we have to look for artifacts in older runs
-      - name: Download coreboot toolchains from older run
-        if: startsWith(matrix.dockerfile, 'coreboot') && steps.artifacts-toolchains-current.outputs.found_artifact != 'true'
-        uses: dawidd6/action-download-artifact@v6
-        with:
-          name: ${{ steps.cache-key.outputs.CACHE_KEY }}-.*
-          name_is_regexp: true
-          search_artifacts: true
+        continue-on-error: true
+        run: |
+          for arch in "amd64" "arm64"; do
+            wget --continue --tries=3 "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar";
+            wget --continue --tries=3 "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar.sha256";
+            sha256sum -c "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar.sha256";
+          done
 
       - name: Prepare toolchains
         if: startsWith(matrix.dockerfile, 'coreboot')
         run: |
           mkdir -p docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}
-          for f in ${{ steps.cache-key.outputs.CACHE_KEY }}-*-xgcc/*.tar; do
+          for f in ${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-*-xgcc/*.tar; do
             ARCH=$( basename "${f}" | sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
             echo "extracting ${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
             mkdir -p "${f}.dir/"
             tar -xf "${f}" -C "${f}.dir/"
             mv "${f}.dir/coreboot/util/crossgcc/${ARCH}-xgcc" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-${ARCH}"
-          done
-      - name: Prepare utils
-        if: startsWith(matrix.dockerfile, 'coreboot')
-        run: |
-          for f in ${{ steps.cache-key.outputs.CACHE_KEY }}-*-utils; do
-            ARCH=$( basename "${f}" | sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
-            echo "${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
-            chmod +rx "${f}"/*
-            mv "${f}" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/utils-${ARCH}"
+            rm -rf "${f}"
           done
 
       - name: Debug list artifacts
@@ -383,10 +352,6 @@ jobs:
         if: startsWith(matrix.dockerfile, 'coreboot')
         run: |
           ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*/bin
-      - name: Debug list utils (amd64)
-        if: startsWith(matrix.dockerfile, 'coreboot')
-        run: |
-          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/utils-*
 
       #============================
       # Build the docker container