ci(docker): distribute build across multiple runners (#286)

Docker build for multiplatform arch is too slow sequentially on the same runner. The build is now distributed on multiple runners. Each architecure image is pushed as a digest and is reconciled into a single image afterwards.
sablierapp · Apr 21, 2024 · 9be1880 · 9be1880
1 parent 28e75c9
commit 9be1880
Show file tree

Hide file tree

Showing 4 changed files with 174 additions and 37 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,13 +14,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v4
+
     - name: Set up Go 1.21
       uses: actions/setup-go@v5
       with:
         go-version: ^1.21
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v4
+        cache-dependency-path: go.sum
 
     - name: Build
       run: go build -v .

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -8,57 +8,165 @@ on:
       - beta
     tags:
       - "v**"
-  pull_request:
+
+env:
+  REGISTRY_IMAGE: acouvreur/sablier
 
 jobs:
   build:
-    name: Docker
+    name: Build Sablier Docker image
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm/v7
+          - linux/arm64
+
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV 
+          
       - name: Checkout
         uses: actions/checkout@v4
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
-            ghcr.io/acouvreur/sablier
-            acouvreur/sablier
+            ghcr.io/${{ env.REGISTRY_IMAGE }}
+            ${{ env.REGISTRY_IMAGE }}
           # generate Docker tags based on the following events/attributes
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
+
+      - name: Go Build Cache for Docker
+        uses: actions/cache@v4
+        with:
+          path: go-build-cache
+          key: ${{ runner.os }}-go-build-cache-${{ hashFiles('**/go.sum') }}
+
+      - name: inject go-build-cache into docker
+        # v1 was composed of two actions: "inject" and "extract".
+        # v2 is unified to a single action.
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          cache-map: |
+            {
+              "go-build-cache": "/root/.cache/go-build"
+            }
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
           buildkitd-flags: --debug
+
       - name: Login to ghcr.io
         uses: docker/login-action@v3 
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3 
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        id: docker_build
+
+      - name: Build and push by digest
+        id: build
         uses: docker/build-push-action@v5
         with:
           context: .
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+            # type=image,name=ghcr.io/${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
           build-args: |
             BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
             VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
-            REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+            REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+      
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"          
+      
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Merge and Publish multiarch
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ghcr.io/${{ env.REGISTRY_IMAGE }}
+            ${{ env.REGISTRY_IMAGE }}
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+      
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3 
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
+      
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   build:
-    name: Build docker image once and share it to E2E jobs
+    name: Build Sablier docker image once and share it to E2E jobs
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -33,16 +33,17 @@ jobs:
           path: /tmp/sablier.tar
 
   traefik:
-    name: Build Sablier for Traefik middleware
+    name: Build Traefik Sablier Plugin
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Go 1.21
         uses: actions/setup-go@v5
         with:
           go-version: 1.21
-
-      - name: Checkout code
-        uses: actions/checkout@v4
+          cache-dependency-path: plugins/traefik/go.sum
 
       - name: Build
         run: cd plugins/traefik && go build -v .
@@ -59,13 +60,16 @@ jobs:
       matrix:
         provider: [docker, docker_swarm, kubernetes]
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Go 1.21
         uses: actions/setup-go@v5
         with:
           go-version: 1.21
-
-      - name: Checkout code
-        uses: actions/checkout@v4
+          cache-dependency-path: |
+            go.sum
+            plugins/traefik/go.sum
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -91,13 +95,15 @@ jobs:
       matrix:
         provider: [docker, docker_swarm] # , kubernetes]
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Go 1.21
         uses: actions/setup-go@v5
         with:
           go-version: 1.21
-
-      - name: Checkout code
-        uses: actions/checkout@v4
+          cache-dependency-path: |
+            go.sum
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -117,6 +123,25 @@ jobs:
       - name: Test ${{ matrix.provider }}
         run: cd plugins/nginx/e2e/${{ matrix.provider }} && bash ./run.sh
 
+  caddy:
+    name: Build Caddy Sablier Plugin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+          cache-dependency-path: plugins/caddy/go.sum
+
+      - name: Build
+        run: cd plugins/caddy && go build -v .
+
+      - name: Test
+        run: cd plugins/caddy && go test -v ./...
+
   build-caddy:
     name: Build Caddy docker image once and share it to Caddy E2E jobs
     runs-on: ubuntu-latest
@@ -152,13 +177,15 @@ jobs:
       matrix:
         provider: [docker, docker_swarm] # , kubernetes]
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Go 1.21
         uses: actions/setup-go@v5
         with:
           go-version: 1.21
-
-      - name: Checkout code
-        uses: actions/checkout@v4
+          cache-dependency-path: |
+            go.sum
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

diff --git a/Dockerfile b/Dockerfile
@@ -1,25 +1,26 @@
 FROM golang:1.21 AS build
 
-ENV PORT 10000
+WORKDIR /src
+RUN go env -w GOMODCACHE=/root/.cache/go-build
 
-WORKDIR /go/src/sablier
-
-COPY go.mod ./
-COPY go.sum ./
-RUN go mod download
-
-COPY . /go/src/sablier
+# See https://docs.docker.com/build/guide/mounts/#add-bind-mounts for cached builds
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
+COPY . /src
 ARG BUILDTIME
 ARG VERSION
 ARG REVISION
 ARG TARGETOS
 ARG TARGETARCH
-RUN make BUILDTIME=${BUILDTIME} VERSION=${VERSION} GIT_REVISION=${REVISION} ${TARGETOS}/${TARGETARCH}
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    make BUILDTIME=${BUILDTIME} VERSION=${VERSION} GIT_REVISION=${REVISION} ${TARGETOS}/${TARGETARCH}
 
 FROM alpine:3.19.1
 
-COPY --from=build /go/src/sablier/sablier* /etc/sablier/sablier
+COPY --from=build /src/sablier* /etc/sablier/sablier
 COPY docker/sablier.yaml /etc/sablier/sablier.yaml
 
 EXPOSE 10000