use self hosted wolfi runners for pre/postsubmit builds

wolfi-dev · Aug 30, 2023 · 05e1474 · 05e1474
1 parent 6afd553
commit 05e1474
Show file tree

Hide file tree

Showing 4 changed files with 211 additions and 161 deletions.
diff --git a/.github/workflows/.build.yaml b/.github/workflows/.build.yaml
@@ -0,0 +1,126 @@
+name: Reusable build workflow
+
+on:
+  workflow_call:
+    inputs:
+      image:
+        required: true
+        type: string
+      melange-config:
+        required: false
+        type: string
+        default: ''
+      registry:
+        required: false
+        type: string
+        default: ''
+
+jobs:
+  build:
+    runs-on: wolfi-builder-${{ matrix.arch }}
+    if: inputs.melange-config != ''
+    strategy:
+      fail-fast: true
+      matrix:
+        arch: ["x86_64", "aarch64"]
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
+      # TODO: Replace this with custom wolfi image
+      - run: |
+          sudo apt update -y && \
+          sudo apt install unzip curl cmake pkg-config -y
+
+      - id: melange
+        uses: chainguard-dev/actions/melange-build@main
+        with:
+          multi-config: ${{ inputs.melange-config }}
+          empty-workspace: false
+          workdir: images/${{ inputs.image }}
+          sign-with-temporary-key: true
+          archs: ${{ matrix.arch }}
+
+      - name: 'Upload built packages archive to Github Artifacts'
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ matrix.arch }}
+          path: ./packages
+          retention-days: 1 # Low ttl since this is just an intermediary used once
+          if-no-files-found: error
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    if: always() && (needs.build.result == 'success' || needs.build.result == 'skipped')
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
+      # TODO: Replace this with custom wolfi image
+      - run: |
+          sudo apt update -y && \
+          sudo apt install unzip curl git -y
+
+      - uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: '1.5.*'
+          terraform_wrapper: false
+
+      # Setup melange
+      - uses: chainguard-dev/actions/setup-melange@main
+        if: inputs.melange-config != ''
+
+      # Fetch the build stages back down
+      - name: 'Download package archives'
+        if: inputs.melange-config != ''
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.image }}-aarch64
+          path: ./packages
+
+      - name: 'Download package archives'
+        if: inputs.melange-config != ''
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.image }}-x86_64
+          path: ./packages
+
+      # Prepare the index for terraform
+      - if: inputs.melange-config != ''
+        run: |
+          # Generate a single key expected by the terraform build
+          melange keygen melange.rsa
+
+          for arch in "x86_64" "aarch64"; do
+            # Not strictly needed, but might as well while we're here
+            melange sign -k melange.rsa packages/$arch/*.apk
+
+            # Rebuild the index
+            melange index -o packages/$arch/APKINDEX.tar.gz packages/$arch/*.apk
+            melange sign-index --signing-key melange.rsa packages/$arch/APKINDEX.tar.gz
+          done
+
+      # Setup local registry
+      - uses: chainguard-dev/actions/setup-registry@main
+        if: inputs.registry == ''
+        with:
+          port: 5000
+
+      # Auth to GitHub Container Registry (ghcr.io)
+      - name: Login to registry
+        if: inputs.registry != ''
+        run: |
+          set -x
+          echo "${{ github.token }}" | docker login \
+            -u "${{ github.repository_owner }}" \
+            --password-stdin ghcr.io
+
+      # Build and push image using terraform-provider-apko
+      - name: Build image with apko/terraform
+        env:
+          # TF_VAR_target_repository: ghcr.io/${{ github.repository_owner }}/${{ matrix.imageName }}
+          TF_VAR_target_repository: ${{ inputs.registry != '' && inputs.registry || format('localhost:5000/{0}', inputs.image) }}
+        run: |
+          set -x
+          cd images/${{ inputs.image }}/
+          terraform init
+          terraform apply -auto-approve
diff --git a/.github/workflows/presubmit-build.yaml b/.github/workflows/presubmit-build.yaml
@@ -1,81 +1,41 @@
 on:
   pull_request:
-jobs:
-  presubmit-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.generate-matrix.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-      - id: generate-matrix
-        run: |
-          set -x
-          # TODO: set ONLY env var based on changed files
-          matrix="$(ONLY="${{ inputs.only }}" ./hack/matrix.sh)"
-          echo "matrix=${matrix}" >> $GITHUB_OUTPUT
-
-  presubmit-build:
-    runs-on: ubuntu-latest
-    needs: presubmit-matrix
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.presubmit-matrix.outputs.matrix) }}
-    permissions:
-      id-token: write
-      packages: write
-      contents: read
-    steps:
-
-      # Setup required tooling etc.
-      - name: Setup QEMU
-        if: ${{ matrix.melangeConfig != '' }}
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2.2.0
-      - uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_version: '1.3.*'
-          terraform_wrapper: false
-
-      # Checkout this repo's source code
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-
-      # Build custom package using melange (if specified)
-      - id: melange
-        if: ${{ matrix.melangeConfig != '' }}
-        uses: chainguard-dev/actions/melange-build@main
-        with:
-          multi-config: ${{ matrix.melangeConfig }}
-          empty-workspace: false
-          workdir: images/${{ matrix.imageName }}
-          sign-with-temporary-key: true
-          archs: x86_64,aarch64
 
-      # Setup local registry
-      - uses: chainguard-dev/actions/setup-registry@main
-        with:
-          port: 5000
-
-      # Build and push image using terraform-provider-apko
-      - name: Build image with apko/terraform
-        env:
-          TF_VAR_target_repository: localhost:5000/${{ matrix.imageName }}
-        run: |
-          set -x
-          cd images/${{ matrix.imageName }}/
-          terraform init
-          terraform apply -auto-approve
-
-  presubmit-roundup:
-    needs:
-      - presubmit-build
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # v3.0.3
-
-      - if: ${{ env.WORKFLOW_CONCLUSION == 'success' }}
-        working-directory: /tmp
-        run: echo ${{ env.WORKFLOW_CONCLUSION }} && exit 0
-
-      - if: ${{ env.WORKFLOW_CONCLUSION == 'failure' }}
-        working-directory: /tmp
-        run: echo ${{ env.WORKFLOW_CONCLUSION }} && exit 1
+jobs:
+  alpine-base:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: alpine-base
+
+  apko:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: apko
+      melange-config: configs/latest.melange.yaml
+
+  gcc-musl:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: gcc-musl
+
+  melange:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: melange
+      melange-config: configs/latest.melange.yaml
+
+  musl-dynamic:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: musl-dynamic
+
+  sdk:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: sdk
+      melange-config: configs/latest.melange.yaml
+
+  wolfictl:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: wolfictl
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -5,75 +5,56 @@ on:
   schedule:
     - cron: '0 1 * * *'
   workflow_dispatch:
-    inputs:
-      only:
-        description: 'Specific image name to build'
-        type: string
-        required: false
-        default: ''
+
 concurrency: release
-jobs:
-  generate-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.generate-matrix.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-      - id: generate-matrix
-        run: |
-          set -x
-          # TODO: set ONLY env var based on changed files
-          matrix="$(ONLY="${{ inputs.only }}" ./hack/matrix.sh)"
-          echo "matrix=${matrix}" >> $GITHUB_OUTPUT
-  build:
-    runs-on: ubuntu-latest
-    needs: generate-matrix
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
-    permissions:
-      id-token: write
-      packages: write
-      contents: read
-    steps:
 
-      # Setup required tooling etc.
-      - name: Setup QEMU
-        if: ${{ matrix.melangeConfig != '' }}
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2.2.0
-      - uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_version: '1.3.*'
-          terraform_wrapper: false
+permissions:
+  id-token: write
+  packages: write
+  contents: read
+
+jobs:
+  alpine-base:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: alpine-base
+      registry: ghcr.io/wolfi-dev/alpine-base
 
-      # Checkout this repo's source code
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+  apko:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: apko
+      melange-config: configs/latest.melange.yaml
+      registry: ghcr.io/wolfi-dev/apko
 
-      # Build custom package using melange (if specified)
-      - id: melange
-        if: ${{ matrix.melangeConfig != '' }}
-        uses: chainguard-dev/actions/melange-build@main
-        with:
-          multi-config: ${{ matrix.melangeConfig }}
-          empty-workspace: false
-          workdir: images/${{ matrix.imageName }}
-          sign-with-temporary-key: true
-          archs: x86_64,aarch64
+  gcc-musl:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: gcc-musl
+      registry: ghcr.io/wolfi-dev/gcc-musl
 
-      # Auth to GitHub Container Registry (ghcr.io)
-      - name: Login to registry
-        run: |
-          set -x
-          echo "${{ github.token }}" | docker login \
-            -u "${{ github.repository_owner }}" \
-            --password-stdin ghcr.io
+  melange:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: melange
+      melange-config: configs/latest.melange.yaml
+      registry: ghcr.io/wolfi-dev/melange
+
+  musl-dynamic:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: musl-dynamic
+      registry: ghcr.io/wolfi-dev/musl-dynamic
+
+  sdk:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: sdk
+      melange-config: configs/latest.melange.yaml
+      registry: ghcr.io/wolfi-dev/sdk
 
-      # Build and push image using terraform-provider-apko
-      - name: Build image with apko/terraform
-        env:
-          TF_VAR_target_repository: ghcr.io/${{ github.repository_owner }}/${{ matrix.imageName }}
-        run: |
-          set -x
-          cd images/${{ matrix.imageName }}/
-          terraform init
-          terraform apply -auto-approve
+  wolfictl:
+    uses: ./.github/workflows/.build.yaml
+    with:
+      image: wolfictl
+      registry: ghcr.io/wolfi-dev/wolfictl
diff --git a/hack/matrix.sh b/hack/matrix.sh