Skip to content

Commit

Permalink
use self hosted wolfi runners for pre/postsubmit builds
Browse files Browse the repository at this point in the history
  • Loading branch information
joshrwolf committed Aug 30, 2023
1 parent 6afd553 commit ce737ff
Show file tree
Hide file tree
Showing 4 changed files with 221 additions and 152 deletions.
126 changes: 126 additions & 0 deletions .github/workflows/.build.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
name: Reusable build workflow

on:
workflow_call:
inputs:
image:
required: true
type: string
melange-config:
required: false
type: string
default: ''
registry:
required: false
type: string
default: ''

jobs:
build:
runs-on: wolfi-builder-${{ matrix.arch }}
if: inputs.melange-config != ''
strategy:
fail-fast: true
matrix:
arch: ["x86_64", "aarch64"]
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0

# TODO: Replace this with custom wolfi image
- run: |
sudo apt update -y && \
sudo apt install unzip curl cmake pkg-config -y
- id: melange
uses: chainguard-dev/actions/melange-build@main
with:
multi-config: ${{ inputs.melange-config }}
empty-workspace: false
workdir: images/${{ inputs.image }}
sign-with-temporary-key: true
archs: ${{ matrix.arch }}

- name: 'Upload built packages archive to Github Artifacts'
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.image }}-${{ matrix.arch }}
path: ./packages
retention-days: 1 # Low ttl since this is just an intermediary used once
if-no-files-found: error

publish:
runs-on: ubuntu-latest
needs: build
if: always() && (needs.build.result == 'success' || needs.build.result == 'skipped')
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0

# TODO: Replace this with custom wolfi image
- run: |
sudo apt update -y && \
sudo apt install unzip curl git -y
- uses: hashicorp/setup-terraform@v2
with:
terraform_version: '1.5.*'
terraform_wrapper: false

# Setup melange
- uses: chainguard-dev/actions/setup-melange@main
if: inputs.melange-config != ''

# Fetch the build stages back down
- name: 'Download package archives'
if: inputs.melange-config != ''
uses: actions/download-artifact@v3
with:
name: ${{ inputs.image }}-aarch64
path: ./packages

- name: 'Download package archives'
if: inputs.melange-config != ''
uses: actions/download-artifact@v3
with:
name: ${{ inputs.image }}-x86_64
path: ./packages

# Prepare the index for terraform
- if: inputs.melange-config != ''
run: |
# Generate a single key expected by the terraform build
melange keygen melange.rsa
for arch in "x86_64" "aarch64"; do
# Not strictly needed, but might as well while we're here
melange sign -k melange.rsa packages/$arch/*.apk
# Rebuild the index
melange index -o packages/$arch/APKINDEX.tar.gz packages/$arch/*.apk
melange sign-index --signing-key melange.rsa packages/$arch/APKINDEX.tar.gz
done
# Setup local registry
- uses: chainguard-dev/actions/setup-registry@main
if: inputs.registry == ''
with:
port: 5000

# Auth to GitHub Container Registry (ghcr.io)
- name: Login to registry
if: inputs.registry != ''
run: |
set -x
echo "${{ github.token }}" | docker login \
-u "${{ github.repository_owner }}" \
--password-stdin ghcr.io
# Build and push image using terraform-provider-apko
- name: Build image with apko/terraform
env:
# TF_VAR_target_repository: ghcr.io/${{ github.repository_owner }}/${{ matrix.imageName }}
TF_VAR_target_repository: ${{ inputs.registry != '' && inputs.registry || format('localhost:5000/{0}', inputs.image) }}
run: |
set -x
cd images/${{ inputs.image }}/
terraform init
terraform apply -auto-approve
116 changes: 38 additions & 78 deletions .github/workflows/presubmit-build.yaml
Original file line number Diff line number Diff line change
@@ -1,81 +1,41 @@
on:
pull_request:
jobs:
presubmit-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.generate-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- id: generate-matrix
run: |
set -x
# TODO: set ONLY env var based on changed files
matrix="$(ONLY="${{ inputs.only }}" ./hack/matrix.sh)"
echo "matrix=${matrix}" >> $GITHUB_OUTPUT
presubmit-build:
runs-on: ubuntu-latest
needs: presubmit-matrix
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.presubmit-matrix.outputs.matrix) }}
permissions:
id-token: write
packages: write
contents: read
steps:

# Setup required tooling etc.
- name: Setup QEMU
if: ${{ matrix.melangeConfig != '' }}
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2.2.0
- uses: hashicorp/setup-terraform@v2
with:
terraform_version: '1.3.*'
terraform_wrapper: false

# Checkout this repo's source code
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0

# Build custom package using melange (if specified)
- id: melange
if: ${{ matrix.melangeConfig != '' }}
uses: chainguard-dev/actions/melange-build@main
with:
multi-config: ${{ matrix.melangeConfig }}
empty-workspace: false
workdir: images/${{ matrix.imageName }}
sign-with-temporary-key: true
archs: x86_64,aarch64

# Setup local registry
- uses: chainguard-dev/actions/setup-registry@main
with:
port: 5000

# Build and push image using terraform-provider-apko
- name: Build image with apko/terraform
env:
TF_VAR_target_repository: localhost:5000/${{ matrix.imageName }}
run: |
set -x
cd images/${{ matrix.imageName }}/
terraform init
terraform apply -auto-approve
presubmit-roundup:
needs:
- presubmit-build
runs-on: ubuntu-latest
if: always()
steps:
- uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # v3.0.3

- if: ${{ env.WORKFLOW_CONCLUSION == 'success' }}
working-directory: /tmp
run: echo ${{ env.WORKFLOW_CONCLUSION }} && exit 0

- if: ${{ env.WORKFLOW_CONCLUSION == 'failure' }}
working-directory: /tmp
run: echo ${{ env.WORKFLOW_CONCLUSION }} && exit 1
jobs:
alpine-base:
uses: ./.github/workflows/.build.yaml
with:
image: alpine-base

apko:
uses: ./.github/workflows/.build.yaml
with:
image: apko
melange-config: configs/latest.melange.yaml

gcc-musl:
uses: ./.github/workflows/.build.yaml
with:
image: gcc-musl

melange:
uses: ./.github/workflows/.build.yaml
with:
image: melange
melange-config: configs/latest.melange.yaml

musl-dynamic:
uses: ./.github/workflows/.build.yaml
with:
image: musl-dynamic

sdk:
uses: ./.github/workflows/.build.yaml
with:
image: sdk
melange-config: configs/latest.melange.yaml

wolfictl:
uses: ./.github/workflows/.build.yaml
with:
image: wolfictl
113 changes: 47 additions & 66 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,75 +5,56 @@ on:
schedule:
- cron: '0 1 * * *'
workflow_dispatch:
inputs:
only:
description: 'Specific image name to build'
type: string
required: false
default: ''

concurrency: release
jobs:
generate-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.generate-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- id: generate-matrix
run: |
set -x
# TODO: set ONLY env var based on changed files
matrix="$(ONLY="${{ inputs.only }}" ./hack/matrix.sh)"
echo "matrix=${matrix}" >> $GITHUB_OUTPUT
build:
runs-on: ubuntu-latest
needs: generate-matrix
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
permissions:
id-token: write
packages: write
contents: read
steps:

# Setup required tooling etc.
- name: Setup QEMU
if: ${{ matrix.melangeConfig != '' }}
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v2.2.0
- uses: hashicorp/setup-terraform@v2
with:
terraform_version: '1.3.*'
terraform_wrapper: false
permissions:
id-token: write
packages: write
contents: read

jobs:
alpine-base:
uses: ./.github/workflows/.build.yaml
with:
image: alpine-base
registry: ghcr.io/wolfi-dev/alpine-base

# Checkout this repo's source code
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
apko:
uses: ./.github/workflows/.build.yaml
with:
image: apko
melange-config: configs/latest.melange.yaml
registry: ghcr.io/wolfi-dev/apko

# Build custom package using melange (if specified)
- id: melange
if: ${{ matrix.melangeConfig != '' }}
uses: chainguard-dev/actions/melange-build@main
with:
multi-config: ${{ matrix.melangeConfig }}
empty-workspace: false
workdir: images/${{ matrix.imageName }}
sign-with-temporary-key: true
archs: x86_64,aarch64
gcc-musl:
uses: ./.github/workflows/.build.yaml
with:
image: gcc-musl
registry: ghcr.io/wolfi-dev/gcc-musl

# Auth to GitHub Container Registry (ghcr.io)
- name: Login to registry
run: |
set -x
echo "${{ github.token }}" | docker login \
-u "${{ github.repository_owner }}" \
--password-stdin ghcr.io
melange:
uses: ./.github/workflows/.build.yaml
with:
image: melange
melange-config: configs/latest.melange.yaml
registry: ghcr.io/wolfi-dev/melange

musl-dynamic:
uses: ./.github/workflows/.build.yaml
with:
image: musl-dynamic
registry: ghcr.io/wolfi-dev/musl-dynamic

sdk:
uses: ./.github/workflows/.build.yaml
with:
image: sdk
melange-config: configs/latest.melange.yaml
registry: ghcr.io/wolfi-dev/sdk

# Build and push image using terraform-provider-apko
- name: Build image with apko/terraform
env:
TF_VAR_target_repository: ghcr.io/${{ github.repository_owner }}/${{ matrix.imageName }}
run: |
set -x
cd images/${{ matrix.imageName }}/
terraform init
terraform apply -auto-approve
wolfictl:
uses: ./.github/workflows/.build.yaml
with:
image: wolfictl
registry: ghcr.io/wolfi-dev/wolfictl
18 changes: 10 additions & 8 deletions hack/matrix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@

set -e
matrix='{"include":[]}'
for name in `find images -mindepth 1 -maxdepth 1 -type d | sed 's|images/||' | sort | xargs`; do
[[ "${ONLY}" == "" || "${ONLY}" == "${name}" ]] || continue
entry='{imageName: "'${name}'"}'
melange_config="$(cd images/${name} && find . -name '*.melange.yaml' | sed 's|./||')"
if [[ "${melange_config}" != "" ]]; then
entry="{imageName: \"${name}\", melangeConfig: \"${melange_config}\"}"
fi
matrix="$(echo "${matrix}" | jq -c ".include += [${entry}]")"
for name in $(find images -mindepth 1 -maxdepth 1 -type d | sed 's|images/||' | sort | xargs); do
for arch in "x86_64" "aarch64"; do
[[ "${ONLY}" == "" || "${ONLY}" == "${name}" ]] || continue
entry='{imageName: "'${name}'", arch: "'${arch}'"}'
melange_config="$(cd images/${name} && find . -name '*.melange.yaml' | sed 's|./||')"
if [[ "${melange_config}" != "" ]]; then
entry="{imageName: \"${name}\", melangeConfig: \"${melange_config}\", arch: \"${arch}\"}"
fi
matrix="$(echo "${matrix}" | jq -c ".include += [${entry}]")"
done
done
echo "${matrix}"

0 comments on commit ce737ff

Please sign in to comment.