Reusable workflow testing

.github/workflows/base-glibc-busybox-bash.yaml

@@ -19,7 +19,7 @@ jobs:
       # The base image is not intended to change often and should be used with
       # version tags or checksum IDs, but not via "latest".
       MAJOR_VERSION: 3
-      MINOR_VERSION: 0
+      MINOR_VERSION: 1
       IMAGE_NAME: base-glibc-busybox-bash
       BUSYBOX_VERSION: '1.36.1'
       DEBIAN_VERSION: '12.2'
@@ -28,7 +28,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       with:
         platforms: arm64
 
@@ -44,15 +44,19 @@ jobs:
           ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }}
           latest
         '
+        # Adds image and tags to outputs which can be used in later steps.
         printf %s\\n \
           "image=${image_name}" \
           "tags=$( echo ${tags} )" \
           >> $GITHUB_OUTPUT
 
+        # Create manifest (which is considered arch-independent)
         for tag in ${tags} ; do
           buildah manifest create "${image_name}:${tag}"
         done
 
+        # --iidfile prints the built image ID to the specified file. This is
+        # used so we can refer to the image in later steps.
         iidfile="$( mktemp )"
         buildah bud \
           --iidfile="${iidfile}" \
@@ -71,6 +75,7 @@ jobs:
           image_id="$( cat "${iidfile}" )"
           rm "${iidfile}"
 
+          # Extract various package info and version info to store as labels
           container="$( buildah from "${image_id}" )"
           run() { buildah run "${container}" "${@}" ; }
           deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )"
@@ -80,6 +85,7 @@ jobs:
           bash="$( run bash --version | sed '1!d' )"
           buildah rm "${container}"
 
+          # Store package/version info as labels for the image
           container="$( buildah from "${image_id}" )"
           buildah config \
             --label=glibc="${glibc}" \
@@ -89,8 +95,11 @@ jobs:
             --label=pkg-list="${pkg_list}" \
             "${container}"
 
+          # Store the new image (now with labels)
           image_id="$( buildah commit "${container}" )"
           buildah rm "${container}"
+
+          # image tag includes arch; then added to manifest which does not include arch
           for tag in ${tags} ; do
             buildah tag \
               "${image_id}" \
@@ -104,6 +113,8 @@ jobs:
     - name: Test
       run: |
         image='${{ steps.build.outputs.image }}'
+
+        # Extract image ids from manifest to test.
         ids="$(
           for tag in ${{ steps.build.outputs.tags }} ; do
             buildah manifest inspect "${image}:${tag}" \
@@ -114,6 +125,8 @@ jobs:
                 done
           done
           )"
+
+        # See Dockerfile.test for actual tests run
         ids="$( printf %s "${ids}" | sort -u )"
         for id in ${ids} ; do
           podman history "${id}"
@@ -127,7 +140,7 @@ jobs:
     - name: Check Tags
       run: |
         # FIX upstream: Quay.io does not support immutable images currently.
-        #               => Try to use the REST API to check for duplicate tags.
+        #               => Try to use the REST API to check for duplicate tags and exit if they exist
         response="$(
           curl -sL \
             'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/'

.github/workflows/base-glibc-debian-bash.yaml

@@ -19,15 +19,15 @@ jobs:
       # The base image is not intended to change often and should be used with
       # version tags or checksum IDs, but not via "latest".
       MAJOR_VERSION: 3
-      MINOR_VERSION: 0
+      MINOR_VERSION: 1
       IMAGE_NAME: base-glibc-debian-bash
       DEBIAN_VERSION: '12.2'
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       with:
         platforms: arm64
 
@@ -43,15 +43,19 @@ jobs:
           ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }}
           latest
         '
+        # Adds image and tags to outputs which can be used in later steps.
         printf %s\\n \
           "image=${image_name}" \
           "tags=$( echo ${tags} )" \
           >> $GITHUB_OUTPUT
 
+        # Create manifest (which is considered arch-independent)
         for tag in ${tags} ; do
           buildah manifest create "${image_name}:${tag}"
         done
 
+        # --iidfile prints the built image ID to the specified file. This is
+        # used so we can refer to the image in later steps.
         for arch in amd64 arm64 ; do
           iidfile="$( mktemp )"
           buildah bud \
@@ -61,6 +65,7 @@ jobs:
           image_id="$( cat "${iidfile}" )"
           rm "${iidfile}"
 
+          # Extract various package info and version info to store as labels
           container="$( buildah from "${image_id}" )"
           run() { buildah run "${container}" "${@}" ; }
           deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )"
@@ -70,6 +75,7 @@ jobs:
           bash="$( run bash --version | sed '1!d' )"
           buildah rm "${container}"
 
+          # Store package/version info as labels for the image
           container="$( buildah from "${image_id}" )"
           buildah config \
             --label=glibc="${glibc}" \
@@ -79,8 +85,11 @@ jobs:
             --label=pkg-list="${pkg_list}" \
             "${container}"
 
+          # Store the new image (now with labels)
           image_id="$( buildah commit "${container}" )"
           buildah rm "${container}"
+
+          # image tag includes arch; then added to manifest which does not include arch
           for tag in ${tags} ; do
             buildah tag \
               "${image_id}" \
@@ -94,6 +103,8 @@ jobs:
     - name: Test
       run: |
         image='${{ steps.build.outputs.image }}'
+
+        # Extract image ids from manifest to test.
         ids="$(
           for tag in ${{ steps.build.outputs.tags }} ; do
             buildah manifest inspect "${image}:${tag}" \
@@ -104,6 +115,8 @@ jobs:
                 done
           done
           )"
+
+        # See Dockerfile.test for actual tests run
         ids="$( printf %s "${ids}" | sort -u )"
         for id in ${ids} ; do
           podman history "${id}"
@@ -117,7 +130,7 @@ jobs:
     - name: Check Tags
       run: |
         # FIX upstream: Quay.io does not support immutable images currently.
-        #               => Try to use the REST API to check for duplicate tags.
+        #               => Try to use the REST API to check for duplicate tags and exit if they exist
         response="$(
           curl -sL \
             'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/'

.github/workflows/bioconda-utils-build-env-cos7.yml

@@ -0,0 +1,212 @@
+name: 'Build & Push: bioconda-utils-build-env-cos7'
+on:
+  push:
+    branches:
+    - test-push
+    paths:
+    - images/bioconda-utils-build-env-cos7/**
+    - .github/workflows/bioconda-utils-build-env-cos7.yml
+  pull_request:
+    paths:
+    - images/bioconda-utils-build-env-cos7/**
+    - .github/workflows/bioconda-utils-build-env-cos7.yml
+
+jobs:
+  build:
+    name: Build & Push
+    runs-on: ubuntu-22.04
+    env:
+      # The base image is not intended to change often and should be used with
+      # version tags or checksum IDs, but not via "latest".
+      MAJOR_VERSION: 3
+      MINOR_VERSION: 1
+      IMAGE_NAME: bioconda-utils-build-env-cos7
+
+    steps:
+    - name: Checkout bioconda-containers
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Checkout bioconda-utils
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        repository: 'bioconda/bioconda-utils'
+        path: 'images/bioconda-utils-build-env-cos7/bioconda-utils'
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: arm64
+
+    - name: Build
+      id: build
+      run: |
+        set -xeu
+        cd 'images/${{ env.IMAGE_NAME }}'
+
+        image_name='${{ env.IMAGE_NAME }}'
+        tags='
+          ${{ env.MAJOR_VERSION }}
+          ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }}
+          latest
+        '
+        # Adds image and tags to outputs which can be used in later steps.
+        printf %s\\n \
+          "image=${image_name}" \
+          "tags=$( echo ${tags} )" \
+          >> $GITHUB_OUTPUT
+
+        # Create manifest (which is considered arch-independent)
+        for tag in ${tags} ; do
+          buildah manifest create "${image_name}:${tag}"
+        done
+
+        # Due to different nomenclature used by conda-forge and buildah, we
+        # need to map archs to base images.
+        for arch_and_image in \
+          "amd64=quay.io/condaforge/linux-anvil-cos7-x86_64" \
+          "arm64=quay.io/condaforge/linux-anvil-aarch64";
+        do
+          # Unpack archs and base images
+          arch=$(echo $arch_and_image | cut -f1 -d "=")
+          base_image=$(echo $arch_and_image | cut -f2 -d "=")
+
+          # --iidfile prints the built image ID to the specified file. This is
+          # used so we can refer to the image in later steps.
+          iidfile="$( mktemp )"
+          buildah bud \
+            --arch="${arch}" \
+            --iidfile="${iidfile}" \
+            --build-arg=base_image="$base_image"
+          image_id="$( cat "${iidfile}" )"
+          rm "${iidfile}"
+
+          # Extract various package info and version info to store as labels
+          container="$( buildah from "${image_id}" )"
+          run() { buildah run "${container}" "${@}" ; }
+          deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )"
+          pkg_list="$( run cat /.pkg.lst | tr '\n' '|' | sed 's/|$//' )"
+          glibc="$( run sh -c 'exec "$( find -xdev -name libc.so.6 -print -quit )"' | sed '1!d' )"
+          bash="$( run bash --version | sed '1!d' )"
+          bioconda_utils="$( run sh -c '. /opt/conda/etc/profile.d/conda.sh && conda activate base && bioconda-utils --version' | rev | cut -f1 -d " " | rev )"
+          buildah rm "${container}"
+
+          # Store package/version info as labels for the image
+          container="$( buildah from "${image_id}" )"
+          buildah config \
+            --label=glibc="${glibc}" \
+            --label=bash="${bash}" \
+            --label=deb-list="${deb_list}" \
+            --label=pkg-list="${pkg_list}" \
+            --label=bioconda-utils="${bioconda_utils}" \
+            "${container}"
+
+          # Store the new image (now with labels)
+          image_id="$( buildah commit "${container}" )"
+          buildah rm "${container}"
+
+          # image tag includes arch; then added to manifest which does not include arch
+          for tag in ${tags} ; do
+            buildah tag \
+              "${image_id}" \
+              "${image_name}:${tag}-${arch}"
+            buildah manifest add \
+              "${image_name}:${tag}" \
+              "${image_id}"
+          done
+        done
+
+    - name: Test
+      run: |
+        image='${{ steps.build.outputs.image }}'
+
+        # Extract image ids from manifest to test.
+        ids="$(
+          for tag in ${{ steps.build.outputs.tags }} ; do
+            buildah manifest inspect "${image}:${tag}" \
+              | jq -r '.manifests[]|.digest' \
+              | while read id ; do
+                  buildah images --format '{{.ID}}{{.Digest}}' \
+                  | sed -n "s/${id}//p"
+                done
+          done
+          )"
+
+        # See Dockerfile.test for actual tests run
+        ids="$( printf %s "${ids}" | sort -u )"
+        for id in ${ids} ; do
+          podman history "${id}"
+          buildah bud \
+            --build-arg=base="${id}" \
+            --file=Dockerfile.test \
+            "images/${image}"
+        done
+        buildah rmi --prune || true
+
+    - name: Check Tags
+      run: |
+        # FIX upstream: Quay.io does not support immutable images currently.
+        #               => Try to use the REST API to check for duplicate tags and exit if they exist
+        response="$(
+          curl -sL \
+            'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/'
+          )"
+
+        existing_tags="$(
+          printf %s "${response}" \
+            | jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name'
+          )" \
+          || {
+            printf %s\\n \
+              'Could not get list of image tags.' \
+              'Does the repository exist on Quay.io?' \
+              'Quay.io REST API response was:' \
+              "${response}"
+            exit 1
+          }
+        for tag in ${{ steps.build.outputs.tags }} ; do
+          case "${tag}" in
+            latest | '${{ env.MAJOR_VERSION }}' ) ;;
+            * )
+              if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then
+                printf 'Tag %s already exists!\n' "${tag}"
+                exit 1
+              fi
+          esac
+        done
+
+    - if: ${{ github.ref == 'refs/heads/test-push' }}
+      name: Push
+      uses: redhat-actions/push-to-registry@v2
+      with:
+        image: ${{ steps.build.outputs.image }}
+        tags: ${{ steps.build.outputs.tags }}
+        registry: ${{ secrets.QUAY_BIOCONDA_REPO }}
+        username: ${{ secrets.QUAY_BIOCONDA_USERNAME }}
+        password: ${{ secrets.QUAY_BIOCONDA_TOKEN }}
+
+    - if: ${{ github.ref == 'refs/heads/test-push' }}
+      name: Test Pushed
+      run: |
+        image='${{ env.IMAGE_NAME }}'
+        ids="$(
+          for tag in ${{ steps.build.outputs.tags }} ; do
+            buildah manifest inspect "${image}:${tag}" \
+              | jq -r '.manifests[]|.digest' \
+              | while read id ; do
+                  buildah images --format '{{.ID}}{{.Digest}}' \
+                  | sed -n "s/${id}//p"
+                done
+          done
+          )"
+        ids="$( printf %s "${ids}" | sort -u )"
+        for id in ${ids} ; do
+          podman history "${id}"
+          buildah bud \
+            --build-arg=base="${id}" \
+            --file=Dockerfile.test \
+            "images/${image}"
+        done
+        buildah rmi --prune || true

.github/workflows/create-env.yaml

@@ -17,14 +17,14 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       MAJOR_VERSION: 3
-      MINOR_VERSION: 0
+      MINOR_VERSION: 1
       IMAGE_NAME: create-env
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       with:
         platforms: arm64
 

images/build-env/Dockerfile → images/bioconda-utils-build-env-cos7/Dockerfile

@@ -1,9 +1,6 @@
-# Specify the base image to support multi-arch images, such as
-# - 'quay.io/condaforge/linux-anvil-aarch64' for Linux aarch64
-# - 'quay.io/condaforge/linux-anvil-cos7-x86_64' for Linux x86_64
-ARG BASE_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64
+ARG base_image
 
-FROM ${BASE_IMAGE} as base
+FROM ${base_image} as base
 
 # Copy over C.UTF-8 locale from our base image to make it consistently available during build.
 COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.utf8 /usr/lib/locale/C.utf8

images/bioconda-utils-build-env-cos7/Dockerfile.test

@@ -0,0 +1,6 @@
+ARG base
+FROM "${base}"
+RUN . /opt/conda/etc/profile.d/conda.sh \
+  && conda activate base \
+  && bioconda-utils --version \
+  && bioconda-utils --help