in_tail: reintroduced the old ignore_older behavior as opt-in

.github/workflows/call-build-images.yaml

@@ -15,7 +15,8 @@ on:
       registry:
         description: The registry to push container images to.
         type: string
-        required: true
+        required: false
+        default: ghcr.io
       username:
         description: The username for the registry.
         type: string
@@ -69,25 +70,33 @@ jobs:
           replace-with: "$1"
           flags: "g"
 
-  # This is the intended approach to multi-arch image and all the other checks scanning,
-  # signing, etc only trigger from this.
-  call-build-images:
-    needs:
-      - call-build-images-meta
-    name: Multiarch container images to GHCR
-    runs-on: ubuntu-latest
-    environment: ${{ inputs.environment }}
+  # Taken from https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
+  # We split this out to make it easier to restart just one of them if it fails and do all in parallel
+  call-build-single-arch-container-images:
+    # Allow us to continue to create a manifest if we want
+    continue-on-error: true
     permissions:
       contents: read
       packages: write
-    outputs:
-      production-digest: ${{ steps.build_push.outputs.digest }}
-      debug-digest: ${{ steps.debug_build_push.outputs.digest }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - amd64
+          - arm64
+          - arm/v7
+        target:
+          - production
+          - debug
+    name: ${{ matrix.platform }}/${{ matrix.target }} container image build
+    # Use GitHub Actions ARM hosted runners
+    runs-on: ${{ (contains(matrix.platform, 'arm') && 'ubuntu-22.04-arm') || 'ubuntu-latest' }}
     steps:
-      - name: Checkout code for modern style builds
+      - name: Checkout code
         uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
+          token: ${{ secrets.token }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -99,37 +108,108 @@ jobs:
         uses: docker/login-action@v3
         with:
           registry: ${{ inputs.registry }}
-          username: ${{ inputs.username }}
+          username: ${{ github.actor }}
           password: ${{ secrets.token }}
 
-      - name: Extract metadata from Github
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ inputs.registry }}/${{ inputs.image }}
-          tags: |
-            raw,${{ inputs.version }}
-            raw,${{ needs.call-build-images-meta.outputs.major-version }}
-            raw,latest
-
-      - name: Build the production images
-        id: build_push
+      - name: Build and push by digest the standard ${{ matrix.target }} image
+        id: build
         uses: docker/build-push-action@v6
         with:
+          # Use path context rather than Git context as we want local files
           file: ./dockerfiles/Dockerfile
           context: .
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64, linux/arm64, linux/arm/v7, linux/s390x
-          target: production
+          target: ${{ matrix.target }}
+          outputs: type=image,name=${{ inputs.registry }}/${{ inputs.image }},push-by-digest=true,name-canonical=true,push=true
+          platforms: linux/${{ matrix.platform }}
           # Must be disabled to provide legacy format images from the registry
           provenance: false
           push: true
           load: false
           build-args: |
             FLB_NIGHTLY_BUILD=${{ inputs.unstable }}
             RELEASE_VERSION=${{ inputs.version }}
+            WAMR_BUILD_TARGET=${{ (contains(matrix.platform, 'arm/v7') && 'ARMV7') || '' }}
+
+      - name: Export ${{ matrix.target }} digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+        shell: bash
+
+      - name: Upload ${{ matrix.target }} digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.target }}-digests-${{ (contains(matrix.platform, 'arm/v7') && 'arm-v7') || matrix.platform }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # Take the digests and produce a multi-arch manifest from them.
+  call-build-container-image-manifests:
+    permissions:
+      contents: read
+      packages: write
+    name: Upload multi-arch container image manifests
+    runs-on: ubuntu-latest
+    needs:
+      - call-build-images-meta
+      - call-build-single-arch-container-images
+    outputs:
+      version: ${{ steps.meta.outputs.version }}
+    steps:
+      - name: Extract metadata from Github
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ inputs.registry }}/${{ inputs.image }}
+          tags: |
+            raw,${{ inputs.version }}
+            raw,${{ needs.call-build-images-meta.outputs.major-version }}
+            raw,latest
+
+      - name: Download production digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: production-digests-*
+          path: /tmp/production-digests
+          merge-multiple: true
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.token }}
+
+      - name: Create production manifest
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ inputs.registry }}/${{ inputs.image }}@sha256:%s ' *)
+        shell: bash
+        working-directory: /tmp/production-digests
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ inputs.registry }}/${{ inputs.image }}:${{ steps.meta.outputs.version }}
+        shell: bash
+
+  # Take the digests and produce a multi-arch manifest from them.
+  call-build-debug-container-image-manifests:
+    permissions:
+      contents: read
+      packages: write
+    name: Upload debug multi-arch container image manifests
+    runs-on: ubuntu-latest
+    needs:
+      - call-build-images-meta
+      - call-build-single-arch-container-images
+    outputs:
+      version: ${{ steps.debug-meta.outputs.version }}
+    steps:
       - id: debug-meta
         uses: docker/metadata-action@v5
         with:
@@ -139,28 +219,39 @@ jobs:
             raw,${{ needs.call-build-images-meta.outputs.major-version }}-debug
             raw,latest-debug
 
-      - name: Build the debug multi-arch images
-        id: debug_build_push
-        uses: docker/build-push-action@v6
+      - name: Download debug digests
+        uses: actions/download-artifact@v4
         with:
-          file: ./dockerfiles/Dockerfile
-          context: .
-          tags: ${{ steps.debug-meta.outputs.tags }}
-          labels: ${{ steps.debug-meta.outputs.labels }}
-          platforms: linux/amd64, linux/arm64, linux/arm/v7, linux/s390x
-          # Must be disabled to provide legacy format images from the registry
-          provenance: false
-          target: debug
-          push: true
-          load: false
-          build-args: |
-            FLB_NIGHTLY_BUILD=${{ inputs.unstable }}
-            RELEASE_VERSION=${{ inputs.version }}
+          pattern: debug-digests-*
+          path: /tmp/debug-digests
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.token }}
+
+      - name: Create debug manifest
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ inputs.registry }}/${{ inputs.image }}@sha256:%s ' *)
+        shell: bash
+        working-directory: /tmp/debug-digests
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ inputs.registry }}/${{ inputs.image }}:${{ steps.debug-meta.outputs.version }}
+        shell: bash
 
   call-build-images-generate-schema:
     needs:
       - call-build-images-meta
-      - call-build-images
+      - call-build-container-image-manifests
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     permissions:
@@ -190,7 +281,7 @@ jobs:
   call-build-images-scan:
     needs:
       - call-build-images-meta
-      - call-build-images
+      - call-build-container-image-manifests
     name: Trivy + Dockle image scan
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
@@ -225,7 +316,8 @@ jobs:
   call-build-images-sign:
     needs:
       - call-build-images-meta
-      - call-build-images
+      - call-build-container-image-manifests
+      - call-build-debug-container-image-manifests
     name: Deploy and sign multi-arch container image manifests
     permissions:
       contents: read
@@ -246,13 +338,13 @@ jobs:
         #
         # We use recursive signing on the manifest to cover all the images.
         run: |
-          cosign sign --recursive \
+          cosign sign --recursive --force \
             -a "repo=${{ github.repository }}" \
             -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
             -a "ref=${{ github.sha }}" \
             -a "release=${{ inputs.version }}" \
-            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \
-            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}"
+            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-container-image-manifests.outputs.version }}" \
+            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-debug-container-image-manifests.outputs.version }}"
         shell: bash
         # Ensure we move on to key-based signing as well
         continue-on-error: true
@@ -265,13 +357,13 @@ jobs:
         # The key needs to cope with newlines
         run: |
           echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key
-          cosign sign --key /tmp/my_cosign.key --recursive \
+          cosign sign --key /tmp/my_cosign.key --recursive --force \
             -a "repo=${{ github.repository }}" \
             -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
             -a "ref=${{ github.sha }}" \
             -a "release=${{ inputs.version }}" \
-            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \
-            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}"
+            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-container-image-manifests.outputs.version }}" \
+            "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-debug-container-image-manifests.outputs.version }}"
           rm -f /tmp/my_cosign.key
         shell: bash
         continue-on-error: true

.github/workflows/call-build-linux-packages.yaml

@@ -106,7 +106,7 @@ jobs:
     name: ${{ matrix.distro }} package build and stage to S3
     environment: ${{ inputs.environment }}
     # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-22.04 usage.
-    runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-22.04' }}
+    runs-on: ${{ ((contains(matrix.distro, 'arm' ) || contains(matrix.distro, 'raspbian')) && (github.repository == 'fluent/fluent-bit') && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }}
     permissions:
       contents: read
     strategy:
@@ -120,16 +120,17 @@ jobs:
         with:
           ref: ${{ inputs.ref }}
 
-      - name: Set up Actuated mirror
-        if: contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit')
-        uses: self-actuated/hub-mirror@master
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      # - name: Set up Actuated mirror
+      #   if: (contains(matrix.distro, 'arm' ) || contains(matrix.distro, 'raspbian')) && (github.repository == 'fluent/fluent-bit')
+      #   uses: self-actuated/hub-mirror@master
 
       - name: Set up Docker Buildx
+        # if: ${{ !(contains(matrix.distro, 'arm' ) || contains(matrix.distro, 'raspbian')) && (github.repository == 'fluent/fluent-bit') }}
         uses: docker/setup-buildx-action@v3
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
       - name: Replace all special characters with dashes
         id: formatted_distro
         run: |

.github/workflows/call-build-macos.yaml

@@ -76,6 +76,8 @@ jobs:
         config:
           - name: "Apple Silicon macOS runner"
             runner: macos-14
+          - name: "Intel macOS runner"
+            runner: macos-14-large
 
     steps:
       - name: Checkout repository

.github/workflows/call-run-integration-test.yaml

@@ -192,7 +192,7 @@ jobs:
           bats-version: 1.9.0
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.11.0
+        uses: helm/kind-action@v1.12.0
         with:
           node_image: kindest/node:${{ matrix.k8s-release }}
           cluster_name: kind

.github/workflows/call-test-images.yaml

@@ -76,7 +76,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        arch: [ linux/amd64, linux/arm64, linux/arm/v7, linux/s390x ]
+        arch: [ linux/amd64, linux/arm64, linux/arm/v7 ]
         include:
           # Rather than extract the specific central arch we just provide it
           - arch: linux/amd64
@@ -85,8 +85,6 @@ jobs:
             expected: arm64
           - arch: linux/arm/v7
             expected: arm
-          - arch: linux/s390x
-            expected: s390x
     steps:
       - name: Log in to the Container registry
         uses: docker/login-action@v3
@@ -184,7 +182,7 @@ jobs:
           ref: ${{ inputs.ref }}
 
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.11.0
+        uses: helm/kind-action@v1.12.0
 
       - name: Set up Helm
         uses: azure/setup-helm@v4

.github/workflows/cron-unstable-build.yaml

@@ -13,8 +13,8 @@ on:
   # Run nightly build at this time, bit of trial and error but this seems good.
   schedule:
     - cron: "0 6 * * *" # master build
-    - cron: "0 12 * * *" # 3.0 build
-    - cron: "0 18 * * *" # 3.1 build
+    - cron: "0 12 * * *" # 3.2 build
+    - cron: "0 18 * * *" # master build
 
 # We do not want a new unstable build to run whilst we are releasing the current unstable build.
 concurrency: unstable-build-release
@@ -55,13 +55,13 @@ jobs:
       - name: 3.0 run
         if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *'
         run: |
-          echo "cron_branch=3.0" >> $GITHUB_ENV
+          echo "cron_branch=3.2" >> $GITHUB_ENV
         shell: bash
 
-      - name: 3.1 run
+      - name: master run
         if: github.event_name == 'schedule' && github.event.schedule=='0 18 * * *'
         run: |
-          echo "cron_branch=3.1" >> $GITHUB_ENV
+          echo "cron_branch=master" >> $GITHUB_ENV
         shell: bash
 
       - name: Output the branch to use