More Openstack packaging fixups [master]

release/packaging/docker-build-images/docker-bake.hcl

@@ -6,6 +6,11 @@ variable "ARCH" {
     default = "amd64"
 }
 
+# The default Ubuntu stream
+variable "STREAM" {
+    default = "noble"
+}
+
 variable "UID" {
     default = 1000
 }
@@ -23,30 +28,25 @@ group "default" {
     targets = ["ubuntu", "centos"]
 }
 
-# All ubuntu images
-group "ubuntu" {
-  targets = ["focal", "jammy", "noble"]
-}
-
 # All centos images
 group "centos" {
     targets = ["centos7"]
 }
 
-# Ubuntu builds
-target "focal" {
-  dockerfile = "ubuntu-focal-build.Dockerfile.${ARCH}"
-  tags = ["calico-build/focal"]
-}
-
-target "jammy" {
-  dockerfile = "ubuntu-jammy-build.Dockerfile.${ARCH}"
-  tags = ["calico-build/jammy"]
-}
+# All Ubuntu builds - in one big matrix, using one Dockerfile
+target "ubuntu" {
+  name = "ubuntu-${STREAM}-${ARCH}"
+  dockerfile = "ubuntu.Dockerfile"
+  matrix = {
+    STREAM = ["focal", "jammy", "noble"]
+    ARCH = ["amd64"]
+  }
+  args = {
+    STREAM = STREAM
+    ARCH = ARCH
 
-target "noble" {
-  dockerfile = "ubuntu-noble-build.Dockerfile.${ARCH}"
-  tags = ["calico-build/noble"]
+  }
+  tags = ["calico-build/${STREAM}"]
 }
 
 # CentOS builds

release/packaging/docker-build-images/ubuntu.Dockerfile

@@ -0,0 +1,25 @@
+# Set our build args (with defaults) 
+ARG STREAM="noble"
+ARG ARCH="amd64"
+
+# We use our build args here to specify which image to start from
+FROM --platform=linux/${ARCH} ubuntu:${STREAM}
+
+# The `FROM` line 'consumes' the build args, so we have to bring them
+# back into scope again for some reason
+ARG STREAM
+ARG ARCH
+
+# We don't really need this but here we are
+LABEL org.opencontainers.image.authors="Daniel Fox <[email protected]>"
+
+# Run our command; we do some mount magic:
+#    1. use cache mounts for apt directories, keyed on stream/arch
+#    2. bind-mount the install deps script in so it doesn't live in the image
+RUN \
+    --mount=type=cache,sharing=locked,id=${STREAM}-${ARCH}-archives,target=/var/cache/apt/archives \
+    --mount=type=cache,sharing=locked,id=${STREAM}-${ARCH}-lists,target=/var/lib/apt/lists \
+    --mount=type=bind,source=install-ubuntu-build-deps,target=/install-ubuntu-build-deps \
+    /install-ubuntu-build-deps
+
+WORKDIR /code

release/packaging/utils/make-packages.sh

@@ -51,9 +51,6 @@ for package_type in "$@"; do
         debver=${FORCE_VERSION_DEB:-$(git_version_to_deb "${version}")}
         debver=$(strip_v "${debver}")
 
-        # Current time in Debian changelog format; e.g. Wed, 02
-        # Mar 2016 14:08:51 +0000.
-        timestamp=$(date "+%a, %d %b %Y %H:%M:%S %z")
         for series in focal jammy noble; do
                 if ${release}; then
                         changelog_message="${NAME} v${debver} (from Git commit ${sha})."
@@ -116,7 +113,7 @@ EOF
             fi
             echo
 
-        } | sed -i '/^%changelog/ r /dev/stdin' ${rpm_spec}
+        } | sed -i '/^%changelog/ r /dev/stdin' "${rpm_spec}"
         fi
 
         elversions=7
@@ -126,7 +123,7 @@ EOF
         [ -n "$imageid" ] && ${DOCKER_RUN_RM} -e "EL_VERSION=el${elversion}" \
             -e FORCE_VERSION="${FORCE_VERSION}" \
             -e RPM_TAR_ARGS="${RPM_TAR_ARGS}" \
-            $imageid /rpm/build-rpms
+            "$imageid" /rpm/build-rpms
         done
 
         cat <<-EOF

release/packaging/utils/publish-debs.sh

@@ -1,8 +1,12 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e # Exit immediately on fail
+set -u # Fail immediately on undefined variable use
 
 REPO_NAME=${REPO_NAME:-master}
 test -n "$SECRET_KEY"
 
+outputdir=$(readlink -f "$(dirname $0)/../output")
 rootdir=$(git rev-parse --show-toplevel)
 keydir=$(mktemp -t -d calico-publish-debs.XXXXXX)
 cp -a "$SECRET_KEY" "${keydir}/key"
@@ -17,28 +21,50 @@ else
     # -ti to docker-run, and $SECRET_KEY must not require a pass phrase.
     interactive=
 fi
-docker run --rm ${interactive} -v "${rootdir}:/code" -v "${keydir}:/keydir" -w /code/release/packaging/output calico-build/focal /bin/sh -c "gpg --import --batch < /keydir/key && debsign -k'*@' --re-sign *_*_source.changes"
 
 for series in focal jammy noble; do
+    echo "Processing uploads for ${series}"
+    echo
     # Get the packages and versions that already exist in the PPA, so we can avoid
     # uploading the same package and version as already exist.  (As they would be rejected
     # anyway by Launchpad.)
-    sources_url="http://ppa.launchpad.net/project-calico/${REPO_NAME}/ubuntu/dists/${series}/main/source/Sources.gz"
-    existing_packages=$(wget -q -O - ${sources_url} | gzip -d | awk '/^Package:/{printf("%s_", $2);} /^Version:/{sub(/^1:/,"", $2); print $2;}')
-    echo "Existing source packages for ${series} in project-calico/${REPO_NAME} are:"
-    echo "${existing_packages}"
-
-    # Use the Distribution header to map changes files to Ubuntu versions, as some of our
-    # packages don't include the Ubuntu version name in the changes file name.
-    # shellcheck disable=SC2013
-    for changes_file in $(grep -l "Distribution: ${series}" ./*_source.changes); do
-        already_exists=false
-        for existing in ${existing_packages}; do
-            if [ "${changes_file}" = "${existing}_source.changes" ]; then
-                already_exists=true
-                break
-            fi
+    sources_url="https://ppa.launchpadcontent.net/project-calico/${REPO_NAME}/ubuntu/dists/${series}/main/source/Sources.gz"
+    if curl -fs -I "${sources_url}" > /dev/null; then
+        existing_packages=$(curl -fsSL "${sources_url}" | zcat | awk '/^Package:/{printf("%s_", $2);} /^Version:/{sub(/^[0-9]:/,"", $2); print $2;}')
+        mapfile -t existing_packages <<< "${existing_packages}"
+        for existing_package in "${existing_packages[@]}"; do
+            # echo "Launchpad has sources for ${existing_package} already, marking to skip" | ts "[check ${series} ${existing_package}]"
+            touch "${outputdir}/${existing_package}_source.ppa.previously-uploaded"
         done
-        ${already_exists} || docker run --rm -v "${rootdir}:/code" -w /code/release/packaging/output calico-build/${series} dput -u "ppa:project-calico/${REPO_NAME}" "${changes_file}" | ts "[upload $series]"
+    fi
+
+    # Loop through our `changes` files, one each for each source package we're uploading
+    # sign them, and then upload the file via dput.
+    #
+    # Another modernization that we probably don't need - use `find` to list files, delimit on nulls,
+    # have bash read each entry null-delimited (-d), with escaped backslashes (-r), into $changes_file
+    # so that we don't have to worry about spaces in filenames destroying our script.
+    find "${outputdir}" -name "*-${series}_source.changes" -print0 | while read -r -d $'\0' changes_file; do
+        filename=$(basename "${changes_file}")
+        package_name="${filename%%_*}"
+        if test -f "${outputdir}/${existing_package}_source.ppa.upload"; then
+            echo "Upload was already completed, skipping" | ts "[upload ${series} ${package_name}]"
+        elif test -f "${outputdir}/${existing_package}_source.ppa.previously-uploaded"; then
+            echo "Launchpad already has this source package, skipping" | ts "[upload ${series} ${package_name}]"
+        else
+            # Ensure we sign our source packages
+            docker run --rm ${interactive} \
+                -v "${rootdir}:/code" \
+                -v "${keydir}:/keydir" \
+                -w /code/release/packaging/output calico-build/${series} \
+                /bin/sh -c "gpg --quiet --import --batch < /keydir/key && debsign -k'*@' --no-re-sign ${filename}" | ts  "[sign ${series} ${package_name}]"
+
+            # Upload the packages to Launchpad via dput
+            docker run --rm \
+                -v "${rootdir}:/code" \
+                -w /code/release/packaging/output \
+                calico-build/${series} \
+                dput -u "ppa:project-calico/${REPO_NAME}" "${filename}" | ts "[upload ${series} ${package_name}]"
+        fi
     done
 done