Add "dev" builds of buildkit

Clamped to officially update once per week, but written such that a BUILDKIT_COMMIT build-arg can build any commit.

.libs/git.sh

@@ -85,3 +85,49 @@ github-file-commit() {
 		unix: ($unix | tonumber),
 	}'
 }
+
+github-clamp-commit() {
+	local repo="$1" # "containerd/containerd"
+	local date="${2:-last monday 00:00:00}" # something date(1) can parse
+	local ref="${3:-HEAD}" # ref to start walking backwards from
+
+	local clampUnix
+	clampUnix="$(date --utc --date "$date" '+%s')" || return 1
+
+	local commit= date unix
+	while [ -z "$commit" ]; do
+		local commits
+		commits="$(
+			wget -qO- --header 'Accept: application/json' "https://github.com/$repo/commits/$ref.atom" \
+				| jq -r '
+					.payload.commitGroups[].commits[]
+					| first([ .committedDate, .authoredDate ] | sort | reverse[]) as $date
+					| "\(.oid) \($date)"
+					| @sh
+				'
+		)"
+		eval "local commitDates=( $commits ) commitDate"
+		[ "${#commitDates[@]}" -gt 0 ] || return 1 # TODO error message?
+		for commitDate in "${commitDates[@]}"; do
+			ref="${commitDate%%[[:space:]]*}"
+			date="${commitDate#$ref[[:space:]]}"
+			[ "$ref" != "$date" ] || return 1 # TODO error message?
+			unix="$(date --utc --date "$date" '+%s')" || return 1 # TODO error message?
+			if [ "$unix" -le "$clampUnix" ]; then
+				commit="$ref"
+				break 2
+			fi
+		done
+	done
+	[ -n "$commit" ] || return 1 # TODO error message?
+	[ -n "$date" ] || return 1
+	[ -n "$unix" ] || return 1
+
+	echo >&2 "github $repo: $commit ($date -- @$unix)"
+
+	jq -nc --arg commit "$commit" --arg date "$date" --arg unix "$unix" '{
+		version: $commit,
+		$date,
+		unix: ($unix | tonumber),
+	}'
+}

buildkit/Dockerfile.dev

@@ -0,0 +1,92 @@
+#
+# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh"
+#
+# PLEASE DO NOT EDIT IT DIRECTLY.
+#
+
+FROM --platform=$BUILDPLATFORM golang:1.22 AS build
+
+# https://github.com/moby/buildkit/commits/HEAD
+ARG BUILDKIT_COMMIT=38a47dbbc69d4640a052a662611aece3427164c0
+# https://github.com/moby/buildkit/tree/38a47dbbc69d4640a052a662611aece3427164c0 -- 2024-11-28T20:16:34.000+01:00 (@1732821394)
+ENV BUILDKIT_COMMIT $BUILDKIT_COMMIT
+
+COPY \
+	argsescaped.patch \
+	containerd-arm64-v8.patch \
+	git-no-submodules.patch \
+	mips64le.patch \
+	noclip.patch \
+	nolint.patch \
+	/tmp/buildkit-patches/
+
+WORKDIR /app
+
+RUN set -eux; \
+	git init --quiet; \
+	git config advice.detachedHead false; \
+	git fetch --depth 1 'https://github.com/moby/buildkit.git' "$BUILDKIT_COMMIT:"; \
+	git checkout FETCH_HEAD; \
+	for patch in /tmp/buildkit-patches/*.patch; do \
+		git apply --verbose "$patch"; \
+	done; \
+	git diff --color
+
+ENV CGO_ENABLED 0
+
+ARG TARGETOS TARGETARCH TARGETVARIANT
+ENV GOOS=$TARGETOS GOARCH=$TARGETARCH VARIANT=$TARGETVARIANT
+
+RUN set -eux; \
+	case "$GOARCH/$VARIANT" in \
+		amd64/*) export GOAMD64="$VARIANT" ;; \
+		arm/*) export GOARM="${VARIANT#v}" ;; \
+		arm64/v8) ;; \
+		*) [ -z "$VARIANT" ] ;; \
+	esac; \
+	go env | grep -E 'OS=|ARCH=|AMD64=|ARM64=|ARM='; \
+	\
+# https://github.com/moby/buildkit/blob/v0.13.2/Dockerfile#L93
+	pkg='github.com/moby/buildkit'; \
+	ldflags=" \
+		-d -w \
+		-X '$pkg/version.Version=dev-tianon' \
+		-X '$pkg/version.Revision=$BUILDKIT_COMMIT+tianon-patches' \
+		-X '$pkg/version.Package=$pkg' \
+	"; \
+	go build -o /buildkitd -trimpath -ldflags "$ldflags" ./cmd/buildkitd; \
+	go build -o /buildctl -trimpath -ldflags "$ldflags" ./cmd/buildctl; \
+	\
+# https://github.com/moby/buildkit/blob/v0.14.0/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile#L21
+	pkg='github.com/moby/buildkit/frontend/dockerfile/cmd/dockerfile-frontend'; \
+	ldflags=" \
+		-d -w \
+		-X '$pkg/version.Version=dev-tianon' \
+		-X '$pkg/version.Revision=$BUILDKIT_COMMIT+tianon-patches' \
+		-X '$pkg/version.Package=$pkg' \
+	"; \
+	go build -o /dockerfile-frontend -trimpath -ldflags "$ldflags" ./frontend/dockerfile/cmd/dockerfile-frontend
+
+FROM --platform=$TARGETPLATFORM infosiftr/moby
+
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends git; \
+	rm -rf /var/lib/apt/lists/*
+
+COPY --from=build --link /buildkitd /buildctl /dockerfile-frontend /usr/local/bin/
+COPY buildkitd-entrypoint.sh /usr/local/bin/
+
+VOLUME /var/lib/buildkit
+
+# https://github.com/docker/buildx/issues/484#issuecomment-749352728
+ENV BUILDKIT_STEP_LOG_MAX_SIZE -1
+ENV BUILDKIT_STEP_LOG_MAX_SPEED -1
+
+# https://github.com/moby/buildkit/blob/v0.14.0/frontend/gateway/gateway.go#L309
+# https://github.com/moby/buildkit/blob/v0.14.0/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile#L35-L36
+LABEL moby.buildkit.frontend.network.none="true"
+LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts"
+
+ENTRYPOINT ["buildkitd-entrypoint.sh"]
+CMD []

buildkit/Dockerfile.template

@@ -4,8 +4,8 @@ FROM --platform=$BUILDPLATFORM golang:{{ .go.version }} AS build
 
 {{ if env.variant == "dev" then ( -}}
 # https://github.com/moby/buildkit/commits/HEAD
-# https://github.com/moby/buildkit/tree/{{ .version }}
 ARG BUILDKIT_COMMIT={{ .version }}
+# https://github.com/moby/buildkit/tree/{{ .version }} -- {{ .date }} (@{{ .unix | tostring }})
 ENV BUILDKIT_COMMIT $BUILDKIT_COMMIT
 {{ ) else ( -}}
 # https://github.com/moby/buildkit/releases

buildkit/gsl.sh

@@ -23,7 +23,8 @@ exec jq -r '
 			Builder: "buildkit",
 		},
 		(
-			(
+			.dev as $dev
+			| (
 				[
 					.variants[] as $variant
 					| {
@@ -48,7 +49,14 @@ exec jq -r '
 				| .value as $version
 				| .value = (
 					if $variant == "dev" then
-						[]
+						[
+							$dev.unix
+							| strftime("dev-%Y%m%d-%H%M%S"),
+								strftime("dev-%Y%m%d"),
+								"dev-\($version)",
+								"dev-\($version[:7])",
+								empty
+						]
 					else
 						$version
 						| [ scan("(?:[.-]+|^)[^.-]*") ]

buildkit/versions.json

@@ -10,7 +10,8 @@
     "",
     "rc",
     "0.16",
-    "0.13"
+    "0.13",
+    "dev"
   ],
   "rc": {
     "commit": "95d190ef4f18b57c717eaad703b67cb2be781ebb",
@@ -38,5 +39,13 @@
     "go": {
       "version": "1.21"
     }
+  },
+  "dev": {
+    "version": "38a47dbbc69d4640a052a662611aece3427164c0",
+    "date": "2024-11-28T20:16:34.000+01:00",
+    "unix": 1732821394,
+    "go": {
+      "version": "1.22"
+    }
   }
 }

buildkit/versions.sh

@@ -14,8 +14,7 @@ variants=(
 	'0.16'
 	'0.13'
 
-	# TODO add this back when I figure out a clean way to do something more akin to a "weekly snapshot" or something so it doesn't have an update every single day
-	#'dev'
+	'dev'
 )
 
 json='{}'
@@ -52,7 +51,7 @@ for variant in "${variants[@]}"; do
 			;;
 
 		dev)
-			bk="$(git-ref-commit 'https://github.com/moby/buildkit.git' 'HEAD')"
+			bk="$(github-clamp-commit 'moby/buildkit')"
 			;;
 
 		*) echo >&2 "error: unknown variant: '$variant'"; exit 1 ;;

containerd/versions.sh

@@ -16,7 +16,7 @@ variants=(
 	'1.7'
 	'1.6'
 
-	# TODO add this when I figure out a clean way to do something more akin to a "weekly snapshot" or something so it doesn't have an update every single day (see also "buildkit")
+	# TODO update Dockerfile.template with the necessary bits to build containerd from source (see buildkit/Dockerfile.template)
 	#'dev'
 )
 
@@ -81,6 +81,11 @@ for variant in "${variants[@]}"; do
 				versions_hooks+=( hook_no-prereleases )
 				;;
 
+			(dev)
+				github-clamp-commit 'containerd/containerd'
+				exit # kill the subshell (no "git-tags" invocation)
+				;;
+
 			(*) echo >&2 "error: unknown variant: '$variant'"; exit 1 ;;
 		esac