POC pushing & pulling trunk from an OSI registry

Modify `trunk.mk` to build JSON files required to build [OCI image
manifests] and an [OCI image index], including new `make` variables
`TITLE`, `DESCRIPTION`, `VENDOR`, `URL`, and `REPO_URL`.

Add `push_trunk`, a shell script to take those JSON files and use `oras` to
build the images and manifests and push them to a registry.

Add `docker_compose.yml` to spin up a PGXN tools image for AMD64 and
running Postgres 16, as well as [zot], a vendor-neutral OCI registry.

Finally, modify `install_trunk` to download an artifact from a registry
address, rather than build from a local file. Use `oras` to fetch the
image for the current platform. There's no support for distinguishing
between artifacts built for different Postgres versions, but there are
annotations in the image index that would allow it.

  [OCI image manifests]: https://github.com/opencontainers/image-spec/blob/main/manifest.md
  [OCI image index]: https://github.com/opencontainers/image-spec/blob/main/image-index.md
  [zot]: https://github.com/project-zot/zot

.gitignore

@@ -9,6 +9,7 @@ regression.diffs
 regression.out
 /sql/semver--?.??.?.sql
 /semver-*
+/semver_annotations.json
 /latest-changes.md
 /semver_binary_copy.bin
 /.vscode

Makefile

@@ -6,6 +6,8 @@ DISTVERSION  = $(shell grep -m 1 '[[:space:]]\{3\}"version":' META.json | \
                sed -e 's/[[:space:]]*"version":[[:space:]]*"\([^"]*\)",\{0,1\}/\1/')
 
 MODULEDIR    = $(EXTENSION)
+DESCRIPTION  = A Postgres data type for the Semantic Version format with support for btree and hash indexing.
+REPO_URL     = https://github.com/theory/pg-semver
 DATA         = $(wildcard sql/*--*.sql)
 DOCS         = $(wildcard doc/*.mmd)
 TESTS        = $(wildcard test/sql/*.sql)

docker-compose.yml

@@ -0,0 +1,57 @@
+version: "3.9"
+name: trunking
+
+# Start a Zot service for pushing and pulling OCI images, and a
+# pgxn/pgxn-tools image for building and testing Linux/AMD binaries.
+#
+# docker compose up -d
+# docker compose exec linux bash
+# make clean && make && make trunk
+#
+# On macOS:
+#   make clean && make && make trunk
+#   ./push_trunk localhost:5000/theory/semver semver-0.32.1+pg16-darwin-23.5.0-arm64 semver-0.32.1+pg16-linux-amd64
+#   clean -dfx --exclude=.vscode
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'  -exec rm -rf "{}" \;
+#
+#   ./install_trunk localhost:5000/theory/semver:v1
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
+#
+# Back on Linux
+#   ./install_trunk zot:5000/theory/semver:v1
+#   find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
+
+# Name the network for all of the services to join.
+networks:
+  default:
+    name: pgxnnet
+
+services:
+  zot:
+    image: ghcr.io/project-zot/zot-linux-arm64:latest
+    container_name: zot
+    ports:
+      - 5000:5000
+    hostname: zot
+
+  linux:
+    image: pgxn/pgxn-tools
+    platform: "linux/amd64"
+    container_name: linux
+    hostname: linux
+    working_dir: /work
+    volumes:
+      - ".:/work"
+    # Install oras, then start Postgres 16 and install rsync and jq
+    entrypoint:
+      - "/bin/sh"
+      - -ecx
+      - |
+          cd /tmp
+          curl -LO "https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz"
+          mkdir -p oras-install/
+          tar -zxf oras_1.2.0_*.tar.gz -C oras-install/
+          sudo mv oras-install/oras /usr/local/bin/
+          rm -rf oras_1.2.0_*.tar.gz oras-install/
+          pg-start 16 rsync jq
+          tail -f /dev/null

install_trunk

@@ -22,18 +22,27 @@ install_trunk() {
     fi
 
     # Determine the platform.
-    local my_os my_osv my_arch
+    local my_os my_osv my_arch platform
     my_os=$(uname | tr '[:upper:]' '[:lower:]')
     my_osv=$(uname -r)
     my_arch="$(uname -m)"
     if [ "$my_arch" == "x86_64" ]; then my_arch=amd64; fi
+    if [ "$my_os" == "darwin" ]; then
+        platform="$my_os/$my_arch:$my_osv"
+    else
+        platform="$my_os/$my_arch"
+    fi
+
+    # Download.
+    local file
+    file=$(basename "$(oras pull --no-tty --plain-http  --format 'go-template={{(first .files).path}}' --platform "$platform" "$trunk")")
 
     # Unpack.
-    printf "Unpacking %s\n" "$trunk"
-    tar zxf "$trunk"
+    printf "Unpacking %s\n" "$file"
+    tar zxf "$file"
 
     # Go into the directory
-    local dir="${trunk%.*}"
+    local dir="${file%.*}"
     cd "$dir" || exit
 
     # Verify the checksums.

push_trunk

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# POC for publishing Trunk packages to an OCI repository with an image index
+# to allow pulling a platform-specific binary. Requires:
+#
+# * oras
+# * jq
+# * zot: docker run -d -p 5000:5000 --name oras-quickstart ghcr.io/project-zot/zot-linux-arm64:latest
+#
+# Inspired by the Homebrew implementation of the pattern as referenced in
+# https://github.com/oras-project/oras/issues/237, and with thanks to the
+# denizens of the #oras and #zot channels on the CNCF Slack.
+
+trap 'exit' ERR
+set -E
+
+# OCI types to create.
+ARTIFACT_TYPE=application/vnd.pgxn.trunk.layer.v1
+MEDIA_TYPE=application/vnd.oci.image.layer.v1.tar+gzip
+CONFIG_TYPE=application/vnd.oci.image.config.v1+json
+OCI_DIR=oci_dir
+INDEX_FILE=image_index.json
+
+push_image() {
+    # Push the image into the OCI layout directory $OCI_DIR.
+    local trunk=$1
+    oras push --no-tty \
+        --oci-layout "$OCI_DIR" \
+        --artifact-type "${ARTIFACT_TYPE}" \
+        --config "${trunk}_config.json":"$CONFIG_TYPE" \
+        --format go-template='{{.digest}}' \
+        --annotation-file "${trunk}_annotations.json" \
+        "$trunk.trunk":"$MEDIA_TYPE"
+}
+
+make_manifest() {
+    local trunk=$1
+    local digest=$2
+    local anno platform
+
+    # Extract just the pgxn.trunk annotations.
+    anno=$(jq -c \
+        '.["$manifest"] | with_entries(select(.key | startswith("org.pgxn.trunk.")))' \
+        "${trunk}_annotations.json"
+    )
+
+    # Extract the platform config.
+    platform=$(jq \
+        'pick(.os, .["os.version"], .architecture)| with_entries(select(.value |. !=null and . != ""))' \
+        "$trunk"_config.json
+    )
+
+    # Create and return the image manifest.
+    oras manifest fetch --oci-layout "$OCI_DIR@${digest}" --descriptor \
+        | jq --argjson anno "$anno" --argjson platform "$platform" \
+        '{
+            mediaType: .mediaType,
+            size: .size,
+            digest: .digest,
+            platform: $platform,
+            annotations: $anno
+        }'
+}
+
+write_index() {
+    darwin_manifest=$1
+    linux_manifest=$2
+
+    # Build the image index with the two manifests.
+    jq -n --argjson linux "$linux_manifest" \
+        --argjson darwin "$darwin_manifest" \
+        --argjson annotations "$(cat semver_annotations.json)" \
+        '{
+            schemaVersion: 2,
+            mediaType: "application/vnd.oci.image.index.v1+json",
+            manifests: [$linux, $darwin],
+            annotations: $annotations
+        }' > "$INDEX_FILE"
+}
+
+push_trunk() {
+    # Only testing for Darwin and Linux rn.
+    local repo=$1
+    local darwin_trunk=$2
+    local linux_trunk=$3
+
+    if [ -z "$repo" ] || [ -z "$darwin_trunk" ] || [ -z "$linux_trunk" ]; then
+        printf "Usage:\n\n  %s REPO DARWIN.trunk LINUX.trunk\n" "$0"
+        exit 1
+    fi
+
+    # Push the images and grab the resulting digests.
+    darwin_digest=$(push_image "$darwin_trunk")
+    linux_digest=$(push_image "$linux_trunk")
+
+    # Create the image manifests.
+    darwin_manifest=$(make_manifest "$darwin_trunk" "$darwin_digest")
+    linux_manifest=$(make_manifest "$linux_trunk" "$linux_digest")
+
+    # Write out and push the image index.
+    write_index "$darwin_manifest" "$linux_manifest"
+    oras manifest push --oci-layout ./"$OCI_DIR":image-index "$INDEX_FILE"
+
+    # Push everything from the local layout to the remote registry.
+    oras cp --from-oci-layout ./"$OCI_DIR":image-index --to-plain-http "${repo}:v1"
+
+    # View the remote image index manifest.
+    oras manifest get --plain-http "${repo}:v1" | jq
+
+    # Cleanup.
+    rm -rf "$OCI_DIR" "$INDEX_FILE"
+}
+
+push_trunk "$@"

trunk.mk

@@ -1,10 +1,16 @@
 # Trunk packaging. To use, create a standard PGXS Makefile for your extension.
-# The only extra variables are DISTVERSION, LICENSE, and LANGUAGE.
+# The only extra variables are DISTVERSION, TITLE, DESCRIPTION, LICENSE,
+# LANGUAGE, VENDOR, URL, and REPO_URL.
 #
 # ``` make
 # DISTVERSION  = 1.2.1
+# TITLE        = Bike
+# DESCRIPTION  = A bicycle inside your database.
 # LICENSE      = mit
 # LANGUAGE     = c
+# VENDOR       = PGXN
+# URL          = https://pgxn.org/dist/bike
+# REPO_URL     = https://github.com/pgxn/bike
 #
 # EXTENSION    = bike
 # MODULEDIR    = $(EXTENSION)
@@ -38,6 +44,9 @@
 
 LAUNGUAGE      ?= c
 LICENSE        ?= PostgreSQL
+TITLE          ?= $(EXTENSION)
+VENDOR         ?= PGXN
+URL            ?= $(REPO_URL)
 
 pkg_arch       := $(shell uname -m)
 ifeq ($(pkg_arch),x86_64)
@@ -64,7 +73,52 @@ pkg_info_files ?= $(wildcard README* readme* Readme* LICENSE* license* License*
 EXTRA_CLEAN    += $(EXTENSION)-$(DISTVERSION)+*/
 
 # Phony target to create the trunk and OCI JSON files.
-trunk: $(pkg).trunk
+trunk: $(pkg).trunk $(pkg)_config.json $(pkg)_annotations.json $(EXTENSION)_annotations.json
+
+# Use jq to create the OCI image index annoations.
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
+$(EXTENSION)_annotations.json:
+	jq -n \
+	    --arg org.opencontainers.image.created "$$(date +%Y-%m-%dT%TZ)" \
+		--arg org.opencontainers.image.licenses "$(LICENSE)" \
+		--arg org.opencontainers.image.title "$(TITLE)" \
+		--arg org.opencontainers.image.description "$(DESCRIPTION)" \
+		--arg org.opencontainers.image.source "$(REPO_URL)" \
+		--arg org.opencontainers.image.vendor "$(VENDOR)" \
+		--arg org.opencontainers.image.ref.name "$(DISTVERSION)" \
+		--arg org.opencontainers.image.version "$(DISTVERSION)" \
+		--arg org.opencontainers.image.url "$(URL)" \
+		'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' > $@
+
+# Use jq to create the OCI image configuration.
+$(pkg)_config.json:
+	@jq -n \
+		--arg os "$(PORTNAME)" \
+		--arg os.version "$(pkg_os_ver)" \
+		--arg architecture "$(pkg_arch)" \
+		--arg created "$$(date +%Y-%m-%dT%TZ)" \
+		'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' > $@
+
+# Use jq to create the OCI image manifest annotations.
+$(pkg)_annotations.json: $(pkg).trunk $(pkg)_config.json
+	@anno=$$(jq -n \
+	    --arg org.opencontainers.image.created "$$(date +%Y-%m-%dT%TZ)" \
+		--arg org.opencontainers.image.title "$(pkg).trunk" \
+		--arg org.opencontainers.image.licenses "$(LICENSE)" \
+		--arg org.opencontainers.image.description "$(DESCRIPTION)" \
+		--arg org.opencontainers.image.source "$(REPO_URL)" \
+		--arg org.opencontainers.image.vendor "$(VENDOR)" \
+		--arg org.opencontainers.image.ref.name "$(DISTVERSION)" \
+		--arg org.opencontainers.image.version "$(DISTVERSION)" \
+		--arg org.opencontainers.image.url "$(URL)" \
+		--arg org.pgxn.trunk.pg.version "$(VERSION)" \
+		--arg org.pgxn.trunk.pg.major "$(MAJORVERSION)" \
+		--arg org.pgxn.trunk.pg.version_num "$(VERSION_NUM)" \
+		--arg org.pgxn.trunk.version "0.1.0" \
+		'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' \
+	) && jq -n \
+		--argjson '$$manifest' "$$anno" \
+		'$$ARGS.named' > $@
 
 $(pkg).trunk: package
 	tar zcvf $@ $(pkg_dir)