Implement better "versions" library

.libs/deb-repo.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
+# TODO implement "hooks.sh"
+
 deb-repo() {
 	[ -n "$uri" ]
 	[ -n "$suite" ]

.libs/git.sh

@@ -1,31 +1,46 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
+_libsDir="$(dirname "$BASH_SOURCE")"
+source "$_libsDir/hooks.sh"
+unset _libsDir
+
 git-tags() {
 	local repo="$1"; shift
 	[ -n "$repo" ]
 
-	local tag
-	# prometheus had "2.35.0-retract" 😩
-	tag="$(
-		git ls-remote --tags "$repo" \
-			| cut -d/ -f3 \
-			| cut -d^ -f1 \
-			| grep -vE -- '-(rc|alpha|beta|retract)|^nightly$|[+][a-zA-Z]+$' \
-			| sort -Vu \
-			| tail -1
-	)"
-	local version="${tag#v}"
+	local reponame="${repo%.git}"
+	reponame="${reponame#git://}"
+	reponame="${reponame#http://}"
+	reponame="${reponame#https://}"
 
-	local base; base="$(basename "$repo")"
-	echo >&2 "git $base: $version"
+	local tags
+	tags="$(git ls-remote --tags --sort=-version:refname "$repo")" || return "$?"
+
+	local json
+	json="$(jq <<<"$tags" -sR '
+		rtrimstr("\n")
+		| split("\n")
+		| map(
+			capture("^(?<commit>[a-f0-9]+)\t(?<ref>refs/tags/(?<tag>v(?<version>[0-9]+.*?)|.*?)(?:\\^{})?)$")
+			| .version //= .tag
+		)
+		# (ab)using that "git ls-remote --sort=-version:refname" will list "annotated" tags before their unannotated counterparts, so they will be the ref we return here
+		| reduce .[] as $i ({}; .[$i.version] //= $i)
+		| [ .[] ]
+	')" || return "$?"
+
+	local jsons
+	jsons="$(jq <<<"$json" -r 'map(tojson | @sh) | join(" ")')" || return "$?"
+	eval "jsons=( $jsons )"
 
-	jq -nc --arg tag "$tag" --arg version "$version" '
-		{
-			version: $version,
-			tag: $tag,
+	(
+		versions_loop_setvars() {
+			json="$1"
+			version="$(jq <<<"$json" -r '.version')" || return "$?"
 		}
-	'
+		versions_loop 'git' "$reponame" "${jsons[@]}"
+	)
 }
 
 git-ref-commit() {

.libs/hooks.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# usage:
+#   versions_loop_setvars() { ...; } # something that takes one one loop var and sets "version" and "json" appropriately
+#   versions_loop 'static-type' "$identifier" "${loopers[@]}"
+#   unset versions_loop_setvars # optional, but good practice -- alternatively, define/invoke in a subshell
+declare -ag versions_hooks # a list of function names
+versions_loop() {
+	local type="$1"; shift
+	local identifier="$1"; shift
+
+	local loop version json found=
+	for loop; do
+		versions_loop_setvars "$loop" || return "$?"
+
+		local extra versions_hook
+		for versions_hook in "${versions_hooks[@]}"; do
+			if ! extra="$("$versions_hook" "$type" "$identifier" "$version" "$json")"; then
+				echo >&2 "skipping $type $identifier: $version (${versions_hook#hook_})"
+				continue 2
+			fi
+			if [ -n "$extra" ]; then
+				json="$(jq <<<"$json" --argjson extra "$extra" '. += $extra')" || return "$?"
+			fi
+		done
+
+		found=1
+		break
+	done
+
+	if [ -z "$found" ]; then
+		echo >&2 "$type $identifier: not found!"
+		return 1
+	fi
+
+	echo >&2 "$type $identifier: $version"
+
+	printf '%s\n' "$json"
+}
+
+# usage:
+#   versions_hooks+=( hook_no-prereleases )
+hook_no-prereleases() {
+	case "$3" in
+		*[0-9.-]rc* | *[0-9.-]alpha* | *[0-9.-]beta* | *[0-9.-]rc* | *[0-9.-]dev*) return 1 ;;
+	esac
+}

.libs/pypi.sh

@@ -1,32 +1,51 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
+_libsDir="$(dirname "$BASH_SOURCE")"
+source "$_libsDir/hooks.sh"
+unset _libsDir
+
 pypi() {
 	local package="$1"; shift
 	[ -n "$package" ]
 
-	local json version
-	json="$(wget -qO- "https://pypi.org/pypi/$package/json")"
-	version="$(jq <<<"$json" -r '.info.version')" # TODO some way to only get versions matching a particular string? ("1.14.x" etc. instead of just latest)
+	local json
+	json="$(wget -qO- --header 'Accept: application/vnd.pypi.simple.v1+json' "https://pypi.org/simple/$package/")" || return "$?"
 
-	echo >&2 "pip $package: $version"
+	local versions
+	versions="$(jq <<<"$json" -r '.versions | reverse | map(@sh) | join(" ")')" || return "$?"
+	eval "versions=( $versions )"
 
-	jq <<<"$json" -c '
-		.info
-		| (
-			.classifiers
-			| map(
-				match("^(?:Programming Language :: )?Python :: ([0-9]+[.][0-9]+)$")
-				| .captures[0].string
-				| split(".")
-				| map(tonumber)
-			)
-			| sort[-1]
-			| join(".")
-		) as $python
-		| {
-			version: .version,
-			python: { version: $python },
+	(
+		local pypi
+		versions_loop_setvars() {
+			version="$1"
+			pypi="$(wget -qO- "https://pypi.org/pypi/$package/$version/json")" || return "$?"
+			json="$(jq -n --arg version "$version" '{ version: $version }')" || return "$?"
+		}
+		hook_pypi-no-yanked() {
+			local yanked
+			yanked="$(jq <<<"$pypi" '.info.yanked')" || return "$?"
+			[ "$yanked" = 'false' ]
+		}
+		hook_pypi-add-python-version() {
+			jq <<<"$pypi" -c '
+				.info.classifiers
+				| map(
+					capture("^(?:Programming Language :: )?Python :: (?<python>[0-9]+[.][0-9]+)$")
+					| .python
+				)
+				| sort_by(
+					split(".")
+					| map(tonumber)
+				)
+				| .[-1] // empty
+				| {
+					python: { version: . },
+				}
+			'
 		}
-	'
+		versions_hooks=( hook_pypi-no-yanked hook_pypi-add-python-version "${versions_hooks[@]}" )
+		versions_loop 'pypi' "$package" "${versions[@]}"
+	)
 }

backblaze-b2/versions.json

@@ -1,6 +1,6 @@
 {
+  "version": "3.9.0",
   "python": {
     "version": "3.10"
-  },
-  "version": "3.9.0"
+  }
 }

backblaze-b2/versions.sh

@@ -7,6 +7,8 @@ dir="$(readlink -ve "$BASH_SOURCE")"
 dir="$(dirname "$dir")"
 source "$dir/../.libs/pypi.sh"
 
+versions_hooks+=( hook_no-prereleases )
+
 json="$(pypi 'b2')"
 
-jq <<<"$json" -S . > versions.json
+jq <<<"$json" '.' > versions.json

beets/versions.json

@@ -1,6 +1,6 @@
 {
+  "version": "1.6.0",
   "python": {
     "version": "3.9"
-  },
-  "version": "1.6.0"
+  }
 }

beets/versions.sh

@@ -7,6 +7,8 @@ dir="$(readlink -ve "$BASH_SOURCE")"
 dir="$(dirname "$dir")"
 source "$dir/../.libs/pypi.sh"
 
+versions_hooks+=( hook_no-prereleases )
+
 json="$(pypi 'beets')"
 
-jq <<<"$json" -S . > versions.json
+jq <<<"$json" '.' > versions.json

containerd/versions.json

@@ -1,9 +1,15 @@
 {
-  "dind": {
-    "version": "d58df1fc6c866447ce2cd129af10e5b507705624"
-  },
+  "commit": "7880925980b188f4c97b462f709d0db8e8962aff",
+  "ref": "refs/tags/v1.7.3^{}",
+  "tag": "v1.7.3",
+  "version": "1.7.3",
   "runc": {
+    "commit": "ccaecfcbc907d70a7aa870a6650887b901b25b82",
+    "ref": "refs/tags/v1.1.9^{}",
+    "tag": "v1.1.9",
     "version": "1.1.9"
   },
-  "version": "1.7.3"
+  "dind": {
+    "version": "d58df1fc6c866447ce2cd129af10e5b507705624"
+  }
 }

containerd/versions.sh

@@ -7,12 +7,13 @@ dir="$(readlink -ve "$BASH_SOURCE")"
 dir="$(dirname "$dir")"
 source "$dir/../.libs/git.sh"
 
+versions_hooks+=( hook_no-prereleases )
+
 containerd="$(git-tags 'https://github.com/containerd/containerd.git')"
 runc="$(git-tags 'https://github.com/opencontainers/runc.git')"
 dind="$(github-file-commit 'moby/moby' 'HEAD' 'hack/dind')"
 
-jq <<<"$containerd" --argjson runc "$runc" --argjson dind "$dind" -S '
+jq <<<"$containerd" --argjson runc "$runc" --argjson dind "$dind" '
 	.runc = $runc
 	| .dind = $dind
-	| del(.tag, .runc.tag)
 ' > versions.json

diffoscope/versions.json

@@ -1,6 +1,6 @@
 {
+  "version": "247",
   "python": {
     "version": "3.10"
-  },
-  "version": "247"
+  }
 }

diffoscope/versions.sh

@@ -7,6 +7,8 @@ dir="$(readlink -ve "$BASH_SOURCE")"
 dir="$(dirname "$dir")"
 source "$dir/../.libs/pypi.sh"
 
+versions_hooks+=( hook_no-prereleases )
+
 json="$(pypi 'diffoscope')"
 
-jq <<<"$json" -S . > versions.json
+jq <<<"$json" '.' > versions.json

docker-master/versions.json

@@ -1,9 +1,9 @@
 {
-  "buildx": {
-    "version": "a97e1641a4df2dbede2f766e23f3e1e5d214283a"
-  },
+  "version": "fa517bb4206090f6af39225f5f8473efe520c3af",
   "cli": {
     "version": "cdabfa2aa55a3560d0fe5f63074571afae840ab3"
   },
-  "version": "fa517bb4206090f6af39225f5f8473efe520c3af"
+  "buildx": {
+    "version": "a97e1641a4df2dbede2f766e23f3e1e5d214283a"
+  }
 }

docker-master/versions.sh

@@ -11,7 +11,7 @@ moby="$(git-ref-commit 'https://github.com/moby/moby.git' 'HEAD')"
 cli="$(git-ref-commit 'https://github.com/docker/cli.git' 'HEAD')"
 buildx="$(git-ref-commit 'https://github.com/docker/buildx.git' 'HEAD')"
 
-jq <<<"$moby" --argjson cli "$cli" --argjson buildx "$buildx" -S '
+jq <<<"$moby" --argjson cli "$cli" --argjson buildx "$buildx" '
 	. += {
 		cli: $cli,
 		buildx: $buildx,

duplicity/versions.json

@@ -1,6 +1,6 @@
 {
+  "version": "1.2.3",
   "python": {
     "version": "3.11"
-  },
-  "version": "1.2.3"
+  }
 }

duplicity/versions.sh

@@ -7,7 +7,17 @@ dir="$(readlink -ve "$BASH_SOURCE")"
 dir="$(dirname "$dir")"
 source "$dir/../.libs/pypi.sh"
 
-json="$(pypi 'duplicity')"
+versions_hooks+=( hook_no-prereleases )
+
+json="$(
+	hook_bad-duplicity-versions() {
+		case "$3" in
+			2.0.0 | 2.0.2) return 1 ;; # https://gitlab.com/duplicity/duplicity/-/issues/746
+		esac
+	}
+	versions_hooks+=( hook_bad-duplicity-versions )
+	pypi 'duplicity'
+)"
 # TODO b2sdk pinning
 
-jq <<<"$json" -S . > versions.json
+jq <<<"$json" '.' > versions.json

github-pages/versions.json

@@ -1,6 +1,6 @@
 {
+  "version": "228",
   "ruby": {
     "version": "2.7"
-  },
-  "version": "228"
+  }
 }

github-pages/versions.sh

@@ -9,17 +9,14 @@ json="$(wget -qO- 'https://pages.github.com/versions.json')"
 json="$(jq <<<"$json" -c '
 	{
 		version: ."github-pages",
-		ruby: {
-			version: (
-				.ruby
-				| match("^([0-9]+[.][0-9]+)[.]")
-				| .captures[0].string
-			),
-		},
+		ruby: (
+			.ruby
+			| capture("^(?<version>[0-9]+[.][0-9]+)[.]")
+		),
 	}
 ')"
 
 version="$(jq <<<"$json" -r '.version')"
 echo >&2 "github-pages: $version"
 
-jq <<<"$json" -S . > versions.json
+jq <<<"$json" '.' > versions.json

handbrake/versions.json

@@ -1,3 +1,6 @@
 {
+  "commit": "04413a27e6d616cddd98c2c6468aca2bf91b87b5",
+  "ref": "refs/tags/1.6.1^{}",
+  "tag": "1.6.1",
   "version": "1.6.1"
 }

handbrake/versions.sh

@@ -7,6 +7,8 @@ dir="$(readlink -ve "$BASH_SOURCE")"
 dir="$(dirname "$dir")"
 source "$dir/../.libs/git.sh"
 
+versions_hooks+=( hook_no-prereleases )
+
 json="$(git-tags 'https://github.com/HandBrake/HandBrake.git')"
 
-jq <<<"$json" -S 'del(.tag)' > versions.json
+jq <<<"$json" '.' > versions.json