use a wheel server as a cache for bootstrapping

.github/workflows/test.yaml

@@ -84,6 +84,7 @@ jobs:
           - bootstrap_build_tags
           - bootstrap_prerelease
           - bootstrap_constraints
+          - bootstrap_cache
           - build
           - build_order
           - build_steps

.mergify.yml

@@ -31,6 +31,7 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, bootstrap_build_tags, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_settings, ubuntu-latest)
@@ -56,6 +57,8 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest)
           - check-success=e2e (3.12, 1.75, build, macos-latest)
           - check-success=e2e (3.12, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, build_order, macos-latest)

e2e/test_bootstrap_build_tags.sh

@@ -20,6 +20,8 @@ EXPECTED_FILES="
 $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl
 "
 
+start_local_wheel_server
+
 pass=true
 for pattern in $EXPECTED_FILES; do
   if [ ! -f "${pattern}" ]; then
@@ -38,10 +40,10 @@ fromager \
   --wheels-repo="$OUTDIR/wheels-repo" \
   --work-dir="$OUTDIR/work-dir" \
   --settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
-  bootstrap 'stevedore==5.2.0'
+  bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
 
-if ! grep -q "stevedore: have wheel version 5.2.0: $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1" "$OUTDIR/bootstrap.log"; then
-  echo "FAIL: Did not find log message have wheel version in $OUTDIR/bootstrap.log" 1>&2
+if ! grep -q "stevedore: found built wheel on cache server" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Did not find log message found built wheel on cache server in $OUTDIR/bootstrap.log" 1>&2
   pass=false
 fi
 
@@ -55,7 +57,7 @@ fromager \
   --wheels-repo="$OUTDIR/wheels-repo" \
   --work-dir="$OUTDIR/work-dir" \
   --settings-dir="$SCRIPTDIR/changelog_settings-2" \
-  bootstrap 'stevedore==5.2.0'
+  bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
 
 EXPECTED_FILES="
 $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl

e2e/test_bootstrap_cache.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test bootstrap while taking advantage of the cache wheel server
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+pass=true
+
+# run fromager once to build wheels that can be used by a local wheel server
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap 'stevedore==5.2.0'
+
+start_local_wheel_server
+rm -rf "$OUTDIR/sdists-repo"
+rm -rf "$OUTDIR/work-dir"
+
+# run fromager with the cache wheel server pointing to the local wheel server
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  --no-cleanup \
+  bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+"
+
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+rm -rf "$OUTDIR/sdists-repo"
+rm -rf "$OUTDIR/work-dir"
+rm -rf "$OUTDIR/wheels-repo"
+
+# run fromager with the cache wheel server pointing to the pypi server
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  --no-cleanup \
+  bootstrap --cache-wheel-server-url="https://pypi.org/simple" 'stevedore==5.2.0'
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+"
+
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+NOT_EXPECTED_FILES="
+$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt
+"
+
+for pattern in $NOT_EXPECTED_FILES; do
+  if [ -f "${pattern}" ]; then
+    echo "Found $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+EXPECTED_DIR="
+$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info
+$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info
+$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info
+"
+
+for pattern in $EXPECTED_DIR; do
+  if [ -d "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass