support bootstrapping from git checkouts of projects

.github/workflows/test.yaml

@@ -85,6 +85,8 @@ jobs:
           - bootstrap_prerelease
           - bootstrap_constraints
           - bootstrap_cache
+          - bootstrap_git_url
+          - bootstrap_git_url_tag
           - build
           - build_order
           - build_steps
@@ -140,7 +142,7 @@ jobs:
 
       - name: Run tests
         run: ./e2e/test_${{ matrix.test-script }}.sh
-      
+
       - name: Upload logs for debugging
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4

.mergify.yml

@@ -31,6 +31,8 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_git_url, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_git_url_tag, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest)
@@ -39,6 +41,7 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, download_sequence, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, elfdeps, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, extra_metadata, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, meson, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, migrate_graph, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, optimize_build, ubuntu-latest)
@@ -48,7 +51,6 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest)
@@ -59,6 +61,10 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_extras, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_extras, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_git_url, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_git_url, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, build, macos-latest)
@@ -75,6 +81,8 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, elfdeps, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, extra_metadata, macos-latest)
           - check-success=e2e (3.12, 1.75, extra_metadata, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, lint_requirements, macos-latest)
+          - check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, meson, macos-latest)
           - check-success=e2e (3.12, 1.75, meson, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, migrate_graph, macos-latest)
@@ -93,8 +101,6 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, report_missing_dependency, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, rust_vendor, macos-latest)
           - check-success=e2e (3.12, 1.75, rust_vendor, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, lint_requirements, macos-latest)
-          - check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest)
           - "-draft"
 
           # At least 1 reviewer

e2e/test_bootstrap_git_url.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Test bootstrapping from a requirement with a git+https URL witout specifying a
+# version tag.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+GIT_REPO_URL="https://opendev.org/openstack/stevedore.git"
+
+fromager \
+  --debug \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap "stevedore @ git+${GIT_REPO_URL}"
+
+find "$OUTDIR/wheels-repo/" -name '*.whl'
+find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
+ls "$OUTDIR"/work-dir/*/build.log || true
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+
+$OUTDIR/bootstrap.log
+$OUTDIR/fromager-errors.log
+
+$OUTDIR/work-dir/pbr-*/build.log
+$OUTDIR/work-dir/setuptools-*/build.log
+$OUTDIR/work-dir/stevedore-*/build.log
+"
+
+pass=true
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass

e2e/test_bootstrap_git_url_tag.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Test bootstrapping from a requirement with a git+https URL and specifying a
+# version tag.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+GIT_REPO_URL="https://opendev.org/openstack/stevedore.git"
+
+fromager \
+  --debug \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap "stevedore @ git+${GIT_REPO_URL}@5.2.0"
+
+find "$OUTDIR/wheels-repo/" -name '*.whl'
+find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
+ls "$OUTDIR"/work-dir/*/build.log || true
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+
+$OUTDIR/bootstrap.log
+$OUTDIR/fromager-errors.log
+
+$OUTDIR/work-dir/pbr-*/build.log
+$OUTDIR/work-dir/setuptools-*/build.log
+$OUTDIR/work-dir/stevedore-*/build.log
+"
+
+pass=true
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass

src/fromager/gitutils.py

@@ -0,0 +1,63 @@
+import logging
+import pathlib
+from urllib.parse import urlparse
+
+from packaging.requirements import Requirement
+
+from fromager import context, external_commands
+
+logger = logging.getLogger(__name__)
+
+
+def git_clone(
+    *,
+    ctx: context.WorkContext,
+    req: Requirement,
+    output_dir: pathlib.Path,
+    repo_url: str,
+    tag: str | None = None,
+    ref: str | None = None,
+    submodules: bool | list[str] = False,
+) -> pathlib.Path:
+    """Clone a git repository"""
+    if tag is not None and ref is not None:
+        raise ValueError("tag and ref are mutually exclusive")
+
+    # Create a clean URL without any credentials for logging
+    parsed_url = urlparse(repo_url)
+    clean_url = parsed_url._replace(netloc=parsed_url.hostname or "").geturl()
+    logger.info(
+        "%s: cloning %s, tag %r, ref %r, submodules %r, into %s",
+        req.name,
+        clean_url,
+        tag,
+        ref,
+        submodules,
+        output_dir,
+    )
+    cmd: list[str] = ["git", "clone"]
+    if tag is not None:
+        # --branch works with branches and tags, but not with commits
+        cmd.extend(["--branch", tag, "--depth", "1"])
+    if submodules:
+        if isinstance(submodules, list):
+            for pathspec in submodules:
+                cmd.append(f"--recurse-submodules={pathspec}")
+        else:
+            # all submodules
+            cmd.append("--recurse-submodules")
+        if tag is not None:
+            cmd.append("--shallow-submodules")
+    cmd.extend([repo_url, str(output_dir)])
+    external_commands.run(cmd, network_isolation=False)
+
+    # --branch only works with names, so we have to checkout the reference we
+    # actually want if it is not a name
+    if ref is not None:
+        external_commands.run(
+            ["git", "checkout", "--recurse-submodules", "--force", ref],
+            cwd=str(output_dir),
+            network_isolation=False,
+        )
+
+    return output_dir