Skip to content

Commit 3773074

Browse files
dhellmanndhellmann
committed
support bootstrapping from git checkouts of projects
Support git+https URLs in the input requirements list. "Resolve" the version by cloning the repo at the specified reference and generating the metadata to extract the version string. Rely on the git clone instead of downloading a source distribution or allowing an override to specify where the source should come from. Rely on the git clone with preparing source, and ignore any overrides. Force sdists to be generated using the PEP-517 approach. Add e2e tests for cases with and without tags in the URLs. Signed-off-by: Doug Hellmann <[email protected]>
1 parent 8614dd2 commit 3773074

File tree

6 files changed

+396
-32
lines changed

6 files changed

+396
-32
lines changed

‎.github/workflows/test.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ jobs:
8585
- bootstrap_prerelease
8686
- bootstrap_constraints
8787
- bootstrap_cache
88+
- bootstrap_git_url
89+
- bootstrap_git_url_tag
8890
- build
8991
- build_order
9092
- build_steps
@@ -140,7 +142,7 @@ jobs:
140142

141143
- name: Run tests
142144
run: ./e2e/test_${{ matrix.test-script }}.sh
143-
145+
144146
- name: Upload logs for debugging
145147
if: ${{ failure() }}
146148
uses: actions/upload-artifact@v4

‎.mergify.yml

+9-3
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ pull_request_rules:
3131
- check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest)
3232
- check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
3333
- check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest)
34+
- check-success=e2e (3.11, 1.75, bootstrap_git_url, ubuntu-latest)
35+
- check-success=e2e (3.11, 1.75, bootstrap_git_url_tag, ubuntu-latest)
3436
- check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
3537
- check-success=e2e (3.11, 1.75, build, ubuntu-latest)
3638
- check-success=e2e (3.11, 1.75, build_order, ubuntu-latest)
@@ -39,6 +41,7 @@ pull_request_rules:
3941
- check-success=e2e (3.11, 1.75, download_sequence, ubuntu-latest)
4042
- check-success=e2e (3.11, 1.75, elfdeps, ubuntu-latest)
4143
- check-success=e2e (3.11, 1.75, extra_metadata, ubuntu-latest)
44+
- check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest)
4245
- check-success=e2e (3.11, 1.75, meson, ubuntu-latest)
4346
- check-success=e2e (3.11, 1.75, migrate_graph, ubuntu-latest)
4447
- check-success=e2e (3.11, 1.75, optimize_build, ubuntu-latest)
@@ -48,7 +51,6 @@ pull_request_rules:
4851
- check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
4952
- check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest)
5053
- check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest)
51-
- check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest)
5254
- check-success=e2e (3.12, 1.75, bootstrap, macos-latest)
5355
- check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest)
5456
- check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest)
@@ -59,6 +61,10 @@ pull_request_rules:
5961
- check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest)
6062
- check-success=e2e (3.12, 1.75, bootstrap_extras, macos-latest)
6163
- check-success=e2e (3.12, 1.75, bootstrap_extras, ubuntu-latest)
64+
- check-success=e2e (3.12, 1.75, bootstrap_git_url, macos-latest)
65+
- check-success=e2e (3.12, 1.75, bootstrap_git_url, ubuntu-latest)
66+
- check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, macos-latest)
67+
- check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, ubuntu-latest)
6268
- check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
6369
- check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
6470
- check-success=e2e (3.12, 1.75, build, macos-latest)
@@ -75,6 +81,8 @@ pull_request_rules:
7581
- check-success=e2e (3.12, 1.75, elfdeps, ubuntu-latest)
7682
- check-success=e2e (3.12, 1.75, extra_metadata, macos-latest)
7783
- check-success=e2e (3.12, 1.75, extra_metadata, ubuntu-latest)
84+
- check-success=e2e (3.12, 1.75, lint_requirements, macos-latest)
85+
- check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest)
7886
- check-success=e2e (3.12, 1.75, meson, macos-latest)
7987
- check-success=e2e (3.12, 1.75, meson, ubuntu-latest)
8088
- check-success=e2e (3.12, 1.75, migrate_graph, macos-latest)
@@ -93,8 +101,6 @@ pull_request_rules:
93101
- check-success=e2e (3.12, 1.75, report_missing_dependency, ubuntu-latest)
94102
- check-success=e2e (3.12, 1.75, rust_vendor, macos-latest)
95103
- check-success=e2e (3.12, 1.75, rust_vendor, ubuntu-latest)
96-
- check-success=e2e (3.12, 1.75, lint_requirements, macos-latest)
97-
- check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest)
98104
- "-draft"
99105

100106
# At least 1 reviewer

‎e2e/test_bootstrap_git_url.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
# Test bootstrapping from a requirement with a git+https URL witout specifying a
4+
# version tag.
5+
6+
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
7+
source "$SCRIPTDIR/common.sh"
8+
9+
GIT_REPO_URL="https://opendev.org/openstack/stevedore.git"
10+
11+
fromager \
12+
--debug \
13+
--log-file="$OUTDIR/bootstrap.log" \
14+
--error-log-file="$OUTDIR/fromager-errors.log" \
15+
--sdists-repo="$OUTDIR/sdists-repo" \
16+
--wheels-repo="$OUTDIR/wheels-repo" \
17+
--work-dir="$OUTDIR/work-dir" \
18+
--settings-dir="$SCRIPTDIR/changelog_settings" \
19+
bootstrap "stevedore @ git+${GIT_REPO_URL}"
20+
21+
find "$OUTDIR/wheels-repo/" -name '*.whl'
22+
find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
23+
ls "$OUTDIR"/work-dir/*/build.log || true
24+
25+
EXPECTED_FILES="
26+
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
27+
$OUTDIR/wheels-repo/downloads/pbr-*.whl
28+
$OUTDIR/wheels-repo/downloads/stevedore-*.whl
29+
30+
$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
31+
$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
32+
33+
$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
34+
$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
35+
$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
36+
37+
$OUTDIR/work-dir/build-order.json
38+
$OUTDIR/work-dir/constraints.txt
39+
40+
$OUTDIR/bootstrap.log
41+
$OUTDIR/fromager-errors.log
42+
43+
$OUTDIR/work-dir/pbr-*/build.log
44+
$OUTDIR/work-dir/setuptools-*/build.log
45+
$OUTDIR/work-dir/stevedore-*/build.log
46+
"
47+
48+
pass=true
49+
for pattern in $EXPECTED_FILES; do
50+
if [ ! -f "${pattern}" ]; then
51+
echo "Did not find $pattern" 1>&2
52+
pass=false
53+
fi
54+
done
55+
56+
$pass

‎e2e/test_bootstrap_git_url_tag.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
# Test bootstrapping from a requirement with a git+https URL and specifying a
4+
# version tag.
5+
6+
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
7+
source "$SCRIPTDIR/common.sh"
8+
9+
GIT_REPO_URL="https://opendev.org/openstack/stevedore.git"
10+
11+
fromager \
12+
--debug \
13+
--log-file="$OUTDIR/bootstrap.log" \
14+
--error-log-file="$OUTDIR/fromager-errors.log" \
15+
--sdists-repo="$OUTDIR/sdists-repo" \
16+
--wheels-repo="$OUTDIR/wheels-repo" \
17+
--work-dir="$OUTDIR/work-dir" \
18+
--settings-dir="$SCRIPTDIR/changelog_settings" \
19+
bootstrap "stevedore @ git+${GIT_REPO_URL}@5.2.0"
20+
21+
find "$OUTDIR/wheels-repo/" -name '*.whl'
22+
find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
23+
ls "$OUTDIR"/work-dir/*/build.log || true
24+
25+
EXPECTED_FILES="
26+
$OUTDIR/wheels-repo/downloads/setuptools-*.whl
27+
$OUTDIR/wheels-repo/downloads/pbr-*.whl
28+
$OUTDIR/wheels-repo/downloads/stevedore-*.whl
29+
30+
$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
31+
$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
32+
33+
$OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
34+
$OUTDIR/sdists-repo/builds/setuptools-*.tar.gz
35+
$OUTDIR/sdists-repo/builds/pbr-*.tar.gz
36+
37+
$OUTDIR/work-dir/build-order.json
38+
$OUTDIR/work-dir/constraints.txt
39+
40+
$OUTDIR/bootstrap.log
41+
$OUTDIR/fromager-errors.log
42+
43+
$OUTDIR/work-dir/pbr-*/build.log
44+
$OUTDIR/work-dir/setuptools-*/build.log
45+
$OUTDIR/work-dir/stevedore-*/build.log
46+
"
47+
48+
pass=true
49+
for pattern in $EXPECTED_FILES; do
50+
if [ ! -f "${pattern}" ]; then
51+
echo "Did not find $pattern" 1>&2
52+
pass=false
53+
fi
54+
done
55+
56+
$pass

‎src/fromager/gitutils.py

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import logging
2+
import pathlib
3+
from urllib.parse import urlparse
4+
5+
from packaging.requirements import Requirement
6+
7+
from fromager import context, external_commands
8+
9+
logger = logging.getLogger(__name__)
10+
11+
12+
def git_clone(
13+
*,
14+
ctx: context.WorkContext,
15+
req: Requirement,
16+
output_dir: pathlib.Path,
17+
repo_url: str,
18+
tag: str | None = None,
19+
ref: str | None = None,
20+
submodules: bool | list[str] = False,
21+
) -> pathlib.Path:
22+
"""Clone a git repository"""
23+
if tag is not None and ref is not None:
24+
raise ValueError("tag and ref are mutually exclusive")
25+
26+
# Create a clean URL without any credentials for logging
27+
parsed_url = urlparse(repo_url)
28+
clean_url = parsed_url._replace(netloc=parsed_url.hostname or "").geturl()
29+
logger.info(
30+
"%s: cloning %s, tag %r, ref %r, submodules %r, into %s",
31+
req.name,
32+
clean_url,
33+
tag,
34+
ref,
35+
submodules,
36+
output_dir,
37+
)
38+
cmd: list[str] = ["git", "clone"]
39+
if tag is not None:
40+
# --branch works with branches and tags, but not with commits
41+
cmd.extend(["--branch", tag, "--depth", "1"])
42+
if submodules:
43+
if isinstance(submodules, list):
44+
for pathspec in submodules:
45+
cmd.append(f"--recurse-submodules={pathspec}")
46+
else:
47+
# all submodules
48+
cmd.append("--recurse-submodules")
49+
if tag is not None:
50+
cmd.append("--shallow-submodules")
51+
cmd.extend([repo_url, str(output_dir)])
52+
external_commands.run(cmd, network_isolation=False)
53+
54+
# --branch only works with names, so we have to checkout the reference we
55+
# actually want if it is not a name
56+
if ref is not None:
57+
external_commands.run(
58+
["git", "checkout", "--recurse-submodules", "--force", ref],
59+
cwd=str(output_dir),
60+
network_isolation=False,
61+
)
62+
63+
return output_dir

0 commit comments

Comments
 (0)