From 8dc4093f967de12acf1f58df0fbbdcba7a95dc01 Mon Sep 17 00:00:00 2001 From: Shubh Bapna Date: Tue, 12 Nov 2024 11:39:09 -0500 Subject: [PATCH] use a wheel server as a cache for bootstrapping by using a cache wheel server url, fromager can download already built wheels as and when needed to speed up the bootstrapping process by avoiding to actually build those wheels. Plus, if those wheels were built by fromager itself, then it can extract the build requirements files from it instead of recomputing them again. Signed-off-by: Shubh Bapna --- .github/workflows/test.yaml | 1 + .mergify.yml | 3 + e2e/test_bootstrap_build_tags.sh | 10 +- e2e/test_bootstrap_cache.sh | 126 ++++++++++++++++++++++ src/fromager/bootstrapper.py | 166 ++++++++++++++++++++--------- src/fromager/commands/bootstrap.py | 12 ++- src/fromager/dependencies.py | 10 +- src/fromager/wheels.py | 39 ++++--- 8 files changed, 296 insertions(+), 71 deletions(-) create mode 100755 e2e/test_bootstrap_cache.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b322f4a7..665c9021 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -84,6 +84,7 @@ jobs: - bootstrap_build_tags - bootstrap_prerelease - bootstrap_constraints + - bootstrap_cache - build - build_order - build_steps diff --git a/.mergify.yml b/.mergify.yml index 8a48cf97..b9f1cd4e 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -31,6 +31,7 @@ pull_request_rules: - check-success=e2e (3.11, 1.75, bootstrap_build_tags, ubuntu-latest) - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest) - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest) + - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest) - check-success=e2e (3.11, 1.75, build, ubuntu-latest) - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest) - check-success=e2e (3.11, 1.75, build_settings, ubuntu-latest) @@ -56,6 +57,8 @@ pull_request_rules: - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest) - check-success=e2e (3.12, 1.75, bootstrap_constraints, macos-latest) - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest) + - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest) + - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest) - check-success=e2e (3.12, 1.75, build, macos-latest) - check-success=e2e (3.12, 1.75, build, ubuntu-latest) - check-success=e2e (3.12, 1.75, build_order, macos-latest) diff --git a/e2e/test_bootstrap_build_tags.sh b/e2e/test_bootstrap_build_tags.sh index 29e55c96..0f8ebc1d 100755 --- a/e2e/test_bootstrap_build_tags.sh +++ b/e2e/test_bootstrap_build_tags.sh @@ -20,6 +20,8 @@ EXPECTED_FILES=" $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl " +start_local_wheel_server + pass=true for pattern in $EXPECTED_FILES; do if [ ! -f "${pattern}" ]; then @@ -38,10 +40,10 @@ fromager \ --wheels-repo="$OUTDIR/wheels-repo" \ --work-dir="$OUTDIR/work-dir" \ --settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \ - bootstrap 'stevedore==5.2.0' + bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0' -if ! grep -q "stevedore: have wheel version 5.2.0: $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1" "$OUTDIR/bootstrap.log"; then - echo "FAIL: Did not find log message have wheel version in $OUTDIR/bootstrap.log" 1>&2 +if ! grep -q "stevedore: found built wheel on cache server" "$OUTDIR/bootstrap.log"; then + echo "FAIL: Did not find log message found built wheel on cache server in $OUTDIR/bootstrap.log" 1>&2 pass=false fi @@ -55,7 +57,7 @@ fromager \ --wheels-repo="$OUTDIR/wheels-repo" \ --work-dir="$OUTDIR/work-dir" \ --settings-dir="$SCRIPTDIR/changelog_settings-2" \ - bootstrap 'stevedore==5.2.0' + bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0' EXPECTED_FILES=" $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl diff --git a/e2e/test_bootstrap_cache.sh b/e2e/test_bootstrap_cache.sh new file mode 100755 index 00000000..71c24ce7 --- /dev/null +++ b/e2e/test_bootstrap_cache.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test bootstrap while taking advantage of the cache wheel server + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" +pass=true + +# run fromager once to build wheels that can be used by a local wheel server +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$SCRIPTDIR/changelog_settings" \ + bootstrap 'stevedore==5.2.0' + +start_local_wheel_server +rm -rf "$OUTDIR/sdists-repo" +rm -rf "$OUTDIR/work-dir" + +# run fromager with the cache wheel server pointing to the local wheel server +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$SCRIPTDIR/changelog_settings" \ + --no-cleanup \ + bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0' + +EXPECTED_FILES=" +$OUTDIR/wheels-repo/downloads/setuptools-*.whl +$OUTDIR/wheels-repo/downloads/pbr-*.whl +$OUTDIR/wheels-repo/downloads/stevedore-*.whl + +$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz +$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz +$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz + +$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt +$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt +$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt + +$OUTDIR/work-dir/build-order.json +$OUTDIR/work-dir/constraints.txt +" + +for pattern in $EXPECTED_FILES; do + if [ ! -f "${pattern}" ]; then + echo "Did not find $pattern" 1>&2 + pass=false + fi +done + +$pass + +rm -rf "$OUTDIR/sdists-repo" +rm -rf "$OUTDIR/work-dir" +rm -rf "$OUTDIR/wheels-repo" + +# run fromager with the cache wheel server pointing to the pypi server +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$SCRIPTDIR/changelog_settings" \ + --no-cleanup \ + bootstrap --cache-wheel-server-url="https://pypi.org/simple" 'stevedore==5.2.0' + +EXPECTED_FILES=" +$OUTDIR/wheels-repo/downloads/setuptools-*.whl +$OUTDIR/wheels-repo/downloads/pbr-*.whl +$OUTDIR/wheels-repo/downloads/stevedore-*.whl + +$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz +$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz +$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz + +$OUTDIR/work-dir/build-order.json +$OUTDIR/work-dir/constraints.txt +" + +for pattern in $EXPECTED_FILES; do + if [ ! -f "${pattern}" ]; then + echo "Did not find $pattern" 1>&2 + pass=false + fi +done + +$pass + +NOT_EXPECTED_FILES=" +$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt +$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt +$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt +" + +for pattern in $NOT_EXPECTED_FILES; do + if [ -f "${pattern}" ]; then + echo "Found $pattern" 1>&2 + pass=false + fi +done + +$pass + +EXPECTED_DIR=" +$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info +$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info +$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info +" + +for pattern in $EXPECTED_DIR; do + if [ -d "${pattern}" ]; then + echo "Did not find $pattern" 1>&2 + pass=false + fi +done + +$pass diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 5489c69c..dac401d7 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -6,6 +6,7 @@ import pathlib import shutil import typing +from urllib.parse import urlparse from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name @@ -14,6 +15,7 @@ from . import ( build_environment, dependencies, + external_commands, finders, progress, resolver, @@ -36,10 +38,12 @@ def __init__( ctx: context.WorkContext, progressbar: progress.Progressbar | None = None, prev_graph: DependencyGraph | None = None, + cache_wheel_server_url: str | None = None, ) -> None: self.ctx = ctx self.progressbar = progressbar or progress.Progressbar(None) self.prev_graph = prev_graph + self.cache_wheel_server_url = cache_wheel_server_url self.why: list[tuple[RequirementType, Requirement, Version]] = [] # Push items onto the stack as we start to resolve their # dependencies so at the end we have a list of items that need to @@ -68,14 +72,14 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: pbi = self.ctx.package_build_info(req) if pbi.pre_built: resolved_version, wheel_url, wheel_filename, unpack_dir = ( - self._handle_prebuilt_sources(req, req_type) + self._resolve_and_download_prebuilt(req, req_type) ) # Remember that this is a prebuilt wheel, and where we got it. source_url = wheel_url source_url_type = str(SourceType.PREBUILT) else: resolved_version, source_url, source_filename, source_url_type = ( - self._handle_sources(req, req_type) + self._resolve_and_download_source(req, req_type) ) self._add_to_graph(req, req_type, resolved_version, source_url) @@ -101,19 +105,32 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: build_env = None sdist_root_dir = None if not pbi.pre_built: - ( - sdist_root_dir, - unpack_dir, - build_dependencies, - ) = self._prepare_source(req, source_filename, resolved_version) - - found_wheel_filename = self._is_wheel_built(req, resolved_version) - if not found_wheel_filename: + unpacked_cached_wheel, cached_wheel_filename = ( + self._download_wheel_from_cache(req, resolved_version) + ) + if not unpacked_cached_wheel: + sdist_root_dir = sources.prepare_source( + ctx=self.ctx, + req=req, + source_filename=source_filename, + version=resolved_version, + ) + unpack_dir = sdist_root_dir.parent + else: + sdist_root_dir = unpacked_cached_wheel + unpack_dir = unpacked_cached_wheel.parent + + # need to call this function irrespective of whether we had the wheel cached + # so that the build dependencies can be bootstrapped + build_dependencies = self._prepare_build_dependencies(req, sdist_root_dir) + + # skip building even if it is a non-fromager built wheel + if not cached_wheel_filename: wheel_filename, build_env = self._build( req, resolved_version, sdist_root_dir, build_dependencies ) else: - wheel_filename = found_wheel_filename + wheel_filename = cached_wheel_filename self._add_to_build_order( req=req, @@ -149,26 +166,6 @@ def _explain(self) -> str: for req_type, req, resolved_version in reversed(self.why) ) - def _is_wheel_built( - self, req: Requirement, resolved_version: Version - ) -> pathlib.Path | None: - pbi = self.ctx.package_build_info(req) - - # FIXME: This is a bit naive, but works for most wheels, including - # our more expensive ones, and there's not a way to know the - # actual name without doing most of the work to build the wheel. - wheel_filename = finders.find_wheel( - downloads_dir=self.ctx.wheels_downloads, - req=req, - dist_version=str(resolved_version), - build_tag=pbi.build_tag(resolved_version), - ) - if wheel_filename: - logger.info( - f"{req.name}: have wheel version {resolved_version}: {wheel_filename}" - ) - return wheel_filename - def _build( self, req: Requirement, @@ -217,17 +214,9 @@ def _build( ) return wheel_filename, build_env - def _prepare_source( - self, req: Requirement, source_filename: pathlib.Path, resolved_version: Version - ) -> tuple[pathlib.Path, pathlib.Path, set[Requirement]]: - sdist_root_dir = sources.prepare_source( - ctx=self.ctx, - req=req, - source_filename=source_filename, - version=resolved_version, - ) - unpack_dir = sdist_root_dir.parent - + def _prepare_build_dependencies( + self, req: Requirement, sdist_root_dir: pathlib.Path + ) -> set[Requirement]: build_system_dependencies = dependencies.get_build_system_dependencies( ctx=self.ctx, req=req, sdist_root_dir=sdist_root_dir ) @@ -253,11 +242,9 @@ def _prepare_source( ) return ( - sdist_root_dir, - unpack_dir, build_system_dependencies | build_backend_dependencies - | build_sdist_dependencies, + | build_sdist_dependencies ) def _handle_build_requirements( @@ -276,7 +263,7 @@ def _handle_build_requirements( build_environment.maybe_install(self.ctx, dep, build_type, str(resolved)) self.progressbar.update() - def _handle_sources( + def _resolve_and_download_source( self, req: Requirement, req_type: RequirementType ) -> tuple[Version, str, pathlib.Path, str]: source_url, resolved_version = self._resolve_source_with_history( @@ -294,7 +281,7 @@ def _handle_sources( return (resolved_version, source_url, source_filename, source_url_type) - def _handle_prebuilt_sources( + def _resolve_and_download_prebuilt( self, req: Requirement, req_type: RequirementType ) -> tuple[Version, str, pathlib.Path, pathlib.Path]: logger.info(f"{req.name}: {req_type} requirement {req} uses a pre-built wheel") @@ -316,11 +303,84 @@ def _handle_prebuilt_sources( logger.info(f"{req.name}: updating temporary mirror with pre-built wheel") shutil.copy(wheel_filename, dest_name) server.update_wheel_mirror(self.ctx) - unpack_dir = self.ctx.work_dir / f"{req.name}-{resolved_version}" - if not unpack_dir.exists(): - unpack_dir.mkdir() + unpack_dir = self._create_unpack_dir(req, resolved_version) return (resolved_version, wheel_url, wheel_filename, unpack_dir) + def _download_wheel_from_cache( + self, req: Requirement, resolved_version: Version + ) -> tuple[pathlib.Path | None, pathlib.Path | None]: + if not self.cache_wheel_server_url: + return None, None + logger.info( + f"{req.name}: checking if wheel was already uploaded to {self.cache_wheel_server_url}" + ) + try: + wheel_url, _ = resolver.resolve( + ctx=self.ctx, + req=Requirement(f"{req.name}=={resolved_version}"), + sdist_server_url=self.cache_wheel_server_url, + include_sdists=False, + include_wheels=True, + ) + wheelfile_name = pathlib.Path(urlparse(wheel_url).path) + pbi = self.ctx.package_build_info(req) + expected_build_tag = pbi.build_tag(resolved_version) + dist_name, dist_version, build_tag, _ = wheels.extract_info_from_wheel_file( + req, wheelfile_name + ) + if expected_build_tag and expected_build_tag != build_tag: + logger.info( + f"{req.name}: found wheel for {resolved_version} in cache but build tag does not match. Got {build_tag} but expected {expected_build_tag}" + ) + return None, None + + cached_wheel = wheels.download_wheel( + req=req, wheel_url=wheel_url, output_directory=self.ctx.wheels_downloads + ) + server.update_wheel_mirror(self.ctx) + logger.info(f"{req.name}: found built wheel on cache server") + unpack_dir = self._create_unpack_dir(req, resolved_version) + # unpack the wheel + external_commands.run( + ["wheel", "unpack", str(cached_wheel), "--dest", unpack_dir], + cwd=unpack_dir, + ) + + dist_filename = f"{dist_name}-{dist_version}" + metadata_dir = unpack_dir / dist_filename / f"{dist_filename}.dist-info" + + # prepare files cached by fromager by placing them in the parent directory of the return path + try: + shutil.copy( + metadata_dir + / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{dependencies.BUILD_SYSTEM_REQ_FILE_NAME}", + unpack_dir / dependencies.BUILD_SYSTEM_REQ_FILE_NAME, + ) + shutil.copy( + metadata_dir + / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{dependencies.BUILD_BACKEND_REQ_FILE_NAME}", + unpack_dir / dependencies.BUILD_BACKEND_REQ_FILE_NAME, + ) + shutil.copy( + metadata_dir + / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{dependencies.BUILD_SDIST_REQ_FILE_NAME}", + unpack_dir / dependencies.BUILD_SDIST_REQ_FILE_NAME, + ) + logger.info(f"{req.name}: extracted build requirements from wheel") + return unpack_dir / dist_filename, cached_wheel + except Exception: + # implies that the wheel server hosted non-fromager built wheels + logger.info( + f"{req.name}: could not extract build requirements from wheel" + ) + shutil.rmtree(unpack_dir) + return None, cached_wheel + except Exception: + logger.info( + f"{req.name}: did not find wheel for {resolved_version} in {self.cache_wheel_server_url}" + ) + return None, None + def _resolve_source_with_history( self, req: Requirement, @@ -465,6 +525,12 @@ def _resolve_from_version_source( ) return None + def _create_unpack_dir(self, req: Requirement, resolved_version: Version): + unpack_dir = self.ctx.work_dir / f"{req.name}-{resolved_version}" + if not unpack_dir.exists(): + unpack_dir.mkdir() + return unpack_dir + def _cleanup( self, req: Requirement, diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index f8ec88ff..ed5b0fbf 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -67,12 +67,19 @@ def _get_requirements_from_args( type=clickext.ClickPath(), help="graph file produced from a previous bootstrap", ) +@click.option( + "-c", + "--cache-wheel-server-url", + "cache_wheel_server_url", + help="url to a wheel server from where fromager can download the wheels that it has built before", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( wkctx: context.WorkContext, requirements_files: list[pathlib.Path], previous_bootstrap_file: pathlib.Path | None, + cache_wheel_server_url: str | None, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -81,6 +88,7 @@ def bootstrap( and optional version constraints. """ + logger.info(f"cache wheel server url: {cache_wheel_server_url}") to_build = _get_requirements_from_args(toplevel, requirements_files) if not to_build: raise RuntimeError( @@ -133,7 +141,9 @@ def bootstrap( ) with progress.progress_context(total=len(to_build)) as progressbar: - bt = bootstrapper.Bootstrapper(wkctx, progressbar, prev_graph) + bt = bootstrapper.Bootstrapper( + wkctx, progressbar, prev_graph, cache_wheel_server_url + ) for req in to_build: bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL) diff --git a/src/fromager/dependencies.py b/src/fromager/dependencies.py index c5a093ac..7b7af384 100644 --- a/src/fromager/dependencies.py +++ b/src/fromager/dependencies.py @@ -18,6 +18,10 @@ logger = logging.getLogger(__name__) +BUILD_SYSTEM_REQ_FILE_NAME = "build-system-requirements.txt" +BUILD_BACKEND_REQ_FILE_NAME = "build-backend-requirements.txt" +BUILD_SDIST_REQ_FILE_NAME = "build-sdist-requirements.txt" + def get_build_system_dependencies( *, @@ -30,7 +34,7 @@ def get_build_system_dependencies( ) pbi = ctx.package_build_info(req) - build_system_req_file = sdist_root_dir.parent / "build-system-requirements.txt" + build_system_req_file = sdist_root_dir.parent / BUILD_SYSTEM_REQ_FILE_NAME if build_system_req_file.exists(): logger.info( f"{req.name}: loading build system dependencies from {build_system_req_file.name}" @@ -95,7 +99,7 @@ def get_build_backend_dependencies( ) pbi = ctx.package_build_info(req) - build_backend_req_file = sdist_root_dir.parent / "build-backend-requirements.txt" + build_backend_req_file = sdist_root_dir.parent / BUILD_BACKEND_REQ_FILE_NAME if build_backend_req_file.exists(): logger.info( f"{req.name}: loading build backend dependencies from {build_backend_req_file.name}" @@ -152,7 +156,7 @@ def get_build_sdist_dependencies( ) pbi = ctx.package_build_info(req) - build_sdist_req_file = sdist_root_dir.parent / "build-sdist-requirements.txt" + build_sdist_req_file = sdist_root_dir.parent / BUILD_SDIST_REQ_FILE_NAME if build_sdist_req_file.exists(): logger.info( f"{req.name}: loading build sdist dependencies from {build_sdist_req_file.name}" diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 1577957d..e1ab7c9a 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -16,7 +16,8 @@ import elfdeps import tomlkit from packaging.requirements import Requirement -from packaging.utils import canonicalize_name, parse_wheel_filename +from packaging.tags import Tag +from packaging.utils import BuildTag, canonicalize_name, parse_wheel_filename from packaging.version import Version from . import external_commands, overrides, requirements_file, resolver, sources @@ -29,6 +30,7 @@ FROMAGER_BUILD_SETTINGS = "fromager-build-settings" FROMAGER_ELF_PROVIDES = "fromager-elf-provides.txt" FROMAGER_ELF_REQUIRES = "fromager-elf-requires.txt" +FROMAGER_BUILD_REQ_PREFIX = "fromager" def _extra_metadata_elfdeps( @@ -99,6 +101,24 @@ def _extra_metadata_elfdeps( return elfinfos +def extract_info_from_wheel_file( + req: Requirement, wheel_file: pathlib.Path +) -> tuple[str, Version, BuildTag, frozenset[Tag]]: + # parse_wheel_filename normalizes the dist name, however the dist-info + # directory uses the verbatim distribution name from the wheel file. + # Packages with upper case names like "MarkupSafe" are affected. + dist_name_normalized, dist_version, build_tag, wheel_tags = parse_wheel_filename( + wheel_file.name + ) + dist_name = wheel_file.name.split("-", 1)[0] + if dist_name_normalized != canonicalize_name(dist_name): + # sanity check, should never fail + raise ValueError( + f"{req.name}: {dist_name_normalized} does not match {dist_name}" + ) + return (dist_name, dist_version, build_tag, wheel_tags) + + def default_add_extra_metadata_to_wheels( ctx: context.WorkContext, req: Requirement, @@ -120,18 +140,9 @@ def add_extra_metadata_to_wheels( wheel_file: pathlib.Path, ) -> pathlib.Path: pbi = ctx.package_build_info(req) - # parse_wheel_filename normalizes the dist name, however the dist-info - # directory uses the verbatim distribution name from the wheel file. - # Packages with upper case names like "MarkupSafe" are affected. - dist_name_normalized, dist_version, _, wheel_tags = parse_wheel_filename( - wheel_file.name + dist_name, dist_version, _, wheel_tags = extract_info_from_wheel_file( + req, wheel_file ) - dist_name = wheel_file.name.split("-", 1)[0] - if dist_name_normalized != canonicalize_name(dist_name): - # sanity check, should never fail - raise ValueError( - f"{req.name}: {dist_name_normalized} does not match {dist_name}" - ) dist_filename = f"{dist_name}-{dist_version}" extra_data_plugin = overrides.find_override_method( @@ -181,7 +192,9 @@ def add_extra_metadata_to_wheels( req_files = sdist_root_dir.parent.glob("*-requirements.txt") for req_file in req_files: - shutil.copy(req_file, dist_info_dir / f"fromager-{req_file.name}") + shutil.copy( + req_file, dist_info_dir / f"{FROMAGER_BUILD_REQ_PREFIX}-{req_file.name}" + ) if any(tag.platform != "all" for tag in wheel_tags): # platlib wheel