From 19c45fbfa2a37f5aef70d57773432f11dec7d3e4 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 16 Jul 2024 23:30:58 -0700 Subject: [PATCH] Add `--scie` option to produce native PEX exes. (#2466) You can now specify `--scie {eager,lazy}` when building a PEX file and one or more additional native executable PEX scies will be produced along side the PEX file. These PEX scies will contain a portable CPython interpreter from [Python Standalone Builds][PBS] in the `--scie eager` case and will instead fetch a portable CPython interpreter just in time on first boot on a given machine if needed in the `--scie lazy` case. Although Pex will pick the target platforms and target portable CPython interpreter version automatically, if more control is desired over which platforms are targeted and which Python version is used, then `--scie-platform`, `--scie-pbs-release`, and `--scie-python-version` can be specified. Closes #636 Closes #1007 Closes #2096 [PBS]: https://github.com/indygreg/python-build-standalone --- .github/workflows/ci.yml | 3 + CHANGES.md | 17 ++ dtox.sh | 14 +- pex/bin/pex.py | 45 ++- pex/pex_bootstrapper.py | 2 +- pex/platforms.py | 2 +- pex/resolve/resolver_configuration.py | 15 + pex/scie/__init__.py | 213 ++++++++++++++ pex/scie/configure-binding.py | 30 ++ pex/scie/model.py | 259 +++++++++++++++++ pex/scie/science.py | 363 +++++++++++++++++++++++ pex/targets.py | 17 +- pex/variables.py | 7 +- pex/venv/installer.py | 2 + pex/version.py | 2 +- tests/integration/scie/__init__.py | 2 + tests/integration/scie/test_pex_scie.py | 371 ++++++++++++++++++++++++ tox.ini | 2 + 18 files changed, 1351 insertions(+), 15 deletions(-) create mode 100644 pex/scie/__init__.py create mode 100644 pex/scie/configure-binding.py create mode 100644 pex/scie/model.py create mode 100644 pex/scie/science.py create mode 100644 tests/integration/scie/__init__.py create mode 100644 tests/integration/scie/test_pex_scie.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cf23e4c..5c849752a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ env: # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391. _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10" _PEX_PEXPECT_TIMEOUT: 10 + # We have integration tests that exercise `--scie` support and these can trigger downloads from + # GitHub Releases that needed elevated rate limit quota, which this gives. + SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }} concurrency: group: CI-${{ github.ref }} # Queue on all branches and tags, but only cancel overlapping PR burns. diff --git a/CHANGES.md b/CHANGES.md index fc44c3984..d35564b33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Release Notes +## 2.11.0 + +This release adds support for creating native PEX executables that +contain their own hermetic CPython interpreter courtesy of +[Python Standalone Builds][PBS] and the [Science project][scie]. + +You can now specify `--scie {eager,lazy}` when building a PEX file and +one or more native executable PEX scies will be produced (one for each +platform the PEX supports). These PEX scies are single file +executables that look and behave like traditional PEXes, but unlike +PEXes they can run on a machine with no Python interpreter available. + +[PBS]: https://github.com/indygreg/python-build-standalone +[scie]: https://github.com/a-scie + +* Add `--scie` option to produce native PEX exes. (#2466) + ## 2.10.1 This release fixes a long-standing bug in Pex parsing of editable diff --git a/dtox.sh b/dtox.sh index 8e258634c..c3d306943 100755 --- a/dtox.sh +++ b/dtox.sh @@ -117,11 +117,21 @@ if [[ -n "${TERM:-}" ]]; then ) fi +if [[ -f "${HOME}/.netrc" ]]; then + DOCKER_ARGS+=( + --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" + ) +fi + +if [[ -d "${HOME}/.ssh" ]]; then + DOCKER_ARGS+=( + --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" + ) +fi + exec docker run \ --rm \ --volume pex-tmp:/tmp \ - --volume "${HOME}/.netrc:${CONTAINER_HOME}/.netrc" \ - --volume "${HOME}/.ssh:${CONTAINER_HOME}/.ssh" \ --volume "pex-root:${CONTAINER_HOME}/.pex" \ --volume pex-caches:/development/pex_dev \ --volume "${ROOT}:/development/pex" \ diff --git a/pex/bin/pex.py b/pex/bin/pex.py index c60aa1627..22925f4ad 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -16,7 +16,7 @@ from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser from textwrap import TextWrapper -from pex import dependency_configuration, pex_warnings +from pex import dependency_configuration, pex_warnings, scie from pex.argparse import HandleBoolAction from pex.commands.command import ( GlobalConfigurationError, @@ -29,6 +29,7 @@ from pex.dist_metadata import Requirement from pex.docs.command import serve_html_docs from pex.enum import Enum +from pex.fetcher import URLFetcher from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints from pex.layout import Layout, ensure_installed @@ -56,6 +57,7 @@ from pex.resolve.resolver_options import create_pip_configuration from pex.resolve.resolvers import Unsatisfiable, sorted_requirements from pex.result import Error, ResultError, catch, try_ +from pex.scie import ScieConfiguration from pex.targets import Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast @@ -314,6 +316,8 @@ def configure_clp_pex_options(parser): ), ) + scie.register_options(group) + group.add_argument( "--always-write-cache", dest="always_write_cache", @@ -1233,6 +1237,27 @@ def do_main( cmdline, # type: List[str] env, # type: Dict[str, str] ): + scie_options = scie.extract_options(options) + if scie_options and not options.pex_name: + raise ValueError( + "You must specify `-o`/`--output-file` to use `{scie_options}`.".format( + scie_options=scie.render_options(scie_options) + ) + ) + scie_configuration = None # type: Optional[ScieConfiguration] + if scie_options: + scie_configuration = scie_options.create_configuration(targets=targets) + if not scie_configuration: + raise ValueError( + "You selected `{scie_options}`, but none of the selected targets have " + "compatible interpreters that can be embedded to form a scie:\n{targets}".format( + scie_options=scie.render_options(scie_options), + targets="\n".join( + target.render_description() for target in targets.unique_targets() + ), + ) + ) + with TRACER.timed("Building pex"): pex_builder = build_pex( requirement_configuration=requirement_configuration, @@ -1276,6 +1301,24 @@ def do_main( verbose=options.seed == Seed.VERBOSE, ) print(seed_info) + if scie_configuration: + url_fetcher = URLFetcher( + network_configuration=resolver_configuration.network_configuration, + password_entries=resolver_configuration.repos_configuration.password_entries, + handle_file_urls=True, + ) + with TRACER.timed("Building scie(s)"): + for scie_info in scie.build( + configuration=scie_configuration, pex_file=pex_file, url_fetcher=url_fetcher + ): + log( + "Saved PEX scie for CPython {version} on {platform} to {scie}".format( + version=scie_info.target.version_str, + platform=scie_info.platform, + scie=os.path.relpath(scie_info.file), + ), + V=options.verbosity, + ) else: if not _compatible_with_current_platform(interpreter, targets.platforms): log("WARNING: attempting to run PEX with incompatible platforms!", V=1) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index a097736fa..e3609efc9 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -314,7 +314,7 @@ def gather_constraints(): path=( os.pathsep.join(ENV.PEX_PYTHON_PATH) if ENV.PEX_PYTHON_PATH - else os.getenv("PATH") + else os.getenv("PATH", "(The PATH is empty!)") ) ) ) diff --git a/pex/platforms.py b/pex/platforms.py index f43950e87..a1693347d 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -29,7 +29,7 @@ def _normalize_platform(platform): # type: (str) -> str - return platform.replace("-", "_").replace(".", "_") + return platform.lower().replace("-", "_").replace(".", "_") @attr.s(frozen=True) diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 83badd4fb..61f788fcd 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -199,9 +199,24 @@ class PexRepositoryConfiguration(object): network_configuration = attr.ib(default=NetworkConfiguration()) # type: NetworkConfiguration transitive = attr.ib(default=True) # type: bool + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return ReposConfiguration() + @attr.s(frozen=True) class LockRepositoryConfiguration(object): parse_lock = attr.ib() # type: Callable[[], Union[Lockfile, Error]] lock_file_path = attr.ib() # type: str pip_configuration = attr.ib() # type: PipConfiguration + + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return self.pip_configuration.repos_configuration + + @property + def network_configuration(self): + # type: () -> NetworkConfiguration + return self.pip_configuration.network_configuration diff --git a/pex/scie/__init__.py b/pex/scie/__init__.py new file mode 100644 index 000000000..3e168c9d8 --- /dev/null +++ b/pex/scie/__init__.py @@ -0,0 +1,213 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +from argparse import Namespace, _ActionsContainer + +from pex.compatibility import urlparse +from pex.fetcher import URLFetcher +from pex.orderedset import OrderedSet +from pex.pep_440 import Version +from pex.scie import science +from pex.scie.model import ( + ScieConfiguration, + ScieInfo, + ScieOptions, + SciePlatform, + ScieStyle, + ScieTarget, +) +from pex.scie.science import SCIENCE_RELEASES_URL, SCIENCE_REQUIREMENT +from pex.typing import TYPE_CHECKING, cast +from pex.variables import ENV, Variables + +if TYPE_CHECKING: + from typing import Iterator, Optional, Tuple, Union + + +__all__ = ( + "ScieConfiguration", + "ScieInfo", + "SciePlatform", + "ScieStyle", + "ScieTarget", + "build", + "extract_options", + "register_options", + "render_options", +) + + +def register_options(parser): + # type: (_ActionsContainer) -> None + + parser.add_argument( + "--scie", + "--par", + dest="scie_style", + default=None, + type=ScieStyle.for_value, + choices=ScieStyle.values(), + help=( + "Create one or more native executable scies from your PEX that include a portable " + "CPython interpreter along with your PEX making for a truly hermetic PEX that can run " + "on machines with no Python installed at all. If your PEX has multiple targets, " + "whether `--platform`s, `--complete-platform`s or local interpreters in any " + "combination, then one PEX scie will be made for each platform, selecting the latest " + "compatible portable CPython interpreter. Note that only CPython>=3.8 is supported. If " + "you'd like to explicitly control the target platforms or the exact portable CPython " + "selected, see `--scie-platform`, `--scie-pbs-release` and `--scie-python-version`. " + "Specifying `--scie {lazy}` will fetch the portable CPython interpreter just in time " + "on first boot of the PEX scie on a given machine if needed. The URL(s) to fetch the " + "portable CPython interpreter from can be customized by exporting the " + "PEX_BOOTSTRAP_URLS environment variable pointing to a json file with the format: " + '`{{"ptex": {{: , ...}}}}` where the file names should match those ' + "found via `SCIE=inspect | jq .ptex` with appropriate replacement URLs. " + "Specifying `--scie {eager}` will embed the portable CPython interpreter in your PEX " + "scie making for a larger file, but requiring no internet access to boot. If you have " + "customization needs not addressed by the Pex `--scie*` options, consider using " + "`science` to build your scies (which is what Pex uses behind the scenes); see: " + "https://science.scie.app.".format(lazy=ScieStyle.LAZY, eager=ScieStyle.EAGER) + ), + ) + parser.add_argument( + "--scie-platform", + dest="scie_platforms", + default=[], + action="append", + type=SciePlatform.parse, + choices=[ + platform + for platform in SciePlatform.values() + if platform not in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64) + ], + help=( + "The platform to produce the native PEX scie executable for. Can be specified multiple " + "times. You can use a value of 'current' to select the current platform. If left " + "unspecified, the platforms implied by the targets selected to build the PEX with are " + "used. Those targets are influenced by the current interpreter running Pex as well as " + "use of `--python`, `--interpreter-constraint`, `--platform` or `--complete-platform` " + "options." + ), + ) + parser.add_argument( + "--scie-pbs-release", + dest="scie_pbs_release", + default=None, + type=str, + help=( + "The Python Standalone Builds release to use. Currently releases are dates of the form " + "YYYYMMDD, e.g.: '20240713'. See their GitHub releases page at " + "https://github.com/indygreg/python-build-standalone/releases to discover available " + "releases. If left unspecified the latest release is used. N.B.: The latest lookup is " + "cached for 5 days. To force a fresh lookup you can remove the cache at " + "/science/downloads." + ), + ) + parser.add_argument( + "--scie-python-version", + dest="scie_python_version", + default=None, + type=Version, + help=( + "The portable CPython version to select. Can be either in `.` form; " + "e.g.: '3.11', or else fully specified as `..`; e.g.: '3.11.3'. " + "If you don't specify this option, Pex will do its best to guess appropriate portable " + "CPython versions. N.B.: Python Standalone Builds does not provide all patch versions; " + "so you should check their releases at " + "https://github.com/indygreg/python-build-standalone/releases if you wish to pin down " + "to the patch level." + ), + ) + parser.add_argument( + "--scie-science-binary", + dest="scie_science_binary", + default=None, + type=str, + help=( + "The file path of a `science` binary or a URL to use to fetch the `science` binary " + "when there is no `science` on the PATH with a version matching {science_requirement}. " + "Pex uses the official `science` releases at {science_releases_url} by default.".format( + science_requirement=SCIENCE_REQUIREMENT, science_releases_url=SCIENCE_RELEASES_URL + ) + ), + ) + + +def render_options(options): + # type: (ScieOptions) -> str + + args = ["--scie", str(options.style)] + for platform in options.platforms: + args.append("--scie-platform") + args.append(str(platform)) + if options.pbs_release: + args.append("--scie-pbs-release") + args.append(options.pbs_release) + if options.python_version: + args.append("--scie-python-version") + args.append(".".join(map(str, options.python_version))) + if options.science_binary_url: + args.append("--scie-science-binary") + args.append(options.science_binary_url) + return " ".join(args) + + +def extract_options(options): + # type: (Namespace) -> Optional[ScieOptions] + + if not options.scie_style: + return None + + python_version = None # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + if options.scie_python_version: + if ( + not options.scie_python_version.parsed_version.release + or len(options.scie_python_version.parsed_version.release) < 2 + ): + raise ValueError( + "Invalid Python version: '{python_version}'.\n" + "Must be in the form `.` or `..`".format( + python_version=options.scie_python_version + ) + ) + python_version = cast( + "Union[Tuple[int, int], Tuple[int, int, int]]", + options.scie_python_version.parsed_version.release, + ) + if python_version < (3, 8): + raise ValueError( + "Invalid Python version: '{python_version}'.\n" + "Scies are built using Python Standalone Builds which only supports Python >=3.8.\n" + "To find supported Python versions, you can browse the releases here:\n" + " https://github.com/indygreg/python-build-standalone/releases".format( + python_version=options.scie_python_version + ) + ) + + science_binary_url = options.scie_science_binary + if science_binary_url: + url_info = urlparse.urlparse(options.scie_science_binary) + if not url_info.scheme and url_info.path and os.path.isfile(url_info.path): + science_binary_url = "file://{path}".format(path=os.path.abspath(url_info.path)) + + return ScieOptions( + style=options.scie_style, + platforms=tuple(OrderedSet(options.scie_platforms)), + pbs_release=options.scie_pbs_release, + python_version=python_version, + science_binary_url=science_binary_url, + ) + + +def build( + configuration, # type: ScieConfiguration + pex_file, # type: str + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> Iterator[ScieInfo] + + return science.build(configuration, pex_file, url_fetcher=url_fetcher, env=env) diff --git a/pex/scie/configure-binding.py b/pex/scie/configure-binding.py new file mode 100644 index 000000000..8f3b1c049 --- /dev/null +++ b/pex/scie/configure-binding.py @@ -0,0 +1,30 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import print_function + +import os +import sys + + +def write_bindings( + env_file, # type: str + installed_pex_dir, # type: str +): + # type: (...) -> None + with open(env_file, "a") as fp: + print("PYTHON=" + sys.executable, file=fp) + print("PEX=" + os.path.realpath(os.path.join(installed_pex_dir, "__main__.py")), file=fp) + + +if __name__ == "__main__": + write_bindings( + env_file=os.environ["SCIE_BINDING_ENV"], + installed_pex_dir=( + # The zipapp case: + os.environ["_PEX_SCIE_INSTALLED_PEX_DIR"] + # The --venv case: + or os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable))) + ), + ) + sys.exit(0) diff --git a/pex/scie/model.py b/pex/scie/model.py new file mode 100644 index 000000000..07e46c6a8 --- /dev/null +++ b/pex/scie/model.py @@ -0,0 +1,259 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import itertools +import os +import platform +from collections import defaultdict + +from pex.enum import Enum +from pex.platforms import Platform +from pex.targets import Targets +from pex.third_party.packaging import tags # noqa +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, Optional, Set, Tuple, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +class ScieStyle(Enum["ScieStyle.Value"]): + class Value(Enum.Value): + pass + + LAZY = Value("lazy") + EAGER = Value("eager") + + +class _CurrentPlatform(object): + def __get__(self, obj, objtype=None): + # type: (...) -> SciePlatform.Value + if not hasattr(self, "_current"): + system = platform.system().lower() + machine = platform.machine().lower() + if "linux" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.LINUX_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.LINUX_X86_64 + elif "darwin" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.MACOS_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.MACOS_X86_64 + elif "windows" == system: + if machine in ("aarch64", "arm64"): + self._current = SciePlatform.WINDOWS_AARCH64 + elif machine in ("amd64", "x86_64"): + self._current = SciePlatform.WINDOWS_X86_64 + if not hasattr(self, "_current"): + raise ValueError( + "The current operating system / machine pair is not supported!: " + "{system} / {machine}".format(system=system, machine=machine) + ) + return self._current + + +class SciePlatform(Enum["SciePlatform.Value"]): + class Value(Enum.Value): + @property + def extension(self): + # type: () -> str + return ( + ".exe" + if self in (SciePlatform.WINDOWS_AARCH64, SciePlatform.WINDOWS_X86_64) + else "" + ) + + def binary_name(self, binary_name): + # type: (str) -> str + return "{binary_name}{extension}".format( + binary_name=binary_name, extension=self.extension + ) + + def qualified_binary_name(self, binary_name): + # type: (str) -> str + return "{binary_name}-{platform}{extension}".format( + binary_name=binary_name, platform=self, extension=self.extension + ) + + def qualified_file_name(self, file_name): + # type: (str) -> str + stem, ext = os.path.splitext(file_name) + return "{stem}-{platform}{ext}".format(stem=stem, platform=self, ext=ext) + + LINUX_AARCH64 = Value("linux-aarch64") + LINUX_X86_64 = Value("linux-x86_64") + MACOS_AARCH64 = Value("macos-aarch64") + MACOS_X86_64 = Value("macos-x86_64") + WINDOWS_AARCH64 = Value("windows-x86_64") + WINDOWS_X86_64 = Value("windows-aarch64") + CURRENT = _CurrentPlatform() + + @classmethod + def parse(cls, value): + # type: (str) -> SciePlatform.Value + return cls.CURRENT if "current" == value else cls.for_value(value) + + +@attr.s(frozen=True) +class ScieTarget(object): + platform = attr.ib() # type: SciePlatform.Value + python_version = attr.ib() # type: Union[Tuple[int, int], Tuple[int, int, int]] + pbs_release = attr.ib(default=None) # type: Optional[str] + + @property + def version_str(self): + # type: () -> str + return ".".join(map(str, self.python_version)) + + +@attr.s(frozen=True) +class ScieInfo(object): + style = attr.ib() # type: ScieStyle.Value + target = attr.ib() # type: ScieTarget + file = attr.ib() # type: str + + @property + def platform(self): + # type: () -> SciePlatform.Value + return self.target.platform + + @property + def python_version(self): + # type: () -> Union[Tuple[int, int], Tuple[int, int, int]] + return self.target.python_version + + +@attr.s(frozen=True) +class ScieOptions(object): + style = attr.ib(default=ScieStyle.LAZY) # type: ScieStyle.Value + platforms = attr.ib(default=()) # type: Tuple[SciePlatform.Value, ...] + pbs_release = attr.ib(default=None) # type: Optional[str] + python_version = attr.ib( + default=None + ) # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + science_binary_url = attr.ib(default=None) # type: Optional[str] + + def create_configuration(self, targets): + # type: (Targets) -> ScieConfiguration + return ScieConfiguration.from_targets(self, targets) + + +@attr.s(frozen=True) +class ScieConfiguration(object): + @classmethod + def from_tags( + cls, + options, # type: ScieOptions + tags, # type: Iterable[tags.Tag] + ): + # type: (...) -> ScieConfiguration + return cls._from_platforms( + options=options, platforms=tuple(Platform.from_tag(tag) for tag in tags) + ) + + @classmethod + def from_targets( + cls, + options, # type: ScieOptions + targets, # type: Targets + ): + # type: (...) -> ScieConfiguration + return cls._from_platforms( + options=options, + platforms=tuple(target.platform for target in targets.unique_targets()), + ) + + @classmethod + def _from_platforms( + cls, + options, # type: ScieOptions + platforms, # type: Iterable[Platform] + ): + # type: (...) -> ScieConfiguration + + python_version = options.python_version + python_versions_by_platform = defaultdict( + set + ) # type: DefaultDict[SciePlatform.Value, Set[Union[Tuple[int, int], Tuple[int, int, int]]]] + for plat in platforms: + if python_version: + plat_python_version = python_version + elif len(plat.version_info) < 2: + continue + else: + # Here were guessing an available PBS CPython version. Since a triple is unlikely to + # hit, we just use major / minor. If the user wants control they can specify + # options.python_version via `--scie-python-version`. + plat_python_version = cast( + "Union[Tuple[int, int], Tuple[int, int, int]]", plat.version_info + )[:2] + + # We use Python Build Standalone to create scies, and we know it does not support + # CPython<3.8. + if plat_python_version < (3, 8): + continue + + # We use Python Build Standalone to create scies, and we know it only provides CPython + # interpreters. + if plat.impl not in ("py", "cp"): + continue + + platform_str = plat.platform + is_aarch64 = "arm64" in platform_str or "aarch64" in platform_str + is_x86_64 = "amd64" in platform_str or "x86_64" in platform_str + if not is_aarch64 ^ is_x86_64: + continue + + if "linux" in platform_str: + scie_platform = ( + SciePlatform.LINUX_AARCH64 if is_aarch64 else SciePlatform.LINUX_X86_64 + ) + elif "mac" in platform_str: + scie_platform = ( + SciePlatform.MACOS_AARCH64 if is_aarch64 else SciePlatform.MACOS_X86_64 + ) + elif "win" in platform_str: + scie_platform = ( + SciePlatform.WINDOWS_AARCH64 if is_aarch64 else SciePlatform.WINDOWS_X86_64 + ) + else: + continue + + python_versions_by_platform[scie_platform].add(plat_python_version) + + for explicit_platform in options.platforms: + if explicit_platform not in python_versions_by_platform: + if options.python_version: + python_versions_by_platform[explicit_platform] = {options.python_version} + else: + python_versions_by_platform[explicit_platform] = set( + itertools.chain.from_iterable(python_versions_by_platform.values()) + ) + if options.platforms: + for configured_platform in tuple(python_versions_by_platform): + if configured_platform not in options.platforms: + python_versions_by_platform.pop(configured_platform, None) + + scie_targets = tuple( + ScieTarget( + platform=scie_platform, + pbs_release=options.pbs_release, + python_version=max(python_versions), + ) + for scie_platform, python_versions in sorted(python_versions_by_platform.items()) + ) + return cls(options=options, targets=tuple(scie_targets)) + + options = attr.ib() # type: ScieOptions + targets = attr.ib() # type: Tuple[ScieTarget, ...] + + def __len__(self): + # type: () -> int + return len(self.targets) diff --git a/pex/scie/science.py b/pex/scie/science.py new file mode 100644 index 000000000..509358941 --- /dev/null +++ b/pex/scie/science.py @@ -0,0 +1,363 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import re +import shutil +import subprocess +from collections import OrderedDict +from subprocess import CalledProcessError + +from pex.atomic_directory import atomic_directory +from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open +from pex.compatibility import shlex_quote +from pex.exceptions import production_assert +from pex.fetcher import URLFetcher +from pex.hashing import Sha256 +from pex.layout import Layout +from pex.pep_440 import Version +from pex.pex_info import PexInfo +from pex.result import Error, try_ +from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget +from pex.third_party.packaging.specifiers import SpecifierSet +from pex.third_party.packaging.version import InvalidVersion +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING, cast +from pex.util import CacheHelper +from pex.variables import ENV, Variables, unzip_dir_relpath + +if TYPE_CHECKING: + from typing import Any, Dict, Iterator, Optional, Union, cast + + import attr # vendor:skip + import toml # vendor:skip +else: + from pex.third_party import attr, toml + + +@attr.s(frozen=True) +class Manifest(object): + target = attr.ib() # type: ScieTarget + path = attr.ib() # type: str + + def binary_name(self, binary_name): + # type: (str) -> str + return self.target.platform.binary_name(binary_name) + + def qualified_binary_name(self, binary_name): + # type: (str) -> str + return self.target.platform.qualified_binary_name(binary_name) + + +SCIENCE_RELEASES_URL = "https://github.com/a-scie/lift/releases" +MIN_SCIENCE_VERSION = Version("0.3.0") +SCIENCE_REQUIREMENT = SpecifierSet("~={min_version}".format(min_version=MIN_SCIENCE_VERSION)) + + +def _science_binary_url(suffix=""): + # type: (str) -> str + return "{science_releases_url}/download/v{version}/{binary}{suffix}".format( + science_releases_url=SCIENCE_RELEASES_URL, + version=MIN_SCIENCE_VERSION.raw, + binary=SciePlatform.CURRENT.qualified_binary_name("science-fat"), + suffix=suffix, + ) + + +PTEX_VERSION = "1.1.1" +SCIE_JUMP_VERSION = "1.1.1" + + +def create_manifests( + configuration, # type: ScieConfiguration + name, # type: str + pex_info, # type: PexInfo + layout, # type: Layout.Value +): + # type: (...) -> Iterator[Manifest] + + pex_root = "{scie.bindings}/pex_root" + if pex_info.venv: + # We let the configure-binding calculate the venv dir at runtime since it depends on the + # interpreter executing the venv PEX. + installed_pex_dir = "" + elif layout is Layout.LOOSE: + installed_pex_dir = "{pex}" + else: + production_assert(pex_info.pex_hash is not None) + pex_hash = cast(str, pex_info.pex_hash) + installed_pex_dir = os.path.join(pex_root, unzip_dir_relpath(pex_hash)) + + env_default = { + "PEX_ROOT": pex_root, + } + + lift = { + "name": name, + "ptex": { + "id": "ptex", + "version": PTEX_VERSION, + "argv1": "{scie.env.PEX_BOOTSTRAP_URLS={scie.lift}}", + }, + "scie_jump": {"version": SCIE_JUMP_VERSION}, + "files": [{"name": "configure-binding.py"}, {"name": "pex"}], + "commands": [ + { + "env": {"default": env_default}, + "exe": "{scie.bindings.configure:PYTHON}", + "args": ["{scie.bindings.configure:PEX}"], + } + ], + "bindings": [ + { + "env": { + "default": env_default, + "remove_exact": ["PATH"], + "remove_re": ["PEX_.*"], + "replace": { + "PEX_INTERPRETER": "1", + "_PEX_SCIE_INSTALLED_PEX_DIR": installed_pex_dir, + # We can get a warning about too-long script shebangs, but this is not + # relevant since we above run the PEX via python and not via shebang. + "PEX_EMIT_WARNINGS": "0", + }, + }, + "name": "configure", + "exe": "#{cpython:python}", + "args": ["{pex}", "{configure-binding.py}"], + } + ], + } # type: Dict[str, Any] + + for target in configuration.targets: + manifest_path = os.path.join( + safe_mkdtemp(), + target.platform.qualified_file_name("{name}-lift.toml".format(name=name)), + ) + with safe_open(manifest_path, "w") as fp: + toml.dump( + { + "lift": dict( + lift, + platforms=[target.platform.value], + interpreters=[ + { + "id": "cpython", + "provider": "PythonBuildStandalone", + "release": target.pbs_release, + "version": target.version_str, + "lazy": configuration.options.style is ScieStyle.LAZY, + } + ], + ) + }, + fp, + ) + yield Manifest(target=target, path=manifest_path) + + +def _science_dir( + env, # type: Variables + *components # type: str +): + # type: (...) -> str + return os.path.join(env.PEX_ROOT, "scies", "science", MIN_SCIENCE_VERSION.raw, *components) + + +def _science_binary_names(): + # type: () -> Iterator[str] + yield SciePlatform.CURRENT.binary_name("science-fat") + yield SciePlatform.CURRENT.qualified_binary_name("science-fat") + yield SciePlatform.CURRENT.binary_name("science") + yield SciePlatform.CURRENT.qualified_binary_name("science") + + +def _is_compatible_science_binary( + binary, # type: str + source=None, # type: Optional[str] +): + # type: (...) -> Union[Version, Error] + try: + version = Version( + subprocess.check_output(args=[binary, "--version"]).decode("utf-8").strip() + ) + except (CalledProcessError, InvalidVersion) as e: + return Error( + "Failed to determine --version of science binary at {source}: {err}".format( + source=source or binary, err=e + ) + ) + else: + if version.raw in SCIENCE_REQUIREMENT: + return version + return Error( + "The science binary at {source} is version {version} which does not match Pex's " + "science requirement of {science_requirement}.".format( + source=source or binary, + version=version.raw, + science_requirement=SCIENCE_REQUIREMENT, + ) + ) + + +def _path_science(): + # type: () -> Optional[str] + for path_element in os.environ.get("PATH", os.defpath).split(os.pathsep): + for binary in ( + os.path.join(path_element, binary_name) for binary_name in _science_binary_names() + ): + if not is_exe(binary): + continue + if isinstance(_is_compatible_science_binary(binary), Error): + continue + return binary + return None + + +def _ensure_science( + url_fetcher=None, # type: Optional[URLFetcher] + science_binary_url=None, # type: Optional[str] + env=ENV, # type: Variables +): + # type: (...) -> str + + target_dir = _science_dir(env, "bin") + with atomic_directory(target_dir=target_dir) as atomic_dir: + if not atomic_dir.is_finalized(): + target_science = os.path.join(atomic_dir.work_dir, "science") + path_science = _path_science() + if path_science: + shutil.copy(path_science, target_science) + else: + fetcher = url_fetcher or URLFetcher() + with open(target_science, "wb") as write_fp, fetcher.get_body_stream( + science_binary_url or _science_binary_url() + ) as read_fp: + shutil.copyfileobj(read_fp, write_fp) + chmod_plus_x(target_science) + + if science_binary_url: + custom_science_binary_version = try_( + _is_compatible_science_binary(target_science, source=science_binary_url) + ) + TRACER.log( + "Using custom science binary from {source} with version {version}.".format( + source=science_binary_url, version=custom_science_binary_version.raw + ) + ) + else: + # Since we used the canonical GitHub Releases URL, we know a checksum file is + # available we can use to verify. + science_sha256_url = _science_binary_url(".sha256") + with fetcher.get_body_stream(science_sha256_url) as fp: + expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ") + actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256) + if expected_sha256 != actual_sha256: + raise ValueError( + "The science binary downloaded from {science_binary_url} does not " + "match the expected SHA-256 fingerprint recorded in " + "{science_sha256_url}.\n" + "Expected {expected_sha256} but found {actual_sha256}.".format( + science_binary_url=science_binary_url, + science_sha256_url=science_sha256_url, + expected_sha256=expected_sha256, + actual_sha256=actual_sha256, + ) + ) + return os.path.join(target_dir, "science") + + +class ScienceError(Exception): + """Indicates an error executing science.""" + + +def build( + configuration, # type: ScieConfiguration + pex_file, # type: str + url_fetcher=None, # type: Optional[URLFetcher] + env=ENV, # type: Variables +): + # type: (...) -> Iterator[ScieInfo] + + science = _ensure_science( + url_fetcher=url_fetcher, + science_binary_url=configuration.options.science_binary_url, + env=env, + ) + name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE) + pex_info = PexInfo.from_pex(pex_file) + layout = Layout.identify(pex_file) + use_platform_suffix = len(configuration.targets) > 1 + + errors = OrderedDict() # type: OrderedDict[Manifest, str] + for manifest in create_manifests(configuration, name, pex_info, layout): + args = [science, "--cache-dir", _science_dir(env, "cache")] + if env.PEX_VERBOSE: + args.append("-{verbosity}".format(verbosity="v" * env.PEX_VERBOSE)) + dest_dir = os.path.dirname(os.path.abspath(pex_file)) + args.extend( + [ + "lift", + "--file", + "pex={pex_file}".format(pex_file=pex_file), + "--file", + "configure-binding.py={configure_binding}".format( + configure_binding=os.path.join( + os.path.dirname(__file__), "configure-binding.py" + ) + ), + "build", + "--dest-dir", + dest_dir, + ] + ) + if use_platform_suffix: + args.append("--use-platform-suffix") + args.append(manifest.path) + with open(os.devnull, "wb") as devnull: + process = subprocess.Popen(args=args, stdout=devnull, stderr=subprocess.PIPE) + _, stderr = process.communicate() + if process.returncode != 0: + saved_manifest = os.path.relpath( + os.path.join(dest_dir, os.path.basename(manifest.path)) + ) + shutil.copy(manifest.path, saved_manifest) + errors[manifest] = ( + "Command `{command}` failed with exit code {exit_code} (saved lift manifest to " + "{saved_manifest} for inspection):\n{stderr}" + ).format( + command=" ".join(shlex_quote(arg) for arg in args[:-1] + [saved_manifest]), + exit_code=process.returncode, + saved_manifest=saved_manifest, + stderr=stderr.decode("utf-8").strip(), + ) + else: + yield ScieInfo( + style=configuration.options.style, + target=manifest.target, + file=os.path.join( + dest_dir, + manifest.qualified_binary_name(name) + if use_platform_suffix + else manifest.binary_name(name), + ), + ) + + if errors: + raise ScienceError( + "Failed to build {count} {scies}:\n\n{errors}".format( + count=len(errors), + scies=pluralize(errors, "scie"), + errors="\n\n".join( + "{index}. For CPython {version} on {platform}: {err}".format( + index=index, + platform=manifest.target.platform, + version=manifest.target.version_str, + err=err, + ) + for index, (manifest, err) in enumerate(errors.items(), start=1) + ), + ) + ) diff --git a/pex/targets.py b/pex/targets.py index 511e05904..37c02f229 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -44,13 +44,14 @@ def binary_name(self, version_components=2): @property def python_version(self): - # type: () -> Optional[Tuple[int, int]] + # type: () -> Optional[Union[Tuple[int, int], Tuple[int, int, int]]] + python_full_version = self.marker_environment.python_full_version + if python_full_version: + return cast("Tuple[int, int, int]", tuple(map(int, python_full_version.split(".")))[:3]) python_version = self.marker_environment.python_version - return ( - cast("Tuple[int, int]", tuple(map(int, python_version.split(".")))[:2]) - if python_version - else None - ) + if python_version: + return cast("Tuple[int, int]", tuple(map(int, python_version.split(".")))[:2]) + return None @property def supported_tags(self): @@ -181,8 +182,8 @@ def binary_name(self, version_components=2): @property def python_version(self): - # type: () -> Tuple[int, int] - return self.interpreter.identity.version[:2] + # type: () -> Tuple[int, int, int] + return self.interpreter.identity.version[:3] @property def is_foreign(self): diff --git a/pex/variables.py b/pex/variables.py index 41ff3b287..f952eede0 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -789,6 +789,11 @@ def _expand_pex_root(pex_root): return os.path.expanduser(Variables.PEX_ROOT.value_or(ENV, fallback=fallback)) +def unzip_dir_relpath(pex_hash): + # type: (str) -> str + return os.path.join("unzipped_pexes", pex_hash) + + def unzip_dir( pex_root, # type: str pex_hash, # type: str @@ -796,7 +801,7 @@ def unzip_dir( ): # type: (...) -> str pex_root = _expand_pex_root(pex_root) if expand_pex_root else pex_root - return os.path.join(pex_root, "unzipped_pexes", pex_hash) + return os.path.join(pex_root, unzip_dir_relpath(pex_hash)) def venv_dir( diff --git a/pex/venv/installer.py b/pex/venv/installer.py index c0c6be2d1..b6b759c8d 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -703,6 +703,8 @@ def sys_executable_paths(): "_PEX_DEP_CONFIG_FILE", # This is used as an experiment knob for atomic_directory locking. "_PEX_FILE_LOCK_STYLE", + # This is used in the scie binding command for ZIPAPP PEXes. + "_PEX_SCIE_INSTALLED_PEX_DIR", ) ] if ignored_pex_env_vars: diff --git a/pex/version.py b/pex/version.py index 9c06c95ad..be6b005a1 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.10.1" +__version__ = "2.11.0" diff --git a/tests/integration/scie/__init__.py b/tests/integration/scie/__init__.py new file mode 100644 index 000000000..87fb2ed9a --- /dev/null +++ b/tests/integration/scie/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/tests/integration/scie/test_pex_scie.py b/tests/integration/scie/test_pex_scie.py new file mode 100644 index 000000000..34b95ec23 --- /dev/null +++ b/tests/integration/scie/test_pex_scie.py @@ -0,0 +1,371 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import glob +import json +import os.path +import re +import subprocess +import sys +from typing import Optional + +import pytest + +from pex.common import is_exe +from pex.layout import Layout +from pex.orderedset import OrderedSet +from pex.scie import SciePlatform, ScieStyle +from pex.targets import LocalInterpreter +from pex.typing import TYPE_CHECKING +from testing import IS_PYPY, PY_VER, make_env, run_pex_command + +if TYPE_CHECKING: + from typing import Any, Iterable, List + + +@pytest.mark.parametrize( + "scie_style", [pytest.param(style, id=str(style)) for style in ScieStyle.values()] +) +@pytest.mark.parametrize( + "layout", [pytest.param(layout, id=str(layout)) for layout in Layout.values()] +) +@pytest.mark.parametrize( + "execution_mode_args", + [ + pytest.param([], id="ZIPAPP"), + pytest.param(["--venv"], id="VENV"), + pytest.param(["--sh-boot"], id="ZIPAPP-sh-boot"), + pytest.param(["--venv", "--sh-boot"], id="VENV-sh-boot"), + ], +) +def test_basic( + tmpdir, # type: Any + scie_style, # type: ScieStyle.Value + layout, # type: Layout.Value + execution_mode_args, # type: List[str] +): + # type: (...) -> None + + pex = os.path.join(str(tmpdir), "cowsay.pex") + result = run_pex_command( + args=[ + "cowsay==5.0", + "-c", + "cowsay", + "-o", + pex, + "--scie", + str(scie_style), + "--layout", + str(layout), + ] + + execution_mode_args + ) + if PY_VER < (3, 8) or IS_PYPY: + result.assert_failure( + expected_error_re=r".*^{message}$".format( + message=re.escape( + "You selected `--scie {style}`, but none of the selected targets have " + "compatible interpreters that can be embedded to form a scie:\n" + "{target}".format( + style=scie_style, target=LocalInterpreter.create().render_description() + ) + ) + ), + re_flags=re.DOTALL | re.MULTILINE, + ) + return + if PY_VER >= (3, 13): + result.assert_failure( + expected_error_re=( + r".*" + r"^Failed to build 1 scie:$" + r".*" + r"^Provider: No released assets found for release [0-9]{{8}} Python {version} " + r"of flavor install_only\.$".format(version=".".join(map(str, PY_VER))) + ), + re_flags=re.DOTALL | re.MULTILINE, + ) + return + result.assert_success() + + scie = os.path.join(str(tmpdir), "cowsay") + assert b"| PAR! |" in subprocess.check_output(args=[scie, "PAR!"], env=make_env(PATH=None)) + + +def test_multiple_platforms(tmpdir): + # type: (Any) -> None + + def create_scies( + output_dir, # type: str + extra_args=(), # type: Iterable[str] + ): + pex = os.path.join(output_dir, "cowsay.pex") + run_pex_command( + args=[ + "cowsay==5.0", + "-c", + "cowsay", + "-o", + pex, + "--scie", + "lazy", + "--platform", + "linux-aarch64-cp-39-cp39", + "--platform", + "linux-x86_64-cp-310-cp310", + "--platform", + "macosx-10.9-arm64-cp-311-cp311", + "--platform", + "macosx-10.9-x86_64-cp-312-cp312", + ] + + list(extra_args) + ).assert_success() + + python_version_by_platform = { + SciePlatform.LINUX_AARCH64: "3.9", + SciePlatform.LINUX_X86_64: "3.10", + SciePlatform.MACOS_AARCH64: "3.11", + SciePlatform.MACOS_X86_64: "3.12", + } + assert SciePlatform.CURRENT in python_version_by_platform + + def assert_platforms( + output_dir, # type: str + expected_platforms, # type: Iterable[SciePlatform.Value] + ): + # type: (...) -> None + + all_output_files = set( + path + for path in os.listdir(output_dir) + if os.path.isfile(os.path.join(output_dir, path)) + ) + for platform in OrderedSet(expected_platforms): + python_version = python_version_by_platform[platform] + binary = platform.qualified_binary_name("cowsay") + assert binary in all_output_files + all_output_files.remove(binary) + scie = os.path.join(output_dir, binary) + assert is_exe(scie), "Expected --scie build to produce a {binary} binary.".format( + binary=binary + ) + if platform is SciePlatform.CURRENT: + assert b"| PEX-scie wabbit! |" in subprocess.check_output( + args=[scie, "PEX-scie wabbit!"], env=make_env(PATH=None) + ) + assert ( + python_version + == subprocess.check_output( + args=[ + scie, + "-c", + "import sys; print('.'.join(map(str, sys.version_info[:2])))", + ], + env=make_env(PEX_INTERPRETER=1), + ) + .decode("utf-8") + .strip() + ) + assert {"cowsay.pex"} == all_output_files, ( + "Expected one output scie for each platform plus the original cowsay.pex. All expected " + "scies were found, but the remaining files are: {remaining_files}".format( + remaining_files=all_output_files + ) + ) + + all_platforms_output_dir = os.path.join(str(tmpdir), "all-platforms") + create_scies(output_dir=all_platforms_output_dir) + assert_platforms( + output_dir=all_platforms_output_dir, + expected_platforms=( + SciePlatform.LINUX_AARCH64, + SciePlatform.LINUX_X86_64, + SciePlatform.MACOS_AARCH64, + SciePlatform.MACOS_X86_64, + ), + ) + + # Now restrict the PEX's implied natural platform set of 4 down to 2 or 3 using + # `--scie-platform`. + restricted_platforms_output_dir = os.path.join(str(tmpdir), "restricted-platforms") + create_scies( + output_dir=restricted_platforms_output_dir, + extra_args=[ + "--scie-platform", + "current", + "--scie-platform", + str(SciePlatform.LINUX_AARCH64), + "--scie-platform", + str(SciePlatform.LINUX_X86_64), + ], + ) + assert_platforms( + output_dir=restricted_platforms_output_dir, + expected_platforms=( + SciePlatform.CURRENT, + SciePlatform.LINUX_AARCH64, + SciePlatform.LINUX_X86_64, + ), + ) + + +PRINT_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))" + + +skip_if_pypy = pytest.mark.skipif(IS_PYPY, reason="PyPy targeted PEXes do not support --scie.") + + +@skip_if_pypy +def test_specified_interpreter(tmpdir): + # type: (Any) -> None + + pex = os.path.join(str(tmpdir), "empty.pex") + run_pex_command( + args=[ + "-o", + pex, + "--scie", + "lazy", + # We pick a specific version that is not in the latest release but is known to provide + # distributions for all platforms Pex tests run on. + "--scie-pbs-release", + "20221002", + "--scie-python-version", + "3.10.7", + ], + ).assert_success() + + assert ( + ".".join(map(str, sys.version_info[:3])) + == subprocess.check_output(args=[pex, "-c", PRINT_VERSION_SCRIPT]).decode("utf-8").strip() + ) + + scie = os.path.join(str(tmpdir), "empty") + assert b"3.10.7\n" == subprocess.check_output(args=[scie, "-c", PRINT_VERSION_SCRIPT]) + + +@skip_if_pypy +def test_specified_science_binary(tmpdir): + # type: (Any) -> None + + pex_root = os.path.join(str(tmpdir), "pex_root") + scie = os.path.join(str(tmpdir), "cowsay") + run_pex_command( + args=[ + "--pex-root", + pex_root, + "cowsay==6.0", + "-c", + "cowsay", + "--scie", + "lazy", + "--scie-python-version", + "3.12", + "-o", + scie, + "--scie-science-binary", + # N.B.: This custom version is both lower than the latest available version (0.4.2 + # at the time of writing) and higher than the minimum supported version of 0.3.0; so + # we can prove we downloaded the custom version via this URL by checking the version + # below since our next floor bump will be from 0.3.0 to at least 0.4.3. + "https://github.com/a-scie/lift/releases/download/v0.4.0/{binary}".format( + binary=SciePlatform.CURRENT.qualified_binary_name("science") + ), + ], + env=make_env(PATH=None), + ).assert_success() + + assert b"| Alternative SCIENCE Facts! |" in subprocess.check_output( + args=[scie, "-t", "Alternative SCIENCE Facts!"] + ) + + science_binaries = glob.glob(os.path.join(pex_root, "scies", "science", "*", "bin", "science")) + assert 1 == len(science_binaries) + science = science_binaries[0] + assert "0.4.0" == subprocess.check_output(args=[science, "--version"]).decode("utf-8").strip() + + +@skip_if_pypy +def test_custom_lazy_urls(tmpdir): + # type: (Any) -> None + + scie = os.path.join(str(tmpdir), "empty") + run_pex_command( + args=[ + "-o", + scie, + "--scie", + "lazy", + "--scie-pbs-release", + "20221002", + "--scie-python-version", + "3.10.7", + ], + ).assert_success() + + assert b"3.10.7\n" == subprocess.check_output(args=[scie, "-c", PRINT_VERSION_SCRIPT]) + + pex_bootstrap_urls = os.path.join(str(tmpdir), "pex_bootstrap_urls.json") + + def make_20221002_3_10_7_file(platform): + # type: (str) -> str + return "cpython-3.10.7+20221002-{platform}-install_only.tar.gz".format(platform=platform) + + def make_20240415_3_10_14_url(platform): + # type: (str) -> str + return ( + "https://github.com/indygreg/python-build-standalone/releases/download/20240415/" + "cpython-3.10.14+20240415-{platform}-install_only.tar.gz".format(platform=platform) + ) + + with open(pex_bootstrap_urls, "w") as fp: + json.dump( + { + "ptex": { + make_20221002_3_10_7_file(platform): make_20240415_3_10_14_url(platform) + for platform in ( + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ) + } + }, + fp, + ) + + process = subprocess.Popen( + args=[scie, "-c", PRINT_VERSION_SCRIPT], + env=make_env( + PEX_BOOTSTRAP_URLS=pex_bootstrap_urls, SCIE_BASE=os.path.join(str(tmpdir), "nce") + ), + stderr=subprocess.PIPE, + ) + _, stderr = process.communicate() + assert 0 != process.returncode, ( + "Expected PEX_BOOTSTRAP_URLS to be used and the resulting fetched interpreter distribution " + "to fail its digest check." + ) + + expected_platform = None # type: Optional[str] + if SciePlatform.CURRENT is SciePlatform.LINUX_AARCH64: + expected_platform = "aarch64-unknown-linux-gnu" + elif SciePlatform.CURRENT is SciePlatform.LINUX_X86_64: + expected_platform = "x86_64-unknown-linux-gnu" + elif SciePlatform.CURRENT is SciePlatform.MACOS_AARCH64: + expected_platform = "aarch64-apple-darwin" + elif SciePlatform.CURRENT is SciePlatform.MACOS_X86_64: + expected_platform = "x86_64-apple-darwin" + assert expected_platform is not None + + assert re.match( + r"^.*Population of work directory failed: The tar\.gz destination .*{expected_file_name} " + r"of size \d+ had unexpected hash: [a-f0-9]{{64}}$.*".format( + expected_file_name=re.escape(make_20221002_3_10_7_file(expected_platform)) + ), + stderr.decode("utf-8"), + flags=re.DOTALL | re.MULTILINE, + ), stderr.decode("utf-8") diff --git a/tox.ini b/tox.ini index d1d129e5b..4664152f6 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,8 @@ passenv = SSH_AUTH_SOCK # Needed for pexpect tests. TERM + # Needed to prevent hitting rate limits on GitHub Releases APIs in `--scie` integration tests. + SCIENCE_AUTH_API_GITHUB_COM_BEARER setenv = pip20: _PEX_PIP_VERSION=20.3.4-patched pip22_2: _PEX_PIP_VERSION=22.2.2