From e13f1681e8ec595fcf5c4a7edb83754b17b68fac Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 13 Jul 2024 20:57:32 -0700 Subject: [PATCH] Fix editable requirement parsing. (#2464) Previously, editable requirements in requirements files were not parsed properly by Pex. Although they did not trigger parse errors, PEXes created from editable requirements would fail to import those requirements at runtime despite the editable project distribution being embedded in the PEX file. Fixes #2410 --- CHANGES.md | 9 ++++ docs-requirements.txt | 6 ++- pex/requirements.py | 11 ++-- pex/version.py | 2 +- tests/integration/test_issue_2410.py | 77 ++++++++++++++++++++++++++++ tests/test_requirements.py | 18 +++++-- 6 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_issue_2410.py diff --git a/CHANGES.md b/CHANGES.md index 338ec953e..fc44c3984 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,14 @@ # Release Notes +## 2.10.1 + +This release fixes a long-standing bug in Pex parsing of editable +requirements. This bug caused PEXes containing local editable project +requirements to fail to import those local editable projects despite +the fact the PEX itself contained them. + +* Fix editable requirement parsing. (#2464) + ## 2.10.0 This release adds support for injecting requirements into the isolated diff --git a/docs-requirements.txt b/docs-requirements.txt index 6c95b3098..fba2375d8 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -2,4 +2,8 @@ furo httpx myst-parser[linkify] sphinx -sphinx-simplepdf \ No newline at end of file +sphinx-simplepdf + +# The 0.11.0 release removes deprecated API parameters which breaks weasyprint (62.3 depends on +# `pydyf>=0.10.0`) which is a dependency of sphinx-simplepdf. +pydyf<0.11.0 diff --git a/pex/requirements.py b/pex/requirements.py index 4c72a2ddd..925f076be 100644 --- a/pex/requirements.py +++ b/pex/requirements.py @@ -418,7 +418,9 @@ def _try_parse_pip_local_formats( directory_name, requirement_parts = match.groups() stripped_path = os.path.join(os.path.dirname(path), directory_name) abs_stripped_path = ( - os.path.join(basepath, stripped_path) if basepath else os.path.abspath(stripped_path) + os.path.join(basepath, stripped_path) + if basepath and not os.path.isabs(stripped_path) + else os.path.abspath(stripped_path) ) if not os.path.exists(abs_stripped_path): return None @@ -650,8 +652,11 @@ def parse_requirements( yield requirement continue - # Skip empty lines, comment lines and all other Pip options. - if not processed_text or processed_text.startswith("-"): + # Skip empty lines, comment lines and all Pip global options. + if not processed_text or ( + processed_text.startswith("-") + and not re.match(r"^(?:-e|--editable)\s.*", processed_text) + ): continue # Only requirement lines remain. diff --git a/pex/version.py b/pex/version.py index 403f9092a..9c06c95ad 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.10.0" +__version__ = "2.10.1" diff --git a/tests/integration/test_issue_2410.py b/tests/integration/test_issue_2410.py new file mode 100644 index 000000000..2d26ba506 --- /dev/null +++ b/tests/integration/test_issue_2410.py @@ -0,0 +1,77 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import subprocess +from textwrap import dedent + +from colors import colors + +from pex.common import safe_open +from pex.typing import TYPE_CHECKING +from testing import run_pex_command + +if TYPE_CHECKING: + from typing import Any + + +def test_pex_with_editable(tmpdir): + # type: (Any) -> None + + project_dir = os.path.join(str(tmpdir), "project") + with safe_open(os.path.join(project_dir, "example.py"), "w") as fp: + fp.write( + dedent( + """\ + import sys + + import colors + + + def colorize(*messages): + return colors.green(" ".join(messages)) + + + if __name__ == "__main__": + print(colorize(*sys.argv[1:])) + sys.exit(0) + """ + ) + ) + with safe_open(os.path.join(project_dir, "setup.py"), "w") as fp: + fp.write( + dedent( + """\ + from setuptools import setup + + + setup( + name="example", + version="0.1.0", + py_modules=["example"], + ) + """ + ) + ) + + requirements = os.path.join(project_dir, "requirements.txt") + with safe_open(requirements, "w") as fp: + fp.write( + dedent( + """\ + ansicolors==1.1.8 + -e file://{project_dir} + """ + ).format(project_dir=project_dir) + ) + + pex = os.path.join(str(tmpdir), "pex") + run_pex_command(args=["-r", requirements, "-m", "example", "-o", pex]).assert_success() + output = ( + subprocess.check_output(args=[pex, "A", "wet", "duck", "flies", "at", "night!"]) + .decode("utf-8") + .strip() + ) + assert colors.green("A wet duck flies at night!") == output, output diff --git a/tests/test_requirements.py b/tests/test_requirements.py index 3caa45fd5..db9bb3348 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -51,7 +51,7 @@ def chroot(): curdir = os.getcwd() try: os.chdir(chroot) - yield chroot + yield os.path.realpath(chroot) finally: os.chdir(curdir) @@ -278,8 +278,8 @@ def test_parse_requirements_stress(chroot): hg+http://hg.example.com/MyProject@da39a3ee5e6b\\ #egg=AnotherProject[extra,more] ; python_version=="3.9.*"&subdirectory=foo/bar - ftp://a/${PROJECT_NAME}-1.0.tar.gz - http://a/${PROJECT_NAME}-1.0.zip + ftp://a/${{PROJECT_NAME}}-1.0.tar.gz + http://a/${{PROJECT_NAME}}-1.0.zip https://a/numpy-1.9.2-cp34-none-win32.whl https://a/numpy-1.9.2-cp34-none-win32.whl;\\ python_version=="3.4.*" and sys_platform=='win32' @@ -295,8 +295,14 @@ def test_parse_requirements_stress(chroot): # Wheel with local version http://download.pytorch.org/whl/cpu/torch-1.12.1%2Bcpu-cp310-cp310-linux_x86_64.whl + + # Editable + -e file://{chroot}/extra/a/local/project + --editable file://{chroot}/extra/a/local/project/ + -e ./another/local/project + --editable ./another/local/project/ """ - ) + ).format(chroot=chroot) ) touch("extra/pyproject.toml") touch("extra/a/local/project/pyproject.toml") @@ -470,6 +476,10 @@ def test_parse_requirements_stress(chroot): url="http://download.pytorch.org/whl/cpu/torch-1.12.1%2Bcpu-cp310-cp310-linux_x86_64.whl", specifier="==1.12.1+cpu", ), + local_req(path=os.path.join(chroot, "extra/a/local/project"), editable=True), + local_req(path=os.path.join(chroot, "extra/a/local/project"), editable=True), + local_req(path=os.path.join(chroot, "extra/another/local/project"), editable=True), + local_req(path=os.path.join(chroot, "extra/another/local/project"), editable=True), url_req( project_name="numpy", url=os.path.realpath("./downloads/numpy-1.9.2-cp34-none-win32.whl"),