From 03ee96a7e6dfcf1ec31917022a69b810e0ab5d8c Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Tue, 7 May 2024 13:02:44 -0400 Subject: [PATCH 1/4] Ensure canonicalization of extras when creating ireqs _compat now exports install_req_from_line and canonicalize_ireq --- piptools/_compat/__init__.py | 4 ++++ piptools/_compat/pip_compat.py | 20 +++++++++++++++++++- piptools/build.py | 8 ++++++-- piptools/resolver.py | 3 +-- piptools/scripts/compile.py | 3 +-- piptools/utils.py | 3 +-- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index be1397092..ccc61eec3 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -3,8 +3,10 @@ from .pip_compat import ( PIP_VERSION, Distribution, + canonicalize_ireq, create_wheel_cache, get_dev_pkgs, + install_req_from_line, parse_requirements, ) @@ -12,6 +14,8 @@ "PIP_VERSION", "Distribution", "parse_requirements", + "install_req_from_line", "create_wheel_cache", "get_dev_pkgs", + "canonicalize_ireq", ] diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 1d346bdfb..0c4543d6a 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -2,7 +2,7 @@ import optparse from dataclasses import dataclass -from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast +from typing import TYPE_CHECKING, Any, Iterable, Iterator, Set, cast import pip from pip._internal.cache import WheelCache @@ -14,7 +14,11 @@ from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.req import parse_requirements as _parse_requirements +from pip._internal.req.constructors import ( + install_req_from_line as _install_req_from_line, +) from pip._internal.req.constructors import install_req_from_parsed_requirement +from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import Requirement @@ -73,6 +77,19 @@ def file_path(self) -> str: return self._url +def canonicalize_ireq(ireq: InstallRequirement) -> None: + if hasattr(ireq.req, "extras") and ireq.req.extras: + ireq.req.extras = set(map(canonicalize_name, ireq.req.extras)) + if hasattr(ireq, "extras") and ireq.extras: + ireq.extras = set(map(canonicalize_name, ireq.extras)) + + +def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: + ireq = _install_req_from_line(*args, **kwargs) + canonicalize_ireq(ireq) + return ireq + + def parse_requirements( filename: str, session: PipSession, @@ -91,6 +108,7 @@ def parse_requirements( file_link = FileLink(install_req.link.url) file_link._url = parsed_req.requirement install_req.link = file_link + canonicalize_ireq(install_req) yield install_req diff --git a/piptools/build.py b/piptools/build.py index da8345b79..0e0bf4022 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -13,10 +13,12 @@ import build.env import pyproject_hooks from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line, parse_req_from_line +from pip._internal.req.constructors import parse_req_from_line from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement +from ._compat import canonicalize_ireq, install_req_from_line + if sys.version_info >= (3, 11): import tomllib else: @@ -229,13 +231,15 @@ def _prepare_requirements( replaced_package_name = req.replace(package_name, str(package_dir), 1) parts = parse_req_from_line(replaced_package_name, comes_from) - yield InstallRequirement( + ireq = InstallRequirement( parts.requirement, comes_from, link=parts.link, markers=parts.markers, extras=parts.extras, ) + canonicalize_ireq(ireq) + yield ireq def _prepare_build_requirements( diff --git a/piptools/resolver.py b/piptools/resolver.py index 0591985b2..ebb9f0b0c 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -14,7 +14,6 @@ update_env_context_manager, ) from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.resolvelib.base import Candidate from pip._internal.resolution.resolvelib.candidates import ExtrasCandidate from pip._internal.resolution.resolvelib.resolver import Resolver @@ -27,7 +26,7 @@ from piptools.cache import DependencyCache from piptools.repositories.base import BaseRepository -from ._compat import create_wheel_cache +from ._compat import create_wheel_cache, install_req_from_line from .exceptions import PipToolsError from .logging import log from .utils import ( diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 195faa756..2d56c4ca5 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -12,10 +12,9 @@ from build import BuildBackendException from click.utils import LazyFile, safecall from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url -from .._compat import parse_requirements +from .._compat import install_req_from_line, parse_requirements from ..build import ProjectMetadata, build_project_metadata from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError diff --git a/piptools/utils.py b/piptools/utils.py index 667c38801..db79c0073 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -22,7 +22,6 @@ import click from click.utils import LazyFile from pip._internal.req import InstallRequirement -from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.resolvelib.base import Requirement as PipRequirement from pip._internal.utils.misc import redact_auth_from_url from pip._internal.vcs import is_url @@ -33,7 +32,7 @@ from pip._vendor.packaging.version import Version from pip._vendor.pkg_resources import get_distribution -from piptools._compat import PIP_VERSION +from piptools._compat import PIP_VERSION, install_req_from_line from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from piptools.subprocess_utils import run_python_snippet From db1164c62e6b9274b70d4df55d743a79d1657fe9 Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Thu, 2 Nov 2023 18:02:56 -0400 Subject: [PATCH 2/4] Reorganize canonicalize_ireq and PIP_VERSION into utils canonicalize_ireq now returns a copy ireq instead of modifying in place --- piptools/_compat/__init__.py | 6 ------ piptools/_compat/pip_compat.py | 25 +++---------------------- piptools/build.py | 18 +++++++++--------- piptools/resolver.py | 3 ++- piptools/scripts/compile.py | 10 ++++++++-- piptools/utils.py | 24 +++++++++++++++++++++++- tests/conftest.py | 3 ++- 7 files changed, 47 insertions(+), 42 deletions(-) diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index ccc61eec3..cded67769 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -1,21 +1,15 @@ from __future__ import annotations from .pip_compat import ( - PIP_VERSION, Distribution, - canonicalize_ireq, create_wheel_cache, get_dev_pkgs, - install_req_from_line, parse_requirements, ) __all__ = [ - "PIP_VERSION", "Distribution", "parse_requirements", - "install_req_from_line", "create_wheel_cache", "get_dev_pkgs", - "canonicalize_ireq", ] diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 0c4543d6a..4405cc6d4 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -2,9 +2,8 @@ import optparse from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterable, Iterator, Set, cast +from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast -import pip from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution @@ -14,16 +13,9 @@ from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.req import parse_requirements as _parse_requirements -from pip._internal.req.constructors import ( - install_req_from_line as _install_req_from_line, -) from pip._internal.req.constructors import install_req_from_parsed_requirement -from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import Requirement -PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) - # The Distribution interface has changed between pkg_resources and # importlib.metadata, so this compat layer allows for a consistent access # pattern. In pip 22.1, importlib.metadata became the default on Python 3.11 @@ -31,6 +23,8 @@ if TYPE_CHECKING: from pip._internal.metadata.importlib import Distribution as _ImportLibDist +from ..utils import PIP_VERSION, canonicalize_ireq + @dataclass(frozen=True) class Distribution: @@ -77,19 +71,6 @@ def file_path(self) -> str: return self._url -def canonicalize_ireq(ireq: InstallRequirement) -> None: - if hasattr(ireq.req, "extras") and ireq.req.extras: - ireq.req.extras = set(map(canonicalize_name, ireq.req.extras)) - if hasattr(ireq, "extras") and ireq.extras: - ireq.extras = set(map(canonicalize_name, ireq.extras)) - - -def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: - ireq = _install_req_from_line(*args, **kwargs) - canonicalize_ireq(ireq) - return ireq - - def parse_requirements( filename: str, session: PipSession, diff --git a/piptools/build.py b/piptools/build.py index 0e0bf4022..88fc00850 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -17,7 +17,7 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement -from ._compat import canonicalize_ireq, install_req_from_line +from .utils import canonicalize_ireq, install_req_from_line if sys.version_info >= (3, 11): import tomllib @@ -231,15 +231,15 @@ def _prepare_requirements( replaced_package_name = req.replace(package_name, str(package_dir), 1) parts = parse_req_from_line(replaced_package_name, comes_from) - ireq = InstallRequirement( - parts.requirement, - comes_from, - link=parts.link, - markers=parts.markers, - extras=parts.extras, + yield canonicalize_ireq( + InstallRequirement( + parts.requirement, + comes_from, + link=parts.link, + markers=parts.markers, + extras=parts.extras, + ) ) - canonicalize_ireq(ireq) - yield ireq def _prepare_build_requirements( diff --git a/piptools/resolver.py b/piptools/resolver.py index ebb9f0b0c..b6107dcac 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -26,7 +26,7 @@ from piptools.cache import DependencyCache from piptools.repositories.base import BaseRepository -from ._compat import create_wheel_cache, install_req_from_line +from ._compat import create_wheel_cache from .exceptions import PipToolsError from .logging import log from .utils import ( @@ -35,6 +35,7 @@ copy_install_requirement, format_requirement, format_specifier, + install_req_from_line, is_pinned_requirement, is_url_requirement, key_from_ireq, diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 2d56c4ca5..96aa9fa96 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -14,7 +14,7 @@ from pip._internal.req import InstallRequirement from pip._internal.utils.misc import redact_auth_from_url -from .._compat import install_req_from_line, parse_requirements +from .._compat import parse_requirements from ..build import ProjectMetadata, build_project_metadata from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError @@ -22,7 +22,13 @@ from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import BacktrackingResolver, LegacyResolver -from ..utils import dedup, drop_extras, is_pinned_requirement, key_from_ireq +from ..utils import ( + dedup, + drop_extras, + install_req_from_line, + is_pinned_requirement, + key_from_ireq, +) from ..writer import OutputWriter from . import options from .options import BuildTargetT diff --git a/piptools/utils.py b/piptools/utils.py index db79c0073..80a2abda5 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -20,8 +20,12 @@ import tomli as tomllib import click +import pip from click.utils import LazyFile from pip._internal.req import InstallRequirement +from pip._internal.req.constructors import ( + install_req_from_line as _install_req_from_line, +) from pip._internal.resolution.resolvelib.base import Requirement as PipRequirement from pip._internal.utils.misc import redact_auth_from_url from pip._internal.vcs import is_url @@ -30,9 +34,9 @@ from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import get_distribution -from piptools._compat import PIP_VERSION, install_req_from_line from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from piptools.subprocess_utils import run_python_snippet @@ -41,6 +45,8 @@ _T = TypeVar("_T") _S = TypeVar("_S") +PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) + UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"} COMPILE_EXCLUDE_OPTIONS = { "--dry-run", @@ -87,6 +93,22 @@ def comment(text: str) -> str: return click.style(text, fg="green") +def canonicalize_ireq(ireq: InstallRequirement) -> InstallRequirement: + """ + Return a copy of ireq with canonicalized extras strings + """ + ireq = copy_install_requirement( + ireq, extras=set(map(canonicalize_name, ireq.extras)) + ) + if ireq.req: + ireq.req.extras = set(ireq.extras) + return ireq + + +def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: + return canonicalize_ireq(_install_req_from_line(*args, **kwargs)) + + def make_install_requirement( name: str, version: str | Version, ireq: InstallRequirement ) -> InstallRequirement: diff --git a/tests/conftest.py b/tests/conftest.py index cb4632293..7310676a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ from pip._vendor.packaging.version import Version from pip._vendor.pkg_resources import Requirement -from piptools._compat import PIP_VERSION, Distribution +from piptools._compat import Distribution from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound from piptools.locations import DEFAULT_CONFIG_FILE_NAMES @@ -37,6 +37,7 @@ from piptools.repositories.base import BaseRepository from piptools.resolver import BacktrackingResolver, LegacyResolver from piptools.utils import ( + PIP_VERSION, as_tuple, is_url_requirement, key_from_ireq, From f13c20f1068e1ab2b1f1e8633a52c6d42da1c34a Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Thu, 2 Nov 2023 18:49:37 -0400 Subject: [PATCH 3/4] Add test: canonicalize_extras Based on the case in #2004 --- tests/test_cli_compile.py | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index a098e0455..c5031fc27 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -2375,6 +2375,49 @@ def test_combine_different_extras_of_the_same_package( ) +def test_canonicalize_extras(pip_conf, runner, tmp_path, make_package, make_wheel): + """ + Ensure extras are written in a consistent format. + """ + pkgs = [ + make_package( + "fake-sqlalchemy", + version="0.1", + extras_require={"fake-postgresql_psycoPG2BINARY": ["fake-greenlet"]}, + ), + make_package( + "fake-greenlet", + version="0.2", + ), + ] + + dists_dir = tmp_path / "dists" + for pkg in pkgs: + make_wheel(pkg, dists_dir) + + with open("requirements.in", "w") as req_in: + req_in.write("fake-sqlalchemy[FAKE_postgresql-psycopg2binary]\n") + + out = runner.invoke( + cli, + [ + "--output-file", + "-", + "--find-links", + str(dists_dir), + "--no-header", + "--no-emit-options", + "--no-annotate", + "--no-strip-extras", + ], + ) + assert out.exit_code == 0 + assert ( + "fake-sqlalchemy[fake-postgresql-psycopg2binary]==0.1" + in out.stdout.splitlines() + ) + + @pytest.mark.parametrize( ("pkg2_install_requires", "req_in_content", "out_expected_content"), ( From 7a1fadadf655c7be402352ad714341a12aa3c5e3 Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Sun, 5 Nov 2023 00:06:28 -0400 Subject: [PATCH 4/4] Fold canonicalize-ireq-extras logic into copy_install_requirement --- piptools/_compat/pip_compat.py | 5 ++--- piptools/build.py | 4 ++-- piptools/utils.py | 18 +++++------------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 4405cc6d4..d956ab245 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -23,7 +23,7 @@ if TYPE_CHECKING: from pip._internal.metadata.importlib import Distribution as _ImportLibDist -from ..utils import PIP_VERSION, canonicalize_ireq +from ..utils import PIP_VERSION, copy_install_requirement @dataclass(frozen=True) @@ -89,8 +89,7 @@ def parse_requirements( file_link = FileLink(install_req.link.url) file_link._url = parsed_req.requirement install_req.link = file_link - canonicalize_ireq(install_req) - yield install_req + yield copy_install_requirement(install_req) def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache: diff --git a/piptools/build.py b/piptools/build.py index 88fc00850..c19e206b5 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -17,7 +17,7 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement -from .utils import canonicalize_ireq, install_req_from_line +from .utils import copy_install_requirement, install_req_from_line if sys.version_info >= (3, 11): import tomllib @@ -231,7 +231,7 @@ def _prepare_requirements( replaced_package_name = req.replace(package_name, str(package_dir), 1) parts = parse_req_from_line(replaced_package_name, comes_from) - yield canonicalize_ireq( + yield copy_install_requirement( InstallRequirement( parts.requirement, comes_from, diff --git a/piptools/utils.py b/piptools/utils.py index 80a2abda5..8d04f7a79 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -93,20 +93,8 @@ def comment(text: str) -> str: return click.style(text, fg="green") -def canonicalize_ireq(ireq: InstallRequirement) -> InstallRequirement: - """ - Return a copy of ireq with canonicalized extras strings - """ - ireq = copy_install_requirement( - ireq, extras=set(map(canonicalize_name, ireq.extras)) - ) - if ireq.req: - ireq.req.extras = set(ireq.extras) - return ireq - - def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: - return canonicalize_ireq(_install_req_from_line(*args, **kwargs)) + return copy_install_requirement(_install_req_from_line(*args, **kwargs)) def make_install_requirement( @@ -536,6 +524,10 @@ def copy_install_requirement( if "req" not in kwargs: kwargs["req"] = copy.deepcopy(template.req) + kwargs["extras"] = set(map(canonicalize_name, kwargs["extras"])) + if kwargs["req"]: + kwargs["req"].extras = set(kwargs["extras"]) + ireq = InstallRequirement(**kwargs) # If the original_link was None, keep it so. Passing `link` as an