Skip to content

Commit

Permalink
Merge pull request #2013 from AndydeCleyre/bugfix/2004
Browse files Browse the repository at this point in the history
Ensure consistent extras formatting in output
  • Loading branch information
chrysle authored May 13, 2024
2 parents 347fec5 + 7a1fada commit 5330964
Show file tree
Hide file tree
Showing 8 changed files with 82 additions and 20 deletions.
2 changes: 0 additions & 2 deletions piptools/_compat/__init__.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
from __future__ import annotations

from .pip_compat import (
PIP_VERSION,
Distribution,
create_wheel_cache,
get_dev_pkgs,
parse_requirements,
)

__all__ = [
"PIP_VERSION",
"Distribution",
"parse_requirements",
"create_wheel_cache",
Expand Down
8 changes: 3 additions & 5 deletions piptools/_compat/pip_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast

import pip
from pip._internal.cache import WheelCache
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution
Expand All @@ -15,18 +14,17 @@
from pip._internal.req import InstallRequirement
from pip._internal.req import parse_requirements as _parse_requirements
from pip._internal.req.constructors import install_req_from_parsed_requirement
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pkg_resources import Requirement

PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split(".")))

# The Distribution interface has changed between pkg_resources and
# importlib.metadata, so this compat layer allows for a consistent access
# pattern. In pip 22.1, importlib.metadata became the default on Python 3.11
# (and later), but is overridable. `select_backend` returns what's being used.
if TYPE_CHECKING:
from pip._internal.metadata.importlib import Distribution as _ImportLibDist

from ..utils import PIP_VERSION, copy_install_requirement


@dataclass(frozen=True)
class Distribution:
Expand Down Expand Up @@ -91,7 +89,7 @@ def parse_requirements(
file_link = FileLink(install_req.link.url)
file_link._url = parsed_req.requirement
install_req.link = file_link
yield install_req
yield copy_install_requirement(install_req)


def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache:
Expand Down
18 changes: 11 additions & 7 deletions piptools/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@
import build.env
import pyproject_hooks
from pip._internal.req import InstallRequirement
from pip._internal.req.constructors import install_req_from_line, parse_req_from_line
from pip._internal.req.constructors import parse_req_from_line
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import Requirement

from .utils import copy_install_requirement, install_req_from_line

if sys.version_info >= (3, 11):
import tomllib
else:
Expand Down Expand Up @@ -229,12 +231,14 @@ def _prepare_requirements(
replaced_package_name = req.replace(package_name, str(package_dir), 1)
parts = parse_req_from_line(replaced_package_name, comes_from)

yield InstallRequirement(
parts.requirement,
comes_from,
link=parts.link,
markers=parts.markers,
extras=parts.extras,
yield copy_install_requirement(
InstallRequirement(
parts.requirement,
comes_from,
link=parts.link,
markers=parts.markers,
extras=parts.extras,
)
)


Expand Down
2 changes: 1 addition & 1 deletion piptools/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
update_env_context_manager,
)
from pip._internal.req import InstallRequirement
from pip._internal.req.constructors import install_req_from_line
from pip._internal.resolution.resolvelib.base import Candidate
from pip._internal.resolution.resolvelib.candidates import ExtrasCandidate
from pip._internal.resolution.resolvelib.resolver import Resolver
Expand All @@ -36,6 +35,7 @@
copy_install_requirement,
format_requirement,
format_specifier,
install_req_from_line,
is_pinned_requirement,
is_url_requirement,
key_from_ireq,
Expand Down
9 changes: 7 additions & 2 deletions piptools/scripts/compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
from build import BuildBackendException
from click.utils import LazyFile, safecall
from pip._internal.req import InstallRequirement
from pip._internal.req.constructors import install_req_from_line
from pip._internal.utils.misc import redact_auth_from_url

from .._compat import parse_requirements
Expand All @@ -23,7 +22,13 @@
from ..repositories import LocalRequirementsRepository, PyPIRepository
from ..repositories.base import BaseRepository
from ..resolver import BacktrackingResolver, LegacyResolver
from ..utils import dedup, drop_extras, is_pinned_requirement, key_from_ireq
from ..utils import (
dedup,
drop_extras,
install_req_from_line,
is_pinned_requirement,
key_from_ireq,
)
from ..writer import OutputWriter
from . import options
from .options import BuildTargetT
Expand Down
17 changes: 15 additions & 2 deletions piptools/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@
import tomli as tomllib

import click
import pip
from click.utils import LazyFile
from pip._internal.req import InstallRequirement
from pip._internal.req.constructors import install_req_from_line
from pip._internal.req.constructors import (
install_req_from_line as _install_req_from_line,
)
from pip._internal.resolution.resolvelib.base import Requirement as PipRequirement
from pip._internal.utils.misc import redact_auth_from_url
from pip._internal.vcs import is_url
Expand All @@ -31,9 +34,9 @@
from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pkg_resources import get_distribution

from piptools._compat import PIP_VERSION
from piptools.locations import DEFAULT_CONFIG_FILE_NAMES
from piptools.subprocess_utils import run_python_snippet

Expand All @@ -42,6 +45,8 @@
_T = TypeVar("_T")
_S = TypeVar("_S")

PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split(".")))

UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"}
COMPILE_EXCLUDE_OPTIONS = {
"--dry-run",
Expand Down Expand Up @@ -88,6 +93,10 @@ def comment(text: str) -> str:
return click.style(text, fg="green")


def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement:
return copy_install_requirement(_install_req_from_line(*args, **kwargs))


def make_install_requirement(
name: str, version: str | Version, ireq: InstallRequirement
) -> InstallRequirement:
Expand Down Expand Up @@ -515,6 +524,10 @@ def copy_install_requirement(
if "req" not in kwargs:
kwargs["req"] = copy.deepcopy(template.req)

kwargs["extras"] = set(map(canonicalize_name, kwargs["extras"]))
if kwargs["req"]:
kwargs["req"].extras = set(kwargs["extras"])

ireq = InstallRequirement(**kwargs)

# If the original_link was None, keep it so. Passing `link` as an
Expand Down
3 changes: 2 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from pip._vendor.packaging.version import Version
from pip._vendor.pkg_resources import Requirement

from piptools._compat import PIP_VERSION, Distribution
from piptools._compat import Distribution
from piptools.cache import DependencyCache
from piptools.exceptions import NoCandidateFound
from piptools.locations import DEFAULT_CONFIG_FILE_NAMES
Expand All @@ -37,6 +37,7 @@
from piptools.repositories.base import BaseRepository
from piptools.resolver import BacktrackingResolver, LegacyResolver
from piptools.utils import (
PIP_VERSION,
as_tuple,
is_url_requirement,
key_from_ireq,
Expand Down
43 changes: 43 additions & 0 deletions tests/test_cli_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -2375,6 +2375,49 @@ def test_combine_different_extras_of_the_same_package(
)


def test_canonicalize_extras(pip_conf, runner, tmp_path, make_package, make_wheel):
"""
Ensure extras are written in a consistent format.
"""
pkgs = [
make_package(
"fake-sqlalchemy",
version="0.1",
extras_require={"fake-postgresql_psycoPG2BINARY": ["fake-greenlet"]},
),
make_package(
"fake-greenlet",
version="0.2",
),
]

dists_dir = tmp_path / "dists"
for pkg in pkgs:
make_wheel(pkg, dists_dir)

with open("requirements.in", "w") as req_in:
req_in.write("fake-sqlalchemy[FAKE_postgresql-psycopg2binary]\n")

out = runner.invoke(
cli,
[
"--output-file",
"-",
"--find-links",
str(dists_dir),
"--no-header",
"--no-emit-options",
"--no-annotate",
"--no-strip-extras",
],
)
assert out.exit_code == 0
assert (
"fake-sqlalchemy[fake-postgresql-psycopg2binary]==0.1"
in out.stdout.splitlines()
)


@pytest.mark.parametrize(
("pkg2_install_requires", "req_in_content", "out_expected_content"),
(
Expand Down

0 comments on commit 5330964

Please sign in to comment.