Skip to content

Commit

Permalink
Include build-system hook in via comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@apljungquist
apljungquist committed Aug 2, 2023
1 parent aac2425 commit 51af3bb
Show file tree
Hide file tree
Showing 6 changed files with 159 additions and 65 deletions.
11 changes: 10 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,12 +569,14 @@ asgiref==3.5.2
# via django
attrs==22.1.0
# via pytest
backports-zoneinfo==0.2.1
# via django
django==4.1
# via my-cool-django-app (pyproject.toml)
editables==0.3
# via hatchling
hatchling==1.11.1
# via my-cool-django-app (pyproject.toml)
# via my-cool-django-app (pyproject.toml::build-system.requires)
iniconfig==1.1.1
# via pytest
packaging==21.3
Expand All @@ -601,6 +603,13 @@ tomli==2.0.1
# pytest
```

Some build backends may also request build dependencies dynamically using the hooks described in PEP 517 and PEP 660.
This will be indicated in the output with one of the following suffixes:

- `(pyproject.toml::build-system.backend::editable)`
- `(pyproject.toml::build-system.backend::sdist)`
- `(pyproject.toml::build-system.backend::wheel)`

### Other useful tools

- [pipdeptree](https://github.com/tox-dev/pipdeptree) to print the dependency tree of the installed packages.
Expand Down
4 changes: 2 additions & 2 deletions examples/readme/constraints.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# This file is autogenerated by pip-compile with Python 3.9
# This file is autogenerated by pip-compile with Python 3.11
# by the following command:
#
# pip-compile --all-build-deps --all-extras --output-file=constraints.txt --strip-extras pyproject.toml
Expand All @@ -13,7 +13,7 @@ django==4.1
editables==0.3
# via hatchling
hatchling==1.11.1
# via my-cool-django-app (pyproject.toml)
# via my-cool-django-app (pyproject.toml::build-system.requires)
iniconfig==1.1.1
# via pytest
packaging==21.3
Expand Down
36 changes: 24 additions & 12 deletions piptools/scripts/compile.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import collections
import itertools
import os
import shlex
Expand Down Expand Up @@ -49,14 +50,24 @@
METADATA_FILENAMES = frozenset({"setup.py", "setup.cfg", "pyproject.toml"})


def _build_requirements(src_dir: str, distributions: Iterable[str]) -> set[str]:
def _build_requirements(
src_dir: str, src_file: str, distributions: Iterable[str], package_name: str
) -> list[InstallRequirement]:
builder = ProjectBuilder(src_dir, runner=pyproject_hooks.quiet_subprocess_runner)
# It is not clear that it should be possible to use `get_requires_for_build` with
# "editable" but it seems to work in practice.
result = set(builder.build_system_requires)
result = collections.defaultdict(set)
for req in builder.build_system_requires:
result[req].add(f"{package_name} ({src_file}::build-system.requires)")
for dist in distributions:
result.update(builder.get_requires_for_build(dist))
return result
for req in builder.get_requires_for_build(dist):
result[req].add(
f"{package_name} ({src_file}::build-system.backend::{dist})"
)

return [
install_req_from_line(k, comes_from=v) for k, vs in result.items() for v in vs
]


def _get_default_option(option_name: str) -> Any:
Expand Down Expand Up @@ -627,14 +638,15 @@ def cli(
if build_deps_for_distributions:
package_name = metadata.get_all("Name")[0]
constraints.extend(
[
install_req_from_line(
req, comes_from=f"{package_name} ({src_file})"
)
for req in _build_requirements(
src_dir, build_deps_for_distributions
)
]
_build_requirements(
# Build requirements will only be present if a pyproject.toml file exists,
# but if there is also a setup.py file then only that will be explicitly
# processed due to the order of `DEFAULT_REQUIREMENTS_FILES`.
src_dir,
"pyproject.toml",
build_deps_for_distributions,
package_name,
)
)
else:
constraints.extend(
Expand Down
3 changes: 3 additions & 0 deletions tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,9 @@ def fake_dists(tmpdir, make_package, make_wheel):
make_package("small-fake-a", version="0.1"),
make_package("small-fake-b", version="0.2"),
make_package("small-fake-c", version="0.3"),
make_package("small-fake-d", version="0.4"),
make_package("small-fake-e", version="0.5"),
make_package("small-fake-f", version="0.6"),
]
for pkg in pkgs:
make_wheel(pkg, dists_path)
Expand Down
166 changes: 117 additions & 49 deletions tests/test_cli_compile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import subprocess
import sys
from textwrap import dedent
from typing import Iterable
from typing import Any, Iterable
from unittest import mock

import pytest
Expand Down Expand Up @@ -2738,38 +2738,54 @@ def test_all_extras_fail_with_extra(fake_dists, runner, make_module, fname, cont
assert exp in out.stderr


def _parse_deps(text: str, keys: Iterable[str]) -> dict[str, str | None]:
"""Get package name to version mapping from ``pip-compile`` output.
def _parse_deps(text: str, keys: Iterable[str]) -> dict[str, dict[str, Any] | None]:
"""Get structured representation of ``pip-compile`` output.
:param text: output from ``pip-compile``
:param keys: names of expected packages. If not in text the version will be set to ``None``.
"""
result: dict[str, str | None] = {k: None for k in keys}

result: dict[str, dict[str, Any] | None] = {k: None for k in keys}
kwargs: dict[str, Any] = {}
for line in text.splitlines():
pkg_spec = line.split("#")[0].strip()
pkg_name, eq_sep, pkg_version = pkg_spec.partition("==")
not_comment, _, comment = line.partition("#")

pkg_name, eq_sep, pkg_version = not_comment.strip().partition("==")
if eq_sep:
kwargs = {"version": pkg_version}
result[pkg_name] = kwargs
assert not comment
continue

if not eq_sep:
# Beginning of via section
before_via, via_sep, after_via = comment.partition("via")
if via_sep:
kwargs["via"] = set()
if after_via.strip():
kwargs["via"].add(after_via.strip())
continue

result[pkg_name] = pkg_version
# Continuation of via section
if "via" in kwargs and not_comment == " ":
kwargs["via"].add(comment.strip())

return result


class Version:
def __init__(self, expected: str) -> None:
self._pat = expected
class Dependency:
def __init__(self, version: str, via: Iterable[str] | None = None) -> None:
self._pat = version
self._via = None if via is None else set(via)

def __eq__(self, other: object) -> bool:
if not isinstance(other, str):
if not isinstance(other, dict):
return False

return fnmatch.fnmatch(other, self._pat)
return fnmatch.fnmatch(other["version"], self._pat) and (
self._via is None or set(other["via"]) == self._via
)

def __repr__(self) -> str:
return f"<{self.__class__.__name__}@{id(self)}(_pat={self._pat!r})>"
return f"<{self.__class__.__name__}@{id(self)}(_pat={self._pat!r}, via={self._via})>"


# This can be removed when support for python<3.8 is dropped
Expand All @@ -2796,53 +2812,110 @@ def copytree_dirs_exist_ok(
["editable"],
[],
{
"setuptools": Version("*"),
"small-fake-c": Version("0.3"),
"small-fake-d": Version("0.4"),
"setuptools": Dependency("*"),
"small-fake-c": Dependency("0.3"),
"small-fake-d": Dependency("0.4"),
"small-fake-f": Dependency("0.6"),
},
),
(
["sdist"],
[],
{
"setuptools": Version("*"),
"small-fake-a": Version("0.1"),
"small-fake-d": Version("0.4"),
"setuptools": Dependency("*"),
"small-fake-a": Dependency("0.1"),
"small-fake-d": Dependency("0.4"),
"small-fake-f": Dependency("0.6"),
},
),
(
["wheel"],
[],
{
"setuptools": Version("*"),
"small-fake-b": Version("0.2"),
"small-fake-d": Version("0.4"),
"wheel": Version("*"),
"setuptools": Dependency("*"),
"small-fake-b": Dependency("0.2"),
"small-fake-d": Dependency("0.4"),
"small-fake-f": Dependency("0.6"),
"wheel": Dependency("*"),
},
),
(
["editable", "sdist", "wheel"],
[],
{
"setuptools": Version("*"),
"small-fake-a": Version("0.1"),
"small-fake-b": Version("0.2"),
"small-fake-c": Version("0.3"),
"small-fake-d": Version("0.4"),
"wheel": Version("*"),
"setuptools": Dependency("*"),
"small-fake-a": Dependency("0.1"),
"small-fake-b": Dependency("0.2"),
"small-fake-c": Dependency("0.3"),
"small-fake-d": Dependency("0.4"),
"small-fake-f": Dependency("0.6"),
"wheel": Dependency("*"),
},
),
# Test also that the via section is correct
# Since this is rather lengthy and we are concerned mostly with the case where more than
# one build-system hook requests the same dependency we only test it with this last
# parameter.
(
["editable", "sdist", "wheel"],
["--all-extras"],
{
"setuptools": Version("*"),
"small-fake-a": Version("0.1"),
"small-fake-b": Version("0.2"),
"small-fake-c": Version("0.3"),
"small-fake-d": Version("0.4"),
"small-fake-e": Version("0.5"),
"wheel": Version("*"),
"setuptools": Dependency(
"*",
[
"small-fake-with-build-deps (pyproject.toml::build-system.requires)",
],
),
"small-fake-a": Dependency(
"0.1",
[
"small-fake-with-build-deps (pyproject.toml::build-system.backend::sdist)",
],
),
"small-fake-b": Dependency(
"0.2",
[
"small-fake-with-build-deps (pyproject.toml::build-system.backend::wheel)",
],
),
"small-fake-c": Dependency(
"0.3",
[
(
"small-fake-with-build-deps"
" (pyproject.toml::build-system.backend::editable)"
),
],
),
"small-fake-d": Dependency(
"0.4",
[
"small-fake-with-build-deps (setup.py)",
],
),
"small-fake-e": Dependency(
"0.5",
[
"small-fake-with-build-deps (setup.py)",
],
),
"small-fake-f": Dependency(
"0.6",
[
(
"small-fake-with-build-deps"
" (pyproject.toml::build-system.backend::editable)"
),
"small-fake-with-build-deps (pyproject.toml::build-system.backend::sdist)",
"small-fake-with-build-deps (pyproject.toml::build-system.backend::wheel)",
],
),
"wheel": Dependency(
"*",
[
"small-fake-with-build-deps (pyproject.toml::build-system.backend::wheel)",
],
),
},
),
),
Expand All @@ -2862,9 +2935,6 @@ def test_build_distribution(
Test that when one or more ``--build-deps-for`` are given the expected packages are
included.
"""
make_wheel(make_package("small-fake-d", version="0.4"), fake_dists),
make_wheel(make_package("small-fake-e", version="0.5"), fake_dists),

# When used as argument to the runner it is not passed to pip
monkeypatch.setenv("PIP_FIND_LINKS", fake_dists)
src_pkg_path = os.path.join(PACKAGES_PATH, "small_fake_with_build_deps")
Expand All @@ -2876,7 +2946,6 @@ def test_build_distribution(
cli,
base_cmd + [f"--build-deps-for={d}" for d in distributions] + other_options,
)

assert out.exit_code == 0
assert _parse_deps(out.stderr, expected_deps) == expected_deps

Expand All @@ -2889,8 +2958,6 @@ def test_all_build_distributions(
Test that ``--all-build-deps`` is equivalent to specifying every
``--build-deps-for``.
"""
make_wheel(make_package("small-fake-d", version="0.4"), fake_dists),

# When used as argument to the runner it is not passed to pip
monkeypatch.setenv("PIP_FIND_LINKS", fake_dists)
src_pkg_path = os.path.join(PACKAGES_PATH, "small_fake_with_build_deps")
Expand Down Expand Up @@ -2927,13 +2994,14 @@ def test_only_build_distributions(fake_dists, runner, tmp_path, monkeypatch):
Test that ``--build-deps-only`` excludes dependencies other than build dependencies.
"""
expected_deps = {
"setuptools": Version("*"),
"small-fake-a": Version("0.1"),
"small-fake-b": Version("0.2"),
"small-fake-c": Version("0.3"),
"setuptools": Dependency("*"),
"small-fake-a": Dependency("0.1"),
"small-fake-b": Dependency("0.2"),
"small-fake-c": Dependency("0.3"),
"small-fake-d": None,
"small-fake-e": None,
"wheel": Version("*"),
"small-fake-f": Dependency("0.6"),
"wheel": Dependency("*"),
}

# When used as argument to the runner it is not passed to pip
Expand Down
4 changes: 3 additions & 1 deletion tests/test_data/packages/small_fake_with_build_deps/backend/backend.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,17 @@ def get_requires_for_build_sdist(config_settings=None):
result = setuptools.build_meta.get_requires_for_build_sdist(config_settings)
assert result == []
result.append("small-fake-a")
result.append("small-fake-f")
return result


def get_requires_for_build_wheel(config_settings=None):
result = setuptools.build_meta.get_requires_for_build_wheel(config_settings)
assert result == ["wheel"]
result.append("small-fake-b")
result.append("small-fake-f")
return result


def get_requires_for_build_editable(config_settings=None):
return ["small-fake-c"]
return ["small-fake-c", "small-fake-f"]

0 comments on commit 51af3bb

Please sign in to comment.