diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 7098d062a..d3dc1bdb9 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -1,17 +1,25 @@ import optparse +import platform +import re from typing import Callable, Iterable, Iterator, Optional, cast import pip +from pip._internal.exceptions import InstallationError from pip._internal.index.package_finder import PackageFinder +from pip._internal.models.link import Link from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.req import parse_requirements as _parse_requirements from pip._internal.req.constructors import install_req_from_parsed_requirement +from pip._internal.req.req_file import ParsedRequirement from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import Requirement +from ..utils import abs_ireq, copy_install_requirement, fragment_string, working_dir + PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) +file_url_schemes_re = re.compile(r"^((git|hg|svn|bzr)\+)?file:") __all__ = [ "get_build_tracker", @@ -29,11 +37,71 @@ def parse_requirements( options: Optional[optparse.Values] = None, constraint: bool = False, isolated: bool = False, + from_dir: Optional[str] = None, ) -> Iterator[InstallRequirement]: for parsed_req in _parse_requirements( filename, session, finder=finder, options=options, constraint=constraint ): - yield install_req_from_parsed_requirement(parsed_req, isolated=isolated) + # This context manager helps pip locate relative paths specified + # with non-URI (non file:) syntax, e.g. '-e ..' + with working_dir(from_dir): + try: + ireq = install_req_from_parsed_requirement( + parsed_req, isolated=isolated + ) + except InstallationError: + # This can happen when the url is a relpath with a fragment, + # so we try again with the fragment stripped + preq_without_fragment = ParsedRequirement( + requirement=re.sub(r"#[^#]+$", "", parsed_req.requirement), + is_editable=parsed_req.is_editable, + comes_from=parsed_req.comes_from, + constraint=parsed_req.constraint, + options=parsed_req.options, + line_source=parsed_req.line_source, + ) + ireq = install_req_from_parsed_requirement( + preq_without_fragment, isolated=isolated + ) + + # At this point the ireq has two problems: + # - Sometimes the fragment is lost (even without an InstallationError) + # - It's now absolute (ahead of schedule), + # so abs_ireq will not know to apply the _was_relative attribute, + # which is needed for the writer to use the relpath. + + # To account for the first: + if not fragment_string(ireq): + fragment = Link(parsed_req.requirement)._parsed_url.fragment + if fragment: + link_with_fragment = Link( + url=f"{ireq.link.url_without_fragment}#{fragment}", + comes_from=ireq.link.comes_from, + requires_python=ireq.link.requires_python, + yanked_reason=ireq.link.yanked_reason, + cache_link_parsing=ireq.link.cache_link_parsing, + ) + ireq = copy_install_requirement(ireq, link=link_with_fragment) + + a_ireq = abs_ireq(ireq, from_dir) + + # To account for the second, we guess if the path was initially relative and + # set _was_relative ourselves: + bare_path = file_url_schemes_re.sub( + "", parsed_req.requirement.split(" @ ", 1)[-1] + ) + is_win = platform.system() == "Windows" + if is_win: + bare_path = bare_path.lstrip("/") + if ( + a_ireq.link is not None + and a_ireq.link.scheme.endswith("file") + and not bare_path.startswith("/") + ): + if not (is_win and re.match(r"[a-zA-Z]:", bare_path)): + a_ireq._was_relative = True + + yield a_ireq if PIP_VERSION[:2] <= (22, 0): diff --git a/piptools/resolver.py b/piptools/resolver.py index 09263c74b..d2af59ac9 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -705,8 +705,8 @@ def _get_install_requirements( for extras_candidate in extras_candidates: project_name = canonicalize_name(extras_candidate.project_name) ireq = result_ireqs[project_name] - ireq.extras |= extras_candidate.extras - ireq.req.extras |= extras_candidate.extras + ireq.extras = set(ireq.extras) | set(extras_candidate.extras) + ireq.req.extras = set(ireq.req.extras) | set(extras_candidate.extras) return set(result_ireqs.values()) @@ -778,4 +778,13 @@ def _get_install_requirement_from_candidate( if source_ireq is not None and ireq_key not in self.existing_constraints: pinned_ireq._source_ireqs = [source_ireq] + # Preserve _was_relative attribute of local path requirements + if pinned_ireq.link.is_file: + # Install requirement keys may not match + for c in self.constraints: + if pinned_ireq.link == c.link: + if hasattr(c, "_was_relative"): + pinned_ireq._was_relative = True + break + return pinned_ireq diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 9424968ac..ce1bb6f09 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -28,6 +28,7 @@ drop_extras, is_pinned_requirement, key_from_ireq, + working_dir, ) from ..writer import OutputWriter @@ -159,6 +160,24 @@ def _get_default_option(option_name: str) -> Any: "Will be derived from input file otherwise." ), ) +@click.option( + "--write-relative-to-output", + is_flag=True, + default=False, + help=( + "Construct relative paths as relative to the output file's parent. " + "Will be written as relative to the current folder otherwise." + ), +) +@click.option( + "--read-relative-to-input", + is_flag=True, + default=False, + help=( + "Resolve relative paths as relative to the input file's parent. " + "Will be resolved as relative to the current folder otherwise." + ), +) @click.option( "--allow-unsafe/--no-allow-unsafe", is_flag=True, @@ -272,6 +291,8 @@ def cli( upgrade: bool, upgrade_packages: Tuple[str, ...], output_file: Union[LazyFile, IO[Any], None], + write_relative_to_output: bool, + read_relative_to_input: bool, allow_unsafe: bool, strip_extras: bool, generate_hashes: bool, @@ -306,24 +327,27 @@ def cli( ).format(DEFAULT_REQUIREMENTS_FILE) ) + src_files = tuple(src if src == "-" else os.path.abspath(src) for src in src_files) + if not output_file: # An output file must be provided for stdin if src_files == ("-",): raise click.BadParameter("--output-file is required if input is from stdin") - # Use default requirements output file if there is a setup.py the source file - elif os.path.basename(src_files[0]) in METADATA_FILENAMES: - file_name = os.path.join( - os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE - ) # An output file must be provided if there are multiple source files elif len(src_files) > 1: raise click.BadParameter( "--output-file is required if two or more input files are given." ) + # Use default requirements output file if there is only a setup.py source file + elif os.path.basename(src_files[0]) in METADATA_FILENAMES: + file_name = os.path.join( + os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE + ) # Otherwise derive the output file from the source file else: - base_name = src_files[0].rsplit(".", 1)[0] - file_name = base_name + ".txt" + file_name = os.path.splitext(src_files[0])[0] + ".txt" + if file_name == src_files[0]: + file_name += ".txt" output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) @@ -385,6 +409,11 @@ def cli( finder=tmp_repository.finder, session=tmp_repository.session, options=tmp_repository.options, + from_dir=( + os.path.dirname(os.path.abspath(output_file.name)) + if write_relative_to_output + else None + ), ) for ireq in filter(is_pinned_requirement, ireqs): @@ -408,8 +437,7 @@ def cli( if src_file == "-": # pip requires filenames and not files. Since we want to support # piping from stdin, we need to briefly save the input from stdin - # to a temporary file and have pip read that. also used for - # reading requirements from install_requires in setup.py. + # to a temporary file and have pip read that. tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) tmpfile.write(sys.stdin.read()) comes_from = "-r -" @@ -435,13 +463,19 @@ def cli( log.error(str(e)) log.error(f"Failed to parse {os.path.abspath(src_file)}") sys.exit(2) - comes_from = f"{metadata.get_all('Name')[0]} ({src_file})" - constraints.extend( - [ - install_req_from_line(req, comes_from=comes_from) - for req in metadata.get_all("Requires-Dist") or [] - ] - ) + with working_dir(os.path.dirname(os.path.abspath(output_file.name))): + comes_from = ( + f"{metadata.get_all('Name')[0]} ({os.path.relpath(src_file)})" + ) + with working_dir( + os.path.dirname(src_file) if read_relative_to_input else None + ): + constraints.extend( + [ + install_req_from_line(req, comes_from=comes_from) + for req in metadata.get_all("Requires-Dist") or [] + ] + ) else: constraints.extend( parse_requirements( @@ -449,6 +483,9 @@ def cli( finder=repository.finder, session=repository.session, options=repository.options, + from_dir=( + os.path.dirname(src_file) if read_relative_to_input else None + ), ) ) @@ -526,6 +563,7 @@ def cli( find_links=repository.finder.find_links, emit_find_links=emit_find_links, emit_options=emit_options, + write_relative_to_output=write_relative_to_output, ) writer.write( results=results, diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 02ed847fb..bb4897691 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -70,6 +70,15 @@ is_flag=True, help="Ignore package index (only looking at --find-links URLs instead)", ) +@click.option( + "--read-relative-to-input", + is_flag=True, + default=False, + help=( + "Resolve relative paths as relative to the input file's parent. " + "Will be resolved as relative to the current folder otherwise." + ), +) @click.option( "--python-executable", help="Custom python executable path if targeting an environment other than current.", @@ -96,6 +105,7 @@ def cli( extra_index_url: Tuple[str, ...], trusted_host: Tuple[str, ...], no_index: bool, + read_relative_to_input: bool, python_executable: Optional[str], verbose: int, quiet: int, @@ -139,7 +149,17 @@ def cli( # Parse requirements file. Note, all options inside requirements file # will be collected by the finder. requirements = flat_map( - lambda src: parse_requirements(src, finder=finder, session=session), src_files + lambda src: parse_requirements( + src, + finder=finder, + session=session, + from_dir=( + os.path.dirname(os.path.abspath(src)) + if read_relative_to_input + else os.getcwd() + ), + ), + src_files, ) try: diff --git a/piptools/utils.py b/piptools/utils.py index 47f363751..32bf3d0ef 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -3,9 +3,10 @@ import itertools import json import os +import platform import re import shlex -import typing +from contextlib import contextmanager from typing import ( Any, Callable, @@ -23,9 +24,11 @@ import click from click.utils import LazyFile +from pip._internal.models.link import Link from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url +from pip._internal.utils.urls import path_to_url, url_to_path from pip._internal.vcs import is_url from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.specifiers import SpecifierSet @@ -61,9 +64,7 @@ def key_from_ireq(ireq: InstallRequirement) -> str: return key_from_req(ireq.req) -def key_from_req( - req: typing.Union[InstallRequirement, Distribution, Requirement] -) -> str: +def key_from_req(req: Union[InstallRequirement, Distribution, Requirement]) -> str: """Get an all-lowercase version of the requirement's name.""" if hasattr(req, "key"): # from pkg_resources, such as installed dists for pip-sync @@ -109,25 +110,80 @@ def is_url_requirement(ireq: InstallRequirement) -> bool: return bool(ireq.original_link) +def fragment_string(ireq: InstallRequirement, omit_egg: bool = False) -> str: + """ + Return a string like "#egg=pkgname&subdirectory=folder", or "". + """ + if ireq.link is None or not ireq.link._parsed_url.fragment: + return "" + fragment = f"#{ireq.link._parsed_url.fragment.replace(os.path.sep, '/')}" + if omit_egg: + fragment = re.sub(r"[#&]egg=[^#&]+", "", fragment).lstrip("#&") + if fragment: + fragment = f"#{fragment}" + fragment = re.sub(r"\[[^\]]+\]$", "", fragment).lstrip("#") + if fragment: + fragment = f"#{fragment}" + return fragment + + def format_requirement( ireq: InstallRequirement, marker: Optional[Marker] = None, hashes: Optional[Set[str]] = None, + from_dir: Optional[str] = None, ) -> str: """ Generic formatter for pretty printing InstallRequirements to the terminal in a less verbose way than using its `__str__` method. """ - if ireq.editable: - line = f"-e {ireq.link.url}" - elif is_url_requirement(ireq): - line = _build_direct_reference_best_efforts(ireq) - else: + if not is_url_requirement(ireq): # Canonicalize the requirement name # https://packaging.pypa.io/en/latest/utils.html#packaging.utils.canonicalize_name req = copy.copy(ireq.req) req.name = canonicalize_name(req.name) line = str(req) + elif not ireq.link.is_file: + line = ( + f"-e {ireq.link.url}" + if ireq.editable + else _build_direct_reference_best_efforts(ireq) + ) + # pip doesn't support relative paths in git+file scheme urls, + # for which ireq.link.is_file == False + else: + fragment = fragment_string(ireq) + extras = f"[{','.join(sorted(ireq.extras))}]" if ireq.extras else "" + # pip install needs different relpath formats, depending on extras and fragments: + # https://github.com/jazzband/pip-tools/pull/1329#issuecomment-1056409415 + if fragment or not extras: + prefix = "file:" + delimiter = "" + else: + prefix = "" + delimiter = "#" + if not from_dir: + line = ( + f"-e {path_to_url(ireq.local_file_path)}{fragment}{delimiter}{extras}" + if ireq.editable + else _build_direct_reference_best_efforts(ireq) + ) + else: + try: + relpath = os.path.relpath(ireq.local_file_path, from_dir).replace( + os.path.sep, "/" + ) + except ValueError: + # On Windows, a relative path is not always possible (no common ancestor) + line = ( + f"-e {path_to_url(ireq.local_file_path)}{fragment}{delimiter}{extras}" + if ireq.editable + else _build_direct_reference_best_efforts(ireq) + ) + else: + if not prefix and not relpath.startswith("."): + prefix = "./" + line = f"{'-e ' if ireq.editable else ''}{prefix}{relpath}{fragment}{extras}" if marker: line = f"{line} ; {marker}" @@ -154,21 +210,12 @@ def _build_direct_reference_best_efforts(ireq: InstallRequirement) -> str: # If we get here then we have a requirement that supports direct reference. # We need to remove the egg if it exists and keep the rest of the fragments. - direct_reference = f"{ireq.name.lower()} @ {ireq.link.url_without_fragment}" - fragments = [] - - # Check if there is any fragment to add to the URI. - if ireq.link.subdirectory_fragment: - fragments.append(f"subdirectory={ireq.link.subdirectory_fragment}") - - if ireq.link.has_hash: - fragments.append(f"{ireq.link.hash_name}={ireq.link.hash}") - - # Then add the fragments into the URI, if any. - if fragments: - direct_reference += f"#{'&'.join(fragments)}" - - return direct_reference + extras = f"[{','.join(sorted(ireq.extras))}]" if ireq.extras else "" + return ( + f"{canonicalize_name(ireq.name)}{extras} @ " + f"{ireq.link.url_without_fragment}" + f"{fragment_string(ireq, omit_egg=True)}" + ) def format_specifier(ireq: InstallRequirement) -> str: @@ -322,6 +369,70 @@ def get_hashes_from_ireq(ireq: InstallRequirement) -> Set[str]: return result +@contextmanager +def working_dir(folder: Optional[str]) -> Iterator[None]: + """Change the current directory within the context, then change it back.""" + if folder is None: + yield + else: + try: + original_dir = os.getcwd() + # The os and pathlib modules are incapable of returning an absolute path to the + # current directory without also resolving symlinks, so this is the realpath. + # This can be avoided on some systems with, e.g. os.environ["PWD"], but we'll + # not go there if we don't have to. + os.chdir(os.path.abspath(folder)) + yield + finally: + os.chdir(original_dir) + + +def abs_ireq( + ireq: InstallRequirement, from_dir: Optional[str] = None +) -> InstallRequirement: + """ + Return the given InstallRequirement if its source isn't a relative path; + Otherwise, return a new one with the relative path rewritten as absolute. + + In this case, an extra attribute is added: _was_relative, + which is always True when present at all. + """ + # We check ireq.link.scheme rather than ireq.link.is_file, + # to also match +file schemes + if ireq.link is None or not ireq.link.scheme.endswith("file"): + return ireq + + naive_path = ireq.local_file_path or ireq.link.path + if platform.system() == "Windows": + naive_path = naive_path.lstrip("/") + + with working_dir(from_dir): + url = path_to_url(naive_path).replace("%40", "@") + + if ( + os.path.normpath(naive_path).lower() + == os.path.normpath(url_to_path(url)).lower() + ): + return ireq + + abs_url = f"{url}{fragment_string(ireq)}" + if "+" in ireq.link.scheme: + abs_url = f"{ireq.link.scheme.split('+')[0]}+{abs_url}" + + abs_link = Link( + url=abs_url, + comes_from=ireq.link.comes_from, + requires_python=ireq.link.requires_python, + yanked_reason=ireq.link.yanked_reason, + cache_link_parsing=ireq.link.cache_link_parsing, + ) + + a_ireq = copy_install_requirement(ireq, link=abs_link) + a_ireq._was_relative = True + + return a_ireq + + def get_compile_command(click_ctx: click.Context) -> str: """ Returns a normalized compile command depending on cli context. @@ -487,6 +598,19 @@ def copy_install_requirement( if "req" not in kwargs: kwargs["req"] = copy.deepcopy(template.req) + # Copy extras from a new link if appropriate. + if ( + not kwargs["extras"] + and kwargs["link"] + and kwargs["link"]._parsed_url.fragment.endswith("]") + ): + kwargs["extras"] = tuple( + map( + str.strip, + kwargs["link"]._parsed_url.fragment.rsplit("[", 1)[-1][:-1].split(","), + ) + ) + ireq = InstallRequirement(**kwargs) # If the original_link was None, keep it so. Passing `link` as an @@ -495,4 +619,8 @@ def copy_install_requirement( template.original_link if original_link is None else original_link ) + for custom_attr in ("_source_ireqs", "_was_relative"): + if hasattr(template, custom_attr): + setattr(ireq, custom_attr, getattr(template, custom_attr)) + return ireq diff --git a/piptools/writer.py b/piptools/writer.py index 6ab95f8ce..c31d26529 100644 --- a/piptools/writer.py +++ b/piptools/writer.py @@ -1,6 +1,7 @@ import os import re import sys +from contextlib import suppress from itertools import chain from typing import ( BinaryIO, @@ -31,6 +32,7 @@ get_compile_command, key_from_ireq, strip_extras, + working_dir, ) MESSAGE_UNHASHED_PACKAGE = comment( @@ -55,13 +57,38 @@ ) -strip_comes_from_line_re = re.compile(r" \(line \d+\)$") +comes_from_line_re = re.compile( + r"^(?P-[rc]) (?P.+)(?P \(line \d+\))$" +) + +comes_from_line_project_re = re.compile( + r"^(?P.+) \((?P.+(/|\\)(?Psetup\.(py|cfg)|pyproject\.toml))\)$" +) + + +def _comes_from_as_string( + comes_from: Union[str, InstallRequirement], from_dir: Optional[str] = None +) -> str: + if not isinstance(comes_from, str): + return cast(str, canonicalize_name(key_from_ireq(comes_from))) + match = comes_from_line_re.search(comes_from) + if match: + with working_dir(from_dir): + with suppress(ValueError): + return f"{match['opts']} {os.path.relpath(match['path'])}" + # ValueError: it's impossible to construct the relative path + return f"{match['opts']} {match['path']}" -def _comes_from_as_string(comes_from: Union[str, InstallRequirement]) -> str: - if isinstance(comes_from, str): - return strip_comes_from_line_re.sub("", comes_from) - return cast(str, canonicalize_name(key_from_ireq(comes_from))) + match = comes_from_line_project_re.search(comes_from) + if match: + with working_dir(from_dir): + with suppress(ValueError): + return f"{match['name']} ({os.path.relpath(match['path'])})" + # ValueError: it's impossible to construct the relative path + return f"{match['name']} ({match['path']})" + + return comes_from def annotation_style_split(required_by: Set[str]) -> str: @@ -102,6 +129,7 @@ def __init__( find_links: List[str], emit_find_links: bool, emit_options: bool, + write_relative_to_output: bool, ) -> None: self.dst_file = dst_file self.click_ctx = click_ctx @@ -121,6 +149,7 @@ def __init__( self.find_links = find_links self.emit_find_links = emit_find_links self.emit_options = emit_options + self.write_relative_to_output = write_relative_to_output def _sort_key(self, ireq: InstallRequirement) -> Tuple[bool, str]: return (not ireq.editable, key_from_ireq(ireq)) @@ -273,8 +302,18 @@ def _format_requirement( hashes: Optional[Dict[InstallRequirement, Set[str]]] = None, ) -> str: ireq_hashes = (hashes if hashes is not None else {}).get(ireq) - - line = format_requirement(ireq, marker=marker, hashes=ireq_hashes) + out_dir = ( + os.getcwd() + if self.dst_file.name == "-" + else os.path.dirname(os.path.abspath(self.dst_file.name)) + ) + from_dir = out_dir if self.write_relative_to_output else os.getcwd() + line = format_requirement( + ireq, + marker=marker, + hashes=ireq_hashes, + from_dir=(from_dir if hasattr(ireq, "_was_relative") else None), + ) if self.strip_extras: line = strip_extras(line) @@ -285,13 +324,13 @@ def _format_requirement( required_by = set() if hasattr(ireq, "_source_ireqs"): required_by |= { - _comes_from_as_string(src_ireq.comes_from) + _comes_from_as_string(src_ireq.comes_from, from_dir=out_dir) for src_ireq in ireq._source_ireqs if src_ireq.comes_from } if ireq.comes_from: - required_by.add(_comes_from_as_string(ireq.comes_from)) + required_by.add(_comes_from_as_string(ireq.comes_from, from_dir=out_dir)) required_by |= set(getattr(ireq, "_required_by", set())) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 72acd52fe..38bb934eb 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -2,6 +2,7 @@ import shutil import subprocess import sys +from contextlib import suppress from textwrap import dedent from unittest import mock @@ -9,7 +10,7 @@ from pip._internal.utils.urls import path_to_url from piptools.scripts.compile import cli -from piptools.utils import COMPILE_EXCLUDE_OPTIONS +from piptools.utils import COMPILE_EXCLUDE_OPTIONS, working_dir from .constants import MINIMAL_WHEELS_PATH, PACKAGES_PATH @@ -660,6 +661,64 @@ def test_local_file_uri_package( assert dependency in out.stderr +@pytest.mark.parametrize( + ("line", "rewritten_line"), + ( + pytest.param( + "./small_fake_a", + "file:small_fake_a", + id="relative path", + ), + pytest.param( + "./small_fake_with_extras[dev,test]", + "./small_fake_with_extras[dev,test]", + id="relative path with extras", + ), + pytest.param( + "./small_fake_a#egg=name", + "file:small_fake_a#egg=name", + id="relative path with egg", + ), + pytest.param( + "./small_fake_with_extras#egg=name[dev,test]", + "file:small_fake_with_extras#egg=name[dev,test]", + id="relative path with egg and extras", + ), + pytest.param( + f"{PACKAGES_PATH}/small_fake_a", + f"small-fake-a @ {path_to_url(PACKAGES_PATH)}/small_fake_a", + id="absolute path", + ), + pytest.param( + f"{PACKAGES_PATH}/small_fake_with_extras[dev,test]", + ( + "small-fake-with-extras[dev,test] @ " + f"{path_to_url(PACKAGES_PATH)}/small_fake_with_extras" + ), + id="absolute path with extras", + ), + pytest.param( + f"{PACKAGES_PATH}/small_fake_a#egg=test", + f"small-fake-a @ {path_to_url(PACKAGES_PATH)}/small_fake_a", + id="absolute path with egg", + ), + pytest.param( + f"{PACKAGES_PATH}/small_fake_with_extras#egg=name[dev,test]", + ( + "small-fake-with-extras[dev,test] @ " + f"{path_to_url(PACKAGES_PATH)}/small_fake_with_extras" + ), + id="absolute path with egg and extras", + ), + ), +) +def test_local_file_path_package(pip_conf, runner, line, rewritten_line): + with working_dir(PACKAGES_PATH): + out = runner.invoke(cli, ["--output-file", "-", "-"], input=line) + assert out.exit_code == 0 + assert rewritten_line in out.stderr + + def test_relative_file_uri_package(pip_conf, runner): # Copy wheel into temp dir shutil.copy( @@ -675,6 +734,62 @@ def test_relative_file_uri_package(pip_conf, runner): assert "file:small_fake_with_deps-0.1-py2.py3-none-any.whl" in out.stderr +@pytest.mark.parametrize( + ("flags", "expected"), + ( + ( + ("--write-relative-to-output", "--read-relative-to-input"), + "-e file:deeper/fake-setuptools-a", + ), + (("--write-relative-to-output",), "-e file:deeper/fake-setuptools-a"), + ((), "-e file:../deep/deeper/fake-setuptools-a"), + (("--read-relative-to-input",), "-e file:../deep/deeper/fake-setuptools-a"), + ), +) +@pytest.mark.parametrize("relpath_prefix", ("file:", "./")) +def test_read_write_relative( + pip_conf, runner, tmp_path, relpath_prefix, flags, expected +): + """ + Relative paths in output are correct between CWD, output, and local packages, + and relative paths in inputs are correctly resolved. + """ + run_dir = tmp_path / "working" + in_path = tmp_path / "reqs.in" + txt_path = tmp_path / "deep" / "reqs.txt" + pkg_path = tmp_path / "deep" / "deeper" / "fake-setuptools-a" + + run_dir.mkdir(parents=True, exist_ok=True) + pkg_path.mkdir(parents=True, exist_ok=True) + + (pkg_path / "setup.py").write_text( + dedent( + """\ + from setuptools import setup + setup( + name="fake-setuptools-a", + install_requires=["small-fake-a==0.1"] + ) + """ + ) + ) + + with working_dir( + in_path.parent if "--read-relative-to-input" in flags else run_dir + ): + in_path.write_text( + f"-e {relpath_prefix}{os.path.relpath(pkg_path).replace(os.path.sep, '/')}" + ) + with working_dir(run_dir): + out = runner.invoke( + cli, + ["-o", os.path.relpath(txt_path), *flags, os.path.relpath(in_path)], + ) + + assert out.exit_code == 0 + assert expected in out.stderr + + def test_direct_reference_with_extras(runner): with open("requirements.in", "w") as req_in: req_in.write( @@ -682,11 +797,91 @@ def test_direct_reference_with_extras(runner): ) out = runner.invoke(cli, ["-n", "--rebuild"]) assert out.exit_code == 0 - assert "pip-tools @ git+https://github.com/jazzband/pip-tools@6.2.0" in out.stderr + assert ( + "pip-tools[coverage,testing] @ git+https://github.com/jazzband/pip-tools@6.2.0" + in out.stderr + ) assert "pytest==" in out.stderr assert "pytest-cov==" in out.stderr +def test_url_package_with_extras(runner): + with open("requirements.in", "w") as req_in: + req_in.write( + "git+https://github.com/jazzband/pip-tools@6.2.0#[testing,coverage]" + ) + out = runner.invoke(cli, ["-n", "--rebuild"]) + assert out.exit_code == 0 + assert ( + "pip-tools[coverage,testing] @ git+https://github.com/jazzband/pip-tools@6.2.0" + in out.stderr + ) + assert "pytest==" in out.stderr + assert "pytest-cov==" in out.stderr + + +@pytest.mark.parametrize("flags", (("--write-relative-to-output",), ())) +def test_local_editable_vcs_package(runner, tmp_path, make_package, flags): + """ + git+file urls are resolved to use absolute paths, and otherwise remain intact. + """ + name = "fake-git-a" + version = "0.1" + + run_dir = tmp_path / "working" + in_path = tmp_path / "requirements.in" + txt_path = tmp_path / "deep" / "requirements.txt" + + run_dir.mkdir(parents=True, exist_ok=True) + txt_path.parent.mkdir(parents=True, exist_ok=True) + + pkg_dir = make_package(name=name, version=version) + repo_dir = pkg_dir.parents[1] + + with working_dir(repo_dir): + fragment = f"#egg={name}&subdirectory={os.path.relpath(pkg_dir)}" + for cmd in ( + ["git", "init", "-q"], + ["git", "config", "user.name", "pip-tools"], + ["git", "config", "user.email", "pip-tools@example.com"], + ["git", "add", "-A"], + ["git", "commit", "-q", "-m", version], + ["git", "tag", version], + ): + subprocess.run(cmd, check=True) + + line_abs = f"-e git+{path_to_url(repo_dir)}@{version}{fragment}".replace( + os.path.sep, "/" + ) + + with working_dir(run_dir): + line_rel = ( + f"-e git+file:{os.path.relpath(repo_dir)}@{version}{fragment}".replace( + os.path.sep, "/" + ) + ) + + in_path.write_text(line_abs) + + out = runner.invoke( + cli, ["-o", os.path.relpath(txt_path), *flags, os.path.relpath(in_path)] + ) + + assert out.exit_code == 0 + assert line_abs in out.stderr + + in_path.write_text(line_rel) + + with suppress(FileNotFoundError): + txt_path.unlink() + out = runner.invoke( + cli, ["-o", os.path.relpath(txt_path), *flags, os.path.relpath(in_path)] + ) + + assert out.exit_code == 0 + assert line_abs in out.stderr + + def test_input_file_without_extension(pip_conf, runner): """ piptools can compile a file without an extension, @@ -702,6 +897,43 @@ def test_input_file_without_extension(pip_conf, runner): assert os.path.exists("requirements.txt") +def test_input_file_with_txt_extension(pip_conf, runner, tmp_path): + """ + Compile an input file ending in .txt to a separate output file (*.txt.txt), + without overwriting the input file. + """ + in_file = tmp_path / "requirements.txt" + content = "small-fake-a==0.1" + + in_file.write_text(content) + + out = runner.invoke(cli, [str(in_file)]) + + assert out.exit_code == 0 + assert in_file.read_text().strip() == content + assert os.path.exists(f"{in_file}.txt") + + +def test_input_file_without_extension_and_dotted_path(pip_conf, runner, tmp_path): + """ + Compile a file without an extension, in a subdir with a dot, + into an input-adjacent file with .txt as the extension. + """ + dotted_dir = tmp_path / "some.folder" + in_path = tmp_path / dotted_dir / "reqs" + txt_path = tmp_path / dotted_dir / "reqs.txt" + + dotted_dir.mkdir(parents=True, exist_ok=True) + + in_path.write_text("small-fake-a==0.1\n") + + out = runner.invoke(cli, [str(in_path)]) + + assert out.exit_code == 0 + assert "small-fake-a==0.1" in out.stderr + assert txt_path.exists() + + def test_ignore_incompatible_existing_pins(pip_conf, runner): """ Successfully compile when existing output pins conflict with input. @@ -1256,6 +1488,58 @@ def test_annotate_option(pip_conf, runner, options, expected): assert out.stderr == dedent(expected) +@pytest.mark.network +@pytest.mark.parametrize("to_stdout", (True, False)) +@pytest.mark.parametrize( + "flags", + ( + (), + ("--write-relative-to-output",), + ("--read-relative-to-input",), + ("--write-relative-to-output", "--read-relative-to-input"), + ), +) +@pytest.mark.parametrize( + ("input_filename", "input_content", "expected"), + ( + ("requirements.in", "small-fake-a==0.1", " # via -r {}"), + ( + "setup.py", + ( + "from setuptools import setup\n" + "setup(name='fake-setuptools-a', install_requires=['small-fake-a==0.1'])" + ), + " # via fake-setuptools-a ({})", + ), + ), + ids=("requirements.in", "setup.py"), +) +def test_annotation_relative_paths( + runner, tmp_path, input_filename, input_content, expected, flags, to_stdout +): + """ + Annotations referencing files use paths relative to the output file. + """ + run_dir = tmp_path + + in_path = tmp_path / input_filename + in_path.write_text(input_content) + + txt_path = tmp_path / "deeper" / "requirements.txt" + txt_path.parent.mkdir(parents=True, exist_ok=True) + + output = "-" if to_stdout else str(txt_path) + + with working_dir(run_dir): + out = runner.invoke( + cli, + ["--find-links", MINIMAL_WHEELS_PATH, "-o", output, *flags, str(in_path)], + ) + assert out.exit_code == 0 + with working_dir(None if to_stdout else txt_path.parent): + assert expected.format(os.path.relpath(in_path)) in out.stderr + + @pytest.mark.parametrize( ("option", "expected"), ( @@ -2247,3 +2531,16 @@ def test_resolver_reaches_max_rounds(runner): out = runner.invoke(cli, ["--max-rounds", 0]) assert out.exit_code != 0, out + + +@pytest.mark.parametrize("extras", (("dev",), ("test",), ("dev", "test"))) +def test_local_file_uri_with_extras(pip_conf, runner, extras): + """ + Test [extras] notation is included output. + """ + uri = path_to_url(os.path.join(PACKAGES_PATH, "small_fake_with_extras")) + out = runner.invoke( + cli, ["--output-file", "-", "-"], input=f"{uri}#[{','.join(extras)}]" + ) + assert out.exit_code == 0 + assert f"small-fake-with-extras[{','.join(sorted(extras))}] @ " in out.stderr diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index aabfa7c7a..c83c14dbe 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -1,12 +1,15 @@ import os import subprocess import sys +from textwrap import dedent from unittest import mock import pytest +from pip._internal.utils.urls import path_to_url from pip._vendor.packaging.version import Version from piptools.scripts.sync import DEFAULT_REQUIREMENTS_FILE, cli +from piptools.utils import working_dir def test_run_as_module_sync(): @@ -246,6 +249,49 @@ def test_sync_dry_run_returns_non_zero_exit_code(runner): assert out.exit_code == 1 +@pytest.mark.parametrize("written_from_txt", (True, False)) +@pytest.mark.parametrize("relpath_prefix", ("file:", "./")) +def test_sync_relative_path(runner, tmp_path, relpath_prefix, written_from_txt): + """ + Ensure sync finds relative paths successfully. + """ + run_dir = tmp_path / "working" + txt_path = tmp_path / "deep" / "reqs.txt" + pkg_path = tmp_path / "deep" / "deeper" / "fake-setuptools-a" + + run_dir.mkdir(parents=True, exist_ok=True) + pkg_path.mkdir(parents=True, exist_ok=True) + + (pkg_path / "setup.py").write_text( + dedent( + """\ + from setuptools import setup + setup( + name="fake-setuptools-a", + install_requires=["small-fake-a==0.1"] + ) + """ + ) + ) + + if written_from_txt: + write_from_dir = txt_path.parent + cli_args = ["--dry-run", "--read-relative-to-input", str(txt_path)] + else: + write_from_dir = run_dir + cli_args = ["--dry-run", str(txt_path)] + + with working_dir(write_from_dir): + with open(txt_path, "w") as reqs_txt: + reqs_txt.write( + f"{relpath_prefix}{os.path.relpath(pkg_path).replace(os.path.sep, '/')}" + ) + + with working_dir(run_dir): + out = runner.invoke(cli, cli_args) + assert path_to_url(pkg_path) in out.stdout + + @mock.patch("piptools.sync.run") def test_python_executable_option( run, diff --git a/tests/test_data/packages/small_fake_with_extras/setup.py b/tests/test_data/packages/small_fake_with_extras/setup.py new file mode 100644 index 000000000..493958aea --- /dev/null +++ b/tests/test_data/packages/small_fake_with_extras/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name="small_fake_with_extras", + version=0.1, + install_requires=["small-fake-a", "small-fake-b"], + extras_require={ + "dev": ["small-fake-a"], + "test": ["small-fake-b"], + }, +) diff --git a/tests/test_utils.py b/tests/test_utils.py index 27f8cebea..311269544 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,8 @@ import logging import operator import os +import platform +import re import shlex import sys @@ -10,6 +12,7 @@ from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( + abs_ireq, as_tuple, dedup, drop_extras, @@ -25,6 +28,7 @@ key_from_ireq, lookup_table, lookup_table_from_tuples, + working_dir, ) @@ -141,7 +145,18 @@ def test_format_requirement_editable_vcs_with_password(from_editable): def test_format_requirement_editable_local_path(from_editable): ireq = from_editable("file:///home/user/package") - assert format_requirement(ireq) == "-e file:///home/user/package" + assert re.match( + r"-e file:///([a-zA-Z]:/)?home/user/package$", format_requirement(ireq) + ) + + +@pytest.mark.skipif( + platform.system() != "Windows", + reason="Relative paths can only be impossible on Windows", +) +def test_format_requirement_impossible_relative_path_becomes_absolute(from_line): + ireq = abs_ireq(from_line("file:./vendor/package.zip"), ".") + assert format_requirement(ireq, from_dir="z:") == ireq.link.url def test_format_requirement_ireq_with_hashes(from_line): @@ -540,3 +555,36 @@ def test_get_sys_path_for_python_executable(): # not testing for equality, because pytest adds extra paths into current sys.path for path in result: assert path in sys.path + + +@pytest.mark.parametrize( + ("folder_name", "use_abspath"), + (("subfolder", True), ("subfolder", False), (None, False)), +) +def test_working_dir(tmpdir_cwd, folder_name, use_abspath): + if folder_name is not None: + os.mkdir(folder_name) + expected_within = os.path.abspath(folder_name) + else: + expected_within = tmpdir_cwd + if use_abspath: + folder_name = expected_within + + with working_dir(folder_name): + assert os.getcwd() == expected_within + + assert os.getcwd() == tmpdir_cwd + + +def test_local_abs_ireq_preserves_source_ireqs(from_line): + ireq1 = from_line("testone==1.2") + ireq1.comes_from = "xyz" + + ireq2 = from_line("testtwo==2.4") + ireq2.comes_from = "lmno" + + ireq3 = from_line("file:testwithsrcs") + ireq3._source_ireqs = [ireq1, ireq2] + ireq3 = abs_ireq(ireq3, os.getcwd()) + + assert ireq3._source_ireqs == [ireq1, ireq2] diff --git a/tests/test_writer.py b/tests/test_writer.py index 7884a4532..68cba423e 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -47,6 +47,7 @@ def writer(tmpdir_cwd): emit_find_links=True, strip_extras=False, emit_options=True, + write_relative_to_output=False, ) yield writer @@ -68,6 +69,37 @@ def test_format_requirement_annotation(from_line, writer): assert writer._format_requirement(ireq) == "test==1.2\n " + comment("# via xyz") +@pytest.mark.parametrize( + ("comes_from", "expected"), + ( + ("xyz (z:/pyproject.toml)", "# via xyz (z:/pyproject.toml)"), + ("-r z:/pyproject.toml (line 1)", "# via -r z:/pyproject.toml"), + ), +) +def test_format_requirement_annotation_impossible_relative_path( + from_line, writer, comes_from, expected +): + "A relative path can't be constructed across drives on a Windows filesystem." + ireq = from_line("test==1.2") + ireq.comes_from = comes_from + + assert writer._format_requirement(ireq) == "test==1.2\n " + comment(expected) + + +def test_format_requirement_annotation_source_ireqs(from_line, writer): + "Annotations credit an ireq's source_ireq's comes_from attribute." + ireq = from_line("test==1.2") + ireq.comes_from = "xyz" + + ireq2 = from_line("testwithsrc==3.0") + ireq2._source_ireqs = [ireq] + + assert ( + writer._format_requirement(ireq2) + == f"testwithsrc==3.0\n {comment('# via xyz')}" + ) + + def test_format_requirement_annotation_lower_case(from_line, writer): ireq = from_line("Test==1.2") ireq.comes_from = "xyz"