diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py index 5f8fdee3d46..057636d1741 100644 --- a/src/pip/_internal/index/collector.py +++ b/src/pip/_internal/index/collector.py @@ -13,7 +13,6 @@ import urllib.request from dataclasses import dataclass from html.parser import HTMLParser -from optparse import Values from typing import ( Callable, Dict, @@ -407,7 +406,7 @@ def __init__( def create( cls, session: PipSession, - options: Values, + index_group: "IndexGroup", suppress_no_index: bool = False, ) -> "LinkCollector": """ @@ -415,22 +414,8 @@ def create( :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. """ - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index and not suppress_no_index: - logger.debug( - "Ignoring indexes: %s", - ",".join(redact_auth_from_url(url) for url in index_urls), - ) - index_urls = [] - - # Make sure find_links is a list before passing to create(). - find_links = options.find_links or [] - search_scope = SearchScope.create( - find_links=find_links, - index_urls=index_urls, - no_index=options.no_index, - ) + search_scope = index_group.create_search_scope(suppress_no_index=suppress_no_index) link_collector = LinkCollector( session=session, search_scope=search_scope, diff --git a/src/pip/_internal/index/index_group.py b/src/pip/_internal/index/index_group.py new file mode 100644 index 00000000000..712de54eae8 --- /dev/null +++ b/src/pip/_internal/index/index_group.py @@ -0,0 +1,83 @@ +from operator import index +from typing import List, Optional +from optparse import Values +import logging + +from pip._internal.models.search_scope import SearchScope +from pip._internal.utils.misc import redact_auth_from_url + +logger = logging.getLogger(__name__) + +class IndexGroup: + """An index group. + + Index groups are used to represent the possible sources of packages. + In pip, there has long been one implicit IndexGroup: the collection + of options that make up pip's package finder behavior. + + This class makes it simpler to have multiple index groups, which + provides the opportunity to have multiple finders with different + indexes and options, and to prioritize finders to prefer one over + another. + + Within an index group, index urls and find-links are considered + equal priority. Any consistent preference of one or the other is + accidental and should not be relied on. The correct way to prioritize + one index over another is to put the indexes in separate groups. + """ + + def __init__(self, index_urls: List[str], find_links: List[str], no_index: bool, + allow_yanked: bool, format_control: Optional["FormatControl"], + ignore_requires_python: bool, prefer_binary: bool + ) -> None: + self.index_urls = index_urls + self.find_links = find_links + self.no_index = no_index + self.format_control = format_control + self.allow_yanked = allow_yanked + self.ignore_requires_python = ignore_requires_python + self.prefer_binary = prefer_binary + + + @classmethod + def create_( + cls, options: Values, + ) -> "IndexGroup": + """ + Create an IndexGroup object from the given options and session. + + :param options: The options to use. + """ + index_urls = options.get("index_url", []) + if not index_urls: + index_urls = [options.get("extra_index_url", [])] + index_urls = [url for urls in index_urls for url in urls] + + find_links = options.get("find_links", []) + if not find_links: + find_links = options.get("find_links", []) + find_links = [url for urls in find_links for url in urls] + + no_index = options.get("no_index", False) + format_control = options.get("format_control", None) + allow_yanked = options.get("allow_yanked", False) + ignore_requires_python = options.get("ignore_requires_python", False) + prefer_binary = options.get("prefer_binary", False) + + return cls(index_urls, find_links, no_index, allow_yanked, format_control, + ignore_requires_python, prefer_binary) + + def create_search_scope(self, suppress_no_index=False): + index_urls = self.index_urls + if self.no_index and not suppress_no_index: + logger.debug( + "Ignoring indexes: %s", + ",".join(redact_auth_from_url(url) for url in self.index_urls), + ) + index_urls = [] + + return SearchScope.create( + find_links=self.find_links, + index_urls=index_urls, + no_index=self.no_index, + ) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 85628ee5d7a..25d3c71aada 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -6,7 +6,8 @@ import logging import re from dataclasses import dataclass -from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union +from optparse import Values +from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union, Dict from pip._vendor.packaging import specifiers from pip._vendor.packaging.tags import Tag @@ -21,6 +22,7 @@ UnsupportedWheel, ) from pip._internal.index.collector import LinkCollector, parse_links +from pip._internal.index.index_group import IndexGroup from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.format_control import FormatControl from pip._internal.models.link import Link @@ -336,7 +338,7 @@ class CandidatePreferences: @dataclass(frozen=True) class BestCandidateResult: - """A collection of candidates, returned by `PackageFinder.find_best_candidate`. + """A collection of candidates, returned by `jPackageFinder.find_best_candidate`. This class is only intended to be instantiated by CandidateEvaluator's `compute_best_candidate()` method. @@ -564,6 +566,70 @@ def compute_best_candidate( class PackageFinder: + """This finds packages for all configured index groups. + + This implements priority between groups, while preserving the + assumption of index equivalence within a group. + + This achieves priority behavior by iterating over the groups in order, + yielding the first match found. + """ + _package_finders: List["InternalPackageFinder"] + _current_package_finder: int = 0 + + + def __init__(self, package_finders: List["InternalPackageFinder"] ) -> None: + self._package_finders = package_finders + + def create(cls, + link_collector: Optional[LinkCollector] = None, + selection_prefs: Optional[SelectionPreferences] = None, + target_python: Optional[TargetPython] = None, + # Args above are for the InternalPackageFinder constructor. + # Args below are the new constructor that handles multiple + # PackageFinder instances. + options: Values | None = None, + session: "PipSession" | None = None, + ) -> "PackageFinder": + """Create an InternalPackageFinder for each index group.""" + + # This is the old constructor that only handles a single + # PackageFinder - so no priority between indexes + if link_collector is not None and selection_prefs is not None: + return PackageFinder([InternalPackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + target_python=target_python, + )]) + + index_groups = [] + # If no explicit index groups are specified, then create one for + # the --index-url, --extra-index-url, and --find-links options. + index_groups = options.get("index_groups") + if not index_groups: + index_groups = [IndexGroup.create_(options, session)] + + package_finders: Dict[str,"InternalPackageFinder"] = {} + for index_group in index_groups: + link_collector = LinkCollector.create(session, index_group) + selection_prefs = SelectionPreferences( + allow_yanked=index_group.allow_yanked, + format_control=index_group.format_control, + ignore_requires_python=index_group.ignore_requires_python, + prefer_binary=index_group.prefer_binary, + ) + # TODO: should index groups be named, and have the order + # be the list of names? + package_finders[index_group.name] = InternalPackageFinder.create( + link_collector=link_collector, selection_prefs=selection_prefs) + + return PackageFinder([package_finders[name] for name in options.get("index_groups_order") or package_finders.keys()]) + + def __getattr__(self, attr): + """Forward attribute access to the current index group.""" + return getattr(self._index_groups[self._current_index_group], attr) + +class InternalPackageFinder: """This finds packages. This is meant to match easy_install's technique for looking for @@ -615,8 +681,8 @@ def create( link_collector: LinkCollector, selection_prefs: SelectionPreferences, target_python: Optional[TargetPython] = None, - ) -> "PackageFinder": - """Create a PackageFinder. + ) -> "InternalPackageFinder": + """Create a InternalPackageFinder for a single index group. :param selection_prefs: The candidate selection preferences, as a SelectionPreferences object.