diff --git a/news/12453.feature.rst b/news/12453.feature.rst new file mode 100644 index 00000000000..704cd012a90 --- /dev/null +++ b/news/12453.feature.rst @@ -0,0 +1 @@ +Improve performance of resolution of large dependency trees, with more caching. diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 49a1f660ca6..140aaafcb8f 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -352,13 +352,13 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}({self.dist!r})" - def __hash__(self) -> int: - return hash((self.__class__, self.name, self.version)) + def __eq__(self, other: object) -> bool: + if not isinstance(other, AlreadyInstalledCandidate): + return NotImplemented + return self.name == other.name and self.version == other.version - def __eq__(self, other: Any) -> bool: - if isinstance(other, self.__class__): - return self.name == other.name and self.version == other.version - return False + def __hash__(self) -> int: + return hash((self.name, self.version)) @property def project_name(self) -> NormalizedName: diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 781c54fd83a..e36df15d459 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -3,6 +3,7 @@ import logging from typing import ( TYPE_CHECKING, + Callable, Dict, FrozenSet, Iterable, @@ -391,6 +392,7 @@ def find_candidates( incompatibilities: Mapping[str, Iterator[Candidate]], constraint: Constraint, prefers_installed: bool, + is_satisfied_by: Callable[[Requirement, Candidate], bool], ) -> Iterable[Candidate]: # Collect basic lookup information from the requirements. explicit_candidates: Set[Candidate] = set() @@ -456,7 +458,7 @@ def find_candidates( for c in explicit_candidates if id(c) not in incompat_ids and constraint.is_satisfied_by(c) - and all(req.is_satisfied_by(c) for req in requirements[identifier]) + and all(is_satisfied_by(req, c) for req in requirements[identifier]) ) def _make_requirements_from_install_req( diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 315fb9c8902..fb0dd85f112 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -1,5 +1,6 @@ import collections import math +from functools import lru_cache from typing import ( TYPE_CHECKING, Dict, @@ -234,8 +235,10 @@ def _eligible_for_upgrade(identifier: str) -> bool: constraint=constraint, prefers_installed=(not _eligible_for_upgrade(identifier)), incompatibilities=incompatibilities, + is_satisfied_by=self.is_satisfied_by, ) + @lru_cache(maxsize=None) def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool: return requirement.is_satisfied_by(candidate) diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py index 4af4a9f25a6..f980a356f18 100644 --- a/src/pip/_internal/resolution/resolvelib/requirements.py +++ b/src/pip/_internal/resolution/resolvelib/requirements.py @@ -1,3 +1,5 @@ +from typing import Any + from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import NormalizedName, canonicalize_name @@ -17,6 +19,14 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}({self.candidate!r})" + def __hash__(self) -> int: + return hash(self.candidate) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, ExplicitRequirement): + return False + return self.candidate == other.candidate + @property def project_name(self) -> NormalizedName: # No need to canonicalize - the candidate did this @@ -49,6 +59,14 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}({str(self._ireq.req)!r})" + def __eq__(self, other: object) -> bool: + if not isinstance(other, SpecifierRequirement): + return NotImplemented + return str(self._ireq) == str(other._ireq) + + def __hash__(self) -> int: + return hash(str(self._ireq)) + @property def project_name(self) -> NormalizedName: assert self._ireq.req, "Specifier-backed ireq is always PEP 508" @@ -98,12 +116,21 @@ def __init__(self, ireq: InstallRequirement) -> None: self._ireq = install_req_drop_extras(ireq) self._extras = frozenset(canonicalize_name(e) for e in self._ireq.extras) + def __eq__(self, other: object) -> bool: + if not isinstance(other, SpecifierWithoutExtrasRequirement): + return NotImplemented + return str(self._ireq) == str(other._ireq) + + def __hash__(self) -> int: + return hash(str(self._ireq)) + class RequiresPythonRequirement(Requirement): """A requirement representing Requires-Python metadata.""" def __init__(self, specifier: SpecifierSet, match: Candidate) -> None: self.specifier = specifier + self._specifier_string = str(specifier) # for faster __eq__ self._candidate = match def __str__(self) -> str: @@ -112,6 +139,17 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}({str(self.specifier)!r})" + def __hash__(self) -> int: + return hash((self._specifier_string, self._candidate)) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, RequiresPythonRequirement): + return False + return ( + self._specifier_string == other._specifier_string + and self._candidate == other._candidate + ) + @property def project_name(self) -> NormalizedName: return self._candidate.project_name @@ -148,6 +186,14 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}({str(self._name)!r})" + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnsatisfiableRequirement): + return NotImplemented + return self._name == other._name + + def __hash__(self) -> int: + return hash(self._name) + @property def project_name(self) -> NormalizedName: return self._name diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index b8cd13cb566..b7b0395b037 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -23,6 +23,14 @@ # Editables +def _is_satisfied_by(requirement: Requirement, candidate: Candidate) -> bool: + """A helper function to check if a requirement is satisfied by a candidate. + + Used for mocking PipProvider.is_satified_by. + """ + return requirement.is_satisfied_by(candidate) + + @pytest.fixture def test_cases(data: TestData) -> Iterator[List[Tuple[str, str, int]]]: def _data_file(name: str) -> Path: @@ -80,6 +88,7 @@ def test_new_resolver_correct_number_of_matches( {}, Constraint.empty(), prefers_installed=False, + is_satisfied_by=_is_satisfied_by, ) assert sum(1 for _ in matches) == match_count @@ -98,6 +107,7 @@ def test_new_resolver_candidates_match_requirement( {}, Constraint.empty(), prefers_installed=False, + is_satisfied_by=_is_satisfied_by, ) for c in candidates: assert isinstance(c, Candidate)