From b4c369730662fc3ce43af1a5893de2ec29327f40 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 13:23:39 +0100 Subject: [PATCH 01/16] Create static methods from find_all_candidates and the methods it calls --- src/pip/_internal/index/package_finder.py | 335 ++++++++++++++++++---- 1 file changed, 277 insertions(+), 58 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 7f2e04e7c37..2798aab4dc2 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -142,7 +142,8 @@ def __init__( self.project_name = project_name - def evaluate_link(self, link): + @staticmethod + def evaluate_link_static(link_evaluator, link): # type: (Link) -> Tuple[bool, Optional[str]] """ Determine whether a link is a candidate for installation. @@ -153,7 +154,7 @@ def evaluate_link(self, link): the link fails to qualify. """ version = None - if link.is_yanked and not self._allow_yanked: + if link.is_yanked and not link_evaluator._allow_yanked: reason = link.yanked_reason or '' return (False, f'yanked for reason: {reason}') @@ -166,9 +167,9 @@ def evaluate_link(self, link): return (False, 'not a file') if ext not in SUPPORTED_EXTENSIONS: return (False, f'unsupported archive format: {ext}') - if "binary" not in self._formats and ext == WHEEL_EXTENSION: + if "binary" not in link_evaluator._formats and ext == WHEEL_EXTENSION: reason = 'No binaries permitted for {}'.format( - self.project_name) + link_evaluator.project_name) return (False, reason) if "macosx10" in link.path and ext == '.zip': return (False, 'macosx10 one') @@ -177,12 +178,12 @@ def evaluate_link(self, link): wheel = Wheel(link.filename) except InvalidWheelFilename: return (False, 'invalid wheel filename') - if canonicalize_name(wheel.name) != self._canonical_name: + if canonicalize_name(wheel.name) != link_evaluator._canonical_name: reason = 'wrong project name (not {})'.format( - self.project_name) + link_evaluator.project_name) return (False, reason) - supported_tags = self._target_python.get_tags() + supported_tags = link_evaluator._target_python.get_tags() if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -198,28 +199,28 @@ def evaluate_link(self, link): version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. - if "source" not in self._formats and ext != WHEEL_EXTENSION: - reason = f'No sources permitted for {self.project_name}' + if "source" not in link_evaluator._formats and ext != WHEEL_EXTENSION: + reason = f'No sources permitted for {link_evaluator.project_name}' return (False, reason) if not version: version = _extract_version_from_fragment( - egg_info, self._canonical_name, + egg_info, link_evaluator._canonical_name, ) if not version: - reason = f'Missing project version for {self.project_name}' + reason = f'Missing project version for {link_evaluator.project_name}' return (False, reason) - match = self._py_version_re.search(version) + match = link_evaluator._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) - if py_version != self._target_python.py_version: + if py_version != link_evaluator._target_python.py_version: return (False, 'Python version is incorrect') supports_python = _check_link_requires_python( - link, version_info=self._target_python.py_version_info, - ignore_requires_python=self._ignore_requires_python, + link, version_info=link_evaluator._target_python.py_version_info, + ignore_requires_python=link_evaluator._ignore_requires_python, ) if not supports_python: # Return None for the reason text to suppress calling @@ -230,6 +231,95 @@ def evaluate_link(self, link): return (True, version) + def evaluate_link(self, link): + # type: (Link) -> Tuple[bool, Optional[str]] + """ + Determine whether a link is a candidate for installation. + + :return: A tuple (is_candidate, result), where `result` is (1) a + version string if `is_candidate` is True, and (2) if + `is_candidate` is False, an optional string to log the reason + the link fails to qualify. + """ + return LinkEvaluator.evaluate_link_static(self, link) + # version = None + # if link.is_yanked and not self._allow_yanked: + # reason = link.yanked_reason or '' + # return (False, f'yanked for reason: {reason}') + + # if link.egg_fragment: + # egg_info = link.egg_fragment + # ext = link.ext + # else: + # egg_info, ext = link.splitext() + # if not ext: + # return (False, 'not a file') + # if ext not in SUPPORTED_EXTENSIONS: + # return (False, f'unsupported archive format: {ext}') + # if "binary" not in self._formats and ext == WHEEL_EXTENSION: + # reason = 'No binaries permitted for {}'.format( + # self.project_name) + # return (False, reason) + # if "macosx10" in link.path and ext == '.zip': + # return (False, 'macosx10 one') + # if ext == WHEEL_EXTENSION: + # try: + # wheel = Wheel(link.filename) + # except InvalidWheelFilename: + # return (False, 'invalid wheel filename') + # if canonicalize_name(wheel.name) != self._canonical_name: + # reason = 'wrong project name (not {})'.format( + # self.project_name) + # return (False, reason) + + # supported_tags = self._target_python.get_tags() + # if not wheel.supported(supported_tags): + # # Include the wheel's tags in the reason string to + # # simplify troubleshooting compatibility issues. + # file_tags = wheel.get_formatted_file_tags() + # reason = ( + # "none of the wheel's tags ({}) are compatible " + # "(run pip debug --verbose to show compatible tags)".format( + # ', '.join(file_tags) + # ) + # ) + # return (False, reason) + + # version = wheel.version + + # # This should be up by the self.ok_binary check, but see issue 2700. + # if "source" not in self._formats and ext != WHEEL_EXTENSION: + # reason = f'No sources permitted for {self.project_name}' + # return (False, reason) + + # if not version: + # version = _extract_version_from_fragment( + # egg_info, self._canonical_name, + # ) + # if not version: + # reason = f'Missing project version for {self.project_name}' + # return (False, reason) + + # match = self._py_version_re.search(version) + # if match: + # version = version[:match.start()] + # py_version = match.group(1) + # if py_version != self._target_python.py_version: + # return (False, 'Python version is incorrect') + + # supports_python = _check_link_requires_python( + # link, version_info=self._target_python.py_version_info, + # ignore_requires_python=self._ignore_requires_python, + # ) + # if not supports_python: + # # Return None for the reason text to suppress calling + # # _log_skipped_link(). + # return (False, None) + + # logger.debug('Found link %s, version: %s', link, version) + + # return (True, version) + def filter_unallowed_hashes( candidates, # type: List[InstallationCandidate] @@ -719,12 +809,8 @@ def make_link_evaluator(self, project_name): ignore_requires_python=self._ignore_requires_python, ) - def _sort_links(self, links): - # type: (Iterable[Link]) -> List[Link] - """ - Returns elements of links in order, non-egg links first, egg links - second, while eliminating duplicates - """ + @staticmethod + def _sort_links_static(links): eggs, no_eggs = [], [] seen = set() # type: Set[Link] for link in links: @@ -736,24 +822,52 @@ def _sort_links(self, links): no_eggs.append(link) return no_eggs + eggs - def _log_skipped_link(self, link, reason): + def _sort_links(self, links): + # type: (Iterable[Link]) -> List[Link] + """ + Returns elements of links in order, non-egg links first, egg links + second, while eliminating duplicates + """ + return PackageFinder._sort_links_static(links) + # eggs, no_eggs = [], [] + # seen = set() # type: Set[Link] + # for link in links: + # if link not in seen: + # seen.add(link) + # if link.egg_fragment: + # eggs.append(link) + # else: + # no_eggs.append(link) + # return no_eggs + eggs + + @staticmethod + def _log_skipped_link_static(package_finder, link, reason): # type: (Link, str) -> None - if link not in self._logged_links: + if link not in package_finder._logged_links: # Put the link at the end so the reason is more visible and because # the link string is usually very long. logger.debug('Skipping link: %s: %s', reason, link) - self._logged_links.add(link) - - def get_install_candidate(self, link_evaluator, link): - # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] + package_finder._logged_links.add(link) + + def _log_skipped_link(self, link, reason): + # type: (Link, str) -> None + PackageFinder._log_skipped_link_static(self, link, reason) + # if link not in self._logged_links: + # # Put the link at the end so the reason is more visible and because + # # the link string is usually very long. + # logger.debug('Skipping link: %s: %s', reason, link) + # self._logged_links.add(link) + + @staticmethod + def get_install_candidate_static(package_finder, link_evaluator, link): """ If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - is_candidate, result = link_evaluator.evaluate_link(link) + is_candidate, result = LinkEvaluator.evaluate_link_static(link_evaluator, link) if not is_candidate: if result: - self._log_skipped_link(link, reason=result) + PackageFinder._log_skipped_link_static(package_finder, link, reason=result) return None return InstallationCandidate( @@ -762,41 +876,97 @@ def get_install_candidate(self, link_evaluator, link): version=result, ) - def evaluate_links(self, link_evaluator, links): + def get_install_candidate(self, link_evaluator, link): + # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] + """ + If the link is a candidate for install, convert it to an + InstallationCandidate and return it. Otherwise, return None. + """ + return PackageFinder.get_install_candidate_static(self, link_evaluator, link) + # is_candidate, result = link_evaluator.evaluate_link(link) + # if not is_candidate: + # if result: + # self._log_skipped_link(link, reason=result) + # return None + + # return InstallationCandidate( + # name=link_evaluator.project_name, + # link=link, + # version=result, + # ) + + @staticmethod + def evaluate_links_static(package_finder, link_evaluator, links): # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] """ Convert links that are candidates to InstallationCandidate objects. """ candidates = [] - for link in self._sort_links(links): - candidate = self.get_install_candidate(link_evaluator, link) + for link in PackageFinder._sort_links_static(links): + candidate = PackageFinder.get_install_candidate_static(package_finder, link_evaluator, link) if candidate is not None: candidates.append(candidate) return candidates - def process_project_url(self, project_url, link_evaluator): + def evaluate_links(self, link_evaluator, links): + # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] + """ + Convert links that are candidates to InstallationCandidate objects. + """ + return PackageFinder.evaluate_links_static(self, link_evaluator, links) + # candidates = [] + # for link in PackageFinder._sort_links_static(links): + # candidate = PackageFinder.get_install_candidate_static(self, link_evaluator, link) + # if candidate is not None: + # candidates.append(candidate) + + # return candidates + + @staticmethod + def process_project_url_static(package_finder, project_url, link_evaluator): # type: (Link, LinkEvaluator) -> List[InstallationCandidate] logger.debug( 'Fetching project page and analyzing links: %s', project_url, ) - html_page = self._link_collector.fetch_page(project_url) + html_page = package_finder._link_collector.fetch_page(project_url) if html_page is None: return [] page_links = list(parse_links(html_page)) with indent_log(): - package_links = self.evaluate_links( + package_links = PackageFinder.evaluate_links_static( + package_finder, link_evaluator, links=page_links, ) return package_links + def process_project_url(self, project_url, link_evaluator): + # type: (Link, LinkEvaluator) -> List[InstallationCandidate] + return PackageFinder.process_project_url_static(self, project_url, link_evaluator) + # logger.debug( + # 'Fetching project page and analyzing links: %s', project_url, + # ) + # html_page = self._link_collector.fetch_page(project_url) + # if html_page is None: + # return [] + + # page_links = list(parse_links(html_page)) + + # with indent_log(): + # package_links = self.evaluate_links( + # link_evaluator, + # links=page_links, + # ) + + # return package_links + + @staticmethod @functools.lru_cache(maxsize=None) - def find_all_candidates(self, project_name): - # type: (str) -> List[InstallationCandidate] + def find_all_candidates_static(package_finder, link_evaluator, project_name): """Find all available InstallationCandidate for project_name This checks index_urls and find_links. @@ -805,31 +975,33 @@ def find_all_candidates(self, project_name): See LinkEvaluator.evaluate_link() for details on which files are accepted. """ - link_evaluator = self.make_link_evaluator(project_name) - collected_sources = self._link_collector.collect_sources( - project_name=project_name, - candidates_from_page=functools.partial( - self.process_project_url, - link_evaluator=link_evaluator, - ), - ) + # Just add link_collector to the tuple + collected_links = package_finder._link_collector.collect_links(project_name) - page_candidates_it = itertools.chain.from_iterable( - source.page_candidates() - for sources in collected_sources - for source in sources - if source is not None + # Refactored evaluate_links, _sort_links, get_install_candidate, _logged_skipped_link + # dependencies needed from PackageFinder : + # * _logged_link (SHOULD FIND A WAY TO KEEP TRACK OF CHANGED IN _logged_link and apply the changes after cached method) + # and link_evaluator + # * Everything + find_links_versions = PackageFinder.evaluate_links_static( + package_finder, + link_evaluator, + links=collected_links.find_links, ) - page_candidates = list(page_candidates_it) - file_links_it = itertools.chain.from_iterable( - source.file_links() - for sources in collected_sources - for source in sources - if source is not None - ) - file_candidates = self.evaluate_links( + # Only needs link collector from package_finder + page_versions = [] + for project_url in collected_links.project_urls: + package_links = package_finder.process_project_url_static( + package_finder, + project_url, + link_evaluator=link_evaluator, + ) + page_versions.extend(package_links) + + file_versions = PackageFinder.evaluate_links_static( + package_finder, link_evaluator, sorted(file_links_it, reverse=True), ) @@ -841,6 +1013,53 @@ def find_all_candidates(self, project_name): # This is an intentional priority ordering return file_candidates + page_candidates + # @functools.lru_cache(maxsize=None) + def find_all_candidates(self, project_name): + # type: (str) -> List[InstallationCandidate] + """Find all available InstallationCandidate for project_name + + This checks index_urls and find_links. + All versions found are returned as an InstallationCandidate list. + + See LinkEvaluator.evaluate_link() for details on which files + are accepted. + """ + link_evaluator = self.make_link_evaluator(project_name) + + return PackageFinder.find_all_candidates_static(self, link_evaluator, project_name) + # collected_links = self._link_collector.collect_links(project_name) + + # link_evaluator = self.make_link_evaluator(project_name) + + # find_links_versions = self.evaluate_links( + # link_evaluator, + # links=collected_links.find_links, + # ) + + # page_versions = [] + # for project_url in collected_links.project_urls: + # package_links = self.process_project_url( + # project_url, link_evaluator=link_evaluator, + # ) + # page_versions.extend(package_links) + + # file_versions = self.evaluate_links( + # link_evaluator, + # links=collected_links.files, + # ) + # if file_versions: + # file_versions.sort(reverse=True) + # logger.debug( + # 'Local files found: %s', + # ', '.join([ + # url_to_path(candidate.link.url) + # for candidate in file_versions + # ]) + # ) + + # # This is an intentional priority ordering + # return file_versions + find_links_versions + page_versions + def make_candidate_evaluator( self, project_name, # type: str From 89d3329cf5f1489924335fa3e96da7dcccdfae4a Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 15:02:42 +0100 Subject: [PATCH 02/16] Add tuple instead of PackageFinder for parameters --- src/pip/_internal/index/package_finder.py | 94 ++++++----------------- 1 file changed, 24 insertions(+), 70 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 2798aab4dc2..2a10b03c5a7 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -6,6 +6,8 @@ import functools import itertools import logging +import collections + import re from typing import FrozenSet, Iterable, List, Optional, Set, Tuple, Union @@ -48,6 +50,7 @@ Tuple[int, int, int, _BaseVersion, Optional[int], BuildTag] ) +PackageFinderTuple = collections.namedtuple('PackageFinder', 'logged_links link_collector') def _check_link_requires_python( link, # type: Link @@ -665,7 +668,6 @@ def compute_best_candidate( best_candidate=best_candidate, ) - class PackageFinder: """This finds packages. @@ -808,7 +810,14 @@ def make_link_evaluator(self, project_name): allow_yanked=self._allow_yanked, ignore_requires_python=self._ignore_requires_python, ) + + def get_state_as_tuple(self, immutable=False): + logged_links = self._logged_links + if immutable: + logged_links = frozenset(logged_links) + return PackageFinderTuple(logged_links=logged_links, link_collector=self._link_collector) + @staticmethod def _sort_links_static(links): eggs, no_eggs = [], [] @@ -829,29 +838,19 @@ def _sort_links(self, links): second, while eliminating duplicates """ return PackageFinder._sort_links_static(links) - # eggs, no_eggs = [], [] - # seen = set() # type: Set[Link] - # for link in links: - # if link not in seen: - # seen.add(link) - # if link.egg_fragment: - # eggs.append(link) - # else: - # no_eggs.append(link) - # return no_eggs + eggs @staticmethod def _log_skipped_link_static(package_finder, link, reason): # type: (Link, str) -> None - if link not in package_finder._logged_links: + if link not in package_finder.logged_links: # Put the link at the end so the reason is more visible and because # the link string is usually very long. logger.debug('Skipping link: %s: %s', reason, link) - package_finder._logged_links.add(link) + package_finder.logged_links.add(link) def _log_skipped_link(self, link, reason): # type: (Link, str) -> None - PackageFinder._log_skipped_link_static(self, link, reason) + PackageFinder._log_skipped_link_static(get_state_as_tuple(), link, reason) # if link not in self._logged_links: # # Put the link at the end so the reason is more visible and because # # the link string is usually very long. @@ -882,7 +881,7 @@ def get_install_candidate(self, link_evaluator, link): If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - return PackageFinder.get_install_candidate_static(self, link_evaluator, link) + return PackageFinder.get_install_candidate_static(get_state_as_tuple(), link_evaluator, link) # is_candidate, result = link_evaluator.evaluate_link(link) # if not is_candidate: # if result: @@ -914,7 +913,7 @@ def evaluate_links(self, link_evaluator, links): """ Convert links that are candidates to InstallationCandidate objects. """ - return PackageFinder.evaluate_links_static(self, link_evaluator, links) + return PackageFinder.evaluate_links_static(get_state_as_tuple(), link_evaluator, links) # candidates = [] # for link in PackageFinder._sort_links_static(links): # candidate = PackageFinder.get_install_candidate_static(self, link_evaluator, link) @@ -929,7 +928,7 @@ def process_project_url_static(package_finder, project_url, link_evaluator): logger.debug( 'Fetching project page and analyzing links: %s', project_url, ) - html_page = package_finder._link_collector.fetch_page(project_url) + html_page = package_finder.link_collector.fetch_page(project_url) if html_page is None: return [] @@ -946,23 +945,7 @@ def process_project_url_static(package_finder, project_url, link_evaluator): def process_project_url(self, project_url, link_evaluator): # type: (Link, LinkEvaluator) -> List[InstallationCandidate] - return PackageFinder.process_project_url_static(self, project_url, link_evaluator) - # logger.debug( - # 'Fetching project page and analyzing links: %s', project_url, - # ) - # html_page = self._link_collector.fetch_page(project_url) - # if html_page is None: - # return [] - - # page_links = list(parse_links(html_page)) - - # with indent_log(): - # package_links = self.evaluate_links( - # link_evaluator, - # links=page_links, - # ) - - # return package_links + return PackageFinder.process_project_url_static(get_state_as_tuple(), project_url, link_evaluator) @staticmethod @functools.lru_cache(maxsize=None) @@ -976,8 +959,11 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): are accepted. """ + # Need to convert logged_links to normal set, so we can update it. + package_finder = PackageFinderTuple(logged_links=set(package_finder.logged_links), link_collector=package_finder.link_collector) + # Just add link_collector to the tuple - collected_links = package_finder._link_collector.collect_links(project_name) + collected_links = package_finder.link_collector.collect_links(project_name) # Refactored evaluate_links, _sort_links, get_install_candidate, _logged_skipped_link # dependencies needed from PackageFinder : @@ -993,7 +979,7 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): # Only needs link collector from package_finder page_versions = [] for project_url in collected_links.project_urls: - package_links = package_finder.process_project_url_static( + package_links = PackageFinder.process_project_url_static( package_finder, project_url, link_evaluator=link_evaluator, @@ -1025,40 +1011,8 @@ def find_all_candidates(self, project_name): are accepted. """ link_evaluator = self.make_link_evaluator(project_name) - - return PackageFinder.find_all_candidates_static(self, link_evaluator, project_name) - # collected_links = self._link_collector.collect_links(project_name) - - # link_evaluator = self.make_link_evaluator(project_name) - - # find_links_versions = self.evaluate_links( - # link_evaluator, - # links=collected_links.find_links, - # ) - - # page_versions = [] - # for project_url in collected_links.project_urls: - # package_links = self.process_project_url( - # project_url, link_evaluator=link_evaluator, - # ) - # page_versions.extend(package_links) - - # file_versions = self.evaluate_links( - # link_evaluator, - # links=collected_links.files, - # ) - # if file_versions: - # file_versions.sort(reverse=True) - # logger.debug( - # 'Local files found: %s', - # ', '.join([ - # url_to_path(candidate.link.url) - # for candidate in file_versions - # ]) - # ) - - # # This is an intentional priority ordering - # return file_versions + find_links_versions + page_versions + package_finder = self.get_state_as_tuple(True) + return PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) def make_candidate_evaluator( self, From b112c35c2595b31ae91c40a2fd420d951c7d2e20 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 15:28:19 +0100 Subject: [PATCH 03/16] Add LinkEvaluator as tuple --- src/pip/_internal/index/package_finder.py | 133 ++++++---------------- 1 file changed, 33 insertions(+), 100 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 2a10b03c5a7..383631218dc 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -51,6 +51,7 @@ ) PackageFinderTuple = collections.namedtuple('PackageFinder', 'logged_links link_collector') +LinkEvaluatorTuple = collections.namedtuple('LinkEvaluator', 'project_name canonical_name formats target_python allow_yanked ignore_requires_python py_version_re') def _check_link_requires_python( link, # type: Link @@ -145,6 +146,17 @@ def __init__( self.project_name = project_name + def get_state_as_tuple(self): + return LinkEvaluatorTuple( + project_name=self.project_name, + canonical_name=self._canonical_name, + formats=self._formats, + target_python=self._target_python, + allow_yanked=self._allow_yanked, + ignore_requires_python=self._ignore_requires_python, + py_version_re=self._py_version_re + ) + @staticmethod def evaluate_link_static(link_evaluator, link): # type: (Link) -> Tuple[bool, Optional[str]] @@ -157,7 +169,7 @@ def evaluate_link_static(link_evaluator, link): the link fails to qualify. """ version = None - if link.is_yanked and not link_evaluator._allow_yanked: + if link.is_yanked and not link_evaluator.allow_yanked: reason = link.yanked_reason or '' return (False, f'yanked for reason: {reason}') @@ -170,7 +182,7 @@ def evaluate_link_static(link_evaluator, link): return (False, 'not a file') if ext not in SUPPORTED_EXTENSIONS: return (False, f'unsupported archive format: {ext}') - if "binary" not in link_evaluator._formats and ext == WHEEL_EXTENSION: + if "binary" not in link_evaluator.formats and ext == WHEEL_EXTENSION: reason = 'No binaries permitted for {}'.format( link_evaluator.project_name) return (False, reason) @@ -181,12 +193,12 @@ def evaluate_link_static(link_evaluator, link): wheel = Wheel(link.filename) except InvalidWheelFilename: return (False, 'invalid wheel filename') - if canonicalize_name(wheel.name) != link_evaluator._canonical_name: + if canonicalize_name(wheel.name) != link_evaluator.canonical_name: reason = 'wrong project name (not {})'.format( link_evaluator.project_name) return (False, reason) - supported_tags = link_evaluator._target_python.get_tags() + supported_tags = link_evaluator.target_python.get_tags() if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -202,28 +214,28 @@ def evaluate_link_static(link_evaluator, link): version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. - if "source" not in link_evaluator._formats and ext != WHEEL_EXTENSION: + if "source" not in link_evaluator.formats and ext != WHEEL_EXTENSION: reason = f'No sources permitted for {link_evaluator.project_name}' return (False, reason) if not version: version = _extract_version_from_fragment( - egg_info, link_evaluator._canonical_name, + egg_info, link_evaluator.canonical_name, ) if not version: reason = f'Missing project version for {link_evaluator.project_name}' return (False, reason) - match = link_evaluator._py_version_re.search(version) + match = link_evaluator.py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) - if py_version != link_evaluator._target_python.py_version: + if py_version != link_evaluator.target_python.py_version: return (False, 'Python version is incorrect') supports_python = _check_link_requires_python( - link, version_info=link_evaluator._target_python.py_version_info, - ignore_requires_python=link_evaluator._ignore_requires_python, + link, version_info=link_evaluator.target_python.py_version_info, + ignore_requires_python=link_evaluator.ignore_requires_python, ) if not supports_python: # Return None for the reason text to suppress calling @@ -244,84 +256,7 @@ def evaluate_link(self, link): `is_candidate` is False, an optional string to log the reason the link fails to qualify. """ - return LinkEvaluator.evaluate_link_static(self, link) - # version = None - # if link.is_yanked and not self._allow_yanked: - # reason = link.yanked_reason or '' - # return (False, f'yanked for reason: {reason}') - - # if link.egg_fragment: - # egg_info = link.egg_fragment - # ext = link.ext - # else: - # egg_info, ext = link.splitext() - # if not ext: - # return (False, 'not a file') - # if ext not in SUPPORTED_EXTENSIONS: - # return (False, f'unsupported archive format: {ext}') - # if "binary" not in self._formats and ext == WHEEL_EXTENSION: - # reason = 'No binaries permitted for {}'.format( - # self.project_name) - # return (False, reason) - # if "macosx10" in link.path and ext == '.zip': - # return (False, 'macosx10 one') - # if ext == WHEEL_EXTENSION: - # try: - # wheel = Wheel(link.filename) - # except InvalidWheelFilename: - # return (False, 'invalid wheel filename') - # if canonicalize_name(wheel.name) != self._canonical_name: - # reason = 'wrong project name (not {})'.format( - # self.project_name) - # return (False, reason) - - # supported_tags = self._target_python.get_tags() - # if not wheel.supported(supported_tags): - # # Include the wheel's tags in the reason string to - # # simplify troubleshooting compatibility issues. - # file_tags = wheel.get_formatted_file_tags() - # reason = ( - # "none of the wheel's tags ({}) are compatible " - # "(run pip debug --verbose to show compatible tags)".format( - # ', '.join(file_tags) - # ) - # ) - # return (False, reason) - - # version = wheel.version - - # # This should be up by the self.ok_binary check, but see issue 2700. - # if "source" not in self._formats and ext != WHEEL_EXTENSION: - # reason = f'No sources permitted for {self.project_name}' - # return (False, reason) - - # if not version: - # version = _extract_version_from_fragment( - # egg_info, self._canonical_name, - # ) - # if not version: - # reason = f'Missing project version for {self.project_name}' - # return (False, reason) - - # match = self._py_version_re.search(version) - # if match: - # version = version[:match.start()] - # py_version = match.group(1) - # if py_version != self._target_python.py_version: - # return (False, 'Python version is incorrect') - - # supports_python = _check_link_requires_python( - # link, version_info=self._target_python.py_version_info, - # ignore_requires_python=self._ignore_requires_python, - # ) - # if not supports_python: - # # Return None for the reason text to suppress calling - # # _log_skipped_link(). - # return (False, None) - - # logger.debug('Found link %s, version: %s', link, version) - - # return (True, version) + return LinkEvaluator.evaluate_link_static(self.get_state_as_tuple(), link) def filter_unallowed_hashes( @@ -850,7 +785,7 @@ def _log_skipped_link_static(package_finder, link, reason): def _log_skipped_link(self, link, reason): # type: (Link, str) -> None - PackageFinder._log_skipped_link_static(get_state_as_tuple(), link, reason) + PackageFinder._log_skipped_link_static(self.get_state_as_tuple(), link, reason) # if link not in self._logged_links: # # Put the link at the end so the reason is more visible and because # # the link string is usually very long. @@ -881,7 +816,7 @@ def get_install_candidate(self, link_evaluator, link): If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - return PackageFinder.get_install_candidate_static(get_state_as_tuple(), link_evaluator, link) + return PackageFinder.get_install_candidate_static(self.get_state_as_tuple(), link_evaluator.get_state_as_tuple(), link) # is_candidate, result = link_evaluator.evaluate_link(link) # if not is_candidate: # if result: @@ -913,14 +848,7 @@ def evaluate_links(self, link_evaluator, links): """ Convert links that are candidates to InstallationCandidate objects. """ - return PackageFinder.evaluate_links_static(get_state_as_tuple(), link_evaluator, links) - # candidates = [] - # for link in PackageFinder._sort_links_static(links): - # candidate = PackageFinder.get_install_candidate_static(self, link_evaluator, link) - # if candidate is not None: - # candidates.append(candidate) - - # return candidates + return PackageFinder.evaluate_links_static(self.get_state_as_tuple(), link_evaluator.get_state_as_tuple(), links) @staticmethod def process_project_url_static(package_finder, project_url, link_evaluator): @@ -945,7 +873,11 @@ def process_project_url_static(package_finder, project_url, link_evaluator): def process_project_url(self, project_url, link_evaluator): # type: (Link, LinkEvaluator) -> List[InstallationCandidate] - return PackageFinder.process_project_url_static(get_state_as_tuple(), project_url, link_evaluator) + return PackageFinder.process_project_url_static( + self.get_state_as_tuple(), + project_url, + link_evaluator.get_state_as_tuple() + ) @staticmethod @functools.lru_cache(maxsize=None) @@ -999,7 +931,6 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): # This is an intentional priority ordering return file_candidates + page_candidates - # @functools.lru_cache(maxsize=None) def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name @@ -1011,7 +942,9 @@ def find_all_candidates(self, project_name): are accepted. """ link_evaluator = self.make_link_evaluator(project_name) + package_finder = self.get_state_as_tuple(True) + link_evaluator = link_evaluator.get_state_as_tuple() return PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) def make_candidate_evaluator( From 151da196fc719a12a2efddb54d22ba1a5cdfc5a2 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 15:29:32 +0100 Subject: [PATCH 04/16] Add TODO --- src/pip/_internal/index/package_finder.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 383631218dc..66e9bf0d3e0 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -897,11 +897,6 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): # Just add link_collector to the tuple collected_links = package_finder.link_collector.collect_links(project_name) - # Refactored evaluate_links, _sort_links, get_install_candidate, _logged_skipped_link - # dependencies needed from PackageFinder : - # * _logged_link (SHOULD FIND A WAY TO KEEP TRACK OF CHANGED IN _logged_link and apply the changes after cached method) - # and link_evaluator - # * Everything find_links_versions = PackageFinder.evaluate_links_static( package_finder, link_evaluator, @@ -928,6 +923,8 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): paths = [url_to_path(c.link.url) for c in file_candidates] logger.debug("Local files found: %s", ", ".join(paths)) + # TODO: If necessary, return tuple of the candidates and logging_links so this can be updated in original function + # This is an intentional priority ordering return file_candidates + page_candidates From 2c6b177752ba2ad9c6decc8130cc14c840ef9f98 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 15:31:54 +0100 Subject: [PATCH 05/16] Remove commented code --- src/pip/_internal/index/package_finder.py | 39 +++++++---------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 66e9bf0d3e0..4620d4b5c14 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -786,18 +786,10 @@ def _log_skipped_link_static(package_finder, link, reason): def _log_skipped_link(self, link, reason): # type: (Link, str) -> None PackageFinder._log_skipped_link_static(self.get_state_as_tuple(), link, reason) - # if link not in self._logged_links: - # # Put the link at the end so the reason is more visible and because - # # the link string is usually very long. - # logger.debug('Skipping link: %s: %s', reason, link) - # self._logged_links.add(link) @staticmethod def get_install_candidate_static(package_finder, link_evaluator, link): - """ - If the link is a candidate for install, convert it to an - InstallationCandidate and return it. Otherwise, return None. - """ + # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] is_candidate, result = LinkEvaluator.evaluate_link_static(link_evaluator, link) if not is_candidate: if result: @@ -811,30 +803,19 @@ def get_install_candidate_static(package_finder, link_evaluator, link): ) def get_install_candidate(self, link_evaluator, link): - # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] """ If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - return PackageFinder.get_install_candidate_static(self.get_state_as_tuple(), link_evaluator.get_state_as_tuple(), link) - # is_candidate, result = link_evaluator.evaluate_link(link) - # if not is_candidate: - # if result: - # self._log_skipped_link(link, reason=result) - # return None - - # return InstallationCandidate( - # name=link_evaluator.project_name, - # link=link, - # version=result, - # ) + return PackageFinder.get_install_candidate_static( + self.get_state_as_tuple(), + link_evaluator.get_state_as_tuple(), + link + ) @staticmethod def evaluate_links_static(package_finder, link_evaluator, links): # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] - """ - Convert links that are candidates to InstallationCandidate objects. - """ candidates = [] for link in PackageFinder._sort_links_static(links): candidate = PackageFinder.get_install_candidate_static(package_finder, link_evaluator, link) @@ -848,7 +829,11 @@ def evaluate_links(self, link_evaluator, links): """ Convert links that are candidates to InstallationCandidate objects. """ - return PackageFinder.evaluate_links_static(self.get_state_as_tuple(), link_evaluator.get_state_as_tuple(), links) + return PackageFinder.evaluate_links_static( + self.get_state_as_tuple(), + link_evaluator.get_state_as_tuple(), + links + ) @staticmethod def process_project_url_static(package_finder, project_url, link_evaluator): @@ -924,7 +909,7 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): logger.debug("Local files found: %s", ", ".join(paths)) # TODO: If necessary, return tuple of the candidates and logging_links so this can be updated in original function - + # This is an intentional priority ordering return file_candidates + page_candidates From ef1053f9bc1f952f26813bda7faad3d1fa3626b5 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 19:44:11 +0100 Subject: [PATCH 06/16] Refactor find_best_candidate --- src/pip/_internal/index/package_finder.py | 23 +++++++++++++++---- tests/unit/test_resolution_legacy_resolver.py | 4 +++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 4620d4b5c14..867671c5b02 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -924,7 +924,7 @@ def find_all_candidates(self, project_name): are accepted. """ link_evaluator = self.make_link_evaluator(project_name) - + package_finder = self.get_state_as_tuple(True) link_evaluator = link_evaluator.get_state_as_tuple() return PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) @@ -947,8 +947,14 @@ def make_candidate_evaluator( specifier=specifier, hashes=hashes, ) - + + @staticmethod @functools.lru_cache(maxsize=None) + def find_best_candidate_static(package_finder, link_evaluator, candidate_evaluator, project_name): + candidates = PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) + + return candidate_evaluator.compute_best_candidate(candidates) + def find_best_candidate( self, project_name, # type: str @@ -964,13 +970,22 @@ def find_best_candidate( :return: A `BestCandidateResult` instance. """ - candidates = self.find_all_candidates(project_name) + link_evaluator = self.make_link_evaluator(project_name) + + package_finder = self.get_state_as_tuple(True) + link_evaluator = link_evaluator.get_state_as_tuple() + candidate_evaluator = self.make_candidate_evaluator( project_name=project_name, specifier=specifier, hashes=hashes, ) - return candidate_evaluator.compute_best_candidate(candidates) + return PackageFinder.find_best_candidate_static( + package_finder, + link_evaluator, + candidate_evaluator, + project_name + ) def find_requirement(self, req, upgrade): # type: (InstallRequirement, bool) -> Optional[InstallationCandidate] diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 236a4c624e7..3138862b248 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -6,6 +6,7 @@ from pip._internal.exceptions import NoneMetadataError, UnsupportedPythonVersion from pip._internal.req.constructors import install_req_from_line +from pip._internal.index.package_finder import PackageFinder from pip._internal.resolution.legacy.resolver import ( Resolver, _check_dist_requires_python, @@ -178,11 +179,12 @@ class TestYankedWarning: Test _populate_link() emits warning if one or more candidates are yanked. """ def _make_test_resolver(self, monkeypatch, mock_candidates): - def _find_candidates(project_name): + def _find_candidates(package_finder, link_evaluator, project_name): return mock_candidates finder = make_test_finder() monkeypatch.setattr(finder, "find_all_candidates", _find_candidates) + monkeypatch.setattr(PackageFinder, "find_all_candidates_static", _find_candidates) return Resolver( finder=finder, From 0deab39049c0a5ad26d52a6c29053a0c89fca897 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 21:34:12 +0100 Subject: [PATCH 07/16] Add static function for make_candidate_evaluator --- src/pip/_internal/index/package_finder.py | 48 +++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 867671c5b02..2511b637f5f 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -50,7 +50,7 @@ Tuple[int, int, int, _BaseVersion, Optional[int], BuildTag] ) -PackageFinderTuple = collections.namedtuple('PackageFinder', 'logged_links link_collector') +PackageFinderTuple = collections.namedtuple('PackageFinder', 'logged_links link_collector target_python candidate_prefs') LinkEvaluatorTuple = collections.namedtuple('LinkEvaluator', 'project_name canonical_name formats target_python allow_yanked ignore_requires_python py_version_re') def _check_link_requires_python( @@ -751,7 +751,7 @@ def get_state_as_tuple(self, immutable=False): if immutable: logged_links = frozenset(logged_links) - return PackageFinderTuple(logged_links=logged_links, link_collector=self._link_collector) + return PackageFinderTuple(logged_links=logged_links, link_collector=self._link_collector, candidate_prefs=self._candidate_prefs, target_python=self._target_python) @staticmethod def _sort_links_static(links): @@ -877,7 +877,8 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): """ # Need to convert logged_links to normal set, so we can update it. - package_finder = PackageFinderTuple(logged_links=set(package_finder.logged_links), link_collector=package_finder.link_collector) + package_finder = PackageFinderTuple(logged_links=set(package_finder.logged_links), link_collector=package_finder.link_collector, target_python=package_finder.target_python, candidate_prefs=package_finder.candidate_prefs) + # Just add link_collector to the tuple collected_links = package_finder.link_collector.collect_links(project_name) @@ -929,6 +930,23 @@ def find_all_candidates(self, project_name): link_evaluator = link_evaluator.get_state_as_tuple() return PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) + @staticmethod + def make_candidate_evaluator_static( + package_finder, + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + candidate_prefs = package_finder.candidate_prefs + return CandidateEvaluator.create( + project_name=project_name, + target_python=package_finder.target_python, + prefer_binary=candidate_prefs.prefer_binary, + allow_all_prereleases=candidate_prefs.allow_all_prereleases, + specifier=specifier, + hashes=hashes, + ) + def make_candidate_evaluator( self, project_name, # type: str @@ -950,7 +968,14 @@ def make_candidate_evaluator( @staticmethod @functools.lru_cache(maxsize=None) - def find_best_candidate_static(package_finder, link_evaluator, candidate_evaluator, project_name): + def find_best_candidate_static( + package_finder, + link_evaluator, + project_name, + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None + ): + candidate_evaluator = PackageFinder.make_candidate_evaluator_static(package_finder, project_name, specifier, hashes) candidates = PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) return candidate_evaluator.compute_best_candidate(candidates) @@ -975,16 +1000,17 @@ def find_best_candidate( package_finder = self.get_state_as_tuple(True) link_evaluator = link_evaluator.get_state_as_tuple() - candidate_evaluator = self.make_candidate_evaluator( - project_name=project_name, - specifier=specifier, - hashes=hashes, - ) + # candidate_evaluator = self.make_candidate_evaluator( + # project_name=project_name, + # specifier=specifier, + # hashes=hashes, + # ) return PackageFinder.find_best_candidate_static( package_finder, link_evaluator, - candidate_evaluator, - project_name + project_name, + specifier, + hashes ) def find_requirement(self, req, upgrade): From 3f19d5a776200a183821d57105d947a427c08df1 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 13 Mar 2021 21:37:53 +0100 Subject: [PATCH 08/16] make_candidate_evaluator calls static method --- src/pip/_internal/index/package_finder.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 2511b637f5f..0b5e6f4117f 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -956,14 +956,12 @@ def make_candidate_evaluator( # type: (...) -> CandidateEvaluator """Create a CandidateEvaluator object to use. """ - candidate_prefs = self._candidate_prefs - return CandidateEvaluator.create( - project_name=project_name, - target_python=self._target_python, - prefer_binary=candidate_prefs.prefer_binary, - allow_all_prereleases=candidate_prefs.allow_all_prereleases, - specifier=specifier, - hashes=hashes, + package_finder = self.get_state_as_tuple() + return PackageFinder.make_candidate_evaluator_static( + package_finder, + project_name, + specifier, + hashes ) @staticmethod @@ -1000,11 +998,6 @@ def find_best_candidate( package_finder = self.get_state_as_tuple(True) link_evaluator = link_evaluator.get_state_as_tuple() - # candidate_evaluator = self.make_candidate_evaluator( - # project_name=project_name, - # specifier=specifier, - # hashes=hashes, - # ) return PackageFinder.find_best_candidate_static( package_finder, link_evaluator, From 239222a8c133f97bb437471890cc70272865d0ed Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sun, 14 Mar 2021 12:19:29 +0100 Subject: [PATCH 09/16] Resolve linting errors --- src/pip/_internal/index/package_finder.py | 269 ++++++++++++------ tests/unit/test_resolution_legacy_resolver.py | 8 +- 2 files changed, 182 insertions(+), 95 deletions(-) mode change 100644 => 100755 src/pip/_internal/index/package_finder.py mode change 100644 => 100755 tests/unit/test_resolution_legacy_resolver.py diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py old mode 100644 new mode 100755 index 0b5e6f4117f..f2c022ef221 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -6,9 +6,8 @@ import functools import itertools import logging -import collections - import re +from collections import namedtuple from typing import FrozenSet, Iterable, List, Optional, Set, Tuple, Union from pip._vendor.packaging import specifiers @@ -50,8 +49,29 @@ Tuple[int, int, int, _BaseVersion, Optional[int], BuildTag] ) -PackageFinderTuple = collections.namedtuple('PackageFinder', 'logged_links link_collector target_python candidate_prefs') -LinkEvaluatorTuple = collections.namedtuple('LinkEvaluator', 'project_name canonical_name formats target_python allow_yanked ignore_requires_python py_version_re') +PackageFinderTuple = namedtuple( + 'PackageFinderTuple', + """ + logged_links + link_collector + target_python + candidate_prefs + """ +) + +LinkEvaluatorTuple = namedtuple( + 'LinkEvaluatorTuple', + """ + project_name + canonical_name + formats + target_python + allow_yanked + ignore_requires_python + py_version_re + """ +) + def _check_link_requires_python( link, # type: Link @@ -147,19 +167,20 @@ def __init__( self.project_name = project_name def get_state_as_tuple(self): + # type: (LinkEvaluator) -> LinkEvaluatorTuple return LinkEvaluatorTuple( project_name=self.project_name, - canonical_name=self._canonical_name, - formats=self._formats, - target_python=self._target_python, - allow_yanked=self._allow_yanked, + canonical_name=self._canonical_name, + formats=self._formats, + target_python=self._target_python, + allow_yanked=self._allow_yanked, ignore_requires_python=self._ignore_requires_python, py_version_re=self._py_version_re ) - + @staticmethod - def evaluate_link_static(link_evaluator, link): - # type: (Link) -> Tuple[bool, Optional[str]] + def evaluate_link_static(link_evaluator_tuple, link): + # type: (LinkEvaluatorTuple, Link) -> Tuple[bool, Optional[str]] """ Determine whether a link is a candidate for installation. @@ -169,7 +190,7 @@ def evaluate_link_static(link_evaluator, link): the link fails to qualify. """ version = None - if link.is_yanked and not link_evaluator.allow_yanked: + if link.is_yanked and not link_evaluator_tuple.allow_yanked: reason = link.yanked_reason or '' return (False, f'yanked for reason: {reason}') @@ -182,9 +203,9 @@ def evaluate_link_static(link_evaluator, link): return (False, 'not a file') if ext not in SUPPORTED_EXTENSIONS: return (False, f'unsupported archive format: {ext}') - if "binary" not in link_evaluator.formats and ext == WHEEL_EXTENSION: + if "binary" not in link_evaluator_tuple.formats and ext == WHEEL_EXTENSION: reason = 'No binaries permitted for {}'.format( - link_evaluator.project_name) + link_evaluator_tuple.project_name) return (False, reason) if "macosx10" in link.path and ext == '.zip': return (False, 'macosx10 one') @@ -193,12 +214,12 @@ def evaluate_link_static(link_evaluator, link): wheel = Wheel(link.filename) except InvalidWheelFilename: return (False, 'invalid wheel filename') - if canonicalize_name(wheel.name) != link_evaluator.canonical_name: + if canonicalize_name(wheel.name) != link_evaluator_tuple.canonical_name: reason = 'wrong project name (not {})'.format( - link_evaluator.project_name) + link_evaluator_tuple.project_name) return (False, reason) - supported_tags = link_evaluator.target_python.get_tags() + supported_tags = link_evaluator_tuple.target_python.get_tags() if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -214,28 +235,28 @@ def evaluate_link_static(link_evaluator, link): version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. - if "source" not in link_evaluator.formats and ext != WHEEL_EXTENSION: - reason = f'No sources permitted for {link_evaluator.project_name}' + if "source" not in link_evaluator_tuple.formats and ext != WHEEL_EXTENSION: + reason = f'No sources permitted for {link_evaluator_tuple.project_name}' return (False, reason) if not version: version = _extract_version_from_fragment( - egg_info, link_evaluator.canonical_name, + egg_info, link_evaluator_tuple.canonical_name, ) if not version: - reason = f'Missing project version for {link_evaluator.project_name}' + reason = f'Missing project version for {link_evaluator_tuple.project_name}' return (False, reason) - match = link_evaluator.py_version_re.search(version) + match = link_evaluator_tuple.py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) - if py_version != link_evaluator.target_python.py_version: + if py_version != link_evaluator_tuple.target_python.py_version: return (False, 'Python version is incorrect') supports_python = _check_link_requires_python( - link, version_info=link_evaluator.target_python.py_version_info, - ignore_requires_python=link_evaluator.ignore_requires_python, + link, version_info=link_evaluator_tuple.target_python.py_version_info, + ignore_requires_python=link_evaluator_tuple.ignore_requires_python, ) if not supports_python: # Return None for the reason text to suppress calling @@ -583,6 +604,7 @@ def sort_best_candidate( if not candidates: return None best_candidate = max(candidates, key=self._sort_key) + return best_candidate def compute_best_candidate( @@ -603,6 +625,7 @@ def compute_best_candidate( best_candidate=best_candidate, ) + class PackageFinder: """This finds packages. @@ -745,16 +768,28 @@ def make_link_evaluator(self, project_name): allow_yanked=self._allow_yanked, ignore_requires_python=self._ignore_requires_python, ) - + def get_state_as_tuple(self, immutable=False): - logged_links = self._logged_links + # type: (bool) -> PackageFinderTuple + if immutable: - logged_links = frozenset(logged_links) + return PackageFinderTuple( + logged_links=frozenset(self._logged_links), + link_collector=self._link_collector, + candidate_prefs=self._candidate_prefs, + target_python=self._target_python + ) + + return PackageFinderTuple( + logged_links=self._logged_links, + link_collector=self._link_collector, + candidate_prefs=self._candidate_prefs, + target_python=self._target_python + ) - return PackageFinderTuple(logged_links=logged_links, link_collector=self._link_collector, candidate_prefs=self._candidate_prefs, target_python=self._target_python) - @staticmethod def _sort_links_static(links): + # type: (Iterable[Link]) -> List[Link] eggs, no_eggs = [], [] seen = set() # type: Set[Link] for link in links: @@ -775,50 +810,70 @@ def _sort_links(self, links): return PackageFinder._sort_links_static(links) @staticmethod - def _log_skipped_link_static(package_finder, link, reason): - # type: (Link, str) -> None - if link not in package_finder.logged_links: + def _log_skipped_link_static(package_finder_tuple, link, reason): + # type: (PackageFinderTuple, Link, str) -> None + if link not in package_finder_tuple.logged_links: # Put the link at the end so the reason is more visible and because # the link string is usually very long. logger.debug('Skipping link: %s: %s', reason, link) - package_finder.logged_links.add(link) - + package_finder_tuple.logged_links.add(link) + def _log_skipped_link(self, link, reason): # type: (Link, str) -> None PackageFinder._log_skipped_link_static(self.get_state_as_tuple(), link, reason) @staticmethod - def get_install_candidate_static(package_finder, link_evaluator, link): - # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] - is_candidate, result = LinkEvaluator.evaluate_link_static(link_evaluator, link) + def get_install_candidate_static( + package_finder_tuple, # type: PackageFinderTuple + link_evaluator_tuple, # type: LinkEvaluatorTuple + link # type: Link + ): + # type: (...) -> Optional[InstallationCandidate] + is_candidate, result = LinkEvaluator.evaluate_link_static( + link_evaluator_tuple, + link + ) if not is_candidate: if result: - PackageFinder._log_skipped_link_static(package_finder, link, reason=result) + PackageFinder._log_skipped_link_static( + package_finder_tuple, + link, + reason=result + ) return None return InstallationCandidate( - name=link_evaluator.project_name, + name=link_evaluator_tuple.project_name, link=link, version=result, ) def get_install_candidate(self, link_evaluator, link): + # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] """ If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ return PackageFinder.get_install_candidate_static( - self.get_state_as_tuple(), - link_evaluator.get_state_as_tuple(), + self.get_state_as_tuple(), + link_evaluator.get_state_as_tuple(), link ) @staticmethod - def evaluate_links_static(package_finder, link_evaluator, links): - # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] + def evaluate_links_static( + package_finder_tuple, # type: PackageFinderTuple + link_evaluator_tuple, # type: LinkEvaluatorTuple + links # type: Iterable[Link] + ): + # type: (...) -> List[InstallationCandidate] candidates = [] for link in PackageFinder._sort_links_static(links): - candidate = PackageFinder.get_install_candidate_static(package_finder, link_evaluator, link) + candidate = PackageFinder.get_install_candidate_static( + package_finder_tuple, + link_evaluator_tuple, + link + ) if candidate is not None: candidates.append(candidate) @@ -830,18 +885,22 @@ def evaluate_links(self, link_evaluator, links): Convert links that are candidates to InstallationCandidate objects. """ return PackageFinder.evaluate_links_static( - self.get_state_as_tuple(), - link_evaluator.get_state_as_tuple(), + self.get_state_as_tuple(), + link_evaluator.get_state_as_tuple(), links ) @staticmethod - def process_project_url_static(package_finder, project_url, link_evaluator): - # type: (Link, LinkEvaluator) -> List[InstallationCandidate] + def process_project_url_static( + package_finder_tuple, # type: PackageFinderTuple + project_url, # type: Link + link_evaluator_tuple # type: LinkEvaluatorTuple + ): + # type: (...) -> List[InstallationCandidate] logger.debug( 'Fetching project page and analyzing links: %s', project_url, ) - html_page = package_finder.link_collector.fetch_page(project_url) + html_page = package_finder_tuple.link_collector.fetch_page(project_url) if html_page is None: return [] @@ -849,8 +908,8 @@ def process_project_url_static(package_finder, project_url, link_evaluator): with indent_log(): package_links = PackageFinder.evaluate_links_static( - package_finder, - link_evaluator, + package_finder_tuple, + link_evaluator_tuple, links=page_links, ) @@ -859,14 +918,19 @@ def process_project_url_static(package_finder, project_url, link_evaluator): def process_project_url(self, project_url, link_evaluator): # type: (Link, LinkEvaluator) -> List[InstallationCandidate] return PackageFinder.process_project_url_static( - self.get_state_as_tuple(), - project_url, + self.get_state_as_tuple(), + project_url, link_evaluator.get_state_as_tuple() ) - + @staticmethod @functools.lru_cache(maxsize=None) - def find_all_candidates_static(package_finder, link_evaluator, project_name): + def find_all_candidates_static( + package_finder_tuple, # type: PackageFinderTuple + link_evaluator_tuple, # type: LinkEvaluatorTuple + project_name # type: str + ): + # type: (...) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name This checks index_urls and find_links. @@ -877,15 +941,21 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): """ # Need to convert logged_links to normal set, so we can update it. - package_finder = PackageFinderTuple(logged_links=set(package_finder.logged_links), link_collector=package_finder.link_collector, target_python=package_finder.target_python, candidate_prefs=package_finder.candidate_prefs) - + package_finder_tuple = PackageFinderTuple( + logged_links=set(package_finder_tuple.logged_links), + link_collector=package_finder_tuple.link_collector, + target_python=package_finder_tuple.target_python, + candidate_prefs=package_finder_tuple.candidate_prefs + ) # Just add link_collector to the tuple - collected_links = package_finder.link_collector.collect_links(project_name) + collected_links = package_finder_tuple.link_collector.collect_links( + project_name + ) find_links_versions = PackageFinder.evaluate_links_static( - package_finder, - link_evaluator, + package_finder_tuple, + link_evaluator_tuple, links=collected_links.find_links, ) @@ -893,24 +963,22 @@ def find_all_candidates_static(package_finder, link_evaluator, project_name): page_versions = [] for project_url in collected_links.project_urls: package_links = PackageFinder.process_project_url_static( - package_finder, - project_url, - link_evaluator=link_evaluator, + package_finder_tuple, + project_url, + link_evaluator_tuple, ) page_versions.extend(package_links) file_versions = PackageFinder.evaluate_links_static( - package_finder, - link_evaluator, - sorted(file_links_it, reverse=True), + package_finder_tuple, + link_evaluator_tuple, + links=collected_links.files, ) if logger.isEnabledFor(logging.DEBUG) and file_candidates: paths = [url_to_path(c.link.url) for c in file_candidates] logger.debug("Local files found: %s", ", ".join(paths)) - # TODO: If necessary, return tuple of the candidates and logging_links so this can be updated in original function - # This is an intentional priority ordering return file_candidates + page_candidates @@ -926,21 +994,26 @@ def find_all_candidates(self, project_name): """ link_evaluator = self.make_link_evaluator(project_name) - package_finder = self.get_state_as_tuple(True) - link_evaluator = link_evaluator.get_state_as_tuple() - return PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) + package_finder_tuple = self.get_state_as_tuple(True) + link_evaluator_tuple = link_evaluator.get_state_as_tuple() + return PackageFinder.find_all_candidates_static( + package_finder_tuple, + link_evaluator_tuple, + project_name + ) @staticmethod def make_candidate_evaluator_static( - package_finder, - project_name, # type: str - specifier=None, # type: Optional[specifiers.BaseSpecifier] - hashes=None, # type: Optional[Hashes] + package_finder_tuple, # type: PackageFinderTuple + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] ): - candidate_prefs = package_finder.candidate_prefs + # type: (...) -> CandidateEvaluator + candidate_prefs = package_finder_tuple.candidate_prefs return CandidateEvaluator.create( project_name=project_name, - target_python=package_finder.target_python, + target_python=package_finder_tuple.target_python, prefer_binary=candidate_prefs.prefer_binary, allow_all_prereleases=candidate_prefs.allow_all_prereleases, specifier=specifier, @@ -956,28 +1029,38 @@ def make_candidate_evaluator( # type: (...) -> CandidateEvaluator """Create a CandidateEvaluator object to use. """ - package_finder = self.get_state_as_tuple() + package_finder_tuple = self.get_state_as_tuple() return PackageFinder.make_candidate_evaluator_static( - package_finder, + package_finder_tuple, project_name, specifier, hashes ) - + @staticmethod @functools.lru_cache(maxsize=None) def find_best_candidate_static( - package_finder, - link_evaluator, - project_name, - specifier=None, # type: Optional[specifiers.BaseSpecifier] - hashes=None + package_finder_tuple, # type: PackageFinderTuple + link_evaluator_tuple, # type: LinkEvaluatorTuple + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None # type: Optional[Hashes] ): - candidate_evaluator = PackageFinder.make_candidate_evaluator_static(package_finder, project_name, specifier, hashes) - candidates = PackageFinder.find_all_candidates_static(package_finder, link_evaluator, project_name) + # type: (...) -> BestCandidateResult + candidate_evaluator = PackageFinder.make_candidate_evaluator_static( + package_finder_tuple, + project_name, + specifier, + hashes + ) + candidates = PackageFinder.find_all_candidates_static( + package_finder_tuple, + link_evaluator_tuple, + project_name + ) return candidate_evaluator.compute_best_candidate(candidates) - + def find_best_candidate( self, project_name, # type: str @@ -995,12 +1078,12 @@ def find_best_candidate( """ link_evaluator = self.make_link_evaluator(project_name) - package_finder = self.get_state_as_tuple(True) - link_evaluator = link_evaluator.get_state_as_tuple() + package_finder_tuple = self.get_state_as_tuple(True) + link_evaluator_tuple = link_evaluator.get_state_as_tuple() return PackageFinder.find_best_candidate_static( - package_finder, - link_evaluator, + package_finder_tuple, + link_evaluator_tuple, project_name, specifier, hashes diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py old mode 100644 new mode 100755 index 3138862b248..aedb02ba804 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -5,8 +5,8 @@ from pip._vendor import pkg_resources from pip._internal.exceptions import NoneMetadataError, UnsupportedPythonVersion -from pip._internal.req.constructors import install_req_from_line from pip._internal.index.package_finder import PackageFinder +from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.legacy.resolver import ( Resolver, _check_dist_requires_python, @@ -184,7 +184,11 @@ def _find_candidates(package_finder, link_evaluator, project_name): finder = make_test_finder() monkeypatch.setattr(finder, "find_all_candidates", _find_candidates) - monkeypatch.setattr(PackageFinder, "find_all_candidates_static", _find_candidates) + monkeypatch.setattr( + PackageFinder, + "find_all_candidates_static", + _find_candidates + ) return Resolver( finder=finder, From 2f5b9a45ea0384cae387404665b8be8703eb156e Mon Sep 17 00:00:00 2001 From: Martin Li Date: Wed, 3 Mar 2021 11:51:48 +0100 Subject: [PATCH 10/16] Extract logic from find_best_candidate and find_all_candidate to static methods --- src/pip/_internal/index/package_finder.py | 210 ++++++++-------------- 1 file changed, 79 insertions(+), 131 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index f2c022ef221..c8f2dbb685b 100755 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -116,6 +116,81 @@ def _check_link_requires_python( return True +@functools.lru_cache(maxsize=None) +def _find_best_candidate_cached( + self, + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] +): + # type: (...) -> BestCandidateResult + """Find matches for the given project and specifier. + + :param specifier: An optional object implementing `filter` + (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable + versions. + + :return: A `BestCandidateResult` instance, + and cache the result to reduce future call's execution time. + """ + candidates = self.find_all_candidates(project_name) + candidate_evaluator = self.make_candidate_evaluator( + project_name=project_name, + specifier=specifier, + hashes=hashes, + ) + return candidate_evaluator.compute_best_candidate(candidates) + + +@functools.lru_cache(maxsize=None) +def _find_all_candidates_cached( + self, + project_name, # type: str +): + # type: (...) -> List[InstallationCandidate] + """Find all available InstallationCandidate for project_name + + This checks index_urls and find_links. + All versions found are returned as an InstallationCandidate list, + and results are cached to reduce future call's execution time. + + See LinkEvaluator.evaluate_link() for details on which files + are accepted. + """ + collected_links = self._link_collector.collect_links(project_name) + + link_evaluator = self.make_link_evaluator(project_name) + + find_links_versions = self.evaluate_links( + link_evaluator, + links=collected_links.find_links, + ) + + page_versions = [] + for project_url in collected_links.project_urls: + package_links = self.process_project_url( + project_url, link_evaluator=link_evaluator, + ) + page_versions.extend(package_links) + + file_versions = self.evaluate_links( + link_evaluator, + links=collected_links.files, + ) + if file_versions: + file_versions.sort(reverse=True) + logger.debug( + 'Local files found: %s', + ', '.join([ + url_to_path(candidate.link.url) + for candidate in file_versions + ]) + ) + + # This is an intentional priority ordering + return file_versions + find_links_versions + page_versions + + class LinkEvaluator: """ @@ -915,73 +990,6 @@ def process_project_url_static( return package_links - def process_project_url(self, project_url, link_evaluator): - # type: (Link, LinkEvaluator) -> List[InstallationCandidate] - return PackageFinder.process_project_url_static( - self.get_state_as_tuple(), - project_url, - link_evaluator.get_state_as_tuple() - ) - - @staticmethod - @functools.lru_cache(maxsize=None) - def find_all_candidates_static( - package_finder_tuple, # type: PackageFinderTuple - link_evaluator_tuple, # type: LinkEvaluatorTuple - project_name # type: str - ): - # type: (...) -> List[InstallationCandidate] - """Find all available InstallationCandidate for project_name - - This checks index_urls and find_links. - All versions found are returned as an InstallationCandidate list. - - See LinkEvaluator.evaluate_link() for details on which files - are accepted. - """ - - # Need to convert logged_links to normal set, so we can update it. - package_finder_tuple = PackageFinderTuple( - logged_links=set(package_finder_tuple.logged_links), - link_collector=package_finder_tuple.link_collector, - target_python=package_finder_tuple.target_python, - candidate_prefs=package_finder_tuple.candidate_prefs - ) - - # Just add link_collector to the tuple - collected_links = package_finder_tuple.link_collector.collect_links( - project_name - ) - - find_links_versions = PackageFinder.evaluate_links_static( - package_finder_tuple, - link_evaluator_tuple, - links=collected_links.find_links, - ) - - # Only needs link collector from package_finder - page_versions = [] - for project_url in collected_links.project_urls: - package_links = PackageFinder.process_project_url_static( - package_finder_tuple, - project_url, - link_evaluator_tuple, - ) - page_versions.extend(package_links) - - file_versions = PackageFinder.evaluate_links_static( - package_finder_tuple, - link_evaluator_tuple, - links=collected_links.files, - ) - - if logger.isEnabledFor(logging.DEBUG) and file_candidates: - paths = [url_to_path(c.link.url) for c in file_candidates] - logger.debug("Local files found: %s", ", ".join(paths)) - - # This is an intentional priority ordering - return file_candidates + page_candidates - def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name @@ -992,33 +1000,7 @@ def find_all_candidates(self, project_name): See LinkEvaluator.evaluate_link() for details on which files are accepted. """ - link_evaluator = self.make_link_evaluator(project_name) - - package_finder_tuple = self.get_state_as_tuple(True) - link_evaluator_tuple = link_evaluator.get_state_as_tuple() - return PackageFinder.find_all_candidates_static( - package_finder_tuple, - link_evaluator_tuple, - project_name - ) - - @staticmethod - def make_candidate_evaluator_static( - package_finder_tuple, # type: PackageFinderTuple - project_name, # type: str - specifier=None, # type: Optional[specifiers.BaseSpecifier] - hashes=None, # type: Optional[Hashes] - ): - # type: (...) -> CandidateEvaluator - candidate_prefs = package_finder_tuple.candidate_prefs - return CandidateEvaluator.create( - project_name=project_name, - target_python=package_finder_tuple.target_python, - prefer_binary=candidate_prefs.prefer_binary, - allow_all_prereleases=candidate_prefs.allow_all_prereleases, - specifier=specifier, - hashes=hashes, - ) + return _find_all_candidates_cached(project_name) def make_candidate_evaluator( self, @@ -1037,30 +1019,6 @@ def make_candidate_evaluator( hashes ) - @staticmethod - @functools.lru_cache(maxsize=None) - def find_best_candidate_static( - package_finder_tuple, # type: PackageFinderTuple - link_evaluator_tuple, # type: LinkEvaluatorTuple - project_name, # type: str - specifier=None, # type: Optional[specifiers.BaseSpecifier] - hashes=None # type: Optional[Hashes] - ): - # type: (...) -> BestCandidateResult - candidate_evaluator = PackageFinder.make_candidate_evaluator_static( - package_finder_tuple, - project_name, - specifier, - hashes - ) - candidates = PackageFinder.find_all_candidates_static( - package_finder_tuple, - link_evaluator_tuple, - project_name - ) - - return candidate_evaluator.compute_best_candidate(candidates) - def find_best_candidate( self, project_name, # type: str @@ -1074,20 +1032,10 @@ def find_best_candidate( (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable versions. - :return: A `BestCandidateResult` instance. + :return: A `BestCandidateResult` instance, + called from a static function that caches function calls. """ - link_evaluator = self.make_link_evaluator(project_name) - - package_finder_tuple = self.get_state_as_tuple(True) - link_evaluator_tuple = link_evaluator.get_state_as_tuple() - - return PackageFinder.find_best_candidate_static( - package_finder_tuple, - link_evaluator_tuple, - project_name, - specifier, - hashes - ) + return _find_best_candidate_cached(project_name, specifier, hashes) def find_requirement(self, req, upgrade): # type: (InstallRequirement, bool) -> Optional[InstallationCandidate] From fcdfb334f33d92de21b167dafdaba4ae6f17d7ab Mon Sep 17 00:00:00 2001 From: Martin Li Date: Wed, 3 Mar 2021 11:59:05 +0100 Subject: [PATCH 11/16] Add 9084.bugfix.rst --- news/9084.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/9084.bugfix.rst diff --git a/news/9084.bugfix.rst b/news/9084.bugfix.rst new file mode 100644 index 00000000000..282e313a18f --- /dev/null +++ b/news/9084.bugfix.rst @@ -0,0 +1 @@ +Extract the logic from package_finder.find_best_candidate and package_finder.find_all_candidate to static methods. \ No newline at end of file From be6c8d62b6114eecfcc40ffd2f3ae6e73ef9001f Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Wed, 17 Mar 2021 16:06:31 +0100 Subject: [PATCH 12/16] Remove unused code --- src/pip/_internal/index/package_finder.py | 72 +++++++++++++++++++ tests/unit/test_resolution_legacy_resolver.py | 1 - 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index c8f2dbb685b..57250f76292 100755 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -990,6 +990,78 @@ def process_project_url_static( return package_links + def process_project_url(self, project_url, link_evaluator): + # type: (Link, LinkEvaluator) -> List[InstallationCandidate] + return PackageFinder.process_project_url_static( + self.get_state_as_tuple(), + project_url, + link_evaluator.get_state_as_tuple() + ) + + @staticmethod + @functools.lru_cache(maxsize=None) + def find_all_candidates_static( + package_finder_tuple, # type: PackageFinderTuple + link_evaluator_tuple, # type: LinkEvaluatorTuple + project_name # type: str + ): + # type: (...) -> List[InstallationCandidate] + """Find all available InstallationCandidate for project_name + + This checks index_urls and find_links. + All versions found are returned as an InstallationCandidate list. + + See LinkEvaluator.evaluate_link() for details on which files + are accepted. + """ + + # Need to convert logged_links to normal set, so we can update it. + package_finder_tuple = PackageFinderTuple( + logged_links=set(package_finder_tuple.logged_links), + link_collector=package_finder_tuple.link_collector, + target_python=package_finder_tuple.target_python, + candidate_prefs=package_finder_tuple.candidate_prefs + ) + + # Add link_collector to the tuple + collected_links = package_finder_tuple.link_collector.collect_links( + project_name + ) + + find_links_versions = PackageFinder.evaluate_links_static( + package_finder_tuple, + link_evaluator_tuple, + links=collected_links.find_links, + ) + + # Only needs link collector from package_finder + page_versions = [] + for project_url in collected_links.project_urls: + package_links = PackageFinder.process_project_url_static( + package_finder_tuple, + project_url, + link_evaluator_tuple, + ) + page_versions.extend(package_links) + + file_versions = PackageFinder.evaluate_links_static( + package_finder_tuple, + link_evaluator_tuple, + links=collected_links.files, + ) + if file_versions: + file_versions.sort(reverse=True) + logger.debug( + 'Local files found: %s', + ', '.join([ + url_to_path(candidate.link.url) + for candidate in file_versions + ]) + ) + + # This is an intentional priority ordering + return file_versions + find_links_versions + page_versions + def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index aedb02ba804..3bf8512117b 100755 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -183,7 +183,6 @@ def _find_candidates(package_finder, link_evaluator, project_name): return mock_candidates finder = make_test_finder() - monkeypatch.setattr(finder, "find_all_candidates", _find_candidates) monkeypatch.setattr( PackageFinder, "find_all_candidates_static", From e7eaede260ec2e842940e7ae637a6a093df4c03e Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Fri, 19 Mar 2021 16:44:04 +0100 Subject: [PATCH 13/16] Add code style improvements --- src/pip/_internal/index/package_finder.py | 128 ++++++++++++------ tests/unit/test_resolution_legacy_resolver.py | 2 +- 2 files changed, 90 insertions(+), 40 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 57250f76292..fbee8a92532 100755 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -862,28 +862,6 @@ def get_state_as_tuple(self, immutable=False): target_python=self._target_python ) - @staticmethod - def _sort_links_static(links): - # type: (Iterable[Link]) -> List[Link] - eggs, no_eggs = [], [] - seen = set() # type: Set[Link] - for link in links: - if link not in seen: - seen.add(link) - if link.egg_fragment: - eggs.append(link) - else: - no_eggs.append(link) - return no_eggs + eggs - - def _sort_links(self, links): - # type: (Iterable[Link]) -> List[Link] - """ - Returns elements of links in order, non-egg links first, egg links - second, while eliminating duplicates - """ - return PackageFinder._sort_links_static(links) - @staticmethod def _log_skipped_link_static(package_finder_tuple, link, reason): # type: (PackageFinderTuple, Link, str) -> None @@ -895,10 +873,10 @@ def _log_skipped_link_static(package_finder_tuple, link, reason): def _log_skipped_link(self, link, reason): # type: (Link, str) -> None - PackageFinder._log_skipped_link_static(self.get_state_as_tuple(), link, reason) + self._log_skipped_link_static(self.get_state_as_tuple(), link, reason) @staticmethod - def get_install_candidate_static( + def _get_install_candidate_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple link # type: Link @@ -929,22 +907,33 @@ def get_install_candidate(self, link_evaluator, link): If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - return PackageFinder.get_install_candidate_static( + return self._get_install_candidate_static( self.get_state_as_tuple(), link_evaluator.get_state_as_tuple(), link ) @staticmethod - def evaluate_links_static( + def _evaluate_links_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple links # type: Iterable[Link] ): # type: (...) -> List[InstallationCandidate] candidates = [] - for link in PackageFinder._sort_links_static(links): - candidate = PackageFinder.get_install_candidate_static( + + eggs, no_eggs = [], [] + seen = set() # type: Set[Link] + for link in links: + if link not in seen: + seen.add(link) + if link.egg_fragment: + eggs.append(link) + else: + no_eggs.append(link) + links = no_eggs + eggs + for link in links: + candidate = PackageFinder._get_install_candidate_static( package_finder_tuple, link_evaluator_tuple, link @@ -959,14 +948,14 @@ def evaluate_links(self, link_evaluator, links): """ Convert links that are candidates to InstallationCandidate objects. """ - return PackageFinder.evaluate_links_static( + return self._evaluate_links_static( self.get_state_as_tuple(), link_evaluator.get_state_as_tuple(), links ) @staticmethod - def process_project_url_static( + def _process_project_url_static( package_finder_tuple, # type: PackageFinderTuple project_url, # type: Link link_evaluator_tuple # type: LinkEvaluatorTuple @@ -982,7 +971,7 @@ def process_project_url_static( page_links = list(parse_links(html_page)) with indent_log(): - package_links = PackageFinder.evaluate_links_static( + package_links = PackageFinder._evaluate_links_static( package_finder_tuple, link_evaluator_tuple, links=page_links, @@ -992,7 +981,7 @@ def process_project_url_static( def process_project_url(self, project_url, link_evaluator): # type: (Link, LinkEvaluator) -> List[InstallationCandidate] - return PackageFinder.process_project_url_static( + return self._process_project_url_static( self.get_state_as_tuple(), project_url, link_evaluator.get_state_as_tuple() @@ -1000,7 +989,7 @@ def process_project_url(self, project_url, link_evaluator): @staticmethod @functools.lru_cache(maxsize=None) - def find_all_candidates_static( + def _find_all_candidates_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple project_name # type: str @@ -1028,7 +1017,7 @@ def find_all_candidates_static( project_name ) - find_links_versions = PackageFinder.evaluate_links_static( + find_links_versions = PackageFinder._evaluate_links_static( package_finder_tuple, link_evaluator_tuple, links=collected_links.find_links, @@ -1037,14 +1026,14 @@ def find_all_candidates_static( # Only needs link collector from package_finder page_versions = [] for project_url in collected_links.project_urls: - package_links = PackageFinder.process_project_url_static( + package_links = PackageFinder._process_project_url_static( package_finder_tuple, project_url, link_evaluator_tuple, ) page_versions.extend(package_links) - file_versions = PackageFinder.evaluate_links_static( + file_versions = PackageFinder._evaluate_links_static( package_finder_tuple, link_evaluator_tuple, links=collected_links.files, @@ -1072,7 +1061,33 @@ def find_all_candidates(self, project_name): See LinkEvaluator.evaluate_link() for details on which files are accepted. """ - return _find_all_candidates_cached(project_name) + link_evaluator = self.make_link_evaluator(project_name) + + package_finder_tuple = self.get_state_as_tuple(immutable=True) + link_evaluator_tuple = link_evaluator.get_state_as_tuple() + return self._find_all_candidates_static( + package_finder_tuple, + link_evaluator_tuple, + project_name + ) + + @staticmethod + def _make_candidate_evaluator_static( + package_finder_tuple, # type: PackageFinderTuple + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> CandidateEvaluator + candidate_prefs = package_finder_tuple.candidate_prefs + return CandidateEvaluator.create( + project_name=project_name, + target_python=package_finder_tuple.target_python, + prefer_binary=candidate_prefs.prefer_binary, + allow_all_prereleases=candidate_prefs.allow_all_prereleases, + specifier=specifier, + hashes=hashes, + ) def make_candidate_evaluator( self, @@ -1084,12 +1099,36 @@ def make_candidate_evaluator( """Create a CandidateEvaluator object to use. """ package_finder_tuple = self.get_state_as_tuple() - return PackageFinder.make_candidate_evaluator_static( + return self._make_candidate_evaluator_static( + package_finder_tuple, + project_name, + specifier, + hashes + ) + + @staticmethod + @functools.lru_cache(maxsize=None) + def _find_best_candidate_static( + package_finder_tuple, # type: PackageFinderTuple + link_evaluator_tuple, # type: LinkEvaluatorTuple + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None # type: Optional[Hashes] + ): + # type: (...) -> BestCandidateResult + candidate_evaluator = PackageFinder._make_candidate_evaluator_static( package_finder_tuple, project_name, specifier, hashes ) + candidates = PackageFinder._find_all_candidates_static( + package_finder_tuple, + link_evaluator_tuple, + project_name + ) + + return candidate_evaluator.compute_best_candidate(candidates) def find_best_candidate( self, @@ -1107,7 +1146,18 @@ def find_best_candidate( :return: A `BestCandidateResult` instance, called from a static function that caches function calls. """ - return _find_best_candidate_cached(project_name, specifier, hashes) + link_evaluator = self.make_link_evaluator(project_name) + + package_finder_tuple = self.get_state_as_tuple(immutable=True) + link_evaluator_tuple = link_evaluator.get_state_as_tuple() + + return self._find_best_candidate_static( + package_finder_tuple, + link_evaluator_tuple, + project_name, + specifier, + hashes + ) def find_requirement(self, req, upgrade): # type: (InstallRequirement, bool) -> Optional[InstallationCandidate] diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 3bf8512117b..f46e55d11b3 100755 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -185,7 +185,7 @@ def _find_candidates(package_finder, link_evaluator, project_name): finder = make_test_finder() monkeypatch.setattr( PackageFinder, - "find_all_candidates_static", + "_find_all_candidates_static", _find_candidates ) From 28b08a04828cdfc2ea0ba058564ab460a5f3257b Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 20 Mar 2021 11:24:14 +0100 Subject: [PATCH 14/16] Update self._logged_links after calling static function --- src/pip/_internal/index/package_finder.py | 29 ++++++++++++------- tests/unit/test_resolution_legacy_resolver.py | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index fbee8a92532..d541413fa81 100755 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -994,7 +994,7 @@ def _find_all_candidates_static( link_evaluator_tuple, # type: LinkEvaluatorTuple project_name # type: str ): - # type: (...) -> List[InstallationCandidate] + # type: (...) -> Tuple[List[InstallationCandidate], Set[Link]] """Find all available InstallationCandidate for project_name This checks index_urls and find_links. @@ -1004,7 +1004,6 @@ def _find_all_candidates_static( are accepted. """ - # Need to convert logged_links to normal set, so we can update it. package_finder_tuple = PackageFinderTuple( logged_links=set(package_finder_tuple.logged_links), link_collector=package_finder_tuple.link_collector, @@ -1012,7 +1011,6 @@ def _find_all_candidates_static( candidate_prefs=package_finder_tuple.candidate_prefs ) - # Add link_collector to the tuple collected_links = package_finder_tuple.link_collector.collect_links( project_name ) @@ -1023,7 +1021,6 @@ def _find_all_candidates_static( links=collected_links.find_links, ) - # Only needs link collector from package_finder page_versions = [] for project_url in collected_links.project_urls: package_links = PackageFinder._process_project_url_static( @@ -1049,7 +1046,10 @@ def _find_all_candidates_static( ) # This is an intentional priority ordering - return file_versions + find_links_versions + page_versions + return ( + file_versions + find_links_versions + page_versions, + package_finder_tuple.logged_links + ) def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] @@ -1065,12 +1065,17 @@ def find_all_candidates(self, project_name): package_finder_tuple = self.get_state_as_tuple(immutable=True) link_evaluator_tuple = link_evaluator.get_state_as_tuple() - return self._find_all_candidates_static( + + (candidates, logged_links) = self._find_all_candidates_static( package_finder_tuple, link_evaluator_tuple, project_name ) + self._logged_links = logged_links + + return candidates + @staticmethod def _make_candidate_evaluator_static( package_finder_tuple, # type: PackageFinderTuple @@ -1115,20 +1120,20 @@ def _find_best_candidate_static( specifier=None, # type: Optional[specifiers.BaseSpecifier] hashes=None # type: Optional[Hashes] ): - # type: (...) -> BestCandidateResult + # type: (...) -> Tuple[BestCandidateResult, Set[Link]] candidate_evaluator = PackageFinder._make_candidate_evaluator_static( package_finder_tuple, project_name, specifier, hashes ) - candidates = PackageFinder._find_all_candidates_static( + (candidates, logged_links) = PackageFinder._find_all_candidates_static( package_finder_tuple, link_evaluator_tuple, project_name ) - return candidate_evaluator.compute_best_candidate(candidates) + return (candidate_evaluator.compute_best_candidate(candidates), logged_links) def find_best_candidate( self, @@ -1151,7 +1156,7 @@ def find_best_candidate( package_finder_tuple = self.get_state_as_tuple(immutable=True) link_evaluator_tuple = link_evaluator.get_state_as_tuple() - return self._find_best_candidate_static( + (best_candidate, logged_links) = self._find_best_candidate_static( package_finder_tuple, link_evaluator_tuple, project_name, @@ -1159,6 +1164,10 @@ def find_best_candidate( hashes ) + self._logged_links = logged_links + + return best_candidate + def find_requirement(self, req, upgrade): # type: (InstallRequirement, bool) -> Optional[InstallationCandidate] """Try to find a Link matching req diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index f46e55d11b3..2774e157340 100755 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -180,7 +180,7 @@ class TestYankedWarning: """ def _make_test_resolver(self, monkeypatch, mock_candidates): def _find_candidates(package_finder, link_evaluator, project_name): - return mock_candidates + return (mock_candidates, []) finder = make_test_finder() monkeypatch.setattr( From c8da7b9ad54936eb3c9af1336605b9c69f3bb4b3 Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 24 Apr 2021 19:10:11 +0200 Subject: [PATCH 15/16] Use collect_sources instead of collect_links --- src/pip/_internal/index/package_finder.py | 166 ++++++------------ tests/unit/test_resolution_legacy_resolver.py | 2 +- 2 files changed, 57 insertions(+), 111 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index d541413fa81..e5d28fcd826 100755 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -56,6 +56,7 @@ link_collector target_python candidate_prefs + process_project_url """ ) @@ -116,81 +117,6 @@ def _check_link_requires_python( return True -@functools.lru_cache(maxsize=None) -def _find_best_candidate_cached( - self, - project_name, # type: str - specifier=None, # type: Optional[specifiers.BaseSpecifier] - hashes=None, # type: Optional[Hashes] -): - # type: (...) -> BestCandidateResult - """Find matches for the given project and specifier. - - :param specifier: An optional object implementing `filter` - (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable - versions. - - :return: A `BestCandidateResult` instance, - and cache the result to reduce future call's execution time. - """ - candidates = self.find_all_candidates(project_name) - candidate_evaluator = self.make_candidate_evaluator( - project_name=project_name, - specifier=specifier, - hashes=hashes, - ) - return candidate_evaluator.compute_best_candidate(candidates) - - -@functools.lru_cache(maxsize=None) -def _find_all_candidates_cached( - self, - project_name, # type: str -): - # type: (...) -> List[InstallationCandidate] - """Find all available InstallationCandidate for project_name - - This checks index_urls and find_links. - All versions found are returned as an InstallationCandidate list, - and results are cached to reduce future call's execution time. - - See LinkEvaluator.evaluate_link() for details on which files - are accepted. - """ - collected_links = self._link_collector.collect_links(project_name) - - link_evaluator = self.make_link_evaluator(project_name) - - find_links_versions = self.evaluate_links( - link_evaluator, - links=collected_links.find_links, - ) - - page_versions = [] - for project_url in collected_links.project_urls: - package_links = self.process_project_url( - project_url, link_evaluator=link_evaluator, - ) - page_versions.extend(package_links) - - file_versions = self.evaluate_links( - link_evaluator, - links=collected_links.files, - ) - if file_versions: - file_versions.sort(reverse=True) - logger.debug( - 'Local files found: %s', - ', '.join([ - url_to_path(candidate.link.url) - for candidate in file_versions - ]) - ) - - # This is an intentional priority ordering - return file_versions + find_links_versions + page_versions - - class LinkEvaluator: """ @@ -852,14 +778,16 @@ def get_state_as_tuple(self, immutable=False): logged_links=frozenset(self._logged_links), link_collector=self._link_collector, candidate_prefs=self._candidate_prefs, - target_python=self._target_python + target_python=self._target_python, + process_project_url=self.process_project_url ) return PackageFinderTuple( logged_links=self._logged_links, link_collector=self._link_collector, candidate_prefs=self._candidate_prefs, - target_python=self._target_python + target_python=self._target_python, + process_project_url=self.process_project_url ) @staticmethod @@ -992,7 +920,8 @@ def process_project_url(self, project_url, link_evaluator): def _find_all_candidates_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple - project_name # type: str + project_name, # type: str, + candidates_from_page ): # type: (...) -> Tuple[List[InstallationCandidate], Set[Link]] """Find all available InstallationCandidate for project_name @@ -1008,46 +937,50 @@ def _find_all_candidates_static( logged_links=set(package_finder_tuple.logged_links), link_collector=package_finder_tuple.link_collector, target_python=package_finder_tuple.target_python, - candidate_prefs=package_finder_tuple.candidate_prefs + candidate_prefs=package_finder_tuple.candidate_prefs, + process_project_url=package_finder_tuple.process_project_url ) - collected_links = package_finder_tuple.link_collector.collect_links( - project_name + link_evaluator = LinkEvaluator( + project_name=link_evaluator_tuple.project_name, + canonical_name=link_evaluator_tuple.canonical_name, + formats=link_evaluator_tuple.formats, + target_python=link_evaluator_tuple.target_python, + allow_yanked=link_evaluator_tuple.allow_yanked, + ignore_requires_python=link_evaluator_tuple.ignore_requires_python ) - - find_links_versions = PackageFinder._evaluate_links_static( - package_finder_tuple, - link_evaluator_tuple, - links=collected_links.find_links, + collected_sources = package_finder_tuple.link_collector.collect_sources( + project_name=project_name, + candidates_from_page=candidates_from_page, ) - page_versions = [] - for project_url in collected_links.project_urls: - package_links = PackageFinder._process_project_url_static( - package_finder_tuple, - project_url, - link_evaluator_tuple, - ) - page_versions.extend(package_links) + page_candidates_it = itertools.chain.from_iterable( + source.page_candidates() + for sources in collected_sources + for source in sources + if source is not None + ) + page_candidates = list(page_candidates_it) - file_versions = PackageFinder._evaluate_links_static( + file_links_it = itertools.chain.from_iterable( + source.file_links() + for sources in collected_sources + for source in sources + if source is not None + ) + file_candidates = PackageFinder._evaluate_links_static( package_finder_tuple, link_evaluator_tuple, - links=collected_links.files, + sorted(file_links_it, reverse=True), ) - if file_versions: - file_versions.sort(reverse=True) - logger.debug( - 'Local files found: %s', - ', '.join([ - url_to_path(candidate.link.url) - for candidate in file_versions - ]) - ) + + if logger.isEnabledFor(logging.DEBUG) and file_candidates: + paths = [url_to_path(c.link.url) for c in file_candidates] + logger.debug("Local files found: %s", ", ".join(paths)) # This is an intentional priority ordering return ( - file_versions + find_links_versions + page_versions, + file_candidates + page_candidates, package_finder_tuple.logged_links ) @@ -1065,11 +998,15 @@ def find_all_candidates(self, project_name): package_finder_tuple = self.get_state_as_tuple(immutable=True) link_evaluator_tuple = link_evaluator.get_state_as_tuple() - + candidates_from_page = functools.partial( + package_finder_tuple.process_project_url, + link_evaluator=link_evaluator, + ) (candidates, logged_links) = self._find_all_candidates_static( package_finder_tuple, link_evaluator_tuple, - project_name + project_name, + candidates_from_page ) self._logged_links = logged_links @@ -1117,6 +1054,7 @@ def _find_best_candidate_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple project_name, # type: str + candidates_from_page, specifier=None, # type: Optional[specifiers.BaseSpecifier] hashes=None # type: Optional[Hashes] ): @@ -1127,10 +1065,12 @@ def _find_best_candidate_static( specifier, hashes ) + (candidates, logged_links) = PackageFinder._find_all_candidates_static( package_finder_tuple, link_evaluator_tuple, - project_name + project_name, + candidates_from_page ) return (candidate_evaluator.compute_best_candidate(candidates), logged_links) @@ -1148,7 +1088,7 @@ def find_best_candidate( (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable versions. - :return: A `BestCandidateResult` instance, + :return: A `BestCandidateResult` instance, called from a static function that caches function calls. """ link_evaluator = self.make_link_evaluator(project_name) @@ -1156,10 +1096,16 @@ def find_best_candidate( package_finder_tuple = self.get_state_as_tuple(immutable=True) link_evaluator_tuple = link_evaluator.get_state_as_tuple() + + candidates_from_page = functools.partial( + package_finder_tuple.process_project_url, + link_evaluator=link_evaluator, + ) (best_candidate, logged_links) = self._find_best_candidate_static( package_finder_tuple, link_evaluator_tuple, project_name, + candidates_from_page, specifier, hashes ) diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 2774e157340..ab05e8bc0cd 100755 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -179,7 +179,7 @@ class TestYankedWarning: Test _populate_link() emits warning if one or more candidates are yanked. """ def _make_test_resolver(self, monkeypatch, mock_candidates): - def _find_candidates(package_finder, link_evaluator, project_name): + def _find_candidates(package_finder, link_evaluator, project_name, candidates_from_page): return (mock_candidates, []) finder = make_test_finder() From 94a491e80b63e0a113c78fd0d52fd8366c85928a Mon Sep 17 00:00:00 2001 From: Quentin Lee Date: Sat, 24 Apr 2021 19:38:17 +0200 Subject: [PATCH 16/16] Fix linting errors --- news/9084.bugfix.rst | 2 +- src/pip/_internal/index/package_finder.py | 17 ++++------------- tests/unit/test_resolution_legacy_resolver.py | 7 ++++++- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/news/9084.bugfix.rst b/news/9084.bugfix.rst index 282e313a18f..f2fbfc754bf 100644 --- a/news/9084.bugfix.rst +++ b/news/9084.bugfix.rst @@ -1 +1 @@ -Extract the logic from package_finder.find_best_candidate and package_finder.find_all_candidate to static methods. \ No newline at end of file +Extract the logic from package_finder.find_best_candidate and package_finder.find_all_candidate to static methods. diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index e5d28fcd826..f7b2ec5e14a 100755 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -8,7 +8,7 @@ import logging import re from collections import namedtuple -from typing import FrozenSet, Iterable, List, Optional, Set, Tuple, Union +from typing import Callable, FrozenSet, Iterable, List, Optional, Set, Tuple, Union from pip._vendor.packaging import specifiers from pip._vendor.packaging.tags import Tag @@ -920,8 +920,8 @@ def process_project_url(self, project_url, link_evaluator): def _find_all_candidates_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple - project_name, # type: str, - candidates_from_page + project_name, # type: str + candidates_from_page # type: Callable[..., List[InstallationCandidate]] ): # type: (...) -> Tuple[List[InstallationCandidate], Set[Link]] """Find all available InstallationCandidate for project_name @@ -941,14 +941,6 @@ def _find_all_candidates_static( process_project_url=package_finder_tuple.process_project_url ) - link_evaluator = LinkEvaluator( - project_name=link_evaluator_tuple.project_name, - canonical_name=link_evaluator_tuple.canonical_name, - formats=link_evaluator_tuple.formats, - target_python=link_evaluator_tuple.target_python, - allow_yanked=link_evaluator_tuple.allow_yanked, - ignore_requires_python=link_evaluator_tuple.ignore_requires_python - ) collected_sources = package_finder_tuple.link_collector.collect_sources( project_name=project_name, candidates_from_page=candidates_from_page, @@ -1054,7 +1046,7 @@ def _find_best_candidate_static( package_finder_tuple, # type: PackageFinderTuple link_evaluator_tuple, # type: LinkEvaluatorTuple project_name, # type: str - candidates_from_page, + candidates_from_page, # type: Callable[..., List[InstallationCandidate]] specifier=None, # type: Optional[specifiers.BaseSpecifier] hashes=None # type: Optional[Hashes] ): @@ -1096,7 +1088,6 @@ def find_best_candidate( package_finder_tuple = self.get_state_as_tuple(immutable=True) link_evaluator_tuple = link_evaluator.get_state_as_tuple() - candidates_from_page = functools.partial( package_finder_tuple.process_project_url, link_evaluator=link_evaluator, diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index ab05e8bc0cd..60d555f0f26 100755 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -179,7 +179,12 @@ class TestYankedWarning: Test _populate_link() emits warning if one or more candidates are yanked. """ def _make_test_resolver(self, monkeypatch, mock_candidates): - def _find_candidates(package_finder, link_evaluator, project_name, candidates_from_page): + def _find_candidates( + package_finder, + link_evaluator, + project_name, + candidates_from_page + ): return (mock_candidates, []) finder = make_test_finder()