diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f61c710..5983ae29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- micropip now respects the `yanked` flag in the PyPI Simple API. + [#208](https://github.com/pyodide/micropip/pull/208) + ## [0.9.0] - 2024/02/01 ### Fixed diff --git a/micropip/constants.py b/micropip/constants.py index 104a71af..998d7e88 100644 --- a/micropip/constants.py +++ b/micropip/constants.py @@ -1,3 +1,10 @@ FAQ_URLS = { "cant_find_wheel": "https://pyodide.org/en/stable/usage/faq.html#why-can-t-micropip-find-a-pure-python-wheel-for-a-package" } + +# https://github.com/pypa/pip/blob/de44d991024ca8a03e9433ca6178f9a5f661754f/src/pip/_internal/resolution/resolvelib/resolver.py#L164-L167 +YANKED_WARNING_MESSAGE = ( + "The candidate selected for download or install is a " + "yanked version: '%s' candidate (version %s " + "at %s)\nReason for being yanked: %s" +) diff --git a/micropip/package_index.py b/micropip/package_index.py index cafca258..8128f06b 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -163,6 +163,11 @@ def _compatible_wheels( # Size of the file in bytes, if available (PEP 700) # This key is not available in the Simple API HTML response, so this field may be None size = file.get("size") + + # PEP-592: + # yanked can be an arbitrary string (reason) or bool. + yanked_reason = file.get("yanked", False) + yield WheelInfo.from_package_index( name=name, filename=filename, @@ -171,6 +176,7 @@ def _compatible_wheels( sha256=sha256, size=size, core_metadata=core_metadata, + yanked_reason=yanked_reason, ) @classmethod diff --git a/micropip/transaction.py b/micropip/transaction.py index 7367ebc1..db326654 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -2,6 +2,7 @@ import importlib.metadata import logging import warnings +from collections.abc import Iterable from dataclasses import dataclass, field from importlib.metadata import PackageNotFoundError from urllib.parse import urlparse @@ -19,7 +20,7 @@ Requirement, ) from ._vendored.packaging.src.packaging.utils import canonicalize_name -from .constants import FAQ_URLS +from .constants import FAQ_URLS, YANKED_WARNING_MESSAGE from .package import PackageMetadata from .package_index import ProjectInfo from .wheelinfo import WheelInfo @@ -238,6 +239,16 @@ async def _add_requirement_from_package_index(self, req: Requirement): logger.debug("Transaction: Selected wheel: %r", wheel) + if wheel.yanked: + yanked_reason = wheel.yanked_reason if wheel.yanked_reason else "None" + logger.info( + YANKED_WARNING_MESSAGE, + wheel.name, + str(wheel.version), + wheel.url, + yanked_reason, + ) + # Maybe while we were downloading pypi_json some other branch # installed the wheel? satisfied, ver = self.check_version_satisfied(req) @@ -327,6 +338,8 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: reverse=True, ) + yanked_versions: list[list[WheelInfo]] = [] + for ver in candidate_versions: if ver not in releases: warnings.warn( @@ -335,18 +348,27 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: ) continue - best_wheel = None - best_tag_index = float("infinity") + wheels = list(releases[ver]) - wheels = releases[ver] - for wheel in wheels: - tag_index = best_compatible_tag_index(wheel.tags) - if tag_index is not None and tag_index < best_tag_index: - best_wheel = wheel - best_tag_index = tag_index + # If the version is yanked, put it in the end of the candidate list. + # If we can't find a wheel that satisfies the requirement, + # install the yanked version as a last resort. + # when the version is yanked, all wheels are yanked, so we can check only the first wheel. + yanked = wheels and wheels[0].yanked + if yanked: + yanked_versions.append(wheels) + continue + + best_wheel = _find_best_wheel(wheels) if best_wheel is not None: - return wheel + return best_wheel + + for wheels in yanked_versions: + best_wheel = _find_best_wheel(wheels) + + if best_wheel is not None: + return best_wheel raise ValueError( f"Can't find a pure Python 3 wheel for '{req}'.\n" @@ -354,3 +376,15 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: "You can use `await micropip.install(..., keep_going=True)` " "to get a list of all packages with missing wheels." ) + + +def _find_best_wheel(wheels: Iterable[WheelInfo]) -> WheelInfo | None: + best_wheel = None + best_tag_index = float("infinity") + for wheel in wheels: + tag_index = best_compatible_tag_index(wheel.tags) + if tag_index is not None and tag_index < best_tag_index: + best_wheel = wheel + best_tag_index = tag_index + + return best_wheel diff --git a/micropip/wheelinfo.py b/micropip/wheelinfo.py index ebe0c2f4..808f9dfd 100644 --- a/micropip/wheelinfo.py +++ b/micropip/wheelinfo.py @@ -44,6 +44,9 @@ class WheelInfo: sha256: str | None = None size: int | None = None # Size in bytes, if available (PEP 700) core_metadata: DistributionMetadata = None # Wheel's metadata (PEP 658 / PEP-714) + yanked_reason: str | bool = ( + False # Whether the wheel has been yanked and the reason (if given) (PEP-592) + ) # Fields below are only available after downloading the wheel, i.e. after calling `download()`. @@ -61,6 +64,7 @@ def __post_init__(self): ), self.url self._project_name = safe_name(self.name) self.metadata_url = self.url + ".metadata" + self.yanked = bool(self.yanked_reason) @classmethod def from_url(cls, url: str) -> "WheelInfo": @@ -100,6 +104,7 @@ def from_package_index( sha256: str | None, size: int | None, core_metadata: DistributionMetadata = None, + yanked_reason: str | bool = False, ) -> "WheelInfo": """Extract available metadata from response received from package index""" parsed_url = urlparse(url) @@ -116,6 +121,7 @@ def from_package_index( sha256=sha256, size=size, core_metadata=core_metadata, + yanked_reason=yanked_reason, ) async def install(self, target: Path) -> None: diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index d50a8af8..79d1cb6a 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -52,6 +52,26 @@ async def _run(selenium): _run(selenium_standalone_micropip) +@integration_test_only +def test_integration_install_yanked(selenium_standalone_micropip, pytestconfig): + @run_in_pyodide + async def _run(selenium): + import contextlib + import io + + import micropip + + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + # install yanked version + await micropip.install("black==21.11b0", verbose=True) + + captured = buf.getvalue() + assert "The candidate selected for download or install is a" in captured + assert "'black' candidate (version 21.11b0" in captured + + _run(selenium_standalone_micropip) + + @integration_test_only def test_integration_list_basic(selenium_standalone_micropip, pytestconfig): @run_in_pyodide diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 66c3cff0..822dfe0c 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -215,6 +215,47 @@ def test_find_wheel_invalid_version(): assert str(wheel.version) == "0.15.5" +def test_yanked_version(): + from micropip._vendored.packaging.src.packaging.requirements import Requirement + from micropip.transaction import find_wheel + + versions = ["0.0.1", "0.15.5", "0.9.1"] + + # Mark 0.15.5 as yanked + # convert generator --> list and monkeypatch the yanked value + metadata = _pypi_metadata("dummy_module", {v: ["py3"] for v in versions}) + for version in list(metadata.releases): + wheels = list(metadata.releases[version]) + for wheel in wheels: + if str(wheel.version) == "0.15.5": + wheel.yanked = True + + metadata.releases[version] = wheels + + # case 1: yanked version should be skipped and the next best version should be selected + + requirement1 = Requirement("dummy_module") + wheel = find_wheel(metadata, requirement1) + + assert str(wheel.version) == "0.9.1" + + # case 2: yanked version is explicitly requested, so it should be selected + + requirement2 = Requirement("dummy_module==0.15.5") + wheel = find_wheel(metadata, requirement2) + + assert str(wheel.version) == "0.15.5" + + # case 3: yanked version is not explicitly requested, but it is the only version available + # so it should be selected + + requirement3 = Requirement("dummy_module>0.10.0") + + wheel = find_wheel(metadata, requirement3) + + assert str(wheel.version) == "0.15.5" + + _best_tag_test_cases = ( "package, version, incompatible_tags, compatible_tags", # Tests assume that `compatible_tags` is sorted from least to most compatible: