From 67e968580e3cb2f426f64f0a572f2d5dfe4ceec0 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Sat, 15 Feb 2025 11:47:44 +0000 Subject: [PATCH 1/6] Respect yanked flag --- micropip/package_index.py | 7 +++++++ micropip/transaction.py | 40 +++++++++++++++++++++++++++++++-------- micropip/wheelinfo.py | 3 +++ tests/test_transaction.py | 36 +++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/micropip/package_index.py b/micropip/package_index.py index cafca258..7137d5d7 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -163,6 +163,12 @@ def _compatible_wheels( # Size of the file in bytes, if available (PEP 700) # This key is not available in the Simple API HTML response, so this field may be None size = file.get("size") + + # PEP-592: + # yanked can be an arbitrary string (reason) or bool. + # any string is considered as True, so we convert it to bool. + yanked = bool(file.get("yanked", False)) + yield WheelInfo.from_package_index( name=name, filename=filename, @@ -171,6 +177,7 @@ def _compatible_wheels( sha256=sha256, size=size, core_metadata=core_metadata, + yanked=yanked, ) @classmethod diff --git a/micropip/transaction.py b/micropip/transaction.py index 65dca5ec..b1997152 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -2,6 +2,7 @@ import importlib.metadata import logging import warnings +from collections.abc import Iterator from dataclasses import dataclass, field from importlib.metadata import PackageNotFoundError from urllib.parse import urlparse @@ -327,6 +328,8 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: reverse=True, ) + yanked_versions = [] + for ver in candidate_versions: if ver not in releases: warnings.warn( @@ -335,18 +338,27 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: ) continue - best_wheel = None - best_tag_index = float("infinity") + wheels = releases[ver] + + # If the version is yanked, put it in the end of the candidate list. + # If we can't find a wheel that satisfies the requirement, + # install the yanked version as a last resort. + yanked = any(wheel.yanked for wheel in wheels) + if yanked: + yanked_versions.append(ver) + continue + + best_wheel = _find_best_wheel(wheels) + if best_wheel is not None: + return best_wheel + + for ver in yanked_versions: wheels = releases[ver] - for wheel in wheels: - tag_index = best_compatible_tag_index(wheel.tags) - if tag_index is not None and tag_index < best_tag_index: - best_wheel = wheel - best_tag_index = tag_index + best_wheel = _find_best_wheel(wheels) if best_wheel is not None: - return wheel + return best_wheel raise ValueError( f"Can't find a pure Python 3 wheel for '{req}'.\n" @@ -354,3 +366,15 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: "You can use `await micropip.install(..., keep_going=True)` " "to get a list of all packages with missing wheels." ) + + +def _find_best_wheel(wheels: Iterator[WheelInfo]) -> WheelInfo | None: + best_wheel = None + best_tag_index = float("infinity") + for wheel in wheels: + tag_index = best_compatible_tag_index(wheel.tags) + if tag_index is not None and tag_index < best_tag_index: + best_wheel = wheel + best_tag_index = tag_index + + return best_wheel diff --git a/micropip/wheelinfo.py b/micropip/wheelinfo.py index ebe0c2f4..11da9dad 100644 --- a/micropip/wheelinfo.py +++ b/micropip/wheelinfo.py @@ -44,6 +44,7 @@ class WheelInfo: sha256: str | None = None size: int | None = None # Size in bytes, if available (PEP 700) core_metadata: DistributionMetadata = None # Wheel's metadata (PEP 658 / PEP-714) + yanked: bool = False # Whether the wheel has been yanked (PEP-592) # Fields below are only available after downloading the wheel, i.e. after calling `download()`. @@ -100,6 +101,7 @@ def from_package_index( sha256: str | None, size: int | None, core_metadata: DistributionMetadata = None, + yanked: bool = False, ) -> "WheelInfo": """Extract available metadata from response received from package index""" parsed_url = urlparse(url) @@ -116,6 +118,7 @@ def from_package_index( sha256=sha256, size=size, core_metadata=core_metadata, + yanked=yanked, ) async def install(self, target: Path) -> None: diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 66c3cff0..3344fca2 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -215,6 +215,42 @@ def test_find_wheel_invalid_version(): assert str(wheel.version) == "0.15.5" +def test_yanked_version(): + from micropip._vendored.packaging.src.packaging.requirements import Requirement + from micropip.transaction import find_wheel + + versions = ["0.0.1", "0.15.5", "0.9.1"] + + # Mark 0.15.5 as yanked + # convert generator --> list and monkeypatch the yanked value + metadata = _pypi_metadata("dummy_module", {v: ["py3"] for v in versions}) + for version in list(metadata.releases): + wheels = list(metadata.releases[version]) + for wheel in wheels: + if str(wheel.version) == "0.15.5": + wheel.yanked = True + + metadata.releases[version] = wheels + + # yanked version should be skipped and the next best version should be selected + requirement1 = Requirement("dummy_module") + wheel = find_wheel(metadata, requirement1) + + assert str(wheel.version) == "0.9.1" + + requirement2 = Requirement("dummy_module==0.15.5") + wheel = find_wheel(metadata, requirement2) + + # no other compatible version available, so the yanked version should be selected + assert str(wheel.version) == "0.15.5" + + requirement3 = Requirement("dummy_module>0.10.0") + + wheel = find_wheel(metadata, requirement3) + + assert str(wheel.version) == "0.15.5" + + _best_tag_test_cases = ( "package, version, incompatible_tags, compatible_tags", # Tests assume that `compatible_tags` is sorted from least to most compatible: From e2c33bcb43bf1dbf7ae67dab1b723286d2b32493 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Sat, 15 Feb 2025 11:52:23 +0000 Subject: [PATCH 2/6] changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f61c710..ba5ebb4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- micropip now respects the `yanked` flag in the PyPI simple API. + [#208](https://github.com/pyodide/micropip/pull/208) + ## [0.9.0] - 2024/02/01 ### Fixed From d12a5305f2ba4b39d051bea7c624ada0ae1b00bb Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Sun, 16 Feb 2025 09:39:06 +0000 Subject: [PATCH 3/6] fix generator --- micropip/transaction.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index b1997152..7bbb7e58 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -2,7 +2,7 @@ import importlib.metadata import logging import warnings -from collections.abc import Iterator +from collections.abc import Iterable from dataclasses import dataclass, field from importlib.metadata import PackageNotFoundError from urllib.parse import urlparse @@ -328,7 +328,7 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: reverse=True, ) - yanked_versions = [] + yanked_versions: list[list[WheelInfo]] = [] for ver in candidate_versions: if ver not in releases: @@ -338,14 +338,14 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: ) continue - wheels = releases[ver] + wheels = list(releases[ver]) # If the version is yanked, put it in the end of the candidate list. # If we can't find a wheel that satisfies the requirement, # install the yanked version as a last resort. yanked = any(wheel.yanked for wheel in wheels) if yanked: - yanked_versions.append(ver) + yanked_versions.append(wheels) continue best_wheel = _find_best_wheel(wheels) @@ -353,8 +353,7 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: if best_wheel is not None: return best_wheel - for ver in yanked_versions: - wheels = releases[ver] + for wheels in yanked_versions: best_wheel = _find_best_wheel(wheels) if best_wheel is not None: @@ -368,7 +367,7 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: ) -def _find_best_wheel(wheels: Iterator[WheelInfo]) -> WheelInfo | None: +def _find_best_wheel(wheels: Iterable[WheelInfo]) -> WheelInfo | None: best_wheel = None best_tag_index = float("infinity") for wheel in wheels: From 6f169548a0eddf9dcfe9f09a62c8c0136c21ea97 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 20 Feb 2025 19:54:30 +0900 Subject: [PATCH 4/6] Update CHANGELOG.md Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba5ebb4a..5983ae29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- micropip now respects the `yanked` flag in the PyPI simple API. +- micropip now respects the `yanked` flag in the PyPI Simple API. [#208](https://github.com/pyodide/micropip/pull/208) ## [0.9.0] - 2024/02/01 From 09231864f23e397a8634f52461c33324aecc7c8c Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 20 Feb 2025 11:19:41 +0000 Subject: [PATCH 5/6] Address comments --- micropip/constants.py | 7 +++++++ micropip/package_index.py | 5 ++--- micropip/transaction.py | 15 +++++++++++++-- micropip/wheelinfo.py | 9 ++++++--- tests/integration/test_integration.py | 20 ++++++++++++++++++++ tests/test_transaction.py | 9 +++++++-- 6 files changed, 55 insertions(+), 10 deletions(-) diff --git a/micropip/constants.py b/micropip/constants.py index 104a71af..998d7e88 100644 --- a/micropip/constants.py +++ b/micropip/constants.py @@ -1,3 +1,10 @@ FAQ_URLS = { "cant_find_wheel": "https://pyodide.org/en/stable/usage/faq.html#why-can-t-micropip-find-a-pure-python-wheel-for-a-package" } + +# https://github.com/pypa/pip/blob/de44d991024ca8a03e9433ca6178f9a5f661754f/src/pip/_internal/resolution/resolvelib/resolver.py#L164-L167 +YANKED_WARNING_MESSAGE = ( + "The candidate selected for download or install is a " + "yanked version: '%s' candidate (version %s " + "at %s)\nReason for being yanked: %s" +) diff --git a/micropip/package_index.py b/micropip/package_index.py index 7137d5d7..8128f06b 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -166,8 +166,7 @@ def _compatible_wheels( # PEP-592: # yanked can be an arbitrary string (reason) or bool. - # any string is considered as True, so we convert it to bool. - yanked = bool(file.get("yanked", False)) + yanked_reason = file.get("yanked", False) yield WheelInfo.from_package_index( name=name, @@ -177,7 +176,7 @@ def _compatible_wheels( sha256=sha256, size=size, core_metadata=core_metadata, - yanked=yanked, + yanked_reason=yanked_reason, ) @classmethod diff --git a/micropip/transaction.py b/micropip/transaction.py index 92d91ac4..db326654 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -20,7 +20,7 @@ Requirement, ) from ._vendored.packaging.src.packaging.utils import canonicalize_name -from .constants import FAQ_URLS +from .constants import FAQ_URLS, YANKED_WARNING_MESSAGE from .package import PackageMetadata from .package_index import ProjectInfo from .wheelinfo import WheelInfo @@ -239,6 +239,16 @@ async def _add_requirement_from_package_index(self, req: Requirement): logger.debug("Transaction: Selected wheel: %r", wheel) + if wheel.yanked: + yanked_reason = wheel.yanked_reason if wheel.yanked_reason else "None" + logger.info( + YANKED_WARNING_MESSAGE, + wheel.name, + str(wheel.version), + wheel.url, + yanked_reason, + ) + # Maybe while we were downloading pypi_json some other branch # installed the wheel? satisfied, ver = self.check_version_satisfied(req) @@ -343,7 +353,8 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo: # If the version is yanked, put it in the end of the candidate list. # If we can't find a wheel that satisfies the requirement, # install the yanked version as a last resort. - yanked = any(wheel.yanked for wheel in wheels) + # when the version is yanked, all wheels are yanked, so we can check only the first wheel. + yanked = wheels and wheels[0].yanked if yanked: yanked_versions.append(wheels) continue diff --git a/micropip/wheelinfo.py b/micropip/wheelinfo.py index 11da9dad..808f9dfd 100644 --- a/micropip/wheelinfo.py +++ b/micropip/wheelinfo.py @@ -44,7 +44,9 @@ class WheelInfo: sha256: str | None = None size: int | None = None # Size in bytes, if available (PEP 700) core_metadata: DistributionMetadata = None # Wheel's metadata (PEP 658 / PEP-714) - yanked: bool = False # Whether the wheel has been yanked (PEP-592) + yanked_reason: str | bool = ( + False # Whether the wheel has been yanked and the reason (if given) (PEP-592) + ) # Fields below are only available after downloading the wheel, i.e. after calling `download()`. @@ -62,6 +64,7 @@ def __post_init__(self): ), self.url self._project_name = safe_name(self.name) self.metadata_url = self.url + ".metadata" + self.yanked = bool(self.yanked_reason) @classmethod def from_url(cls, url: str) -> "WheelInfo": @@ -101,7 +104,7 @@ def from_package_index( sha256: str | None, size: int | None, core_metadata: DistributionMetadata = None, - yanked: bool = False, + yanked_reason: str | bool = False, ) -> "WheelInfo": """Extract available metadata from response received from package index""" parsed_url = urlparse(url) @@ -118,7 +121,7 @@ def from_package_index( sha256=sha256, size=size, core_metadata=core_metadata, - yanked=yanked, + yanked_reason=yanked_reason, ) async def install(self, target: Path) -> None: diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index d50a8af8..79d1cb6a 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -52,6 +52,26 @@ async def _run(selenium): _run(selenium_standalone_micropip) +@integration_test_only +def test_integration_install_yanked(selenium_standalone_micropip, pytestconfig): + @run_in_pyodide + async def _run(selenium): + import contextlib + import io + + import micropip + + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + # install yanked version + await micropip.install("black==21.11b0", verbose=True) + + captured = buf.getvalue() + assert "The candidate selected for download or install is a" in captured + assert "'black' candidate (version 21.11b0" in captured + + _run(selenium_standalone_micropip) + + @integration_test_only def test_integration_list_basic(selenium_standalone_micropip, pytestconfig): @run_in_pyodide diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 3344fca2..822dfe0c 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -232,18 +232,23 @@ def test_yanked_version(): metadata.releases[version] = wheels - # yanked version should be skipped and the next best version should be selected + # case 1: yanked version should be skipped and the next best version should be selected + requirement1 = Requirement("dummy_module") wheel = find_wheel(metadata, requirement1) assert str(wheel.version) == "0.9.1" + # case 2: yanked version is explicitly requested, so it should be selected + requirement2 = Requirement("dummy_module==0.15.5") wheel = find_wheel(metadata, requirement2) - # no other compatible version available, so the yanked version should be selected assert str(wheel.version) == "0.15.5" + # case 3: yanked version is not explicitly requested, but it is the only version available + # so it should be selected + requirement3 = Requirement("dummy_module>0.10.0") wheel = find_wheel(metadata, requirement3) From 4230dd588e2bd0b055da5998a8402794c127461d Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 20 Feb 2025 11:33:10 +0000 Subject: [PATCH 6/6] [integration]