From b00d2beb17830e8fdad70a18683f152e259de77f Mon Sep 17 00:00:00 2001 From: Alexander Kozhevnikov Date: Sat, 10 Dec 2022 02:00:29 +0000 Subject: [PATCH] Fixed wheel choice by build tag --- CHANGELOG.md | 9 +++ micropip/_micropip.py | 34 +++++++++-- tests/test_micropip.py | 125 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 674e768d..e13e93ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for adding mock packages, for use where something is a dependency and you don't need it, or you need only a limited subset of the package. This is done using `micropip.add_mock_package`, `micropip.remove_mock_package` and `micropip.list_mock_packages`. Packages installed like this will be skipped by dependency resolution when you later install real packages. +### Fixed + +- When multiple compatible builds for a package exist, the best + build is now installed, as determined by the order of tags in + [`packaging.tags.sys_tags`](https://packaging.pypa.io/en/latest/tags.html#packaging.tags.sys_tags). + For example, if a package has two pure Python wheels, one tagged `py30` and + another tagged `py35`, the `py35` wheel will now always get installed. + [#34](https://github.com/pyodide/micropip/pull/34) + ## [0.1.0] - 2022/09/18 Initial standalone release. For earlier release notes, see diff --git a/micropip/_micropip.py b/micropip/_micropip.py index e3fe999f..706facf1 100644 --- a/micropip/_micropip.py +++ b/micropip/_micropip.py @@ -89,13 +89,27 @@ def from_url(url: str) -> "WheelInfo": parsed_url=parsed_url, ) + def best_compatible_tag_index(self) -> int | None: + """Get the index of the first tag in ``packaging.tags.sys_tags()`` that this wheel has. + + Since ``packaging.tags.sys_tags()`` is sorted from most specific ("best") to most + general ("worst") compatibility, this index douples as a priority rank: given two + compatible wheels, the one whose best index is closer to zero should be installed. + + Returns + ------- + ``int | None`` + The index, or ``None`` if this wheel has no compatible tags. + """ + for index, tag in enumerate(sys_tags()): + if tag in self.tags: + return index + return None + def is_compatible(self): if self.filename.endswith("py3-none-any.whl"): return True - for tag in sys_tags(): - if tag in self.tags: - return True - return False + return self.best_compatible_tag_index() is not None def check_compatible(self) -> None: if self.is_compatible(): @@ -268,15 +282,23 @@ def find_wheel(metadata: dict[str, Any], req: Requirement) -> WheelInfo: ) continue + best_wheel = None + best_tag_index = float("infinity") + release = releases[str(ver)] for fileinfo in release: url = fileinfo["url"] if not url.endswith(".whl"): continue wheel = WheelInfo.from_url(url) - if wheel.is_compatible(): + tag_index = wheel.best_compatible_tag_index() + if tag_index is not None and tag_index < best_tag_index: wheel.digests = fileinfo["digests"] - return wheel + best_wheel = wheel + best_tag_index = tag_index + + if best_wheel is not None: + return wheel raise ValueError( f"Can't find a pure Python 3 wheel for '{req}'.\n" diff --git a/tests/test_micropip.py b/tests/test_micropip.py index 1dba9450..0414ac90 100644 --- a/tests/test_micropip.py +++ b/tests/test_micropip.py @@ -481,6 +481,30 @@ async def test_package_with_extra_transitive( assert "depb" not in pkg_list +def _pypi_metadata(package, versions_to_tags): + # Build package release metadata as would be returned from + # https://pypi.org/pypi/{pkgname}/json + # + # `package` is a string containing the package name as + # it would appear in a wheel file name. + # + # `versions` is a mapping with version strings as + # keys and iterables of tag strings as values. + releases = {} + for version, tags in versions_to_tags.items(): + release = [] + for tag in tags: + wheel_name = f"{package}-{version}-{tag}-none-any.whl" + wheel_info = { + "filename": wheel_name, + "url": wheel_name, + "digests": None, + } + release.append(wheel_info) + releases[version] = release + return {"releases": releases} + + def test_last_version_from_pypi(): pytest.importorskip("packaging") from packaging.requirements import Requirement @@ -490,14 +514,7 @@ def test_last_version_from_pypi(): requirement = Requirement("dummy_module") versions = ["0.0.1", "0.15.5", "0.9.1"] - # building metadata as returned from - # https://pypi.org/pypi/{pkgname}/json - releases = {} - for v in versions: - filename = f"dummy_module-{v}-py3-none-any.whl" - releases[v] = [{"filename": filename, "url": filename, "digests": None}] - - metadata = {"releases": releases} + metadata = _pypi_metadata("dummy_module", {v: ["py3"] for v in versions}) # get version number from find_wheel wheel = find_wheel(metadata, requirement) @@ -505,6 +522,78 @@ def test_last_version_from_pypi(): assert str(wheel.version) == "0.15.5" +_best_tag_test_cases = ( + "package, version, incompatible_tags, compatible_tags", + # Tests assume that `compatible_tags` is sorted from least to most compatible: + [ + # Common modern case (pure Python 3-only wheel): + ("hypothesis", "6.60.0", [], ["py3"]), + # Common historical case (pure Python 2-or-3 wheel): + ("attrs", "22.1.0", [], ["py2.py3"]), + # Still simple, less common (separate Python 2 and 3 wheels): + ("raise", "1.1.9", ["py2"], ["py3"]), + # More complicated, rarer cases: + ("compose", "1.4.8", [], ["py2.py30", "py35", "py38"]), + ("with_as_a_function", "1.0.1", ["py20", "py25"], ["py26.py3"]), + ("with_as_a_function", "1.1.0", ["py22", "py25"], ["py26.py30", "py33"]), + ], +) + + +@pytest.mark.parametrize(*_best_tag_test_cases) +def test_best_tag_from_pypi(package, version, incompatible_tags, compatible_tags): + pytest.importorskip("packaging") + from packaging.requirements import Requirement + + from micropip._micropip import find_wheel + + requirement = Requirement(package) + tags = incompatible_tags + compatible_tags + + metadata = _pypi_metadata(package, {version: tags}) + + wheel = find_wheel(metadata, requirement) + + best_tag = tags[-1].split(".")[-1] + "-none-any" + assert best_tag in set(map(str, wheel.tags)) + + +# A newer version with a compatible wheel has higher precedence +# than an older version with a more precisely compatible wheel. +# This test verifies that we didn't break that corner case: +@pytest.mark.parametrize( + "package, old_version, old_tags, new_version, new_tags", + [ + ("compose", "1.1.1", ["py2.py3"], "1.2.0", ["py2.py30", "py35", "py38"]), + ( + "with_as_a_function", + "1.0.1", + ["py20", "py25", "py26.py3"], + "1.1.0", + ["py22", "py25", "py26.py30", "py33"], + ), + ], +) +def test_last_version_and_best_tag_from_pypi( + package, old_version, new_version, old_tags, new_tags +): + pytest.importorskip("packaging") + from packaging.requirements import Requirement + + from micropip._micropip import find_wheel + + requirement = Requirement(package) + + metadata = _pypi_metadata( + package, + {old_version: old_tags, new_version: new_tags}, + ) + + wheel = find_wheel(metadata, requirement) + + assert str(wheel.version) == new_version + + @pytest.mark.asyncio async def test_install_non_pure_python_wheel(): pytest.importorskip("packaging") @@ -936,6 +1025,26 @@ def test_check_compatible(mock_platform, interp, abi, arch, ctx): WheelInfo.from_url(wheel_name).check_compatible() +@pytest.mark.parametrize(*_best_tag_test_cases) +def test_best_compatible_tag(package, version, incompatible_tags, compatible_tags): + from micropip._micropip import WheelInfo + + for tag in incompatible_tags: + wheel_name = f"{package}-{version}-{tag}-none-any.whl" + wheel = WheelInfo.from_url(wheel_name) + assert wheel.best_compatible_tag_index() is None + + wheels = [] + for tag in compatible_tags: + wheel_name = f"{package}-{version}-{tag}-none-any.whl" + wheel = WheelInfo.from_url(wheel_name) + wheels.append(wheel) + + sorted_wheels = sorted(wheels, key=WheelInfo.best_compatible_tag_index) + sorted_wheels.reverse() + assert sorted_wheels == wheels + + @run_in_pyodide() def test_persistent_mock_pyodide(selenium_standalone_micropip): # import sys