MAINT Remove remaining codes that import compat functions directly (#263)

Author: ryanking13

micropip/package_manager.py

@@ -11,7 +11,7 @@
 )
 
 from . import _mock_package, package_index
-from ._compat import CompatibilityLayer, compatibility_layer
+from ._compat import CompatibilityLayer
 from ._utils import get_files_in_distribution, get_root
 from ._vendored.packaging.src.packaging.markers import default_environment
 from .constants import FAQ_URLS
@@ -29,10 +29,7 @@ class PackageManager:
     independent of other instances.
     """
 
-    def __init__(self, compat: type[CompatibilityLayer] | None = None) -> None:
-
-        if compat is None:
-            compat = compatibility_layer
+    def __init__(self, compat: type[CompatibilityLayer]) -> None:
 
         self.index_urls = package_index.DEFAULT_INDEX_URLS[:]
         self.compat_layer: type[CompatibilityLayer] = compat
@@ -239,7 +236,9 @@ async def install(
             # Install PyPI packages
             # detect whether the wheel metadata is from PyPI or from custom location
             # wheel metadata from PyPI has SHA256 checksum digest.
-            await asyncio.gather(*(wheel.install(wheel_base) for wheel in wheels))
+            await asyncio.gather(
+                *(wheel.install(wheel_base, self.compat_layer) for wheel in wheels)
+            )
 
             # Install built-in packages
             if pyodide_packages:

micropip/transaction.py

@@ -332,13 +332,17 @@ async def add_wheel(
         logger.info("Collecting %s%s", wheel.name, specifier)
         logger.info("  Downloading %s", wheel.url.split("/")[-1])
 
-        wheel_download_task = asyncio.create_task(wheel.download(self.fetch_kwargs))
+        wheel_download_task = asyncio.create_task(
+            wheel.download(self.fetch_kwargs, self._compat_layer)
+        )
         if self.deps:
             # Case 1) If metadata file is available,
             #         we can gather requirements without waiting for the wheel to be downloaded.
             if wheel.pep658_metadata_available():
                 try:
-                    await wheel.download_pep658_metadata(self.fetch_kwargs)
+                    await wheel.download_pep658_metadata(
+                        self.fetch_kwargs, self._compat_layer
+                    )
                 except OSError:
                     # If something goes wrong while downloading the metadata,
                     # we have to wait for the wheel to be downloaded.

micropip/wheelinfo.py

@@ -7,12 +7,7 @@
 from typing import Any, Literal
 from urllib.parse import ParseResult, unquote, urlparse
 
-from ._compat import (
-    fetch_bytes,
-    install,
-    loadedPackages,
-    to_js,
-)
+from ._compat import CompatibilityLayer
 from ._utils import best_compatible_tag_index, parse_wheel_filename
 from ._vendored.packaging.src.packaging.requirements import Requirement
 from ._vendored.packaging.src.packaging.tags import Tag
@@ -132,7 +127,9 @@ def from_package_index(
             _best_tag_index=best_tag_index,
         )
 
-    async def install(self, target: Path) -> None:
+    async def install(
+        self, target: Path, compat_layer: type[CompatibilityLayer]
+    ) -> None:
         """
         Install the wheel to the target directory.
 
@@ -148,13 +145,15 @@ async def install(self, target: Path) -> None:
                 "Micropip internal error: attempted to install wheel before downloading it?"
             )
         _validate_sha256_checksum(self._data, self.sha256)
-        await self._install(target)
+        await self._install(target, compat_layer)
 
-    async def download(self, fetch_kwargs: dict[str, Any]):
+    async def download(
+        self, fetch_kwargs: dict[str, Any], compat_layer: type[CompatibilityLayer]
+    ):
         if self._data is not None:
             return
 
-        self._data = await self._fetch_bytes(self.url, fetch_kwargs)
+        self._data = await self._fetch_bytes(self.url, fetch_kwargs, compat_layer)
 
         # The wheel's metadata might be downloaded separately from the wheel itself.
         # If it is not downloaded yet or if the metadata is not available, extract it from the wheel.
@@ -174,14 +173,15 @@ def pep658_metadata_available(self) -> bool:
     async def download_pep658_metadata(
         self,
         fetch_kwargs: dict[str, Any],
+        compat_layer: type[CompatibilityLayer],
     ) -> None:
         """
         Download the wheel's metadata. If the metadata is not available, return None.
         """
         if self.core_metadata is None:
             return None
 
-        data = await self._fetch_bytes(self.metadata_url, fetch_kwargs)
+        data = await self._fetch_bytes(self.metadata_url, fetch_kwargs, compat_layer)
 
         match self.core_metadata:
             case {"sha256": checksum}:  # sha256 checksum available
@@ -204,14 +204,19 @@ def requires(self, extras: set[str]) -> list[Requirement]:
         self._requires = requires
         return requires
 
-    async def _fetch_bytes(self, url: str, fetch_kwargs: dict[str, Any]):
+    async def _fetch_bytes(
+        self,
+        url: str,
+        fetch_kwargs: dict[str, Any],
+        compat_layer: type[CompatibilityLayer],
+    ):
         if self.parsed_url.scheme not in ("https", "http", "emfs", "file"):
             # Don't raise ValueError it gets swallowed
             raise TypeError(
                 f"Cannot download from a non-remote location: {url!r} ({self.parsed_url!r})"
             )
         try:
-            bytes = await fetch_bytes(url, fetch_kwargs)
+            bytes = await compat_layer.fetch_bytes(url, fetch_kwargs)
             return bytes
         except OSError as e:
             if self.parsed_url.hostname in [
@@ -228,7 +233,9 @@ async def _fetch_bytes(self, url: str, fetch_kwargs: dict[str, Any]):
                 ) from e
             raise e
 
-    async def _install(self, target: Path) -> None:
+    async def _install(
+        self, target: Path, compat_layer: type[CompatibilityLayer]
+    ) -> None:
         """
         Install the wheel to the target directory.
         """
@@ -247,15 +254,15 @@ async def _install(self, target: Path) -> None:
                 sorted(x.name for x in self._requires)
             )
 
-        await install(
+        await compat_layer.install(
             # TODO: Probably update install API to accept bytes directly, instead of converting it to JS.
-            to_js(self._data),
+            compat_layer.to_js(self._data),
             self.filename,
             str(target),
             metadata,
         )
 
-        setattr(loadedPackages, self._project_name, wheel_source)
+        setattr(compat_layer.loadedPackages, self._project_name, wheel_source)
 
 
 def _validate_sha256_checksum(data: bytes, expected: str | None = None) -> None:

tests/conftest.py

@@ -240,7 +240,8 @@ def __eq__(self, other):
 
 
 class mock_fetch_cls:
-    def __init__(self):
+    def __init__(self, compat_layer=None):
+        self._compat_layer = compat_layer
         self.releases_map = {}
         self.metadata_map = {}
         self.top_level_map = {}
@@ -351,12 +352,12 @@ def write_file(filename, contents):
 
 
 @pytest.fixture
-def mock_fetch(monkeypatch, mock_importlib):
-    from micropip import package_index, wheelinfo
+def mock_fetch(monkeypatch, mock_importlib, host_compat_layer):
+    from micropip import package_index
 
-    result = mock_fetch_cls()
+    result = mock_fetch_cls(host_compat_layer)
     monkeypatch.setattr(package_index, "query_package", result.query_package)
-    monkeypatch.setattr(wheelinfo, "fetch_bytes", result._fetch_bytes)
+    monkeypatch.setattr(host_compat_layer, "fetch_bytes", result._fetch_bytes)
     return result
 
 
@@ -469,3 +470,15 @@ def host_compat_layer():
     from micropip._compat._compat_not_in_pyodide import CompatibilityNotInPyodide
 
     yield CompatibilityNotInPyodide
+
+
+@pytest.fixture
+def host_package_manager(host_compat_layer):
+    """
+    Fixture to provide a package manager for the host environment.
+    """
+    from micropip.package_manager import PackageManager
+
+    package_manager = PackageManager(compat=host_compat_layer)
+
+    yield package_manager

tests/test_install.py

@@ -264,14 +264,13 @@ async def test_install_pre(
 
 
 @pytest.mark.asyncio
-async def test_fetch_wheel_fail(monkeypatch, wheel_base):
+async def test_fetch_wheel_fail(monkeypatch, wheel_base, host_compat_layer):
     import micropip
-    from micropip import wheelinfo
 
     def _mock_fetch_bytes(arg, *args, **kwargs):
         raise OSError(f"Request for {arg} failed with status 404: Not Found")
 
-    monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes)
+    monkeypatch.setattr(host_compat_layer, "fetch_bytes", _mock_fetch_bytes)
 
     msg = "Access-Control-Allow-Origin"
     with pytest.raises(ValueError, match=msg):
@@ -395,7 +394,9 @@ async def run_test(selenium, url, name, version):
 
 
 @pytest.mark.asyncio
-async def test_custom_index_urls(mock_package_index_json_api, monkeypatch):
+async def test_custom_index_urls(
+    mock_package_index_json_api, monkeypatch, host_compat_layer
+):
     mock_server_fake_package = mock_package_index_json_api(
         pkgs=["fake-pkg-micropip-test"]
     )
@@ -407,9 +408,7 @@ async def _mock_fetch_bytes(url, *args):
         _wheel_url = url
         return b"fake wheel"
 
-    from micropip import wheelinfo
-
-    monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes)
+    monkeypatch.setattr(host_compat_layer, "fetch_bytes", _mock_fetch_bytes)
 
     try:
         await micropip.install(

tests/test_package_manager.py

@@ -2,19 +2,10 @@
 from conftest import mock_fetch_cls
 
 import micropip.package_index as package_index
-from micropip.package_manager import PackageManager
 
 
-def get_test_package_manager() -> PackageManager:
-    package_manager = PackageManager()
-
-    # TODO: inject necessary constructor parameters
-
-    return package_manager
-
-
-def test_set_index_urls():
-    manager = get_test_package_manager()
+def test_set_index_urls(host_package_manager):
+    manager = host_package_manager
 
     default_index_urls = package_index.DEFAULT_INDEX_URLS
     assert manager.index_urls == default_index_urls
@@ -34,8 +25,8 @@ def test_set_index_urls():
 
 
 @pytest.mark.asyncio
-async def test_list_packages(mock_fetch: mock_fetch_cls):
-    manager = get_test_package_manager()
+async def test_list_packages(mock_fetch: mock_fetch_cls, host_package_manager):
+    manager = host_package_manager
 
     dummy = "dummy"
     mock_fetch.add_pkg_version(dummy)
@@ -50,8 +41,10 @@ async def test_list_packages(mock_fetch: mock_fetch_cls):
 
 
 @pytest.mark.asyncio
-async def test_custom_index_url(mock_package_index_json_api, monkeypatch):
-    manager = get_test_package_manager()
+async def test_custom_index_url(
+    mock_package_index_json_api, monkeypatch, host_compat_layer, host_package_manager
+):
+    manager = host_package_manager
 
     mock_server_fake_package = mock_package_index_json_api(
         pkgs=["fake-pkg-micropip-test"]
@@ -64,9 +57,7 @@ async def _mock_fetch_bytes(url, *args):
         _wheel_url = url
         return b"fake wheel"
 
-    from micropip import wheelinfo
-
-    monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes)
+    monkeypatch.setattr(host_compat_layer, "fetch_bytes", _mock_fetch_bytes)
 
     manager.set_index_urls([mock_server_fake_package])
 

tests/test_wheelinfo.py

@@ -51,24 +51,24 @@ def test_from_package_index():
 
 
 @pytest.mark.asyncio
-async def test_download(wheel_catalog):
+async def test_download(wheel_catalog, host_compat_layer):
     pytest_wheel = wheel_catalog.get("pytest")
     wheel = WheelInfo.from_url(pytest_wheel.url)
 
     assert wheel._metadata is None
 
-    await wheel.download({})
+    await wheel.download({}, host_compat_layer)
 
     assert wheel._metadata is not None
 
 
 @pytest.mark.asyncio
-async def test_requires(wheel_catalog, tmp_path):
+async def test_requires(wheel_catalog, tmp_path, host_compat_layer):
     pytest_wheel = wheel_catalog.get("pytest")
     wheel = WheelInfo.from_url(pytest_wheel.url)
-    await wheel.download({})
+    await wheel.download({}, host_compat_layer)
 
-    wheel._install(tmp_path)
+    wheel._install(tmp_path, host_compat_layer)
 
     requirements_default = [str(r.name) for r in wheel.requires(set())]
     assert "pluggy" in requirements_default
@@ -80,7 +80,7 @@ async def test_requires(wheel_catalog, tmp_path):
 
 
 @pytest.mark.asyncio
-async def test_download_pep658_metadata(wheel_catalog):
+async def test_download_pep658_metadata(wheel_catalog, host_compat_layer):
     pytest_wheel = wheel_catalog.get("pytest")
     sha256 = "dummy-sha256"
     size = 1234
@@ -98,7 +98,7 @@ async def test_download_pep658_metadata(wheel_catalog):
 
     assert wheel_with_metadata.pep658_metadata_available()
     assert wheel_with_metadata._metadata is None
-    await wheel_with_metadata.download_pep658_metadata({})
+    await wheel_with_metadata.download_pep658_metadata({}, host_compat_layer)
     assert wheel_with_metadata._metadata is not None
 
     # metadata should be calculated from the metadata file
@@ -119,7 +119,7 @@ async def test_download_pep658_metadata(wheel_catalog):
 
     assert not wheel_without_metadata.pep658_metadata_available()
     assert wheel_without_metadata._metadata is None
-    await wheel_without_metadata.download_pep658_metadata({})
+    await wheel_without_metadata.download_pep658_metadata({}, host_compat_layer)
     assert wheel_without_metadata._metadata is None
 
     # 3) the metadata extracted from the wheel should be the same
@@ -134,14 +134,14 @@ async def test_download_pep658_metadata(wheel_catalog):
     )
 
     assert wheel._metadata is None
-    await wheel.download({})
+    await wheel.download({}, host_compat_layer)
     assert wheel._metadata is not None
 
     assert wheel._metadata.deps == wheel_with_metadata._metadata.deps
 
 
 @pytest.mark.asyncio
-async def test_download_pep658_metadata_checksum(wheel_catalog):
+async def test_download_pep658_metadata_checksum(wheel_catalog, host_compat_layer):
     pytest_wheel = wheel_catalog.get("pytest")
     sha256 = "dummy-sha256"
     size = 1234
@@ -158,7 +158,7 @@ async def test_download_pep658_metadata_checksum(wheel_catalog):
 
     assert wheel._metadata is None
     with pytest.raises(RuntimeError, match="Invalid checksum: expected dummy-sha256"):
-        await wheel.download_pep658_metadata({})
+        await wheel.download_pep658_metadata({}, host_compat_layer)
 
     checksum = "62eb95408ccec185e7a3b8f354a1df1721cd8f463922f5a900c7bf4b69c5a4e8"  # TODO: calculate this from the file
     wheel = WheelInfo.from_package_index(
@@ -172,5 +172,5 @@ async def test_download_pep658_metadata_checksum(wheel_catalog):
     )
 
     assert wheel._metadata is None
-    await wheel.download_pep658_metadata({})
+    await wheel.download_pep658_metadata({}, host_compat_layer)
     assert wheel._metadata is not None