Organizing the compatibility layer (#127)

* feat: organize compat directory

* feat: refactor compat not in pyodide

* feat: restructure _compat module exports

* feat: refactor _compat_in_pyodide

* fix: resolve mypy

Author: RulerOfCakes

micropip/_compat.py

@@ -0,0 +1,48 @@
+import sys
+
+from .compatibility_layer import CompatibilityLayer
+
+compatibility_layer: type[CompatibilityLayer] | None = None
+
+IN_BROWSER = "_pyodide_core" in sys.modules
+
+if IN_BROWSER:
+    from ._compat_in_pyodide import CompatibilityInPyodide
+
+    compatibility_layer = CompatibilityInPyodide
+else:
+    from ._compat_not_in_pyodide import CompatibilityNotInPyodide
+
+    compatibility_layer = CompatibilityNotInPyodide
+
+
+REPODATA_INFO = compatibility_layer.repodata_info()
+
+REPODATA_PACKAGES = compatibility_layer.repodata_packages()
+
+fetch_bytes = compatibility_layer.fetch_bytes
+
+fetch_string_and_headers = compatibility_layer.fetch_string_and_headers
+
+loadedPackages = compatibility_layer.loadedPackages
+
+loadDynlibsFromPackage = compatibility_layer.loadDynlibsFromPackage
+
+loadPackage = compatibility_layer.loadPackage
+
+get_dynlibs = compatibility_layer.get_dynlibs
+
+to_js = compatibility_layer.to_js
+
+
+__all__ = [
+    "REPODATA_INFO",
+    "REPODATA_PACKAGES",
+    "fetch_bytes",
+    "fetch_string_and_headers",
+    "loadedPackages",
+    "loadDynlibsFromPackage",
+    "loadPackage",
+    "get_dynlibs",
+    "to_js",
+]

micropip/_compat/__init__.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
+
+if TYPE_CHECKING:
+    pass
+
+from pyodide._package_loader import get_dynlibs
+from pyodide.ffi import IN_BROWSER, to_js
+from pyodide.http import pyfetch
+
+from .compatibility_layer import CompatibilityLayer
+
+try:
+    import pyodide_js
+    from pyodide_js import loadedPackages, loadPackage
+    from pyodide_js._api import (  # type: ignore[import]
+        loadBinaryFile,
+        loadDynlibsFromPackage,
+    )
+
+    REPODATA_PACKAGES = pyodide_js._api.repodata_packages.to_py()
+    REPODATA_INFO = pyodide_js._api.repodata_info.to_py()
+except ImportError:
+    if IN_BROWSER:
+        raise
+    # Otherwise, this is pytest test collection so let it go.
+
+
+class CompatibilityInPyodide(CompatibilityLayer):
+    @staticmethod
+    def repodata_info() -> dict[str, str]:
+        return REPODATA_INFO
+
+    @staticmethod
+    def repodata_packages() -> dict[str, dict[str, Any]]:
+        return REPODATA_PACKAGES
+
+    @staticmethod
+    async def fetch_bytes(url: str, kwargs: dict[str, str]) -> bytes:
+        parsed_url = urlparse(url)
+        if parsed_url.scheme == "emfs":
+            return Path(parsed_url.path).read_bytes()
+        if parsed_url.scheme == "file":
+            return (await loadBinaryFile(parsed_url.path)).to_bytes()
+
+        return await (await pyfetch(url, **kwargs)).bytes()
+
+    @staticmethod
+    async def fetch_string_and_headers(
+        url: str, kwargs: dict[str, str]
+    ) -> tuple[str, dict[str, str]]:
+        response = await pyfetch(url, **kwargs)
+
+        content = await response.string()
+        headers: dict[str, str] = response.headers
+
+        return content, headers
+
+    loadedPackages = loadedPackages
+
+    get_dynlibs = get_dynlibs
+
+    loadDynlibsFromPackage = loadDynlibsFromPackage
+
+    loadPackage = loadPackage
+
+    to_js = to_js

micropip/_compat/_compat_in_pyodide.py

@@ -0,0 +1,72 @@
+import re
+from pathlib import Path
+from typing import IO, TYPE_CHECKING, Any
+from urllib.request import Request, urlopen
+from urllib.response import addinfourl
+
+from .compatibility_layer import CompatibilityLayer
+
+if TYPE_CHECKING:
+    from ..wheelinfo import PackageData
+
+
+class CompatibilityNotInPyodide(CompatibilityLayer):
+
+    # Vendored from packaging
+    _canonicalize_regex = re.compile(r"[-_.]+")
+
+    class loadedPackages(CompatibilityLayer.loadedPackages):
+        @staticmethod
+        def to_py():
+            return {}
+
+    @staticmethod
+    def repodata_info() -> dict[str, str]:
+        return {}
+
+    @staticmethod
+    def repodata_packages() -> dict[str, dict[str, Any]]:
+        return {}
+
+    @staticmethod
+    def _fetch(url: str, kwargs: dict[str, Any]) -> addinfourl:
+        return urlopen(Request(url, **kwargs))
+
+    @staticmethod
+    async def fetch_bytes(url: str, kwargs: dict[str, Any]) -> bytes:
+        return CompatibilityNotInPyodide._fetch(url, kwargs=kwargs).read()
+
+    @staticmethod
+    async def fetch_string_and_headers(
+        url: str, kwargs: dict[str, Any]
+    ) -> tuple[str, dict[str, str]]:
+        response = CompatibilityNotInPyodide._fetch(url, kwargs=kwargs)
+        headers = {k.lower(): v for k, v in response.headers.items()}
+        return response.read().decode(), headers
+
+    @staticmethod
+    def get_dynlibs(archive: IO[bytes], suffix: str, target_dir: Path) -> list[str]:
+        return []
+
+    @staticmethod
+    async def loadDynlibsFromPackage(
+        pkg_metadata: "PackageData", dynlibs: list[str]
+    ) -> None:
+        pass
+
+    @staticmethod
+    async def loadPackage(names: str | list[str]) -> None:
+        pass
+
+    @staticmethod
+    def to_js(
+        obj: Any,
+        /,
+        *,
+        depth: int = -1,
+        pyproxies=None,
+        create_pyproxies: bool = True,
+        dict_converter=None,
+        default_converter=None,
+    ) -> Any:
+        return obj

micropip/_compat/_compat_not_in_pyodide.py

@@ -0,0 +1,73 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import IO, TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from ..wheelinfo import PackageData
+
+
+class CompatibilityLayer(ABC):
+    """
+    CompatibilityLayer represents the interface that must be implemented for each viable environment.
+
+    All of the following methods / properties must be implemented for use both inside and outside of pyodide.
+    """
+
+    class loadedPackages(ABC):
+        @staticmethod
+        @abstractmethod
+        def to_py():
+            pass
+
+    @staticmethod
+    @abstractmethod
+    def repodata_info() -> dict[str, str]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def repodata_packages() -> dict[str, dict[str, Any]]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    async def fetch_bytes(url: str, kwargs: dict[str, str]) -> bytes:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    async def fetch_string_and_headers(
+        url: str, kwargs: dict[str, Any]
+    ) -> tuple[str, dict[str, str]]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_dynlibs(archive: IO[bytes], suffix: str, target_dir: Path) -> list[str]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    async def loadDynlibsFromPackage(
+        pkg_metadata: "PackageData", dynlibs: list[str]
+    ) -> None:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    async def loadPackage(names: str | list[str]) -> None:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def to_js(
+        obj: Any,
+        /,
+        *,
+        depth: int = -1,
+        pyproxies: Any,
+        create_pyproxies: bool = True,
+        dict_converter: Any,
+        default_converter: Any,
+    ) -> Any:
+        pass