Fix micropip.install to search shared libraries inside the wheel correctly

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04]
-        pyodide-version: [0.22.1, 0.23.4, 0.24.0]
+        pyodide-version: [0.24.1, 0.25.0]
         test-config: [
           {runner: selenium, runtime: chrome, runtime-version: latest },
         ]

micropip/_compat.py

@@ -9,7 +9,7 @@
         fetch_bytes,
         fetch_string_and_headers,
         get_dynlibs,
-        loadDynlib,
+        loadDynlibsFromPackage,
         loadedPackages,
         loadPackage,
         to_js,
@@ -21,7 +21,7 @@
         fetch_bytes,
         fetch_string_and_headers,
         get_dynlibs,
-        loadDynlib,
+        loadDynlibsFromPackage,
         loadedPackages,
         loadPackage,
         to_js,
@@ -33,7 +33,7 @@
     "fetch_bytes",
     "fetch_string_and_headers",
     "loadedPackages",
-    "loadDynlib",
+    "loadDynlibsFromPackage",
     "loadPackage",
     "get_dynlibs",
     "to_js",

micropip/_compat_in_pyodide.py

@@ -9,7 +9,10 @@
     import pyodide_js
     from js import Object
     from pyodide_js import loadedPackages, loadPackage
-    from pyodide_js._api import loadBinaryFile, loadDynlib  # type: ignore[import]
+    from pyodide_js._api import (  # type: ignore[import]
+        loadBinaryFile,
+        loadDynlibsFromPackage,
+    )
 
     REPODATA_PACKAGES = pyodide_js._api.repodata_packages.to_py()
     REPODATA_INFO = pyodide_js._api.repodata_info.to_py()
@@ -49,7 +52,7 @@ async def fetch_string_and_headers(
     "REPODATA_INFO",
     "REPODATA_PACKAGES",
     "loadedPackages",
-    "loadDynlib",
+    "loadDynlibsFromPackage",
     "loadPackage",
     "get_dynlibs",
     "to_js",

micropip/_compat_not_in_pyodide.py

@@ -1,6 +1,9 @@
 import re
 from pathlib import Path
-from typing import IO, Any
+from typing import IO, TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .wheelinfo import PackageData
 
 REPODATA_PACKAGES: dict[str, dict[str, Any]] = {}
 
@@ -31,7 +34,9 @@ async def fetch_string_and_headers(
     return response.read().decode(), headers
 
 
-async def loadDynlib(dynlib: str, is_shared_lib: bool) -> None:
+async def loadDynlibsFromPackage(
+    pkg_metadata: "PackageData", dynlibs: list[str]
+) -> None:
     pass
 
 
@@ -74,7 +79,7 @@ def loadPackage(packages: str | list[str]) -> None:
 
 
 __all__ = [
-    "loadDynlib",
+    "loadDynlibsFromPackage",
     "fetch_bytes",
     "fetch_string_and_headers",
     "REPODATA_INFO",

micropip/wheelinfo.py

@@ -1,11 +1,10 @@
-import asyncio
 import hashlib
 import io
 import json
 import zipfile
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 from urllib.parse import ParseResult, urlparse
 
 from packaging.requirements import Requirement
@@ -15,13 +14,20 @@
 from ._compat import (
     fetch_bytes,
     get_dynlibs,
-    loadDynlib,
+    loadDynlibsFromPackage,
     loadedPackages,
 )
 from ._utils import parse_wheel_filename
 from .metadata import Metadata, safe_name, wheel_dist_info_dir
 
 
+@dataclass
+class PackageData:
+    file_name: str
+    package_type: Literal["shared_library", "package"]
+    shared_library: bool
+
+
 @dataclass
 class WheelInfo:
     """
@@ -185,11 +191,17 @@ def _write_dist_info(self, file: str, content: str) -> None:
     async def _load_libraries(self, target: Path) -> None:
         """
         Compiles shared libraries (WASM modules) in the wheel and loads them.
-        TODO: integrate with pyodide's dynamic library loading mechanism.
         """
         assert self._data
+
+        pkg = PackageData(
+            file_name=self.filename,
+            package_type="package",
+            shared_library=False,
+        )
+
         dynlibs = get_dynlibs(io.BytesIO(self._data), ".whl", target)
-        await asyncio.gather(*map(lambda dynlib: loadDynlib(dynlib, False), dynlibs))
+        await loadDynlibsFromPackage(pkg, dynlibs)
 
 
 def _validate_sha256_checksum(data: bytes, expected: str | None = None) -> None:

tests/conftest.py

@@ -127,10 +127,13 @@ def _register_handler(self, path: Path) -> str:
 
         return self._httpserver.url_for(f"/{path.name}")
 
-    def add_wheel(self, path: Path):
+    def add_wheel(self, path: Path, replace: bool = True):
         name = parse_wheel_filename(path.name)[0]
         url = self._register_handler(path)
 
+        if name in self._wheels and not replace:
+            return
+
         self._wheels[name] = self.Wheel(
             path, name, path.name, name.replace("-", "_"), url
         )
@@ -140,12 +143,19 @@ def get(self, name: str) -> Wheel:
 
 
 @pytest.fixture(scope="session")
-def wheel_catalog():
+def wheel_catalog(pytestconfig):
     """Run a mock server that serves pre-built wheels"""
     with WheelCatalog() as catalog:
         for wheel in TEST_WHEEL_DIR.glob("*.whl"):
             catalog.add_wheel(wheel)
 
+        # Add wheels in the pyodide distribution so we can use it in the test.
+        # This is a workaround to get a wheel build with a same emscripten version that the Pyodide is build with,
+        # But probably we should find a better way that does not depend on Pyodide distribution.
+        dist_dir = Path(pytestconfig.getoption("dist_dir"))
+        for wheel in dist_dir.glob("*.whl"):
+            catalog.add_wheel(wheel, replace=False)
+
         yield catalog
 
 

tests/test_install.py

@@ -387,3 +387,30 @@ async def _mock_fetch_bytes(url, *args):
         pass
 
     assert "fake_pkg_micropip_test-1.0.0-py2.py3-none-any.whl" in _wheel_url
+
+
+def test_install_pkg_with_sharedlib_deps(selenium_standalone_micropip, wheel_catalog):
+    """
+    Test if micropip can locate shared libraries in the wheel file correctly.
+    shapely requires libgeos and it is bundled inside the shapely wheel.
+    If micropip does not locate the shared libraries correctly,
+    it will fail with the message "Didn't expect to load any more file_packager files!"
+
+    TODO: maybe build a wheel for test-only purpose instead of relying on a real package?
+    """
+    selenium = selenium_standalone_micropip
+    numpy_wheel = wheel_catalog.get("numpy")
+    shapely_wheel = wheel_catalog.get("shapely")
+
+    @run_in_pyodide
+    async def run(selenium, numpy_url, shapely_url):
+        import micropip
+
+        await micropip.install(numpy_url)
+        await micropip.install(shapely_url)
+
+        from shapely.geometry import Point
+
+        Point(0, 0)
+
+    run(selenium, numpy_wheel.url, shapely_wheel.url)