diff --git a/CHANGELOG.md b/CHANGELOG.md index af5dbe67..152aa3c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.10.1] - 2024/07/05 + +### Fixed + +- `micropip.freeze()` now updates the URLs inside the lockfile to absolute URLs. + This behavior is consistent with the `lockfileURL` behavior change in Pyodide 0.28.0. + [#241](https://github.com/pyodide/micropip/pull/241) + ## [0.10.0] - 2024/07/02 ### Added diff --git a/micropip/_compat/__init__.py b/micropip/_compat/__init__.py index e8843a42..3580f38a 100644 --- a/micropip/_compat/__init__.py +++ b/micropip/_compat/__init__.py @@ -21,6 +21,8 @@ LOCKFILE_PACKAGES = compatibility_layer.lockfile_packages +lockfile_base_url = compatibility_layer.lockfile_base_url + fetch_bytes = compatibility_layer.fetch_bytes fetch_string_and_headers = compatibility_layer.fetch_string_and_headers @@ -40,4 +42,5 @@ "loadedPackages", "loadPackage", "to_js", + "lockfile_base_url", ] diff --git a/micropip/_compat/_compat_in_pyodide.py b/micropip/_compat/_compat_in_pyodide.py index a3e89322..92adaaa4 100644 --- a/micropip/_compat/_compat_in_pyodide.py +++ b/micropip/_compat/_compat_in_pyodide.py @@ -8,7 +8,7 @@ try: import pyodide_js - from pyodide_js import loadedPackages, loadPackage + from pyodide_js import loadedPackages, loadPackage, lockfileBaseUrl from pyodide_js._api import ( # type: ignore[import] install, loadBinaryFile, @@ -58,3 +58,5 @@ async def fetch_string_and_headers( lockfile_info = LOCKFILE_INFO lockfile_packages = LOCKFILE_PACKAGES + + lockfile_base_url = lockfileBaseUrl diff --git a/micropip/_compat/_compat_not_in_pyodide.py b/micropip/_compat/_compat_not_in_pyodide.py index df067e0c..efc94bd8 100644 --- a/micropip/_compat/_compat_not_in_pyodide.py +++ b/micropip/_compat/_compat_not_in_pyodide.py @@ -84,3 +84,5 @@ def to_js( lockfile_info = {} lockfile_packages = {} + + lockfile_base_url = None diff --git a/micropip/_compat/compatibility_layer.py b/micropip/_compat/compatibility_layer.py index cee77521..c7e4946b 100644 --- a/micropip/_compat/compatibility_layer.py +++ b/micropip/_compat/compatibility_layer.py @@ -19,6 +19,8 @@ def to_py(): lockfile_packages: dict[str, dict[str, Any]] + lockfile_base_url: str | None = None + @staticmethod @abstractmethod async def fetch_bytes(url: str, kwargs: dict[str, str]) -> bytes: diff --git a/micropip/freeze.py b/micropip/freeze.py index cec13e8e..a0d0cbf1 100644 --- a/micropip/freeze.py +++ b/micropip/freeze.py @@ -4,6 +4,7 @@ from copy import deepcopy from importlib.metadata import Distribution from typing import Any +from urllib.parse import urljoin from ._utils import get_dist_info from ._vendored.packaging.src.packaging.requirements import Requirement @@ -11,16 +12,23 @@ def freeze_lockfile( - lockfile_packages: dict[str, dict[str, Any]], lockfile_info: dict[str, str] + lockfile_packages: dict[str, dict[str, Any]], + lockfile_info: dict[str, str], + lockfile_base_url: str | None = None, ) -> str: - return json.dumps(freeze_data(lockfile_packages, lockfile_info)) + return json.dumps(freeze_data(lockfile_packages, lockfile_info, lockfile_base_url)) def freeze_data( - lockfile_packages: dict[str, dict[str, Any]], lockfile_info: dict[str, str] + lockfile_packages: dict[str, dict[str, Any]], + lockfile_info: dict[str, str], + lockfile_base_url: str | None = None, ) -> dict[str, Any]: packages = deepcopy(lockfile_packages) packages.update(load_pip_packages(lockfile_packages)) + if lockfile_base_url is not None: + # Override the base URL for the packages + override_base_url(packages, lockfile_base_url) # Sort packages = dict(sorted(packages.items())) @@ -30,6 +38,19 @@ def freeze_data( } +def override_base_url( + lockfile_packages: dict[str, dict[str, Any]], + lockfile_base_url: str, +): + """ + Updates the relative URLs in the lockfile packages to absolute URLs by appending the base URL. + This assures that when the generated lockfile is deployed separately from the packages, + the URLs will still point to the correct location. + """ + for pkg in lockfile_packages.values(): + pkg["file_name"] = urljoin(lockfile_base_url, pkg["file_name"]) + + def load_pip_packages( lockfile_packages: dict[str, dict[str, Any]], ) -> Iterator[tuple[str, dict[str, Any]]]: diff --git a/micropip/package_manager.py b/micropip/package_manager.py index 208e81cf..b75dbca6 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -320,7 +320,9 @@ def freeze(self) -> str: ``lockFileURL`` of :js:func:`~globalThis.loadPyodide`. """ return freeze_lockfile( - self.compat_layer.lockfile_packages, self.compat_layer.lockfile_info + self.compat_layer.lockfile_packages, + self.compat_layer.lockfile_info, + self.compat_layer.lockfile_base_url, ) def add_mock_package( diff --git a/tests/test_freeze.py b/tests/test_freeze.py index d6fe0deb..3a186558 100644 --- a/tests/test_freeze.py +++ b/tests/test_freeze.py @@ -1,5 +1,6 @@ import pytest from conftest import mock_fetch_cls +from pytest_pyodide import run_in_pyodide @pytest.mark.asyncio @@ -102,3 +103,73 @@ def test_freeze_lockfile_compat( assert package.install_dir == "site" assert not package.unvendored_tests assert package.version == wheel.version + + +def test_override_base_url(): + from micropip.freeze import override_base_url + + lockfile_packages = { + "pkg1": {"file_name": "pkg1-1.0.0-py3-none-any.whl"}, + "pkg2": {"file_name": "pkg2-2.0.0-py3-none-any.whl"}, + "pkg3": {"file_name": "https://other.com/pkg3-3.0.0-py3-none-any.whl"}, + } + base_url = "https://example.com/packages/" + + override_base_url(lockfile_packages, base_url) + + assert ( + lockfile_packages["pkg1"]["file_name"] + == "https://example.com/packages/pkg1-1.0.0-py3-none-any.whl" + ) + assert ( + lockfile_packages["pkg2"]["file_name"] + == "https://example.com/packages/pkg2-2.0.0-py3-none-any.whl" + ) + assert ( + lockfile_packages["pkg3"]["file_name"] + == "https://other.com/pkg3-3.0.0-py3-none-any.whl" + ) + + +def test_url_after_freeze_pyodide(selenium_standalone_micropip): + + @run_in_pyodide + def _run(selenium, prefix): + import json + + from pyodide_js import lockfileBaseUrl + from pyodide_js._api import lockfile_packages + + import micropip + + new_lockfile_str = micropip.freeze() + new_lockfile_packages = json.loads(new_lockfile_str)["packages"] + + orig_lockfile_packages = lockfile_packages.to_py() + + for orig_pkg_name, orig_pkg in orig_lockfile_packages.items(): + assert orig_pkg_name in new_lockfile_packages + + new_pkg = new_lockfile_packages[orig_pkg_name] + + assert new_pkg["name"] == orig_pkg["name"] + assert new_pkg["version"] == orig_pkg["version"] + assert new_pkg["sha256"] == orig_pkg["sha256"] + assert new_pkg["imports"] == orig_pkg["imports"] + assert new_pkg["depends"] == orig_pkg["depends"] + assert new_pkg["install_dir"] == orig_pkg["install_dir"] + assert new_pkg["unvendored_tests"] == orig_pkg["unvendored_tests"] + + # original lockfile will have relative URLs + # TODO: this might change later if packages are served from PyPI + assert not orig_pkg["file_name"].startswith(prefix) + + # new lockfile should have absolute URLs + assert new_pkg["file_name"].startswith(prefix) + assert new_pkg["file_name"].startswith(lockfileBaseUrl) + + assert orig_pkg["file_name"] in new_pkg["file_name"] + + selenium = selenium_standalone_micropip + prefix = ("/",) if selenium.browser == "node" else ("http://", "https://") + _run(selenium, prefix)